diff --git a/.github/android-consumer-test/README.md b/.github/android-consumer-test/README.md new file mode 100644 index 00000000..91e2885a --- /dev/null +++ b/.github/android-consumer-test/README.md @@ -0,0 +1,26 @@ + +# llama-android consumer-test fixture (CI only) + +Minimal AGP app that consumes the `net.ladenthin:llama-android` AAR from `mavenLocal`. +Not a shipped project — it exists so CI proves Android Studio consumption end to end. + +Two CI jobs drive it (`.github/workflows/publish.yml`): + +1. **`package-android-aar`** — `assembleRelease` with R8: validates AAR format, manifest + minSdk merge, `jni/{arm64-v8a,x86_64}` packaging, and the shipped consumer ProGuard + rules (asserts the APK still carries the binding and both `.so` files). +2. **`test-android-emulator`** — boots a KVM-accelerated **x86_64** emulator, adb-pushes a + small GGUF (the already-cached draft model) to + `/data/local/tmp/jllama-test-model.gguf`, and runs `connectedDebugAndroidTest`: + `OnDeviceInferenceTest` loads the binding via `System.loadLibrary` from the AAR's + `jni/x86_64/libjllama.so`, reads the GGUF with the pure-Java `GgufInspector`, and runs + real native inference on the emulator. Tests self-skip when the model was not pushed, + so a local `connectedAndroidTest` against a bare emulator stays green. + +What the emulator job deliberately does NOT cover: arm64 kernels (real-device ABI — the +example app on hardware covers that) and the Adreno/OpenCL flavor (no OpenCL ICD in the +emulator). diff --git a/.github/android-consumer-test/app/build.gradle.kts b/.github/android-consumer-test/app/build.gradle.kts new file mode 100644 index 00000000..e058a978 --- /dev/null +++ b/.github/android-consumer-test/app/build.gradle.kts @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +plugins { + id("com.android.application") +} + +// Injected by CI (-PjllamaVersion=) so the fixture always tests +// the AAR that was just built and published to mavenLocal. +val jllamaVersion: String = providers.gradleProperty("jllamaVersion").get() + +android { + namespace = "net.ladenthin.llama.consumertest" + compileSdk = 35 + + defaultConfig { + applicationId = "net.ladenthin.llama.consumertest" + minSdk = 28 + targetSdk = 35 + versionCode = 1 + versionName = "1.0" + // arm64 = real devices; x86_64 = the CI emulator (and x86_64 Android hardware). + ndk { + abiFilters += listOf("arm64-v8a", "x86_64") + } + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + // Full R8 pass so the AAR's consumer proguard.txt is actually exercised: + // if the shipped rules were broken, the binding would be stripped or the + // build would fail here. + isMinifyEnabled = true + signingConfig = signingConfigs.getByName("debug") + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } +} + +dependencies { + implementation("net.ladenthin:llama-android:$jllamaVersion") + + // On-emulator instrumentation (test-android-emulator CI job). + androidTestImplementation("androidx.test.ext:junit:1.2.1") + androidTestImplementation("androidx.test:runner:1.6.2") +} diff --git a/.github/android-consumer-test/app/src/androidTest/java/net/ladenthin/llama/consumertest/OnDeviceInferenceTest.java b/.github/android-consumer-test/app/src/androidTest/java/net/ladenthin/llama/consumertest/OnDeviceInferenceTest.java new file mode 100644 index 00000000..55230e94 --- /dev/null +++ b/.github/android-consumer-test/app/src/androidTest/java/net/ladenthin/llama/consumertest/OnDeviceInferenceTest.java @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.consumertest; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import net.ladenthin.llama.GgufInspector; +import net.ladenthin.llama.LlamaModel; +import net.ladenthin.llama.parameters.InferenceParameters; +import net.ladenthin.llama.parameters.ModelParameters; +import net.ladenthin.llama.value.GgufMetadata; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * On-emulator smoke of the shipped AAR: the binding must load via + * {@code System.loadLibrary("jllama")} from the APK's native-lib dir (the AAR's + * {@code jni/x86_64/} on the CI emulator), and both the pure-Java GGUF inspector and real + * native inference must work on Android/bionic. The model file is adb-pushed by the + * {@code test-android-emulator} CI job; every test self-skips when it is absent so the + * fixture stays green on a bare emulator. + */ +@RunWith(AndroidJUnit4.class) +public class OnDeviceInferenceTest { + + /** Where the CI job adb-pushes the test GGUF (world-readable in /data/local/tmp). */ + private static final String MODEL_PATH = "/data/local/tmp/jllama-test-model.gguf"; + + private static void assumeModelPresent() { + assumeTrue("test model not pushed to " + MODEL_PATH, new File(MODEL_PATH).canRead()); + } + + @Test + public void ggufInspectorReadsTheModelOnDevice() throws IOException { + assumeModelPresent(); + + GgufMetadata meta = GgufInspector.read(Paths.get(MODEL_PATH)); + + assertTrue("tensor count must be positive, was " + meta.getTensorCount(), meta.getTensorCount() > 0); + assertTrue("architecture key must be present: " + meta, meta.getArchitecture().isPresent()); + } + + @Test + public void nativeInferenceGeneratesTokensOnDevice() { + assumeModelPresent(); + + // Loading LlamaModel exercises System.loadLibrary("jllama") + JNI_OnLoad's + // FindClass resolution against the D8-dexed classes — the load-time surface + // the build-only CI jobs cannot reach. + try (LlamaModel model = new LlamaModel( + new ModelParameters().setModel(MODEL_PATH).setCtxSize(512).setGpuLayers(0))) { + String generated = model.complete(new InferenceParameters("Hello").withNPredict(8)); + + assertFalse("expected generated tokens, got empty output", generated.isEmpty()); + } + } +} diff --git a/.github/android-consumer-test/app/src/main/AndroidManifest.xml b/.github/android-consumer-test/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000..04cf4db8 --- /dev/null +++ b/.github/android-consumer-test/app/src/main/AndroidManifest.xml @@ -0,0 +1,9 @@ + + + + + diff --git a/.github/android-consumer-test/app/src/main/java/net/ladenthin/llama/consumertest/ConsumerSmoke.java b/.github/android-consumer-test/app/src/main/java/net/ladenthin/llama/consumertest/ConsumerSmoke.java new file mode 100644 index 00000000..ce8e9e77 --- /dev/null +++ b/.github/android-consumer-test/app/src/main/java/net/ladenthin/llama/consumertest/ConsumerSmoke.java @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.consumertest; + +import net.ladenthin.llama.LlamaModel; +import net.ladenthin.llama.parameters.InferenceParameters; +import net.ladenthin.llama.parameters.ModelParameters; + +/** + * Compile-time consumer check: references the binding's API surface exactly the way an app + * would, so the AGP build fails if the AAR's classes.jar is broken or its POM dependencies + * (Jackson etc.) do not resolve. No code here runs in CI — the fixture is never installed on a + * device; on-device inference is covered by the example app (separate project). + */ +public final class ConsumerSmoke { + + private ConsumerSmoke() {} + + /** + * Opens a model from an absolute on-device path. + * + * @param modelPath absolute path to a GGUF file on device storage + * @return the loaded model (caller closes) + */ + public static LlamaModel open(String modelPath) { + return new LlamaModel(new ModelParameters().setModel(modelPath).setCtxSize(512)); + } + + /** + * Runs a short completion. + * + * @param model the loaded model + * @param prompt the prompt text + * @return the generated text + */ + public static String prompt(LlamaModel model, String prompt) { + return model.complete(new InferenceParameters(prompt).withNPredict(8)); + } +} diff --git a/.github/android-consumer-test/gradle.properties b/.github/android-consumer-test/gradle.properties new file mode 100644 index 00000000..5687bb11 --- /dev/null +++ b/.github/android-consumer-test/gradle.properties @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT +org.gradle.jvmargs=-Xmx2g +android.useAndroidX=true diff --git a/.github/android-consumer-test/settings.gradle.kts b/.github/android-consumer-test/settings.gradle.kts new file mode 100644 index 00000000..0d12dea9 --- /dev/null +++ b/.github/android-consumer-test/settings.gradle.kts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +// CI fixture (NOT a shipped project): a minimal AGP app that consumes the +// net.ladenthin:llama-android AAR from mavenLocal and runs a full R8 release +// build — proving Android Studio consumption end-to-end (AAR format, manifest +// minSdk merge, jni packaging, consumer proguard rules) without an emulator. +// Driven by the package-android-aar job in .github/workflows/publish.yml. +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } + plugins { + id("com.android.application") version "8.7.3" + } +} + +dependencyResolutionManagement { + repositories { + mavenLocal() // the freshly built llama-android AAR is published here first + google() + mavenCentral() + } +} + +rootProject.name = "llama-android-consumer-test" +include(":app") diff --git a/.github/dockcross/dockcross-android-x86_64 b/.github/dockcross/dockcross-android-x86_64 new file mode 100755 index 00000000..81c9a173 --- /dev/null +++ b/.github/dockcross/dockcross-android-x86_64 @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +DEFAULT_DOCKCROSS_IMAGE=dockcross/android-x86_64:20260515-5fd14ac + +#------------------------------------------------------------------------------ +# Helpers +# +err() { + echo -e >&2 "ERROR: $*\n" +} + +die() { + err "$*" + exit 1 +} + +has() { + # eg. has command update + local kind=$1 + local name=$2 + + type -t $kind:$name | grep -q function +} + +# If OCI_EXE is not already set, search for a container executor (OCI stands for "Open Container Initiative") +if [ -z "$OCI_EXE" ]; then + if which podman >/dev/null 2>/dev/null; then + OCI_EXE=podman + elif which docker >/dev/null 2>/dev/null; then + OCI_EXE=docker + else + die "Cannot find a container executor. Search for docker and podman." + fi +fi + +#------------------------------------------------------------------------------ +# Command handlers +# +command:update-image() { + $OCI_EXE pull $FINAL_IMAGE +} + +help:update-image() { + echo "Pull the latest $FINAL_IMAGE ." +} + +command:update-script() { + if cmp -s <( $OCI_EXE run --rm $FINAL_IMAGE ) $0; then + echo "$0 is up to date" + else + echo -n "Updating $0 ... " + $OCI_EXE run --rm $FINAL_IMAGE > $0 && echo ok + fi +} + +help:update-script() { + echo "Update $0 from $FINAL_IMAGE ." +} + +command:update() { + command:update-image + command:update-script +} + +help:update() { + echo "Pull the latest $FINAL_IMAGE, and then update $0 from that." +} + +command:help() { + if [[ $# != 0 ]]; then + if ! has command $1; then + err \"$1\" is not an dockcross command + command:help + elif ! has help $1; then + err No help found for \"$1\" + else + help:$1 + fi + else + cat >&2 < +ENDHELP + exit 1 + fi +} + +#------------------------------------------------------------------------------ +# Option processing +# +special_update_command='' +while [[ $# != 0 ]]; do + case $1 in + + --) + shift + break + ;; + + --args|-a) + ARG_ARGS="$2" + shift 2 + ;; + + --config|-c) + ARG_CONFIG="$2" + shift 2 + ;; + + --image|-i) + ARG_IMAGE="$2" + shift 2 + ;; + update|update-image|update-script) + special_update_command=$1 + break + ;; + -*) + err Unknown option \"$1\" + command:help + exit + ;; + + *) + break + ;; + + esac +done + +# The precedence for options is: +# 1. command-line arguments +# 2. environment variables +# 3. defaults + +# Source the config file if it exists +DEFAULT_DOCKCROSS_CONFIG=~/.dockcross +FINAL_CONFIG=${ARG_CONFIG-${DOCKCROSS_CONFIG-$DEFAULT_DOCKCROSS_CONFIG}} + +[[ -f "$FINAL_CONFIG" ]] && source "$FINAL_CONFIG" + +# Set the docker image +FINAL_IMAGE=${ARG_IMAGE-${DOCKCROSS_IMAGE-$DEFAULT_DOCKCROSS_IMAGE}} + +# Handle special update command +if [ "$special_update_command" != "" ]; then + case $special_update_command in + + update) + command:update + exit $? + ;; + + update-image) + command:update-image + exit $? + ;; + + update-script) + command:update-script + exit $? + ;; + + esac +fi + +# Set the docker run extra args (if any) +FINAL_ARGS=${ARG_ARGS-${DOCKCROSS_ARGS}} + +# Bash on Ubuntu on Windows +UBUNTU_ON_WINDOWS=$([ -e /proc/version ] && grep -l Microsoft /proc/version || echo "") +# MSYS, Git Bash, etc. +MSYS=$([ -e /proc/version ] && grep -l MINGW /proc/version || echo "") +# CYGWIN +CYGWIN=$([ -e /proc/version ] && grep -l CYGWIN /proc/version || echo "") + +if [ -z "$UBUNTU_ON_WINDOWS" -a -z "$MSYS" -a "$OCI_EXE" != "podman" ]; then + USER_IDS=(-e BUILDER_UID="$( id -u )" -e BUILDER_GID="$( id -g )" -e BUILDER_USER="$( id -un )" -e BUILDER_GROUP="$( id -gn )") +fi + +# Change the PWD when working in Docker on Windows +if [ -n "$UBUNTU_ON_WINDOWS" ]; then + WSL_ROOT="/mnt/" + CFG_FILE=/etc/wsl.conf + if [ -f "$CFG_FILE" ]; then + CFG_CONTENT=$(cat $CFG_FILE | sed -r '/[^=]+=[^=]+/!d' | sed -r 's/\s+=\s/=/g') + eval "$CFG_CONTENT" + if [ -n "$root" ]; then + WSL_ROOT=$root + fi + fi + HOST_PWD=`pwd -P` + HOST_PWD=${HOST_PWD/$WSL_ROOT//} +elif [ -n "$MSYS" ]; then + HOST_PWD=$PWD + HOST_PWD=${HOST_PWD/\//} + HOST_PWD=${HOST_PWD/\//:\/} +elif [ -n "$CYGWIN" ]; then + for f in pwd readlink cygpath ; do + test -n "$(type "${f}" )" || { echo >&2 "Missing functionality (${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="$( cygpath -w "$( readlink -f "$( pwd ;)" ; )" ; )" ; +else + HOST_PWD=$PWD + [ -L $HOST_PWD ] && HOST_PWD=$(readlink $HOST_PWD) +fi + +# Mount Additional Volumes +if [ -z "$SSH_DIR" ]; then + SSH_DIR="$HOME/.ssh" +fi + +HOST_VOLUMES= +if [ -e "$SSH_DIR" -a -z "$MSYS" ]; then + if test -n "${CYGWIN}" ; then + HOST_VOLUMES+="-v $(cygpath -w ${SSH_DIR} ; ):/home/$(id -un)/.ssh" ; + else + HOST_VOLUMES+="-v $SSH_DIR:/home/$(id -un)/.ssh" ; + fi ; +fi + +#------------------------------------------------------------------------------ +# Now, finally, run the command in a container +# +TTY_ARGS= +tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti +CONTAINER_NAME=dockcross_$RANDOM +$OCI_EXE run $TTY_ARGS --name $CONTAINER_NAME \ + -v "$HOST_PWD":/work \ + $HOST_VOLUMES \ + "${USER_IDS[@]}" \ + $FINAL_ARGS \ + $FINAL_IMAGE "$@" +run_exit_code=$? + +# Attempt to delete container +rm_output=$($OCI_EXE rm -f $CONTAINER_NAME 2>&1) +rm_exit_code=$? +if [[ $rm_exit_code != 0 ]]; then + if [[ "$CIRCLECI" == "true" ]] && [[ $rm_output == *"Driver btrfs failed to remove"* ]]; then + : # Ignore error because of https://circleci.com/docs/docker-btrfs-error/ + else + echo "$rm_output" + exit $rm_exit_code + fi +fi + +exit $run_exit_code + +################################################################################ +# +# This image is not intended to be run manually. +# +# To create a dockcross helper script for the +# dockcross/android-x86_64:20240418-88c04a4 image, run: +# +# docker run --rm dockcross/android-x86_64:20240418-88c04a4 > dockcross-android-x86_64-20240418-88c04a4 +# chmod +x dockcross-android-x86_64-20240418-88c04a4 +# +# You may then wish to move the dockcross script to your PATH. +# +################################################################################ diff --git a/.github/run-android-emulator-test.sh b/.github/run-android-emulator-test.sh new file mode 100755 index 00000000..a4c20034 --- /dev/null +++ b/.github/run-android-emulator-test.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT + +# Executed INSIDE reactivecircus/android-emulator-runner's `script:` (emulator booted, +# adb connected). Lives in a file because the runner executes the `script:` input LINE BY +# LINE through sh — multi-line if-blocks are a syntax error there (exit code 2). +# +# Pushes the cached tiny draft model to the emulator (world-readable so the app-uid +# instrumentation can open it) and runs the consumer fixture's instrumented tests. +# A missing model degrades to a warning — the on-device tests then self-skip +# (Assume) instead of failing the fixture. +set -euo pipefail + +if [ -f "models/${DRAFT_MODEL_NAME}" ]; then + adb push "models/${DRAFT_MODEL_NAME}" /data/local/tmp/jllama-test-model.gguf + adb shell chmod 644 /data/local/tmp/jllama-test-model.gguf +else + echo "::warning::draft model missing from the GGUF cache — on-device tests will self-skip" +fi + +VERSION=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version | tail -n1) +echo "Running connectedDebugAndroidTest against llama-android ${VERSION}" +gradle -p .github/android-consumer-test connectedDebugAndroidTest "-PjllamaVersion=${VERSION}" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c8cc9b2d..6477ac03 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -173,6 +173,30 @@ jobs: - name: Build and test llama-langchain4j run: mvn -B --no-transfer-progress -f llama-langchain4j/pom.xml verify + # --------------------------------------------------------------------------- + # Model-free unit tests for the Kotlin coroutines facade (llama-kotlin). + # Pure Kotlin/JVM reactor module; its 6 tests fake the Iterable+AutoCloseable + # seam, so no native library and no model are needed. + # --------------------------------------------------------------------------- + + test-java-llama-kotlin: + name: Build and Test llama-kotlin + needs: startgate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-java@v5 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: temurin + - name: Install parent + core net.ladenthin:llama into the local repo (Java only) + run: > + mvn -B --no-transfer-progress -pl llama -am -DskipTests -Denforcer.skip=true + -Dspotless.check.skip=true -Dspotbugs.skip=true + -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Dgpg.skip=true install + - name: Build and test llama-kotlin + run: mvn -B --no-transfer-progress -f llama-kotlin/pom.xml verify + # --------------------------------------------------------------------------- # Model-backed integration for the langchain4j adapters. Reuses the SAME shared GGUF # cache (populated once by download-models) and the SAME Linux-x86_64 native artifact the @@ -214,6 +238,7 @@ jobs: -Dnet.ladenthin.llama.model.path=models/${REASONING_MODEL_NAME} -Dnet.ladenthin.llama.langchain4j.embedding.model=models/${NOMIC_EMBED_MODEL_NAME} -Dnet.ladenthin.llama.langchain4j.rerank.model=models/${RERANKING_MODEL_NAME} + -Dnet.ladenthin.llama.langchain4j.tool.model=models/${TOOL_MODEL_NAME} # --------------------------------------------------------------------------- # Build the llama.cpp WebUI ONCE, from the same pinned tag CMakeLists.txt fetches, @@ -605,6 +630,38 @@ jobs: name: Linux-Android-aarch64-libraries path: ${{ github.workspace }}/llama/src/main/resources/net/ladenthin/llama/ + crosscompile-android-x86_64: + name: Cross-Compile Android x86_64 + needs: [startgate, build-webui] + runs-on: ubuntu-latest + # Android x86_64 CPU ABI: consumed by the llama-android AAR as jni/x86_64 (so the + # Android emulator — and x86_64 Android devices/Chromebooks — can run the binding) + # and merged into the default JAR's Linux-Android/x86_64 tree via the *-libraries + # glob. Same dockcross + sccache steady-state env as the arm64 job; the wrapper + # pins the same image tag. Fail-loud and in the package/publish needs graphs. + env: + USE_CACHE: ${{ github.event_name != 'workflow_dispatch' || inputs.use_cache }} + SCCACHE_WEBDAV_ENDPOINT: https://cache.depot.dev + SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }} + DOCKCROSS_ARGS: "-e SCCACHE_WEBDAV_ENDPOINT -e SCCACHE_WEBDAV_TOKEN -e USE_CACHE" + steps: + - uses: actions/checkout@v7 + - name: Download shared WebUI assets + uses: actions/download-artifact@v8 + with: + name: webui-generated + path: ${{ github.workspace }}/llama/webui-generated/ + - name: Build libraries + shell: bash + run: | + .github/dockcross/dockcross-android-x86_64 .github/build.sh "-DOS_NAME=Linux-Android -DOS_ARCH=x86_64" + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: Linux-Android-x86_64-libraries + path: ${{ github.workspace }}/llama/src/main/resources/net/ladenthin/llama/ + if-no-files-found: error + crosscompile-android-aarch64-opencl: name: Cross-Compile Android aarch64 (OpenCL/Adreno) needs: [startgate, build-webui] @@ -635,6 +692,213 @@ jobs: name: android-libraries-opencl path: ${{ github.workspace }}/llama/src/main/resources_android_opencl/net/ladenthin/llama/ + # --------------------------------------------------------------------------- + # Android AAR packaging (net.ladenthin:llama-android + llama-android-opencl). + # Plain-Gradle AAR assembly — no AGP and no Android SDK needed to BUILD (an AAR + # is a documented zip; see llama-android/README.md); AGP is only needed to + # CONSUME it, which the consumer smoke test below exercises on the runner's + # preinstalled Android SDK: a minimal app resolves the AAR from mavenLocal and + # runs a full R8 release build (validating AAR format, manifest minSdk merge, + # jni/ packaging, and the shipped consumer proguard rules). Structural checks + # additionally pin the AAR entries and the 16 KB LOAD-segment alignment + # (Google Play requirement for Android 15+ targets). Fail-loud and in the + # publish `needs:` graphs — a broken AAR blocks publishing, same policy as + # every native artifact job. + # --------------------------------------------------------------------------- + + package-android-aar: + name: Package + Validate Android AARs + needs: [crosscompile-android-aarch64, crosscompile-android-x86_64, crosscompile-android-aarch64-opencl] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-java@v5 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: temurin + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: "8.14.3" + - name: Build core jar (byte-identical classes payload for the AAR) + run: > + mvn -B --no-transfer-progress -pl llama -am -DskipTests -Denforcer.skip=true + -Dspotless.check.skip=true -Dspotbugs.skip=true + -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Dgpg.skip=true package + - name: Download Android CPU natives (arm64) + uses: actions/download-artifact@v8 + with: + name: Linux-Android-aarch64-libraries + path: stage/cpu/ + - name: Download Android CPU natives (x86_64) + uses: actions/download-artifact@v8 + with: + name: Linux-Android-x86_64-libraries + path: stage/cpu-x86_64/ + - name: Download Android OpenCL natives + uses: actions/download-artifact@v8 + with: + name: android-libraries-opencl + path: stage/opencl/ + - name: Stage natives for the AAR build + run: | + mkdir -p llama-android/natives/cpu/arm64-v8a llama-android/natives/cpu/x86_64 llama-android/natives/opencl/arm64-v8a + cp stage/cpu/Linux-Android/aarch64/libjllama.so llama-android/natives/cpu/arm64-v8a/ + cp stage/cpu-x86_64/Linux-Android/x86_64/libjllama.so llama-android/natives/cpu/x86_64/ + cp stage/opencl/Linux-Android/aarch64/libjllama.so llama-android/natives/opencl/arm64-v8a/ + - name: Assemble AARs + publish to mavenLocal + run: gradle -p llama-android aarCpu aarOpencl publishToMavenLocal + - name: Validate AAR structure + 16 KB page-size alignment + shell: bash + run: | + set -euo pipefail + VERSION=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version | tail -n1) + for flavor in llama-android llama-android-opencl; do + AAR="llama-android/build/aar/${flavor}-${VERSION}.aar" + echo "== validating $AAR" + test -f "$AAR" + # The CPU AAR is multi-ABI (arm64 devices + x86_64 emulators/devices); + # the OpenCL flavor stays arm64-only (Adreno is Qualcomm ARM hardware). + ABIS="arm64-v8a" + if [ "$flavor" = "llama-android" ]; then ABIS="arm64-v8a x86_64"; fi + ENTRIES="AndroidManifest.xml classes.jar proguard.txt R.txt" + for abi in $ABIS; do ENTRIES="$ENTRIES jni/$abi/libjllama.so"; done + for entry in $ENTRIES; do + unzip -l "$AAR" | grep -q "${entry}$" || { echo "::error::$AAR is missing $entry"; exit 1; } + done + unzip -p "$AAR" AndroidManifest.xml | grep -q 'android:minSdkVersion="28"' \ + || { echo "::error::$AAR manifest lost minSdkVersion 28"; exit 1; } + unzip -p "$AAR" classes.jar > /tmp/aar-classes.jar + unzip -l /tmp/aar-classes.jar | grep -q "net/ladenthin/llama/LlamaModel.class" \ + || { echo "::error::$AAR classes.jar lost LlamaModel"; exit 1; } + if unzip -l /tmp/aar-classes.jar | grep -qE "module-info\.class|net/ladenthin/llama/(Linux|Mac|Windows)/"; then + echo "::error::$AAR classes.jar carries module-info or desktop native resources"; exit 1 + fi + rm -rf /tmp/aar-natives && unzip -o -q -d /tmp/aar-natives "$AAR" "jni/*/libjllama.so" + # Google Play 16 KB page-size requirement (Android 15+ targets): every + # LOAD segment must be aligned to a multiple of 16384. CMake pins + # -Wl,-z,max-page-size=16384 for every Android ABI; this asserts it held. + for so in /tmp/aar-natives/jni/*/libjllama.so; do + for align in $(readelf -lW "$so" | awk '$1=="LOAD"{print $NF}'); do + if [ $(( align % 16384 )) -ne 0 ]; then + echo "::error::$so: LOAD alignment $align is not a multiple of 16384 (16 KB page-size regression)"; exit 1 + fi + done + # dlopen-ability gate: an app consuming the AAR bundles no other native + # libs, so every DT_NEEDED must be a bionic system library (plus the + # vendor ICD libOpenCL.so for the OpenCL flavor). A stray dependency — + # libomp.so / libc++_shared.so once shipped exactly this way — makes + # System.loadLibrary fail on every device with UnsatisfiedLinkError. + # CMake's Android guard (GGML_OPENMP OFF + -static-libstdc++) keeps the + # list clean; this asserts it held. + ALLOWED="libc.so libm.so libdl.so liblog.so libandroid.so" + if [ "$flavor" = "llama-android-opencl" ]; then ALLOWED="$ALLOWED libOpenCL.so"; fi + for needed in $(readelf -dW "$so" | awk '/\(NEEDED\)/{gsub(/[\[\]]/,"",$NF); print $NF}'); do + case " $ALLOWED " in + *" $needed "*) ;; + *) echo "::error::$so: DT_NEEDED '$needed' is not a bionic system library — dlopen would fail on-device"; exit 1 ;; + esac + done + done + done + - name: AGP consumer smoke test (R8 release build from mavenLocal) + shell: bash + run: | + set -euo pipefail + VERSION=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version | tail -n1) + gradle -p .github/android-consumer-test assembleRelease "-PjllamaVersion=${VERSION}" + APK=.github/android-consumer-test/app/build/outputs/apk/release/app-release.apk + test -f "$APK" + for abi in arm64-v8a x86_64; do + unzip -l "$APK" | grep -q "lib/$abi/libjllama.so" \ + || { echo "::error::consumer APK is missing lib/$abi/libjllama.so"; exit 1; } + done + # The AAR's consumer proguard.txt must have carried the binding through R8. + unzip -p "$APK" "classes*.dex" | grep -aq "Lnet/ladenthin/llama/LlamaModel;" \ + || { echo "::error::R8 stripped net.ladenthin.llama.LlamaModel — consumer proguard rules broken"; exit 1; } + - name: Upload AARs + uses: actions/upload-artifact@v7 + with: + name: llama-android-aars + path: llama-android/build/aar/*.aar + if-no-files-found: error + + # --------------------------------------------------------------------------- + # On-emulator runtime validation of the Android AAR: boots a KVM-accelerated + # x86_64 emulator (GitHub Linux runners have KVM; arm64 images cannot run here, + # which is exactly why the AAR carries the jni/x86_64 ABI), publishes the CPU AAR + # to mavenLocal, adb-pushes the already-cached tiny draft model, and runs the + # consumer fixture's connectedDebugAndroidTest — System.loadLibrary from the APK, + # pure-Java GgufInspector on-device, and real native inference on Android/bionic. + # RELEASE GATE (in both publish needs graphs) since PR #298: the job ran + # flake-free through the PR's validation cycle (boot ~30 s, on-device inference + # green), so a broken on-device runtime now blocks publishing — same fail-loud + # policy as every native artifact job. If emulator-boot flakiness ever appears, + # re-run the job first; demote it from the needs graphs only as a last resort. + # --------------------------------------------------------------------------- + + test-android-emulator: + name: Android emulator on-device test (x86_64) + needs: [crosscompile-android-aarch64, crosscompile-android-x86_64, download-models] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-java@v5 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: temurin + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: "8.14.3" + - name: Enable KVM group permissions (GitHub-hosted runner) + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Build core jar (classes payload for the AAR) + run: > + mvn -B --no-transfer-progress -pl llama -am -DskipTests -Denforcer.skip=true + -Dspotless.check.skip=true -Dspotbugs.skip=true + -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Dgpg.skip=true package + - name: Download Android CPU natives (arm64) + uses: actions/download-artifact@v8 + with: + name: Linux-Android-aarch64-libraries + path: stage/cpu/ + - name: Download Android CPU natives (x86_64) + uses: actions/download-artifact@v8 + with: + name: Linux-Android-x86_64-libraries + path: stage/cpu-x86_64/ + - name: Stage natives + publish the CPU AAR to mavenLocal + run: | + mkdir -p llama-android/natives/cpu/arm64-v8a llama-android/natives/cpu/x86_64 + cp stage/cpu/Linux-Android/aarch64/libjllama.so llama-android/natives/cpu/arm64-v8a/ + cp stage/cpu-x86_64/Linux-Android/x86_64/libjllama.so llama-android/natives/cpu/x86_64/ + gradle -p llama-android publishLlamaAndroidPublicationToMavenLocal + - name: Restore shared GGUF model cache (populated by download-models; no re-download) + uses: actions/cache@v6 + with: + path: models/ + key: gguf-models-v1 + - name: Run on-emulator instrumentation (connectedDebugAndroidTest) + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 30 + arch: x86_64 + target: default + disable-animations: true + emulator-options: -no-snapshot -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim + # One line on purpose: the emulator-runner executes `script:` LINE BY LINE via sh, + # so shell control flow must live in the committed helper script. + script: .github/run-android-emulator-test.sh + - name: Upload instrumentation reports (on failure) + if: failure() + uses: actions/upload-artifact@v7 + with: + name: android-emulator-test-reports + path: .github/android-consumer-test/app/build/reports/androidTests/ + if-no-files-found: ignore + # --------------------------------------------------------------------------- # Native build jobs — produce release artifacts + run C++ unit tests # --------------------------------------------------------------------------- @@ -2029,6 +2293,7 @@ jobs: - build-linux-x86_64-vulkan - build-linux-aarch64-vulkan - crosscompile-android-aarch64 + - crosscompile-android-x86_64 - crosscompile-android-aarch64-opencl - build-windows-x86_64 - build-windows-x86 @@ -2205,7 +2470,7 @@ jobs: publish-snapshot: name: Publish Snapshot to Central - needs: [check-snapshot, crosscompile-linux-x86_64-cuda, crosscompile-android-aarch64-opencl, code-style, test-java-llama-langchain4j] + needs: [check-snapshot, crosscompile-linux-x86_64-cuda, crosscompile-android-aarch64-opencl, package-android-aar, test-android-emulator, code-style, test-java-llama-langchain4j, test-java-llama-kotlin] if: needs.check-snapshot.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central @@ -2305,16 +2570,37 @@ jobs: *-SNAPSHOT) echo "OK: -SNAPSHOT version, continuing snapshot deploy." ;; *) echo "::error::Refusing to publish non-SNAPSHOT version '$VERSION' from the snapshot job. Snapshot publishing requires a -SNAPSHOT version; releases go through the v* tag path."; exit 1 ;; esac - # One reactor deploy publishes all three artifacts at the same version: - # net.ladenthin:llama-parent (the pom), :llama (the core jar + classifiers), and - # :llama-langchain4j. The `release` profile (GPG + Central Publishing) is inherited - # from the parent, so every module — including the parent pom — is signed. - - name: Publish snapshot (reactor - parent + llama + llama-langchain4j) + # One reactor deploy publishes all four Maven artifacts at the same version: + # net.ladenthin:llama-parent (the pom), :llama (the core jar + classifiers), + # :llama-langchain4j, and :llama-kotlin. The `release` profile (GPG + Central + # Publishing) is inherited from the parent, so every module — including the + # parent pom — is signed. The Android AARs are published by the separate + # Gradle step below (Maven cannot deploy aar). + - name: Publish snapshot (reactor - parent + llama + llama-langchain4j + llama-kotlin) run: mvn --batch-mode --no-transfer-progress -P release,cuda,vulkan-linux,vulkan-linux-aarch64,opencl-android,windows-msvc,cuda-windows,vulkan-windows,opencl-windows,rocm-linux,rocm-windows,sycl-fp16-linux,sycl-fp32-linux,sycl-windows,opencl-windows-aarch64,openvino-linux,openvino-windows -Dmaven.test.skip=true deploy env: MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + # Android AARs (llama-android, llama-android-opencl): assembled and published by + # the plain-Gradle build in llama-android/ — Maven cannot deploy + # aar. Natives are already on disk from the artifact + # downloads above; the core jar was just built by the reactor deploy. + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: "8.14.3" + - name: Publish Android AAR snapshots (llama-android + llama-android-opencl) + run: | + mkdir -p llama-android/natives/cpu/arm64-v8a llama-android/natives/cpu/x86_64 llama-android/natives/opencl/arm64-v8a + cp llama/src/main/resources/net/ladenthin/llama/Linux-Android/aarch64/libjllama.so llama-android/natives/cpu/arm64-v8a/ + cp llama/src/main/resources/net/ladenthin/llama/Linux-Android/x86_64/libjllama.so llama-android/natives/cpu/x86_64/ + cp llama/src/main/resources_android_opencl/net/ladenthin/llama/Linux-Android/aarch64/libjllama.so llama-android/natives/opencl/arm64-v8a/ + gradle -p llama-android publishAllPublicationsToCentralSnapshotsRepository + env: + CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + CENTRAL_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} + MAVEN_GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - name: Collect signed artifacts run: | mkdir -p signed-snapshot-assets @@ -2322,6 +2608,9 @@ jobs: cp llama/target/*.jar.asc signed-snapshot-assets/ 2>/dev/null || true cp llama-langchain4j/target/*.jar signed-snapshot-assets/ 2>/dev/null || true cp llama-langchain4j/target/*.jar.asc signed-snapshot-assets/ 2>/dev/null || true + cp llama-kotlin/target/*.jar signed-snapshot-assets/ 2>/dev/null || true + cp llama-kotlin/target/*.jar.asc signed-snapshot-assets/ 2>/dev/null || true + cp llama-android/build/aar/*.aar signed-snapshot-assets/ 2>/dev/null || true - uses: actions/upload-artifact@v7 with: name: signed-snapshot-assets @@ -2356,7 +2645,7 @@ jobs: publish-release: name: Publish Release to Central if: needs.check-tag.result == 'success' && inputs.publish_to_central - needs: [check-tag, crosscompile-linux-x86_64-cuda, crosscompile-android-aarch64-opencl, code-style, test-java-llama-langchain4j] + needs: [check-tag, crosscompile-linux-x86_64-cuda, crosscompile-android-aarch64-opencl, package-android-aar, test-android-emulator, code-style, test-java-llama-langchain4j, test-java-llama-kotlin] runs-on: ubuntu-latest environment: maven-central permissions: @@ -2450,12 +2739,43 @@ jobs: # net.ladenthin:llama-parent (the pom), :llama (the core jar + classifiers), and # :llama-langchain4j. The `release` profile (GPG + Central Publishing) is inherited # from the parent, so every module — including the parent pom — is signed. - - name: Publish release (reactor - parent + llama + llama-langchain4j) + - name: Publish release (reactor - parent + llama + llama-langchain4j + llama-kotlin) run: mvn --batch-mode --no-transfer-progress -P release,cuda,vulkan-linux,vulkan-linux-aarch64,opencl-android,windows-msvc,cuda-windows,vulkan-windows,opencl-windows,rocm-linux,rocm-windows,sycl-fp16-linux,sycl-fp32-linux,sycl-windows,opencl-windows-aarch64,openvino-linux,openvino-windows -Dmaven.test.skip=true deploy env: MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + # Android AARs (llama-android, llama-android-opencl): Maven cannot deploy + # aar, so the plain-Gradle build signs + publishes them + # into a local Maven-layout staging repo, which is zipped into a Central + # Portal bundle and uploaded via the Publisher API (publishingType=AUTOMATIC: + # the deployment publishes as soon as portal validation passes). Natives are + # already on disk from the artifact downloads above; the core jar was just + # built by the reactor deploy. + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: "8.14.3" + - name: Publish Android AAR release bundle to Central Portal + shell: bash + run: | + set -euo pipefail + mkdir -p llama-android/natives/cpu/arm64-v8a llama-android/natives/cpu/x86_64 llama-android/natives/opencl/arm64-v8a + cp llama/src/main/resources/net/ladenthin/llama/Linux-Android/aarch64/libjllama.so llama-android/natives/cpu/arm64-v8a/ + cp llama/src/main/resources/net/ladenthin/llama/Linux-Android/x86_64/libjllama.so llama-android/natives/cpu/x86_64/ + cp llama/src/main/resources_android_opencl/net/ladenthin/llama/Linux-Android/aarch64/libjllama.so llama-android/natives/opencl/arm64-v8a/ + gradle -p llama-android publishAllPublicationsToStagingRepository + VERSION=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version | tail -n1) + ( cd llama-android/build/staging-repo && zip -r -q ../central-bundle.zip net -x "*maven-metadata*" ) + TOKEN=$(printf "%s:%s" "$CENTRAL_USERNAME" "$CENTRAL_PASSWORD" | base64 -w0) + curl --fail-with-body -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -F "bundle=@llama-android/build/central-bundle.zip" \ + "https://central.sonatype.com/api/v1/publisher/upload?publishingType=AUTOMATIC&name=llama-android-$VERSION" + env: + CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + CENTRAL_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} + MAVEN_GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - name: Collect signed artifacts run: | mkdir -p signed-release-assets @@ -2463,6 +2783,9 @@ jobs: cp llama/target/*.jar.asc signed-release-assets/ 2>/dev/null || true cp llama-langchain4j/target/*.jar signed-release-assets/ 2>/dev/null || true cp llama-langchain4j/target/*.jar.asc signed-release-assets/ 2>/dev/null || true + cp llama-kotlin/target/*.jar signed-release-assets/ 2>/dev/null || true + cp llama-kotlin/target/*.jar.asc signed-release-assets/ 2>/dev/null || true + cp llama-android/build/aar/*.aar signed-release-assets/ 2>/dev/null || true - uses: actions/upload-artifact@v7 with: name: signed-release-assets diff --git a/.gitignore b/.gitignore index d160476a..02aed140 100644 --- a/.gitignore +++ b/.gitignore @@ -74,4 +74,13 @@ llama/src/main/cpp/llama.cpp/ # Local AI agent tooling (not part of the project) AGENTS.md -.agents/ \ No newline at end of file +.agents/ +# llama-android AAR build (Gradle): build outputs, Gradle caches, and the +# CI-staged native libraries (never committed — same policy as resources_*). +llama-android/build/ +llama-android/.gradle/ +llama-android/natives/ + +# CI Android consumer-test fixture (Gradle/AGP) build outputs +.github/android-consumer-test/**/build/ +.github/android-consumer-test/.gradle/ diff --git a/CLAUDE.md b/CLAUDE.md index 543c6ad5..456d0e65 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Java bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) via JNI, providing a high-level API for LLM inference in Java. The Java layer communicates with a native C++ library through JNI. -Current llama.cpp pinned version: **b9870** +Current llama.cpp pinned version: **b9878** ## Upgrading CUDA Version @@ -376,7 +376,7 @@ needs no extra step here, `build-webui` re-reads the tag and rebuilds the matchi ships no UI): ```bash # needs node/npm + network; embed.cpp is plain C++17 (no npm) -git clone --depth 1 --branch b9870 https://github.com/ggml-org/llama.cpp /tmp/lc +git clone --depth 1 --branch b9878 https://github.com/ggml-org/llama.cpp /tmp/lc ( cd /tmp/lc/tools/ui && npm ci && npm run build \ && ( cd dist && find . -type f -not -path './_gzip/*' \ | while read -r f; do mkdir -p "_gzip/$(dirname "$f")"; gzip -9 -c "$f" > "_gzip/$f"; done ) \ @@ -416,7 +416,7 @@ cache lives in **Depot Cache** over sccache's **WebDAV** backend: - `SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }}` — a Depot **organization** token, stored as the repo secret **`DEPOT_TOKEN`**. -Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b9870`), the +Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b9878`), the ~280 upstream object files are byte-identical every run, so a warm cache recompiles only the *changed* files. Depot's cache is **shared across all branches** (unlike GitHub's per-branch `actions/cache`), so every branch builds incrementally; a `b` version bump @@ -529,6 +529,8 @@ Current patches: | `0003-pr22393-server-add-slot-prompt-similarity-getter-setter.patch` | **Upstream-PR carry** of [ggml-org/llama.cpp#22393](https://github.com/ggml-org/llama.cpp/pull/22393) ("server : add slot_prompt_similarity getter/setter") while it is still open upstream. Purely additive: adds `server_context::get_slot_prompt_similarity()` / `set_slot_prompt_similarity(float)` (`tools/server/server-context.{cpp,h}`) so an embedding/JNI caller can query and tune the slot-selection threshold at runtime without reloading the model. Verbatim copy of the PR — drop it once a pinned `b` includes the change. | | `0004-pr23116-server-per-request-reasoning-budget-tokens.patch` | **Upstream-PR carry** of [ggml-org/llama.cpp#23116](https://github.com/ggml-org/llama.cpp/pull/23116) ("server: honour per-request reasoning_budget_tokens in chat completions"), motivated by java-llama.cpp#140, while it is still open upstream. `oaicompat_chat_params_parse` (`tools/server/server-common.cpp`) only read the Anthropic `thinking_budget_tokens` alias and always wrote the server-level `reasoning_budget_message`, so a per-request `reasoning_budget_tokens` / `reasoning_budget_message` on a chat-completions request was ignored. The patch reads both overrides **before** the generic copy loop (precedence: `reasoning_budget_tokens` > `thinking_budget_tokens` alias > server default) and threads the per-request message through. Carries the upstream `tests/test-chat.cpp` additions verbatim so the patch is submittable as-is; like `0001`'s test/call-site flips they are **applied-but-not-compiled** here (`LLAMA_BUILD_TESTS` is OFF for the FetchContent subproject). Drop it once a pinned `b` includes the change. | | `0005-server-recurrent-near-prompt-end-checkpoints.patch` | **Multi-turn tool-calling perf fix for recurrent/hybrid models (e.g. Granite-4)**, upstream-submittable. In `server_context::update_slots` (`tools/server/server-context.cpp`) the near-prompt-end context checkpoints are gated by `checkpoint_min_step` (default 8192 tokens). An agentic conversation that appends only assistant/tool messages never produces a new user-message checkpoint (`is_user_start`/`is_last_user_message` match `COMMON_CHAT_ROLE_USER` only), so after turn 1 no new checkpoint is ever created and — because recurrent state can only roll back to a checkpoint — **every turn re-prefills the whole conversation tail** (measured on a synthetic granitehybrid model: prefilled tokens grew 901 → 1544 → 2187 → 2830 → 3473 over turns 2–6). The patch (1) exempts near-prompt-end checkpoints from the min-step spacing when the memory can only roll back via checkpoints (`ctx_tgt_seq_rm_type` is `FULL` or `RS` — SWA-only models are unaffected), and (2) skips creating a checkpoint whose position equals the newest one (the last-user-message checkpoint was re-created identically on every turn, flooding the 32-entry list). After the patch each turn restores the previous turn's near-end checkpoint and prefill is constant (~new-turn-sized; 647 tokens/turn in the same measurement, ≈5.4× less prefill at turn 6 and growing with conversation length). Validated output-identical (`temperature=0`) vs. unpatched. Complements — not duplicates — open upstream PRs #24035/#24899/#24891 (they fix checkpoint *invalidation/retention*; this fixes checkpoint *starvation*). Drop once upstream solves agentic checkpoint placement (e.g. a merged role-boundary checkpointing design, cf. #21885 / #22826 discussion). | +| `0007-server-attach-http-frontend.patch` | **Adds `llama_server_attach(argc, argv, server_context&)`** so the `NativeServer` *attach mode* can serve an **already-loaded `LlamaModel`** over the full upstream HTTP frontend — no second model load, no `start_loop()`; the LlamaModel's worker keeps driving the shared `server_context` and the HTTP routes post tasks to its queue (the queue is the synchronization point). Mechanically: (1) extracts the common route table + CORS-proxy/tools blocks out of `llama_server()` into `llama_server_register_common_routes(...)` (shared verbatim, so the entry points cannot drift; returns `false` on tools-setup failure); (2) adds `llama_server_attach`, which parses only the HTTP-side argv via `common_params_parse`, starts `g_stream_sessions` GC + `server_http_context`, registers the common routes plus the non-router resumable-streaming handlers, marks ready immediately (model already loaded), and blocks on the HTTP thread until `llama_server_request_shutdown()` — never calling `common_init()`, backend init, `ctx_server.terminate()` or `llama_backend_free()` (the embedding caller owns those). Applies after `0001`+`0006` (same file); closes the "NativeServer — reuse an already-loaded LlamaModel" TODO. Upstream-submittable ("server: let embedding callers attach the HTTP frontend to an existing server_context"). | +| `0008-server-models-worker-cmd-override.patch` | **Makes router mode usable in-JVM.** The router (`server-models.cpp`) spawns each model worker by re-executing its own binary (`get_server_exec_path()` = `/proc/self/exe` & friends) — inside a JVM that binary is `java`, not a llama-server, so embedded router workers could never start. The patch adds env `LLAMA_SERVER_WORKER_CMD` (whitespace-split; read in `server_model_meta::update_args`) which replaces only the leading binary-path token of the rendered worker args, letting an embedding host relaunch workers through its own bootstrap — e.g. `java -cp app.jar net.ladenthin.llama.server.NativeServer` (each worker is then a fresh JVM running the classic single-model `NativeServer`). Exposed in Java as `NativeServer.setWorkerCommand(String...)` (JNI `setenv`); exercised by `RouterModeIntegrationTest` (Linux CI). Upstream-submittable (also useful for containerized/wrapped deployments). | | `0006-server-embed-native-server-jni.patch` | **Makes `server.cpp`'s `llama_server` embeddable in the JVM** so the `NativeServer` JNI bridge can run the full upstream HTTP server (WebUI included) inside `libjllama` — see "Two server modes" below. b9870 already exposes `int llama_server(int, char**)` (non-static; no `main` in the file), so the patch only adds embedded-mode support: (1) a `g_llama_server_embedded` flag + `llama_server_set_embedded()` / `llama_server_request_shutdown()` (declared in the committed `src/main/cpp/native_server_bridge.h`); (2) skips installing the process-wide SIGINT/SIGTERM handlers when embedded (they would hijack the JVM's); (3) in embedded mode parses the **forwarded** argv via `common_params_parse` instead of `common_params_parse_main` (whose `GetCommandLineW` recovery would pick up `java.exe`'s command line — the same Windows class of bug `0001` fixes). `llama_server_request_shutdown()` mirrors the SIGTERM path (invokes the installed `shutdown_handler` → `ctx_server.terminate()` unblocks `start_loop()`), giving JNI an out-of-band stop since `ctx_server` is loop-local. Applies **after `0001`** (which flips this call site to `common_params_parse_main`), so its context is the post-`0001` tree; regenerate against `0001`+source on a bump. Only touches `tools/server/server.cpp`. | ## OuteTTS build-time extraction (`cmake/generate-tts-upstream.cmake`) @@ -776,7 +778,7 @@ the README. The summary below covers only the optional-model bindings: | `net.ladenthin.llama.vision.image` | `MultimodalIntegrationTest` | committed default `src/test/resources/images/test-image.jpg`; override to any png/jpeg/webp/gif on disk | | `net.ladenthin.llama.audio.model` | `AudioInputIntegrationTest` (llama.cpp discussion #13759) | audio-input model GGUF, e.g. `ultravox-v0_5-llama-3_2-1b.gguf` | | `net.ladenthin.llama.audio.mmproj` | `AudioInputIntegrationTest` | matching audio mmproj/encoder, e.g. `mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf` | -| `net.ladenthin.llama.audio.input` | `AudioInputIntegrationTest` | a `.wav`/`.mp3` clip on disk (no committed default — audio is not committed) | +| `net.ladenthin.llama.audio.input` | `AudioInputIntegrationTest` | committed default `src/test/resources/audios/sample.wav`; override to any `.wav`/`.mp3` on disk | | `net.ladenthin.llama.tts.ttc.model` | `TtsIntegrationTest` | OuteTTS text-to-codes model, e.g. `OuteTTS-0.2-500M-Q4_K_M.gguf` | | `net.ladenthin.llama.tts.vocoder.model` | `TtsIntegrationTest` | matching codes-to-speech vocoder, e.g. `WavTokenizer-Large-75-F16.gguf` | @@ -795,7 +797,7 @@ mvn test -Dtest=MultimodalIntegrationTest \ mvn test -Dtest=AudioInputIntegrationTest \ -Dnet.ladenthin.llama.audio.model=models/ultravox-v0_5-llama-3_2-1b.gguf \ -Dnet.ladenthin.llama.audio.mmproj=models/mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf \ - -Dnet.ladenthin.llama.audio.input=/path/to/speech.wav + -Dnet.ladenthin.llama.audio.input=/path/to/speech.wav # optional: defaults to the committed src/test/resources/audios/sample.wav mvn test -Dtest=TtsIntegrationTest \ -Dnet.ladenthin.llama.tts.ttc.model=models/OuteTTS-0.2-500M-Q4_K_M.gguf \ -Dnet.ladenthin.llama.tts.vocoder.model=models/WavTokenizer-Large-75-F16.gguf @@ -945,7 +947,7 @@ If the local check passes (`BUILD SUCCESS`), the `mvn package` job in - The `server` package is a dedicated top layer in the ArchUnit `layeredArchitecture` rule (the only layer allowed to access the root `Api`); `noInternalJdkImports` carries an explicit exception for the supported `com.sun.net.httpserver` (the exported `jdk.httpserver` module, which `module-info.java` `requires`). See README "OpenAI-compatible HTTP server". **Native layer** (`src/main/cpp/`): -- `jllama.cpp` — JNI implementation bridging Java calls to llama.cpp. ~1,516 lines; 30 native methods (27 `LlamaModel` + 3 `TextToSpeech`). +- `jllama.cpp` — JNI implementation bridging Java calls to llama.cpp. ~1,650 lines; 33 native methods (29 `LlamaModel` + 3 `TextToSpeech` + 1 `LlamaQuantizer`). - `utils.hpp` — Helper utilities (format helpers, argv stripping, token-piece serialisation). - `json_helpers.hpp` — Pure JSON transformation helpers (no JNI, no llama state). Independently unit-testable. - `jni_helpers.hpp` — JNI bridge helpers (handle management + server orchestration). Includes `json_helpers.hpp`. @@ -957,7 +959,7 @@ If the local check passes (`BUILD SUCCESS`), the `mvn package` job in The library exposes **two** ways to serve a model over HTTP, on two different transports. The fat jar's `Main-Class` is `server.ServerLauncher`, a tiny dispatcher: it runs `OpenAiCompatServer` when `--jllama-openai-compat` is present (that marker is stripped, the rest forwarded) and the default `NativeServer` otherwise. Both mains are also runnable directly by class name via `java -cp`. The two modes: 1. **`server.OpenAiCompatServer` (Java transport).** OpenAI/Ollama/Anthropic-compatible JSON API on the JDK's `com.sun.net.httpserver`, driving the compiled server *core* over JNI. Embeddable, no extra dependency, and it can share/reuse a `LlamaModel`. It serves **no** static assets — its `/` route is a 404, so **no WebUI**. It has its own `main` (run via `java -cp net.ladenthin.llama.server.OpenAiCompatServer …`); its CLI (`OpenAiServerCli`) maps a curated flag subset (`-m/-c/-b/-ub/-ngl/-t/-tb/-ctk/-ctv/--jinja/--chat-template-kwargs/--host/--port/--parallel/--mmproj/--api-key/--embedding/--reranking`). -2. **`server.NativeServer` (native transport) — the default fat-jar server (when `--jllama-openai-compat` is absent).** Runs the **full upstream `llama_server`** (via `patches/0006` + `native_server.cpp`) inside `libjllama`, forwarding the raw llama-server argv verbatim — so **every** llama-server flag works and the **embedded WebUI is served** (when the assets are compiled in; CI's released jars have them, local `cmake` builds use the empty-asset stub). It is an **independent lifecycle** (loads its own model from the argv, like `llama-server.exe`; owns the process's llama backend + stderr logging while running), **single-instance per process** (upstream keeps shutdown state in file-scope globals), and **not available on Android** (the `subprocess.h` guard). Reusing an already-loaded `LlamaModel`'s context is a documented TODO. `libjllama` loading anywhere a JVM runs is what makes this "no separate `llama-server.exe`" possible. +2. **`server.NativeServer` (native transport) — the default fat-jar server (when `--jllama-openai-compat` is absent).** Runs the **full upstream `llama_server`** (via `patches/0006` + `native_server.cpp`) inside `libjllama`, forwarding the raw llama-server argv verbatim — so **every** llama-server flag works and the **embedded WebUI is served** (when the assets are compiled in; CI's released jars have them, local `cmake` builds use the empty-asset stub). With the classic constructor it is an **independent lifecycle** (loads its own model from the argv, like `llama-server.exe`; owns the process's llama backend + stderr logging while running); the **attach constructor** (`NativeServer(LlamaModel, String...)`, via `patches/0007`'s `llama_server_attach`) instead serves an **already-loaded `LlamaModel`** — one copy of the weights, the model's worker keeps driving inference, the HTTP routes post to its queue; caller closes the server before the model. **Router mode** (start without a model argument: `--models-dir`, `GET/POST /models`, per-request model selection) works in-JVM after `NativeServer.setWorkerCommand(...)` redirects the worker spawn to a fresh JVM (`patches/0008` — upstream re-execs its own binary, which in a JVM is `java`); the typed `server.RouterClient` (+ `value.RouterModel`, `json.RouterModelsResponseParser`) wraps the model-management endpoints (list/load/unload/await-loaded with fail-fast on failed workers) so callers don't hand-roll HTTP+JSON. Either way it is **single-instance per process** (upstream keeps shutdown state in file-scope globals) and **not available on Android** (the `subprocess.h` guard). `libjllama` loading anywhere a JVM runs is what makes this "no separate `llama-server.exe`" possible. ### Native Helper Architecture @@ -999,7 +1001,14 @@ Functions: `log_level_name`, `format_log_as_json`. *Layer B* (requires upstream server headers in the TU before `jni_helpers.hpp`): orchestration. Includes `json_helpers.hpp` so all bridge helpers can call transforms directly. -- `json_to_jstring_impl` — serialises any `json` value to a JNI string via `dump()`. +- `utf8_to_jstring_impl` — builds a `java.lang.String` from raw standard-UTF-8 bytes via the cached + `String(byte[], "UTF-8")` constructor. **Payload text must never go through `NewStringUTF`**: JNI + specifies *Modified* UTF-8 input there, so standard UTF-8 containing supplementary-plane + characters (every 4-byte emoji) is spec-invalid — Android CheckJNI aborts on it. The mirror of + `parse_jstring`'s `String.getBytes("UTF-8")` input path. +- `json_to_jstring_impl` — serialises any `json` value to a JNI string via upstream + `safe_json_to_str` (dump with `error_handler_t::replace`, so content ending in an incomplete + UTF-8 sequence yields U+FFFD instead of throwing `json::type_error 316`) + `utf8_to_jstring_impl`. - `results_to_jstring_impl` — delegates to `results_to_json` then `json_to_jstring_impl`. - `vec_to_jarray_impl` — generic C++ vector → JNI primitive array. - `embedding_to_jfloat_array_impl` — converts `std::vector` to `jfloatArray`. @@ -1100,7 +1109,8 @@ properties set, so `LlamaEmbeddingsTest`, `MultimodalIntegrationTest`, and `TtsI **run on every platform** rather than self-skipping. `validate-models.{sh,bat}` treats all of these as **required** (a missing model hard-fails the job before tests run, so a download regression can never silently downgrade to a skip). The only model still self-skipping is the -audio-input model (`AudioInputIntegrationTest`) — it has no committed clip and no CI download. +audio-input model (`AudioInputIntegrationTest`) — the prompt clip is committed +(`src/test/resources/audios/sample.wav`) but the audio model + mmproj have no CI download. The shared GGUF cache (`actions/cache`, key `gguf-models-v1`, path `models/`) holds the full set and is populated **once, upfront** by a dedicated **`download-models`** job (`needs: startgate`): it is the single place the ~5 GB set is fetched from HuggingFace (the ten `curl` steps + the @@ -1146,18 +1156,18 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" | File | Tests | Scope | |------|-------|-------| -| `src/test/cpp/test_utils.cpp` | 156 | Upstream helpers: `server_tokens`, `server_grammar_trigger`, `gen_tool_call_id`, `json_value`, `json_get_nested_values`, UTF-8 helpers, `format_response_rerank`, `format_embeddings_response_oaicompat`, `oaicompat_completion_params_parse`, `oaicompat_chat_params_parse`, `are_lora_equal`, `strip_flag_from_argv`, `token_piece_value`, `json_is_array_and_contains_numbers`, `format_oai_sse`, `format_oai_resp_sse`, `format_anthropic_sse` | -| `src/test/cpp/test_server.cpp` | 197 | Upstream result types: `result_timings`, `task_params::to_json()` (incl. `dry_sequence_breakers`, `preserved_tokens`, `timings_per_token`), `completion_token_output`, `server_task_result_cmpl_partial` (non-oaicompat + `to_json_oaicompat` + logprobs + `to_json_oaicompat_chat` + `to_json_anthropic` + dispatcher), `server_task_result_cmpl_final` (non-oaicompat + `to_json_oaicompat` + `to_json_oaicompat_chat` + `to_json_oaicompat_chat_stream` + `to_json_anthropic` + `to_json_anthropic_stream` + tool_calls + dispatcher), `server_task_result_embd`, `server_task_result_rerank`, `server_task_result_metrics`, `server_task_result_slot_save_load`, `server_task_result_slot_erase`, `server_task_result_apply_lora`, `server_task_result_error`, `format_error_response`, `server_task::need_sampling()`, `server_task::n_tokens()`, `server_schema::eval_llama_cmpl_schema()` (parsing pipeline + grammar routing + error paths + per-request `dry_*` and `sse_ping_interval` field round-trips incl. hard-limit + server-default inheritance), `response_fields` projection | +| `src/test/cpp/test_utils.cpp` | 162 | Upstream helpers: `server_tokens`, `server_grammar_trigger`, `gen_tool_call_id`, `json_value`, `json_get_nested_values`, UTF-8 helpers, `format_response_rerank`, `format_embeddings_response_oaicompat`, `oaicompat_completion_params_parse`, `oaicompat_chat_params_parse`, `are_lora_equal`, `strip_flag_from_argv`, `token_piece_value`, `json_is_array_and_contains_numbers`, `format_oai_sse`, `format_oai_resp_sse`, `format_anthropic_sse`, `parse_lora_request` | +| `src/test/cpp/test_server.cpp` | 201 | Upstream result types: `result_timings`, `task_params::to_json()` (incl. `dry_sequence_breakers`, `preserved_tokens`, `timings_per_token`), `completion_token_output`, `server_task_result_cmpl_partial` (non-oaicompat + `to_json_oaicompat` + logprobs + `to_json_oaicompat_chat` + `to_json_anthropic` + dispatcher), `server_task_result_cmpl_final` (non-oaicompat + `to_json_oaicompat` + `to_json_oaicompat_chat` + `to_json_oaicompat_chat_stream` + `to_json_anthropic` + `to_json_anthropic_stream` + tool_calls + dispatcher), `server_task_result_embd`, `server_task_result_rerank`, `server_task_result_metrics`, `server_task_result_slot_save_load`, `server_task_result_slot_erase`, `server_task_result_apply_lora`, `server_task_result_get_lora`, `server_task_result_error`, `format_error_response`, `server_task::need_sampling()`, `server_task::n_tokens()`, `server_schema::eval_llama_cmpl_schema()` (parsing pipeline + grammar routing + error paths + per-request `dry_*` and `sse_ping_interval` field round-trips incl. hard-limit + server-default inheritance), `response_fields` projection | | `src/test/cpp/test_json_helpers.cpp` | 47 | All functions in `json_helpers.hpp`: `get_result_error_message`, `results_to_json`, `rerank_results_to_json`, `parse_encoding_format`, `extract_embedding_prompt`, `is_infill_request`, `parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk` | | `src/test/cpp/test_log_helpers.cpp` | 13 | All functions in `log_helpers.hpp`: `log_level_name`, `format_log_as_json` | -| `src/test/cpp/test_jni_helpers.cpp` | 47 | All functions in `jni_helpers.hpp` using a zero-filled `JNINativeInterface_` mock | +| `src/test/cpp/test_jni_helpers.cpp` | 54 | All functions in `jni_helpers.hpp` using a zero-filled `JNINativeInterface_` mock (incl. the `utf8_to_jstring_impl` byte-array string path: emoji byte-preservation, truncated-UTF-8 replace-not-throw) | | `src/test/cpp/test_tts_wav.cpp` | 2 | The in-memory WAV writer `pcm_to_wav16_bytes` in `tts_wav.hpp` (WAV header/payload + little-endian clamping). The OuteTTS DSP it pairs with is derived from upstream `tts.cpp` and covered end-to-end by the Java `TtsIntegrationTest`, not unit-tested here. | -**Current total: 462 tests (all passing).** +**Current total: 479 tests (all passing).** #### Upstream source location (in CMake build tree) -llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b9870`. +llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b9878`. **GoogleTest** is a separate `BUILD_TESTING`-only FetchContent (`GIT_TAG v1.17.0`), used solely by the `jllama_test` C++ unit-test binary — not by the shipped library, and not coupled to the @@ -1343,9 +1353,11 @@ See [`../workspace/policies/ci-test-diagnostics.md`](../workspace/policies/ci-te ## PIT Mutation Testing See [`../workspace/policies/pit-mutation-testing.md`](../workspace/policies/pit-mutation-testing.md). -Run PIT with the lifecycle prefix — `mvn test-compile org.pitest:pitest-maven:mutationCoverage`. -Repo-specific gotcha: the gate reaches 100% only with the audio fixture present — without it -`value.ContentPart.audioFile(Path)` is uncovered (98%); see policy §4 and `TODO.md`. +Run PIT with the lifecycle prefix — `mvn test-compile org.pitest:pitest-maven:mutationCoverage` +(from the repo root add `-f llama/pom.xml`). The gate is **hermetic** — no model or audio fixture +needed: `ContentPartTest`'s `@TempDir` tests cover `value.ContentPart.audioFile(Path)` (verified +295/295, 0 NO_COVERAGE in a fixture-less sandbox; the former audio-fixture gotcha is resolved, +see `TODO.md`). ## JPMS Module Descriptor @@ -1355,21 +1367,31 @@ keeping it clear of the JPMS module-mode javadoc trap that bit BAF. **Before rai javadoc source level to ≥ 9, read** [`../workspace/policies/jpms-module-descriptor.md`](../workspace/policies/jpms-module-descriptor.md). -## Repository layout — Maven reactor (`llama/` + `llama-langchain4j/`) +## Repository layout — Maven reactor (`llama/` + `llama-langchain4j/` + `llama-kotlin/`) + the `llama-android/` Gradle build The repo root is a thin **aggregator/parent POM** (`net.ladenthin:llama-parent`, -`packaging=pom`) with two modules: +`packaging=pom`) with three modules: - **`llama/`** — the native JNI core (`net.ladenthin:llama`). *All the core sources and build files live here now:* `llama/src/`, `llama/CMakeLists.txt`, `llama/cmake/`, `llama/patches/`, `llama/pom.xml`, `llama/spotbugs-exclude.xml`, `llama/lombok.config`, `llama/.clang-format`. Its published coordinates are unchanged (`net.ladenthin:llama`), so consumers are unaffected. - **`llama-langchain4j/`** — the LangChain4j adapters (see below). +- **`llama-kotlin/`** — the Kotlin coroutines façade (see "Android AAR + Kotlin façade" below). -Both modules inherit the single `` from the parent, so they **ship in lockstep by +All modules inherit the single `` from the parent, so they **ship in lockstep by construction** (no CI guard needed). The parent also holds the shared `release` profile (GPG + -Central Publishing), so one reactor `mvn -P release deploy` signs and publishes all three -artifacts (`llama-parent` pom, `llama`, `llama-langchain4j`) at the same version. +Central Publishing), so one reactor `mvn -P release deploy` signs and publishes all four +Maven artifacts (`llama-parent` pom, `llama`, `llama-langchain4j`, `llama-kotlin`) at the same +version. + +**`llama-android/` is deliberately NOT a reactor module** but a standalone plain-Gradle build +(no AGP, no Android SDK needed to build): Maven cannot produce or deploy an artifact with +`aar` (the only android-maven-plugin is dead), while Gradle's built-in +`maven-publish` can. It stays version-locked anyway — `llama-android/build.gradle.kts` parses +the version and the mirrored dependency versions out of the Maven poms at configure time, so +`mvn versions:set` remains the single bump point (no Gradle-side edit on a bump). See +"Android AAR + Kotlin façade" below. **Consequences for build commands:** the core's cmake/native build runs *in `llama/`*. `.github/build.sh` / `build.bat` `cd` into `llama/` themselves (relative to the script), so CI @@ -1383,11 +1405,15 @@ file shows `cmake -B build` / `src/main/...` / `mvn compile` at the root, read i **Version bump:** the child modules declare **no `` of their own** — their *project* version is inherited from the parent. But each child still hardcodes the parent version inside its `` pointer (Maven requires a literal there — there is **no `${revision}`/CI-friendly -versioning** here), so a version change must be applied to **all three poms in lockstep**: +versioning** here), so a version change must be applied to **all four poms in lockstep**: - `pom.xml` (root) — `` - `llama/pom.xml` — `` - `llama-langchain4j/pom.xml` — `` +- `llama-kotlin/pom.xml` — `` + +(`llama-android/` needs **no** edit — its Gradle build reads the root pom's version at +configure time.) The safe way is `mvn -q versions:set -DnewVersion=X.Y.Z -DgenerateBackupPoms=false` from the repo root (it updates the parent and every child `` reference at once). Changing only the root @@ -1408,6 +1434,9 @@ missed again.) release version now appears in only ~4 spots here, not ~20 — the runtime details live once in the classifier table.) - **`llama-langchain4j/README.md`** — its own `` snippet. +- **`llama-android/README.md`** and **`llama-kotlin/README.md`** — their Gradle dependency + snippets, plus the `llama-android`/`llama-kotlin` snippets in the root README's + "Importing in Android" section. (If single-source ergonomics are wanted, the Maven CI-friendly `${revision}` property + `flatten-maven-plugin` would let a bump touch only the root — @@ -1456,9 +1485,83 @@ Wiring: `-Dnet.ladenthin.llama.langchain4j.{embedding,rerank}.model` / `net.ladenthin.llama.model.path` properties. It is validation-only (not a release gate); a cold cache degrades to a self-skip. -**Open follow-ups** (documented in `llama-langchain4j/README.md`): tool calling -(`ToolSpecification` ↔ jllama `ToolDefinition`), `response_format`/JSON mode, and multimodal -user input (currently flattened to text). +**Mapped** (since 5.0.6): blocking tool calling (`ToolSpecification` ↔ jllama `ToolDefinition` +via the module's own `JsonSchemaElementSerializer` — langchain4j's serializer lives in its +`internal` package, so the module carries a public-API-only recursive walk emitting the same +`$defs`/`#/$defs/…` conventions; tool-call turns round-trip in both directions), +`response_format`/JSON mode (`json_object` + `json_schema` structured output), and multimodal +user input (`ImageContent`/`AudioContent` → `ContentPart` array-form content; needs `--mmproj`). +Streaming (since 5.0.6, second pass): `JllamaStreamingChatModel` now streams over the native +OAI chunk path via the module's `StreamingChunkAssembler` — streamed tool calls +(`onPartialToolCall`/`onCompleteToolCall` + `toolExecutionRequests()` on the final response), +per-token thinking events (`onPartialThinking` + `AiMessage.thinking()`), real finish reason and +token usage. **Open follow-up** (documented in `llama-langchain4j/README.md`): `modelName()` is +ignored (one model per adapter). + +## Android AAR + Kotlin façade (`llama-android/` + `llama-kotlin/`) + +Two consumable Android-facing artifacts, replacing the submodule/NDK source-integration flow as +the recommended path (README "Importing in Android", Option 1): + +- **`net.ladenthin:llama-android`** / **`llama-android-opencl`** — AARs (`aar`) + carrying the core classes + the CI-built `libjllama.so` natives under `jni/` — the CPU AAR is + **multi-ABI** (`arm64-v8a` devices + `x86_64` emulators/Chromebooks, built by the + `crosscompile-android-x86_64` dockcross job whose artifact also merges into the default JAR's + `Linux-Android/x86_64` tree via the `*-libraries` glob; the OpenCL flavor stays arm64-only — + Adreno is Qualcomm ARM hardware), a + `minSdkVersion 28` manifest (AGP enforces the floor on consumers), and consumer R8/ProGuard + rules (`consumer-proguard.txt` → `proguard.txt` in the AAR; keeps `net.ladenthin.llama.**` for + the JNI `FindClass`/Jackson reflection surface). The AAR's `classes.jar` is the + **byte-identical Maven-built core jar** minus the desktop/Android native resource trees + (~70 MB APK bloat otherwise) and `module-info.class` (D8 rejects it); on Android `LlamaLoader` + resolves via `System.loadLibrary("jllama")`, which finds the AAR-installed `.so` — no loader + change was needed. Built by the **standalone plain-Gradle build** in `llama-android/` + (see "Repository layout" for why it is not a Maven module); the POM mirrors the core's + compile-scope deps (jackson/slf4j-api/jspecify/checker-qual, versions parsed from + `llama/pom.xml` — deliberately NOT logback, which is the JVM-only runtime binding). +- **`net.ladenthin:llama-kotlin`** — Maven reactor module; pure-Kotlin (2.2, jvmTarget 1.8) + coroutines façade: `generateFlow`/`generateChatFlow` (cold `Flow`, source closed on + completion/error/cancellation) and `completeSuspend`/`chatSuspend`/`chatCompleteTextSuspend`/ + `embedSuspend` (`completeSuspend` wires coroutine cancellation into the cooperative + `CancellationToken`). The core dep is **provided-scope** so Android consumers pair it with the + AAR instead of transitively pulling the fat desktop JAR. 6 model-free unit tests fake the + `Iterable & AutoCloseable` seam (`closeableIterableFlow`/`withCancellationToken` internals). + +**16 KB page-size invariant (Google Play, Android 15+ targets):** `llama/CMakeLists.txt` pins +`-Wl,-z,max-page-size=16384` in the Android guard block, and the `package-android-aar` CI job +asserts every LOAD segment of the shipped `.so` is 16384-aligned via `readelf` — a dockcross +toolchain bump cannot silently regress Play compatibility. + +**dlopen-ability invariant (bionic-only DT_NEEDED):** the same Android guard block sets +`GGML_OPENMP OFF` (ggml uses its std::thread pool — Android ships no `libomp.so`; same trade +as the Windows-arm64 clang-cl job) and links `-static-libstdc++` (no `libc++_shared.so` +dependency — that runtime only exists when an app packages it itself). Without both, the +dockcross cross-clang emitted `DT_NEEDED` on `libomp.so` + `libc++_shared.so`, which made +`System.loadLibrary("jllama")` fail with `UnsatisfiedLinkError` on every device (caught by the +`test-android-emulator` job; the released 5.0.5 arm64 lib had the same latent defect). The +`package-android-aar` job enforces a per-`.so` `DT_NEEDED` whitelist (`libc.so libm.so libdl.so +liblog.so libandroid.so`, plus `libOpenCL.so` for the OpenCL flavor) via `readelf -dW`, and +`LlamaLoader` now includes the swallowed `System.loadLibrary` message in its +"Directly from .apk/lib (…)" tried-path entry so a future dlopen reason is never invisible. + +**CI (`publish.yml`):** `test-java-llama-kotlin` (model-free unit tests); +`package-android-aar` (needs both Android native jobs) builds the core jar, stages the natives, +assembles both AARs, validates structure (entries, minSdk, classes.jar content, 16 KB alignment), +publishes to mavenLocal, and runs the **AGP consumer smoke test** — the minimal app fixture in +`.github/android-consumer-test/` resolves the AAR from mavenLocal and runs a full R8 +`assembleRelease` on the runner's preinstalled Android SDK (this is what actually validates +AGP/Android Studio consumption). **On-device runtime IS now CI-covered** via +`test-android-emulator`: a KVM-accelerated x86_64 emulator (API 30) runs the fixture's +`connectedDebugAndroidTest` — `System.loadLibrary` from the AAR's `jni/x86_64`, on-device +`GgufInspector`, and real native inference against the adb-pushed cached draft model +(AMD-Llama-135m). The job is a **release gate** (in both +publish `needs:` graphs) since PR #298, after running flake-free through the PR's validation +cycle. arm64 kernels + the Adreno/OpenCL flavor remain out of emulator scope — +the planned example app covers those on hardware. +Both publish jobs `need` these jobs (fail-loud release gating) and publish the AARs via Gradle: +snapshots to the Central snapshots repo (`publishAllPublicationsToCentralSnapshotsRepository`), +releases as a signed Central Portal bundle upload (staging repo → zip → Publisher API). +`llama-kotlin` rides the normal reactor `mvn -P release deploy`. ## Open TODOs diff --git a/README.md b/README.md index 94c13d02..df67388d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) ![Platform](https://img.shields.io/badge/Platform-Linux%20%7C%20macOS%20%7C%20Windows%20%7C%20Android-lightgrey) -[![llama.cpp b9870](https://img.shields.io/badge/llama.cpp-%23b9870-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b9870) +[![llama.cpp b9878](https://img.shields.io/badge/llama.cpp-%23b9878-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b9878) [![JPMS](https://img.shields.io/badge/JPMS-modular%20JAR-25A162)](https://openjdk.org/projects/jigsaw/) ![JUnit](https://img.shields.io/badge/tested%20with-JUnit6-25A162) [![JSpecify](https://img.shields.io/badge/JSpecify-1.0.0%20%40NullMarked-25A162)](https://jspecify.dev) @@ -102,8 +102,10 @@ Inference of Meta's LLaMA model (and others) in pure C/C++. - Text completion (blocking and streaming) with full control over sampling parameters. - OpenAI-compatible **chat completion** with automatic chat-template application, including streaming and tool/function calling support via the upstream server. -- **Embeddings** and **reranking** for retrieval pipelines. +- **Embeddings** (single and native-batched via `embed(Collection)`) and **reranking** for retrieval pipelines. +- **Runtime LoRA adapter control** — list the loaded adapters and change their scales at runtime without reloading the model (`getLoraAdapters()` / `setLoraAdapters(Map)`), the typed counterpart of the upstream `GET`/`POST /lora-adapters` endpoints. - **Text-to-speech** (`TextToSpeech`) over the two-model OuteTTS + WavTokenizer pipeline, returning WAV audio. +- **In-JVM GGUF quantization** (`LlamaQuantizer`) over llama.cpp's `llama_model_quantize` — convert a GGUF to another quantization scheme without shelling out to `llama-quantize`. - **Infilling** (fill-in-the-middle) for code models. - **Tokenize / detokenize** and **JSON-schema → grammar** conversion. - **Raw JSON endpoint handlers** mirroring the upstream llama.cpp HTTP server (`/completions`, `/v1/completions`, `/embeddings`, `/infill`, `/tokenize`, `/detokenize`). @@ -173,7 +175,7 @@ exclusive — and optionally a CPU Windows build. | Classifier | Backend | Target platform | Runtime requirement | |---|---|---|---| -| _(none)_ | CPU | Linux x86-64 / aarch64 / s390x, macOS x86-64 / aarch64, Windows x86-64 / x86 / aarch64 (Ninja Multi-Config + MSVC), Android aarch64 (CPU) | A JDK 8+ JVM. **Linux `aarch64` additionally requires glibc ≥ 2.39** (e.g. Ubuntu 24.04+, Debian 13+) — it is built natively on `ubuntu-24.04-arm`, matching upstream llama.cpp's own ARM binaries; older-glibc ARM hosts (Ubuntu 22.04, Debian 12, RHEL 8/9, Amazon Linux 2023) are not supported. Linux x86-64 keeps a glibc 2.17 floor (manylinux2014). **Windows `aarch64`** (Windows on ARM — Snapdragon X / Surface) is built natively on `windows-11-arm` and ships in the default JAR alongside the x86-64 / x86 natives. | +| _(none)_ | CPU | Linux x86-64 / aarch64 / s390x, macOS x86-64 / aarch64, Windows x86-64 / x86 / aarch64 (Ninja Multi-Config + MSVC), Android aarch64 + x86-64 (CPU) | A JDK 8+ JVM. **Linux `aarch64` additionally requires glibc ≥ 2.39** (e.g. Ubuntu 24.04+, Debian 13+) — it is built natively on `ubuntu-24.04-arm`, matching upstream llama.cpp's own ARM binaries; older-glibc ARM hosts (Ubuntu 22.04, Debian 12, RHEL 8/9, Amazon Linux 2023) are not supported. Linux x86-64 keeps a glibc 2.17 floor (manylinux2014). **Windows `aarch64`** (Windows on ARM — Snapdragon X / Surface) is built natively on `windows-11-arm` and ships in the default JAR alongside the x86-64 / x86 natives. | | `msvc-windows` | CPU (MSVC / Visual Studio generator) | Windows x86-64 and x86 | None beyond a JDK 8+ JVM. Same CPU backend as the default JAR's Windows natives, but compiled with the Visual Studio generator instead of `Ninja Multi-Config`. Both use the same MSVC toolchain (static `/MT` CRT), so they are functionally equivalent — provided as an alternate-toolchain option. | | `cuda13-windows-x86-64` | CUDA 13 | Windows x86-64 with NVIDIA GPU | NVIDIA driver + CUDA 13 Toolkit installed on the host (`cudart64_13.dll`, `cublas64_13.dll`, `cublasLt64_13.dll` resolvable on `PATH`). The runtime libraries are **not bundled** in the JAR; native-library load fails with `UnsatisfiedLinkError` if they are absent. No CPU fallback. | | `vulkan-windows-x86-64` | Vulkan | Windows x86-64 with a Vulkan 1.2+ GPU (NVIDIA / AMD / Intel) | A Vulkan runtime (`vulkan-1.dll`), which current GPU drivers install. No Vulkan SDK is needed at runtime. The most portable Windows GPU option (vendor-independent). | @@ -238,8 +240,10 @@ there. Pick **at most one** classifier (they are mutually exclusive): > [!NOTE] > Android `armeabi-v7a` (32-bit ARM) is **not** published. Only 64-bit -> `aarch64` Android binaries are shipped, both as the CPU-only default JAR -> and as `opencl-android-aarch64`. 32-bit Android devices are unsupported +> Android binaries are shipped: `aarch64` (devices) and `x86_64` +> (emulators, Chromebooks, x86-64 Android hardware) in the CPU-only default +> JAR and the `llama-android` AAR, plus `aarch64` as +> `opencl-android-aarch64`. 32-bit Android devices are unsupported > by the released artifacts; building from source via the > `.github/dockcross/dockcross-android-arm` toolchain is possible but not > wired into CI. @@ -309,7 +313,7 @@ Every `net.ladenthin.llama.*` system property recognised by the library, deep-sc | `net.ladenthin.llama.vision.image` | `llama/src/test/resources/images/test-image.jpg` (a CC-BY-4.0 / MIT-granted photo committed to the repo) | test | `MultimodalIntegrationTest` | Visual prompt image. Any png/jpeg/webp/gif works; the extension drives MIME detection. | | `net.ladenthin.llama.audio.model` | unset (test self-skips) | test | `AudioInputIntegrationTest` (llama.cpp discussion #13759) | Path to an audio-input model GGUF (e.g. Ultravox, Qwen2.5-Omni). | | `net.ladenthin.llama.audio.mmproj` | unset (test self-skips) | test | `AudioInputIntegrationTest` | Matching audio mmproj (encoder) GGUF. | -| `net.ladenthin.llama.audio.input` | unset (test self-skips) | test | `AudioInputIntegrationTest` | `.wav`/`.mp3` audio prompt clip; the extension drives format detection. | +| `net.ladenthin.llama.audio.input` | `src/test/resources/audios/sample.wav` (committed) | test | `AudioInputIntegrationTest` | `.wav`/`.mp3` audio prompt clip; the extension drives format detection. | | `net.ladenthin.llama.tts.ttc.model` | unset (test self-skips) | test | `TtsIntegrationTest` | Path to the OuteTTS text-to-codes GGUF. CI default is `OuteTTS-0.2-500M-Q4_K_M.gguf`. | | `net.ladenthin.llama.tts.vocoder.model` | unset (test self-skips) | test | `TtsIntegrationTest` | Path to the matching codes-to-speech vocoder GGUF. CI default is `WavTokenizer-Large-75-F16.gguf`. | @@ -516,9 +520,32 @@ ModelParameters modelParams = new ModelParameters() .enableEmbedding(); try (LlamaModel model = new LlamaModel(modelParams)) { float[] embedding = model.embed("Embed this sentence"); + // Batch form: one native dispatch for many inputs, results in request order. + List embeddings = model.embed(Arrays.asList("First sentence", "Second sentence")); } ``` +### Runtime LoRA adapter control + +Adapters loaded at model-load time (`addLoraAdapter(...)` / `addLoraScaledAdapter(...)`, optionally +`setLoraInitWithoutApply()` to start disabled) can be listed and re-scaled at runtime without +reloading the model — the typed counterpart of the upstream `GET`/`POST /lora-adapters` endpoints: + +```java +ModelParameters modelParams = new ModelParameters() + .setModel("models/base.gguf") + .addLoraScaledAdapter("models/adapter.gguf", 1.0f); +try (LlamaModel model = new LlamaModel(modelParams)) { + List adapters = model.getLoraAdapters(); // [{id=0, path=..., scale=1.0}] + model.setLoraAdapter(0, 0.5f); // re-scale at runtime + model.setLoraAdapters(Collections.emptyMap()); // disable all adapters +} +``` + +Per the upstream contract, a scale update lists the adapters to keep active — any adapter missing +from the map is set to scale `0` (disabled). The native side clears affected KV caches when the +effective adapter set changes. + ### Text-to-Speech `TextToSpeech` synthesizes audio from text over llama.cpp's OuteTTS pipeline. It is a separate @@ -545,6 +572,18 @@ Compatible GGUFs (the CI test defaults): OuteTTS [`OuteTTS-0.2-500M-GGUF`](https://huggingface.co/second-state/OuteTTS-0.2-500M-GGUF) + [`WavTokenizer`](https://huggingface.co/ggml-org/WavTokenizer). +### GGUF Quantization + +`LlamaQuantizer` converts a GGUF to another quantization scheme in-process (llama.cpp's +`llama_model_quantize` — the `llama-quantize` tool without the separate binary): + +```java +LlamaQuantizer.quantize("model-f16.gguf", "model-q4_k_m.gguf", QuantizationType.Q4_K_M); +// Re-quantizing an already-quantized GGUF degrades quality and must be opted into: +LlamaQuantizer.quantize("model-q8_0.gguf", "model-q4_0.gguf", QuantizationType.Q4_0, + /* threads */ 0, /* allowRequantize */ true); +``` + ### Raw JSON Endpoints For direct access to the upstream llama.cpp server API, the following methods take a JSON request and return @@ -556,6 +595,53 @@ a JSON response, matching the HTTP server's contract: Server state is exposed via `getMetrics()`, `eraseSlot(int)`, `saveSlot(int, String)`, `restoreSlot(int, String)`, and `getModelMeta()`. +### Conversation checkpoints: rewind + fork (`Session`) + +A `Session` can be snapshotted and branched — the KV-cache slot state and the transcript move +together, so native state and history can never drift apart: + +```java +try (Session session = new Session(model, 0, "You are terse.")) { + session.send("My name is Alice."); + SessionCheckpoint cp = session.checkpoint("checkpoints/turn1.bin"); + + session.send("Tell me a joke."); + session.rewind(cp); // undo everything after the checkpoint + session.send("Tell me a story instead."); // retry from the branch point + + // Branch into a second slot (model loaded with setParallel(2)+): + try (Session forked = session.fork(1, "checkpoints/branch.bin")) { + forked.send("Answer as a pirate."); // both sessions continue independently + } +} +``` + +Checkpoint files are caller-managed (KV dumps grow with context usage) and both operations are +rejected while a stream is in progress. For plain transformer models a rewind is also achievable +cheaply by resending a truncated history with `cache_prompt` (prefix reuse); checkpoints make the +branch point exact and are the only reliable rollback for recurrent/hybrid models (e.g. +Granite-4), whose state cannot be recomputed from a prefix. + +### GGUF metadata inspection (no model load) + +`GgufInspector` reads a GGUF's header and key/value table **without loading the model** — pure +Java, no native library, cost independent of file size (parsing stops before the tensor data). +Useful for model pickers and download validators: + +```java +GgufMetadata meta = GgufInspector.read(Paths.get("models/Qwen3-0.6B-Q4_K_M.gguf")); +meta.getArchitecture(); // Optional[qwen3] +meta.getModelName(); // Optional[Qwen3 0.6B] +meta.getParameterCount(); // OptionalLong[751632384] +meta.getContextLength(); // OptionalLong[40960] (.context_length) +meta.getFileType(); // OptionalLong[15] (llama_ftype, cf. QuantizationType) +meta.getChatTemplate(); // Optional[{{- ... }}] +meta.getEntries(); // full decoded key/value table +``` + +Supports GGUF v2/v3, little- and big-endian (auto-detected), and fails loud on v1/corrupt files. +For metadata of an already-loaded model use `getModelMeta()` instead. + ### Prompt and KV Cache Reuse Prompt-prefix reuse is enabled by default in llama.cpp and can be controlled per request with @@ -730,13 +816,71 @@ try (NativeServer server = new NativeServer( } ``` -Differences from `OpenAiCompatServer`: it **loads its own model** from the arguments (an independent -lifecycle, like `llama-server.exe`, not a shared `LlamaModel`), it is **single-instance per +Differences from `OpenAiCompatServer`: with the classic constructor it **loads its own model** from +the arguments (an independent lifecycle, like `llama-server.exe`), it is **single-instance per process**, it serves the **WebUI** (in released jars — local `cmake` builds ship the empty-asset stub, so no UI there), and it is **not available on Android** (the upstream server needs `posix_spawn`). Readiness: poll `GET /health`. No SSL (plain HTTP — bind localhost or front with a TLS proxy). +#### Attach mode — serve an already-loaded `LlamaModel` + +`NativeServer` can also **attach** the full upstream HTTP frontend (routes, WebUI, resumable +streaming) to a `LlamaModel` you already loaded — one copy of the weights, shared between direct +JNI calls and HTTP: + +```java +try (LlamaModel model = new LlamaModel(new ModelParameters().setModel("models/model.gguf")); + NativeServer server = new NativeServer(model, "--host", "127.0.0.1", "--port", "8080").start()) { + // HTTP (incl. WebUI in released jars) and direct Java calls share the same loaded model. + String direct = model.complete(new InferenceParameters("2+2=").withNPredict(4)); + Thread.currentThread().join(); +} +``` + +In attach mode the arguments carry only the HTTP-side flags (`--host`, `--port`, `--api-key`, …; +no `-m`), the server reports healthy immediately (the model is already loaded), and the **caller +keeps ownership of the model** — close the server before the model, never the other way around. + +#### Router mode — multi-model management + +Started **without** a model argument, the upstream server runs in **router mode**: it lists models +from `--models-dir`, loads/unloads them on demand (`GET /models`, `POST /models/load`, +`POST /models/unload`, per-request `"model"` selection) and serves each model from a **worker +subprocess**. Upstream spawns workers by re-executing its own binary — inside a JVM that binary is +`java`, so before starting an embedded router you must point the worker spawn at this library's +bootstrap: + +```java +String javaBin = System.getProperty("java.home") + File.separator + "bin" + File.separator + "java"; +NativeServer.setWorkerCommand(javaBin, "-cp", System.getProperty("java.class.path"), + "net.ladenthin.llama.server.NativeServer"); +try (NativeServer router = new NativeServer( + "--host", "127.0.0.1", "--port", "8080", "--models-dir", "models").start()) { + Thread.currentThread().join(); // each loaded model runs as a fresh worker JVM +} +``` + +Worker-command tokens may not contain whitespace (the value is whitespace-split natively). + +**Typed model management (`RouterClient`).** Instead of hand-rolling HTTP+JSON against the +management endpoints, use `server.RouterClient` — a plain-HTTP typed client (works against the +embedded router above or any external `llama-server` router): + +```java +RouterClient client = new RouterClient(8080); +List models = client.listModels(); // GET /models, typed status per entry +client.loadModel("Qwen3-0.6B-Q4_K_M"); // POST /models/load (non-blocking) +client.awaitModelLoaded("Qwen3-0.6B-Q4_K_M", 240_000L); // poll until LOADED; fails fast if the + // worker died (exit code in the message) +client.unloadModel("Qwen3-0.6B-Q4_K_M"); // POST /models/unload +``` + +`RouterModel` carries the identifier, the lifecycle status +(`UNLOADED`/`LOADING`/`LOADED`/`SLEEPING`/`DOWNLOADING`/`DOWNLOADED`), and the router's +failed-worker marker. Chat requests then select a model per request via the standard +`"model"` field on `POST /v1/chat/completions`. + ### LangChain4j integration A separate artifact, **`net.ladenthin:llama-langchain4j`**, adapts a `LlamaModel` to @@ -832,6 +976,16 @@ Flowable tokens = Flowable.using( ``` #### Kotlin Flow (Android / coroutines) + +Ready-made: the optional [`net.ladenthin:llama-kotlin`](llama-kotlin/README.md) artifact ships +`generateFlow`/`generateChatFlow` extensions (close-on-cancellation included) plus `suspend` +wrappers whose coroutine cancellation is wired to the binding's cooperative `CancellationToken`: + +```kotlin +model.generateChatFlow(params).flowOn(Dispatchers.IO).collect { print(it.text) } +``` + +Hand-rolled equivalent (no extra dependency): ```kotlin fun llama(model: LlamaModel, params: InferenceParameters) = flow { model.generate(params).use { iterable -> @@ -894,7 +1048,36 @@ The `LogLevel` enum values passed to the callback correspond to the native llama > **Minimum Android version: API 28 (Android 9.0 Pie).** Devices running > Android 8.1 (API 27) or earlier are not supported. -You can use this library in Android project. +### Option 1 (recommended): the `llama-android` AAR from Maven Central + +One dependency line in Android Studio — no submodule, no NDK build, no manual ProGuard rules: + +```kotlin +dependencies { + implementation("net.ladenthin:llama-android:5.0.6") + // or, for Qualcomm Adreno GPUs (device must provide an OpenCL ICD): + // implementation("net.ladenthin:llama-android-opencl:5.0.6") + + // optional Kotlin coroutines facade (Flow streaming + suspend wrappers): + implementation("net.ladenthin:llama-kotlin:5.0.6") +} +``` + +The AAR carries the full `net.ladenthin:llama` Java API, the CI-built native libraries for +`arm64-v8a` (devices) **and** `x86_64` (Android Studio emulator, Chromebooks — app bundles +split per ABI so phones download only arm64), both 16 KB page-size compliant, consumer +R8/ProGuard rules (applied automatically), and a manifest `minSdkVersion 28` that AGP +enforces against your app. CI boots an x86_64 emulator and runs real on-device inference +against every AAR build. +Do **not** also depend on the desktop `net.ladenthin:llama` JAR in the same app — the AAR +already contains those classes, and the JAR would drag ~70 MB of desktop natives into your +APK. See [`llama-android/README.md`](llama-android/README.md) and +[`llama-kotlin/README.md`](llama-kotlin/README.md) for details. + +### Option 2 (advanced): build from source inside your app + +Use this only if you need to patch the native layer or build for an ABI this project does +not ship. 1. Add java-llama.cpp as a submodule in your an droid `app` project directory ```shell git submodule add https://github.com/bernardladenthin/java-llama.cpp @@ -956,7 +1139,7 @@ keep class net.ladenthin.llama.** { *; } Forward-looking ideas being tracked for this fork: - **Adopt feature ideas from the Kotlin Llama Stack client.** Candidates (multimodal image input, typed chat messages, async API, batch inference, typed usage/timings) are inventoried with effort estimates in [`docs/feature-investigation-llama-stack-client-kotlin.md`](docs/feature-investigation-llama-stack-client-kotlin.md), derived from [`ogx-ai/llama-stack-client-kotlin`](https://github.com/ogx-ai/llama-stack-client-kotlin). -- **Ship a directly Android-capable artifact.** Building on the existing [Importing in Android](#importing-in-android) flow and the `opencl-android-aarch64` classifier (see [Choosing the right classifier](#choosing-the-right-classifier)), the goal is a first-class Android Maven artifact — including a typed image-input helper for VLMs such as Qwen2.5-VL — so downstream Android projects can drop their dependency on [`ogx-ai/llama-stack-client-kotlin`](https://github.com/ogx-ai/llama-stack-client-kotlin) entirely. +- **Ship a directly Android-capable artifact — DONE.** `net.ladenthin:llama-android` / `llama-android-opencl` (AAR, arm64-v8a, minSdk 28, consumer ProGuard rules, 16 KB page-size compliant) plus the optional `net.ladenthin:llama-kotlin` coroutines façade ship from this repo — see [Importing in Android](#importing-in-android). Typed image input for VLMs is covered by `ContentPart.imageBytes(...)` / `imageFile(...)` (see the multimodal section), so downstream Android projects can drop their dependency on [`ogx-ai/llama-stack-client-kotlin`](https://github.com/ogx-ai/llama-stack-client-kotlin) entirely. A dedicated example app remains a follow-up. - **Resolve all upstream `kherud/java-llama.cpp` open issues.** All 37 open issues at fork time are catalogued with per-issue verdicts in [`docs/history/49be664_open_issues.md`](docs/history/49be664_open_issues.md); fixes land in this fork as they are completed. Vision inputs (issues [#103](docs/history/49be664_open_issues.md#103--vlm-support--image-input-for-multimodal-models) and [#34](docs/history/49be664_open_issues.md#34--support-multimodal-inputs)) are now wired end to end through blocking, typed, streaming, and OpenAI-compatible request surfaces. ## Troubleshooting @@ -993,10 +1176,19 @@ The system's updated C++ runtime will be used instead, resolving the crash. **Bindings / wrappers** +- [kherud/java-llama.cpp](https://github.com/kherud/java-llama.cpp) — the upstream Java binding this project was forked from (see the note at the top of this README); development continues independently here, with the fork-time upstream issues catalogued in [`docs/history/49be664_open_issues.md`](docs/history/49be664_open_issues.md). +- [llamacpp4j](https://github.com/sebicom/llamacpp4j) — alternative Java/JNI binding to llama.cpp (SWIG-generated facade); pre-GGUF, dormant since 2023 but historically the other Java JNI option. +- [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) — the Python llama.cpp binding; the de-facto feature benchmark among llama.cpp bindings (server mode, multimodal, speculative decoding). +- [LLamaSharp](https://github.com/SciSharp/LLamaSharp) — C#/.NET llama.cpp binding with per-backend runtime packages (CPU/CUDA/Vulkan/Metal), the .NET analogue of this project's classifier matrix. +- [node-llama-cpp](https://github.com/withcatai/node-llama-cpp) — Node.js/TypeScript llama.cpp binding (prebuilt binaries, JSON-schema-constrained output, function calling). - [LLaMAndroid](https://github.com/Rattlyy/LLaMAndroid/tree/main/app) — Android app demonstrating usage of llama.cpp bindings. -- [llama-stack-client-kotlin](https://github.com/ogx-ai/llama-stack-client-kotlin) — Kotlin client for the Llama Stack API. +- [llama-stack-client-kotlin](https://github.com/ogx-ai/llama-stack-client-kotlin) — Kotlin client for the Llama Stack API with an ExecuTorch-backed local-inference path (the [`llama-android`](llama-android/) AAR + [`llama-kotlin`](llama-kotlin/) façade cover the same on-device ground natively). - [llama.cpp-android-tutorial](https://github.com/JackZeng0208/llama.cpp-android-tutorial) — Step-by-step tutorial for running llama.cpp on Android. -- [llamacpp4j](https://github.com/sebicom/llamacpp4j) — alternative Java/JNI binding to llama.cpp (SWIG-generated facade); pre-GGUF, dormant since 2023 but historically the other Java JNI option. + +**Other local inference stacks (no llama.cpp JVM binding)** + +- [Ollama](https://github.com/ollama/ollama) — llama.cpp-based local model runner with its own HTTP API and model registry. This project's OpenAI-compatible server implements the Ollama-native API surface (`/api/version`, `/api/tags`, `/api/show`, `/api/chat`, `/api/generate`), so Ollama-speaking clients (e.g. VS Code Copilot's Ollama provider) work against an in-process jllama model. +- [ExecuTorch](https://github.com/pytorch/executorch) — PyTorch's on-device inference runtime (`.pte` models, XNNPACK/NPU delegates); the engine behind `llama-stack-client-kotlin`'s local mode and the main non-llama.cpp alternative for Android on-device inference (GGUF is not supported there — different model format ecosystem). **Pure-Java single-model inference (no JNI / no llama.cpp)** — Alfonso² Peterssen's `*.java` family of standalone, dependency-free Java inference runtimes, one per model architecture. Useful when JNI is unavailable (e.g. some sandboxes / GraalVM native-image scenarios) or when you want a single jar with no native side at all. Different design point from this project, which prioritises GGUF compatibility and llama.cpp performance via JNI. diff --git a/REUSE.toml b/REUSE.toml index 55720a91..56bccff6 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -20,6 +20,7 @@ path = [ "docs/history/CHAT_INTEGRATION_SUMMARY.md", "docs/history/REFACTORING.md", "llama/src/test/resources/images/README.md", + "llama/src/test/resources/audios/README.md", ".github/PULL_REQUEST_TEMPLATE.md", ".github/ISSUE_TEMPLATE/bug_report.md", ".github/ISSUE_TEMPLATE/feature_request.md", @@ -55,6 +56,7 @@ SPDX-License-Identifier = "MIT" path = [ ".github/dockcross/dockcross-android-arm", ".github/dockcross/dockcross-android-arm64", + ".github/dockcross/dockcross-android-x86_64", ".github/dockcross/dockcross-linux-arm64-lts", ".github/dockcross/dockcross-manylinux2014-x64", ".github/dockcross/dockcross-manylinux_2_28-x64", @@ -89,6 +91,13 @@ path = "llama/src/test/resources/images/test-image.jpg" SPDX-FileCopyrightText = "2026 Bernard Ladenthin " SPDX-License-Identifier = "MIT" +# Test audio clip recorded by the project author; granted under MIT for this +# project (see llama/src/test/resources/audios/README.md). +[[annotations]] +path = "llama/src/test/resources/audios/sample.wav" +SPDX-FileCopyrightText = "2026 Bernard Ladenthin " +SPDX-License-Identifier = "MIT" + # JNI header files from Oracle (GPL-2.0-only WITH Classpath-exception-2.0) [[annotations]] path = [ diff --git a/TODO.md b/TODO.md index e6390529..4e23bde7 100644 --- a/TODO.md +++ b/TODO.md @@ -13,320 +13,40 @@ cross-cutting initiative. ## Open — jllama-specific -### NativeServer — reuse an already-loaded `LlamaModel` (open, enhancement) - -`net.ladenthin.llama.server.NativeServer` (the native-transport server mode that runs the full -upstream `llama_server` — WebUI included — inside `libjllama` over JNI) currently loads its **own** -model from the forwarded argv, exactly like running `llama-server.exe`. This is the "independent -lifecycle" v1: simple, and every llama-server flag is forwarded verbatim. - -**Enhancement:** let `NativeServer` optionally attach to an **already-loaded** `LlamaModel`'s -`server_context` instead of loading a second copy of the weights (saves the RAM/VRAM and load time -of a duplicate model when a caller already has a `LlamaModel` open). Feasibility notes from the -initial investigation: - -- The upstream HTTP transport (`server_http_context`) and the route bundle - (`server_routes routes(params, ctx_server)`) only need a reference to a `server_context`. A - `LlamaModel` already owns and drives one (`jllama_context` in `jni_helpers.hpp`), and its JNI - methods already post tasks to that context's queue — so a second driver (the HTTP routes) posting - to the same queue is plausible; the queue is the synchronization point. -- The real work is **lifecycle/ownership**: today `llama_server()` owns the whole flow (parse → - backend init → `ctx_server.load_model` → `start_loop` on its own thread → cleanup). Reuse would - need a *different* entry that skips model loading and the `start_loop`/backend ownership (the - existing `LlamaModel` worker already runs the loop), registers the HTTP routes against the shared - `server_context`, and starts only `server_http_context`. That is a separate, smaller C++ entry - point (not `llama_server`), plus reconciling params (the loaded model's params vs. server params) - and ensuring only one thread drives `update_slots`. -- Logging: `llama_server` calls `common_init()` which routes llama.cpp logging to stderr/file; a - reuse path must not clobber the JNI log callback a `LlamaModel` consumer may rely on. - -Until then, run `NativeServer` standalone (it owns the process's llama backend + logging while -running), or use the Java-transport `OpenAiCompatServer` when sharing a `LlamaModel`. - -### PIT gate not hermetic — `value.ContentPart.audioFile(Path)` (open) - -The PIT mutation gate reaches 100% **only when the audio test fixture is present**. Without it the -run is **98%**: 4 `NO_COVERAGE` mutants in `value.ContentPart.audioFile(Path)` (the null file-name -guard, the `.wav`/`.mp3` extension dispatch, and `Files.readAllBytes`). The only test that exercises -that method is `AudioInputIntegrationTest`, which is model-/fixture-gated and self-skips (`Assume`) -when no audio clip is supplied (`net.ladenthin.llama.audio.input` — no committed default). So any -environment lacking the clip (e.g. a network-restricted sandbox) reds the gate. Fix: add a hermetic -temp-file unit test for `audioFile(Path)` — write a few bytes to a `@TempDir` `*.wav` / `*.mp3` and -assert the format dispatch — mirroring the existing `imageFile(Path)` temp-file tests (PNG/JPG/GIF/ -WEBP), which already make the image path hermetic. See -[`../workspace/policies/pit-mutation-testing.md`](../workspace/policies/pit-mutation-testing.md) §4. - -### Code audit — pre-existing correctness / safety findings (RESOLVED — PRs #258 + #260) - -A multi-area audit (2026-06-20) of the **existing** codebase surfaced 18 correctness/safety findings, -intentionally **split into tiers so each could land as its own small, focused PR**. **All 18 are now -fixed and merged** — Tiers 1–3 in **#258**, the deferred `LlamaLoader` extraction race in **#260** — -with regression tests added in **#261 / #262**. The full per-finding rationale lives in those PRs and -their commits; the concise record below is kept for traceability. Nothing in this section is open -except the optional follow-up noted at the end. - -**`LlamaLoader` native-lib extraction temp-path race — DONE (atomic write + content-reuse).** -`extractFile` now (1) reuses a byte-identical existing copy instead of rewriting it — so it never -replaces a file another JVM has already loaded (which fails on Windows) — and (2) otherwise extracts -to a per-attempt unique temp file and **atomically moves** it into place, so a concurrent loader can -never observe a half-written library. `jllama` is statically linked (`BUILD_SHARED_LIBS OFF`), so the -extracted file is self-contained — no multi-DLL co-location to coordinate. Verified by the -`NativeLibraryLoadSmokeTest` (real extract+load on macOS) + a `resourceMatchesFile` unit test; the -Windows locked-replace path is exercised by CI's Windows jobs. - -**Tier 1 — high impact (#258, `a4325ff`)** - -- **N1** — unhandled C++ exceptions crossing the JNI boundary → JVM abort; every entry point (incl. the - public `LlamaModel.jsonSchemaToGrammar`, plus encode/tokenize/embeddings/rerank/infill/applyTemplate) - now converts the failure to a `LlamaException` instead of crashing the process. -- **N2** — `parse_string_array` null-deref (null element / OOM) + per-iteration JNI local-ref leak. -- **J1** — `close()` / native `delete()` double-free under concurrent close → `synchronized` close. -- **P1** — `ServerMetrics` cumulative token totals truncated `int` → negative → `Timings.promptN` / - `predictedN` widened to `long`. - -**Tier 2 — medium (#258, `a4325ff` + `3e500aa`)** - -- **S1** — unbounded request body → OOM DoS → 16 MiB cap + `Content-Length` pre-check; oversized → HTTP 413. -- **N3** — streaming-reader use-after-free → reader held as a `shared_ptr` and copied out under the lock - before `next()`. -- **J5** — `Session` permanently wedged on an abandoned stream → `Session.cancelStream()` clears the guard - and rolls back the pending user turn. -- **J3** — `LlamaIterator.hasNext` made `volatile` (observed across a cross-thread `cancel()`). -- **N4** — log callback made `noexcept` + non-throwing env lookup (no exception unwinds through llama.cpp - C frames from an unattached thread). - -**Tier 3 — hardening (#258, `ac3ad6d`)** - -- **S3** — constant-time bearer-key comparison (`MessageDigest.isEqual`). -- **S2** — SSE heartbeat pool sized to the core count (one stalled client can't starve other streams). -- **P3** — `ChatMessage.toolCalls` defensively copied + wrapped unmodifiable. -- **NaN/Inf** — non-finite `float`/`double` rejected at `JsonParameters.withScalar` (they would serialize - to the invalid JSON tokens `NaN`/`Infinity`). -- **OSInfo** — armhf-detection `exec()` routed through a drain-and-close helper (no fd leak / pipe-full hang). -- **completeBatch** — `completeBatch` / `completeBatchWithStats` / `chatBatch` join every future before - propagating the first failure (no abandoned in-flight requests). -- **Docs** — `/props` + Ollama discovery routes documented as intentionally-unauthenticated metadata; - `parseProbabilities` documented as last-wins on duplicate token text (use `parseLogprobs` for lossless data). - -**Still open — optional follow-up (lower priority):** full per-process extraction **directory** isolation -+ a `cleanup()` that recursively removes dead-process dirs. Now that writes are atomic and content-checked -this is a tidiness improvement (stops the shared-tmpdir `cleanup()` racing a live peer's flat file), not a -correctness fix — and it still needs the Windows locked-file co-design noted above. - -### OpenAI-compatible HTTP endpoint (shipped; follow-ups open) - -`net.ladenthin.llama.server.OpenAiCompatServer` is the single OpenAI-compatible server (JDK -`com.sun.net.httpserver`, no new dependency, fat-jar `Main-Class`). It exposes the OpenAI routes -`POST /v1/chat/completions` (streaming SSE + non-streaming), `/v1/completions`, `/v1/embeddings`, -`/v1/rerank`, `/infill`, `GET /v1/models`, `GET /health` and `GET /props`, **plus three alternative -protocol surfaces** — Ollama-native (`/api/version`, `/api/tags`, `/api/show`, `/api/chat`, -`/api/generate`), Anthropic Messages (`POST /v1/messages`) and OpenAI Responses (`POST /v1/responses`). -Every route is also reachable without the `/v1` prefix and sits behind a CORS filter. The CLI is parsed -by the testable `OpenAiServerCli`. (Consolidated from PR #240's JDK + streaming server and #242's -NanoHTTPD server; NanoHTTPD + its dependency deleted.) - -**IDE/agent backend hardening — DONE** (from the deep-research investigation -[`docs/feature-investigation-ide-agent-backend.md`](docs/feature-investigation-ide-agent-backend.md); -primary goal: agentic tool-calling with Qwen): - -- Agentic tool-calling verified wire-correct: C++ guard pins `tool_calls.function.arguments` as a JSON - **string** (not object) at b9739 (llama.cpp #20198), plus the existing `finish_reason:"tool_calls"` - test. -- `stream_options.include_usage` forwarded (new `InferenceParameters.withStreamOptions`) so the trailing - usage chunk is emitted, and `OpenAiSseFormatter.ensureUsageCachedTokens` guarantees - `usage.prompt_tokens_details.cached_tokens` (fixes the Copilot custom-endpoint crash, vscode #273482). -- `response_format` (`json_object`/`json_schema`) forwarded for structured outputs. -- `POST /infill` (FIM autocomplete for llama.vscode/Twinny/Tabby/Continue) → native `handleInfill`. -- `POST /v1/rerank` (RAG) → `handleRerank` reshaped to `results`/`data` (`OaiRerankSupport`). -- CORS preflight + `Access-Control-Allow-Origin`; bare-path (no `/v1`) aliases; `cache_prompt=true` - default; `--mmproj` (vision), `--embedding`, `--reranking` CLI flags. -- **Alternative protocol surfaces** (pure translation over the OpenAI core; tool calls reconstructed by - `ToolCallDeltaAccumulator`): **Ollama-native** (`/api/version`, `/api/tags`, `/api/show`, `/api/chat` - with NDJSON streaming, `/api/generate` prompt-completion/FIM — `OllamaApiSupport`; `/api/show` - advertises tools/insert/vision + context length); **Anthropic Messages** (`POST /v1/messages`, SSE - events — `AnthropicApiSupport` + `AnthropicStreamTranslator`); **OpenAI Responses** (`POST - /v1/responses`, SSE events — `ResponsesApiSupport` + `ResponsesStreamTranslator`). -- **`GET /props`** (llama.cpp-native): `default_generation_settings.n_ctx` + `modalities` so autocomplete - clients (llama.vscode) size their context window (`OpenAiSseFormatter.propsJson`). -- Gated **integration round-trips** over a real socket, run in CI's `test-java-linux-x86_64` job, - self-skipping when the model is absent — structural assertions only: - - `OpenAiCompatServerIntegrationTest` (Qwen3-0.6B, chat mode): OpenAI chat (non-stream/stream/tools/ - models) plus Ollama `/api/chat` + discovery, Anthropic `/v1/messages`, OpenAI `/v1/responses` - (non-stream + stream) and `/props`. - - `OpenAiServerEmbeddingsIntegrationTest` (CodeLlama-7B + `enableEmbedding`): `/v1/embeddings` (+ bare - alias). - - `OpenAiServerRerankIntegrationTest` (jina-reranker + `enableReranking`): `/v1/rerank` (sorted - `results`/`data`, `top_n` cap). - - `OpenAiServerCompletionIntegrationTest` (CodeLlama-7B): `/v1/completions`, `/infill`, and Ollama - `/api/generate` (plain + FIM via `suffix`). - -**Open follow-ups (deferred):** - -- **Streaming raw-completion path — IN PROGRESS (no new native method needed).** The earlier premise was - wrong: a streaming raw-completion JNI path **already exists** (`requestCompletion`/`receiveCompletionJson`, - exposed as `LlamaModel.generate(InferenceParameters) → LlamaIterable`), so this is **Java-only server - wiring**, not JNI/C++. Progress: **(a) streaming `POST /v1/completions` — DONE** (`OpenAiRequestMapper` - `toCompletionParameters` + `OpenAiBackend.streamCompletions` driving `generate()` + an - `OpenAiSseFormatter.completionChunk` `text_completion` chunk + the `streamCompletions` SSE handler; - HTTP test green). **Remaining:** (b) **token-streaming Ollama `/api/generate`** (translate the - `text_completion` chunks to NDJSON, mirroring the chat→Ollama translator) and (c) **Continue's native - `POST /completion`** route in the llama.cpp-native streaming shape (`{"content":…,"stop":…}` per chunk). +### LlamaLoader extraction-directory isolation (optional follow-up, low priority) + +Left over from the 2026-06-20 code audit (18/18 findings fixed in PRs #258/#260, regression tests in +#261/#262 — see the Done section): full per-process extraction **directory** isolation + a `cleanup()` +that recursively removes dead-process dirs. Since extraction writes are atomic and content-checked, +this is a tidiness improvement (stops the shared-tmpdir `cleanup()` racing a live peer's flat file), +not a correctness fix — and it needs Windows locked-file co-design. + +### OpenAI-compatible HTTP endpoint — open follow-ups (Java transport; deprioritized) + +The `OpenAiCompatServer` surface itself is shipped (routes, protocol translations, integration +round-trips — see CLAUDE.md "Two server modes"). **Owner priority: the native-transport +`NativeServer` comes first; Java-transport-only items below are deliberately deprioritized.** + +- **Streaming raw-completion remainder:** (a) streaming `POST /v1/completions` is DONE; remaining are + (b) token-streaming Ollama `/api/generate` (translate `text_completion` chunks to NDJSON, mirroring + the chat→Ollama translator) and (c) Continue's native `POST /completion` route in the llama.cpp-native + streaming shape (`{"content":…,"stop":…}` per chunk). Java-only server wiring. - **Future *output* modalities (audio / image) — design note, not yet actionable.** llama.cpp's server - produces **text** (plus embeddings/rerank); it does **not** generate images or audio output, so there is - no engine behind a TTS/image-gen response today and building that API surface now would be dead code. - When/if it becomes real, the integration points are already isolated: a new `OpenAiBackend.stream*` - primitive + an `OpenAiSseFormatter.*Chunk` formatter per modality, wired into a per-route handler — the - exact shape the text `streamCompletions` path now establishes. Two concrete future hooks: (1) llama.cpp's - **OuteTTS** audio path (if it lands in the embedded server) → an `/v1/audio/speech`-style route emitting - audio chunks; (2) routing image/audio generation to an **external** model behind the same server (the - binding would proxy, not generate). Keep `LlamaOutput`/chunk formatters modality-neutral so neither - requires reworking the streaming core. + produces text (plus embeddings/rerank) only; the integration points are isolated (a new + `OpenAiBackend.stream*` primitive + `OpenAiSseFormatter.*Chunk` per modality). Two future hooks: + OuteTTS behind an `/v1/audio/speech`-style route; proxying image/audio generation to an external + model. Keep chunk formatters modality-neutral. - **Incremental tool-call streaming on the alternative surfaces.** Ollama/Anthropic/Responses emit each - tool call *whole* at end-of-stream (reconstructed by `ToolCallDeltaAccumulator`) rather than streaming - argument fragments. Fine for clients that apply tool calls after generation; revisit if a client needs + tool call whole at end-of-stream (`ToolCallDeltaAccumulator`); revisit only if a client needs incremental `input_json_delta` / `function_call_arguments.delta` fidelity. -- **Per-model FIM template registry** (Qwen/CodeLlama/DeepSeek v1&V2/StarCoder2/Codestral) — only needed - if we also expose `/v1/completions`-with-`suffix` FIM; `/infill` (and Ollama `/api/generate` with a - `suffix`) applies the model's FIM tokens server-side, so this is lower value. -- **Multi-model registry.** Only one model id is advertised/served today; serving several would need - multi-model load + lifecycle management. -- **Manual real-client validation.** Gated server-side round-trips now exist for every surface (above). - What remains is manual validation against the actual editor clients — point Copilot's Ollama provider / - a Custom Endpoint, Claude Code, and a Responses client at the running server — since a server-side - round-trip confirms the wire shapes but not each client's own parser. -- **Gemma 4 tool-calling validation.** Confirm the pinned llama.cpp (`b9789`) includes the Gemma 4 - tool-call parser fixes; if not, bump per the upgrade procedure. -- **NativeServer — wire upstream `server.cpp` routes to JNI (in progress; scaffold landed `dd264b2`).** - The upstream HTTP transport (`tools/server/server-http.cpp` + the cpp-httplib backend) is already - compiled into `libjllama`, and a `server.NativeServer` Java scaffold + `NativeServerSmokeTest` landed - in `dd264b2`. **Remaining:** wire the upstream `server.cpp` route table (the one upstream TU still - excluded from the build — it carries `main()` + route wiring) to JNI so the native HTTP server (and the - embedded WebUI) can be started/stopped from Java. This is the **native-transport alternative** to the - JDK-based `OpenAiCompatServer` (which is complete and the primary surface); value is shipping the full - llama.cpp server + WebUI in-process without a separate `llama-server` binary. JNI + C++ work. - -### Windows native classifiers — default flip (Ninja default + MSVC classifier) + CUDA/Vulkan/OpenCL GPU - -**Design decision UPDATED by the owner (supersedes the earlier "MSVC is the permanent default" -note): the default Windows CPU JAR is now the Ninja Multi-Config build, and the MSVC / Visual -Studio build ships as the `msvc-windows` classifier.** Rationale: both generators use the same MSVC -toolchain (`cl.exe`, static `/MT` CRT) on the same runner, so the produced DLLs are functionally -equivalent with identical runtime dependencies — the only difference is build-system plumbing + -sccache caching. Making Ninja the default gives the most-pulled JAR the cache; MSVC stays available -as a classifier. Three Windows GPU classifiers were added at the same time (x86_64 only, all Ninja): -`cuda13-windows-x86-64`, `vulkan-windows-x86-64`, `opencl-windows-x86-64`. - -**Why the cache needs Ninja.** The cache mechanism is the CMake *compiler launcher* -(`-DCMAKE_C_COMPILER_LAUNCHER=sccache`); the Visual Studio generator ignores it entirely, only -Ninja/Makefile generators honor it. Upstream llama.cpp also builds its Windows artifacts with Ninja -Multi-Config + MSVC. - -**What shipped (this branch — pending first CI validation):** -- **CPU build jobs:** `build-windows-x86_64` / `build-windows-x86` are now **Ninja** (default, - artifacts `Windows-{arch}-libraries`); `build-windows-x86_64-msvc` / `build-windows-x86-msvc` are - **MSVC** (artifacts `Windows-{arch}-msvc`). `test-java-windows-x86_64` (default/Ninja) and - `test-java-windows-x86_64-msvc` both load the DLL via JNI and run the full model-backed suite. -- **GPU build jobs (x86_64, Ninja, build the artifact only — runners have no GPU, and a - GPU-linked jllama_test can't be enumerated there; C++ suite runs on the CPU jobs):** - `build-windows-x86_64-cuda` (`Jimver/cuda-toolkit@v0.2.35` CUDA `13.2.0` + `-DGGML_CUDA=ON`), - `build-windows-x86_64-vulkan` (`jakoch/install-vulkan-sdk-action` + `-DGGML_VULKAN=ON`), - `build-windows-x86_64-opencl` (`build_opencl_windows.bat` stages the ICD loader + `-DGGML_OPENCL=ON`). -- **`CMakeLists.txt`** — OS-aware backend routing (CUDA/OpenCL → Windows trees, new Vulkan branch). -- **`.github/build.bat`** — also wraps nvcc with sccache for CUDA builds. -- **`.github/build_opencl_windows.bat`** — new, Windows analogue of `build_opencl_android.sh`. -- **`pom.xml`** — profiles `windows-msvc` / `cuda-windows` / `vulkan-windows` / `opencl-windows` - (classifiers `msvc-windows` / `cuda13-windows-x86-64` / `vulkan-windows-x86-64` / `opencl-windows-x86-64`). -- **`publish.yml`** — the `package` / `publish-snapshot` / `publish-release` jobs download each - non-default artifact into `src/main/resources_windows_{msvc,cuda,vulkan,opencl}/` and activate the - four profiles; all five Windows build jobs are in the `package` `needs:` graph. -- Docs: `README.md` classifier table + `CLAUDE.md` "Windows native classifiers" section. - -**Verification — first CI run done (PR #276, run 28327740376).** Green on the first try: default Ninja -CPU flip (x64+x86), MSVC classifier (x64+x86), and the **OpenCL** GPU job (`build_opencl_windows.bat` -ICD staging works). Two GPU jobs were fixed after the first run: **CUDA** (`Version not available: -13.0.0` → bumped `Jimver/cuda-toolkit` `v0.2.24`→`v0.2.35` + `13.2.0`) and **Vulkan** -(`find_package(Vulkan)` couldn't read the `humbletim` SDK layout → switched to -`jakoch/install-vulkan-sdk-action`). Re-run pending to confirm both fixes. - -**Optional follow-up:** smoke-test that each *published* classifier JAR loads its DLL on a clean -Windows host with the matching GPU driver/toolkit installed. - -**Reference notes:** -- Cache backend is **sccache + Depot WebDAV** (consistent with the other 8 jobs — one token, shared - cross-branch) rather than upstream's per-branch ccache. sccache supports MSVC `cl.exe`; the - Release config emits no debug info, so the `/Zi`→`/Z7` PDB caveat doesn't apply. -- It is **"Ninja Multi-Config"**, not plain Ninja — it keeps multi-config semantics, so - `cmake --build … --config Release` and the config-specific `RUNTIME_OUTPUT_DIRECTORY_RELEASE` - properties behave exactly as under the VS generator; `/MT` runtime and x64-vs-x86 gating unchanged. -- The arch (`x64`/`x86`) comes from `ilammy/msvc-dev-cmd@v1`, not a `-A` flag (Ninja takes no `-A`). - -### Known regression (b9739) — Windows JNI: `common_params_parse` ignores caller argv - -**Status: FIXED via local source patch (`patches/0001-win32-arg-parse-embed-guard.patch`).** Surfaced -while bringing PR #248 green (the b9739 build fixes let the Windows Java jobs run to completion and -exposed this). Applied through the generic `patches/` mechanism (see CLAUDE.md "Local llama.cpp source -patches"), so it covers every C++ build and re-applies on each clean build. - -**Note on the fix shape (count-guard → deterministic removal).** The first patch used fix option 1 -below — the count-guard (override only when the re-derived arg count equals `argc`). It fixed 21/25 -Windows Java tests, but **collided** on the 4 server-integration setups (`OpenAiServerRerank*`, -`OpenAiServerToolCalling*`, `MultimodalIntegrationTest`, `OpenAiCompatServerIntegrationTest`) whose -argv length happened to equal `java.exe`'s, so they kept failing with the same parse error. The patch -was changed to **fix option 2** (drop the override entirely for our build — a JNI library is never the -process, so the override is pure liability), which is deterministic. **As of the b9789 bump the patch -was reshaped into the clean opt-in form intended for upstreaming (fix option 3's core):** -`common_params_parse` now parses exactly the argv it is given, and a new `common_params_parse_main()` -wrapper carries the `GetCommandLineW` UTF-8 recovery that the standalone tools' `main()` opt into. -**The patch now carries the full upstream change (37 files):** the ~34 `common_params_parse(argc, argv, -…)` call sites across `tools/*`, `examples/*` and the `tests/*` programs flip to -`common_params_parse_main()`, plus a `tests/test-arg-parser.cpp` regression case. Embedded callers stay -on `common_params_parse`. Our subproject build compiles only the `arg.{cpp,h}` core -(`LLAMA_BUILD_TOOLS`/`TESTS` OFF), so the flips + test are validated via a one-off tools+tests build -(the new test's asserts pass; `test-arg-parser`'s only red is the live `ggml.ai` download check, which -is sandbox-network). The 37-file patch must be re-verified on each llama.cpp bump (the applier fails -loud). Submit it to llama.cpp and drop the local copy once merged. - -**Symptom.** On **Windows x86_64 only**, every Java test that loads a real model fails in -`LlamaModel.loadModel` (native) with `LlamaException: "Failed to parse model parameters"` -(25 errors in `Java Tests Windows 2025 x86_64`, both the VS *and* Ninja DLLs). macOS and Linux Java -tests pass. The argv we build is platform-neutral (`--model models/.gguf`, relative, forward -slashes — `TestConstants.MODEL_PATH`), so it is **not** the Windows-Ninja build, **not** our argv, -and **not** a path/escaping issue. - -**Root cause (upstream llama.cpp, new in b9739).** `jllama.cpp` (`load_model_impl`, ~line 606) builds -a CLI argv from `ModelParameters` and calls upstream -`common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)`. In b9739, `common/arg.cpp`'s -`common_params_parse` gained a **Windows-only** prologue (arg.cpp:924-931): - -```cpp -bool common_params_parse(int argc, char ** argv, ...) { -#ifdef _WIN32 - auto utf8 = make_utf8_argv(); // = CommandLineToArgvW(GetCommandLineW()) - if (!utf8.ptrs.empty()) { // always non-empty under a JVM - argc = (int) utf8.buf.size(); - argv = utf8.ptrs.data(); // DISCARDS the caller-supplied argv - } -#endif - ... common_params_parse_ex(argc, argv, ctx_arg) ... -} -``` - -It unconditionally replaces the caller's argv with the host **process** command line -(`GetCommandLineW()`). For the standalone `llama-server.exe` this is correct (fixes UTF-8 CLI args). -For an **embedded/JNI** caller the process is **`java.exe`**, whose command line has no `--model`, so -`common_params_parse_ex` fails and `common_params_parse` returns `false` → our "Failed to parse model -parameters". `common_params_parse_ex` is `static`, so we cannot bypass the block by calling the inner -parser. Our JNI already passes correct UTF-8 argv (`GetStringUTFChars`), so the re-derivation is -unnecessary for us. **This is an upstream bug affecting every embedded Windows consumer of -`common_params_parse`.** - -**Fix options (history — option 2 chosen).** (1) guard the block by arg-count — *tried first, it -collided* (see the count-guard note above); (2) **remove the `_WIN32` override for our build — CHOSEN** -(deterministic; our JNI always passes correct UTF-8 argv); (3) file an upstream PR and wait. The patch -re-applies on every llama.cpp bump and the applier fails loud if it stops applying — it is part of the -upgrade checklist. Pre-existing on `main` since #247 (b9682→b9739); independent of the Windows-Ninja -classifier work. **Remaining open item: the upstream PR** (see "Upstream llama.cpp PR" below) so the -local patch can eventually be dropped. +- **Per-model FIM template registry** — only needed if `/v1/completions`-with-`suffix` FIM is exposed; + `/infill` applies the model's FIM tokens server-side, so low value. +- **Multi-model registry (Java transport).** The native surface has this via router mode + + `RouterClient`; the Java `OpenAiCompatServer` still advertises/serves a single model id. +- **Manual real-client validation.** Server-side round-trips exist for every surface; what remains is + pointing the actual editor clients (Copilot Ollama provider / Custom Endpoint, Claude Code, a + Responses client) at a running server, since round-trips confirm wire shapes but not each client's + parser. ### SonarCloud "Security Rating on New Code" gate — PR #248 (open) @@ -365,22 +85,29 @@ workflow in `.github/workflows/`). It contributes to the `mergeable_state: block as `sonarcloud.io`. To triage: open the check's details link from the PR (or allowlist the host), read the 17 findings, then accept policy-OK licenses on the dashboard or adjust the policy. Confirm whether it is a *required* status (if so it blocks merge; if advisory it does not). - -### Upstream llama.cpp PR — drop the local Windows arg-parse patch (open) - -`patches/0001-win32-arg-parse-embed-guard.patch` is a **local** fix re-applied on every build. To drop -it, PR upstream (against #24779): add a `common_params_parse_argv` companion (or a -`common_params_parse` opt-out flag) that trusts the caller's argv — preserving the standalone tools' -UTF-8 fix while letting embedders (JNI, and any FFI binding) pass their own argv. Ship with the -standalone-safe repro (a plain exe that passes a synthetic argv and shows it gets discarded on Windows -because `GetCommandLineW()` returns the host process line). Once merged and the pin is bumped past it, -delete the patch. - -### Branch protection — aarch64 job renamed (open, owner action) - -The native aarch64 switch renamed the check **`Cross-Compile Linux aarch64 (LTS)` → `Build and Test -Linux aarch64`**. If a required status check pinned the old name, repoint it or it will sit pending -forever. +- **Still red on PR #298 (2026-07-05):** the same status ("17 issues found") posts on every head there + too and contributes to its `mergeable_state: blocked`. Same triage path: read the findings on the + scanner's dashboard, accept policy-OK licenses or adjust the policy. + +### Upstream PR submissions — drop the carried patches (open) + +Six of the eight `patches/` are upstream-submittable verbatim; each accepted PR (once the pin is +bumped past it) deletes a patch from the bump checklist. (`0003`/`0004` are carries of already-open +upstream PRs #22393/#23116 — they drop automatically when those merge.) + +- **`0001` Windows arg-parse embed guard** (against #24779): `common_params_parse` trusts the caller's + argv; `common_params_parse_main()` keeps the standalone tools' UTF-8 recovery. Ship with the + standalone-safe repro (synthetic argv discarded on Windows because `GetCommandLineW()` returns the + host process line). +- **`0002` preserve caller load-progress callback** (b9789 regression: server clobbers + `params_base.load_progress_callback`). +- **`0005` recurrent near-prompt-end checkpoints** (agentic checkpoint starvation on recurrent/hybrid + models; complements upstream #24035/#24899/#24891). +- **`0006` embeddable `llama_server`** (no process signal handlers, forwarded-argv parse, out-of-band + shutdown). +- **`0007` `llama_server_attach`** (HTTP frontend on an existing `server_context`). +- **`0008` `LLAMA_SERVER_WORKER_CMD` router worker override** (also useful for containerized/wrapped + deployments). ### llama.cpp upstream feature exposure (queued, deferred by policy) @@ -433,20 +160,27 @@ Feel free to contribute fixes — PRs welcome. `maxRequestBodyBytes` limit (e.g. default 4 MB) and reject oversized requests with `HTTP 413 Content Too Large` before buffering them. -### Feature backlog from similar projects +### Feature backlog from similar projects (remainder: jbang example) -- **Feature backlog from similar projects.** See [`docs/feature-investigation-similar-projects.md`](docs/feature-investigation-similar-projects.md) for the consolidated investigation across the 5 pure-Java sibling runtimes ([llama3.java](https://github.com/mukel/llama3.java), [gemma4.java](https://github.com/mukel/gemma4.java), [gptoss.java](https://github.com/mukel/gptoss.java), [qwen35.java](https://github.com/mukel/qwen35.java), [nemotron3.java](https://github.com/mukel/nemotron3.java)) plus the dormant alternative JNI binding [llamacpp4j](https://github.com/sebicom/llamacpp4j). The doc captures 18 candidate items grouped into cross-cutting themes (UTF-8 streaming boundary safety, thinking-channel router, operator timing line, jbang single-file example, README system-properties table, etc.) and per-repo unique findings (Harmony channel decoder, Qwen empty-`` injection, llama_state_* save/load, llama_adapter_lora_* hot-apply, etc.), each with effort sizing (XS / S / M / L) and a prioritised backlog. - - **Recommended first batch** (items 1, 3, 4, 5): UTF-8 boundary-safe streaming decoder + ~~per-run timing line~~ + one jbang-runnable example + ~~a README system-properties table~~; ~1-2 days total, no JNI changes. - - **DONE so far:** - - README system-properties table (`e36f631`, with two cleanups in `3ae6c81` + `28dc9e6`). - - Per-run timing line (`TimingsLogger` class + wire-in to `CompletionResponseParser` and `ChatResponseParser`; format mirrors what `llama.cpp` CLI prints — `prompt: N tok in X ms (Y tok/s) | gen: … | cache: N | draft: …`; dedicated SLF4J logger `net.ladenthin.llama.timings` so users can suppress it independently; 7 unit tests pin format + pipeline behaviour). - - **Remaining first-batch items:** UTF-8 boundary-safe streaming decoder + jbang example. +The consolidated investigation lives in +[`docs/feature-investigation-similar-projects.md`](docs/feature-investigation-similar-projects.md) +(18 candidates across the 5 pure-Java sibling runtimes + llamacpp4j, with effort sizing). Everything +high-value from it has shipped — README system-properties table, per-run timing line +(`TimingsLogger`), UTF-8 boundary safety (native `utf8_to_jstring_impl` path), runtime LoRA control, +typed batch embeddings, in-JVM router mode, in-JVM GGUF quantization, GGUF metadata inspector, +session fork/rewind (see the Done section). **Remaining:** -### Android distribution: AAR + Kotlin-friendly API + sample app +- **jbang single-file example** (XS-S): a `//DEPS net.ladenthin:llama` one-file runnable demo so new + users can try the binding without a Maven project. +- Further per-repo unique findings in the doc can be pulled on demand; none is currently prioritized. -- **Publish a proper Android AAR alongside the existing JAR-with-resources packaging.** Today java-llama.cpp already cross-compiles the Android arm64 native lib in two flavours (CPU-only, bundled into the main JAR; OpenCL/Adreno under classifier `opencl-android-aarch64`), but both ship as plain Maven JARs that bury `libjllama.so` under `net/ladenthin/llama/Linux-Android/aarch64/`. Android/Gradle consumers expect an `.aar` with an `AndroidManifest.xml`, the native lib under `jni/arm64-v8a/`, and Maven coordinates like `net.ladenthin:llama-android:@aar`. This is the format the [LLaMAndroid](https://github.com/Rattlyy/LLaMAndroid) integration referenced elsewhere in this file has to work around manually. Investigate using `com.android.library` via Gradle in a sibling module, or hand-rolling the AAR layout from the Maven build. Coordinate ABI coverage with any future armv7-a / x86_64 work so the AAR can declare multiple `jniLibs//` entries when those land. +### Android example app (own session; the remaining Android item) -- **Provide a Kotlin-friendly façade + Android sample app.** The pure-Java `LlamaIterable` / `LlamaModel` API works on Android today (LLaMAndroid wraps it in a Kotlin `flow {}` block), but a small first-party Kotlin module — coroutine `Flow` adapters, `suspend` variants of the blocking calls, idiomatic `use {}` resource handling — would lower the integration cost meaningfully and serve as the canonical reference for downstream consumers. Pair it with a minimal sample app (single `Activity`, model picker, streaming text view) under e.g. `examples/android-sample/` so the AAR has an exercised end-to-end path in CI. Treat LLaMAndroid as the prior-art baseline; reuse patterns that already work there. +The AAR + Kotlin façade + multi-ABI (arm64-v8a/x86_64) + emulator CI shipped, and the emulator job is +a release gate (see the Done section / CLAUDE.md "Android AAR + Kotlin façade"). Remaining: a minimal +sample app under e.g. `examples/android-sample/` (single Activity, model picker, streaming text view) +consuming `net.ladenthin:llama-android` + `llama-kotlin` — it validates what the emulator cannot: +real arm64 hardware and the Adreno/OpenCL flavor. Treat LLaMAndroid as prior art. ### GraalVM Native Image evaluation @@ -495,6 +229,34 @@ Feel free to contribute fixes — PRs welcome. ## Done (kept for history) +### 2026-07-05 feature wave (PR #298) + follow-ups + +One-liners for the sections removed from "Open" (full detail: PR #298, CLAUDE.md, git history): + +- **NativeServer attach mode** — `NativeServer(LlamaModel, String...)` via `patches/0007` + (`llama_server_attach`); serves an already-loaded model over the full upstream HTTP frontend. +- **Typed router API** — `server.RouterClient` + `value.RouterModel` + parser; router mode in-JVM via + `patches/0008` + `NativeServer.setWorkerCommand`. +- **GGUF metadata inspector** — pure-Java `GgufInspector` + `value.GgufMetadata` (LE/BE, fail-loud). +- **Session fork/rewind** — `Session.checkpoint/rewind/fork` + `value.SessionCheckpoint`. +- **LangChain4j v1 + streaming** — tool calling, JSON mode, multimodal; streamed tool calls + + per-token thinking via `StreamingChunkAssembler`. +- **UTF-8 JNI path, runtime LoRA control, typed batch embeddings, in-JVM quantizer** (first batches). +- **Android AAR + Kotlin façade + x86_64 ABI + emulator CI** — incl. the dlopen fix (`GGML_OPENMP OFF` + + `-static-libstdc++`, DT_NEEDED whitelist; also fixed the latent 5.0.5 arm64 defect); emulator job + promoted to a release gate; committed audio fixture (`audios/sample.wav`) wired as the + `AudioInputIntegrationTest` default. +- **PIT gate hermeticity** — verified 295/295, 0 NO_COVERAGE with no fixtures; stale gotcha removed. +- **llama.cpp b9870 → b9873 → b9876** — all 8 patches re-verified each step. +- **Windows native classifiers (Ninja default flip + MSVC classifier + CUDA/Vulkan/OpenCL)** — shipped + earlier; docs live in CLAUDE.md "Windows native classifiers". +- **b9739 Windows JNI arg-parse regression** — fixed via `patches/0001`; upstream submission tracked + in "Upstream PR submissions" above. +- **Code audit (18 findings)** — fixed in PRs #258/#260 (+ tests #261/#262); only the optional + extraction-directory isolation remains (own section above). +- **Branch protection aarch64 check rename** — closed as a no-op per owner. + + ### b9739 upgrade + PR #248 (Windows Ninja, native aarch64, patches mechanism) - **llama.cpp b9682 → b9739** (#247, merged) + build fixes: `server-schema.cpp` added to the diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 3eb5b4fe..1796c7f5 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -5,27 +5,32 @@ The maintainer-facing release procedure is **centralized in the workspace repo** java-llama.cpp is a **Maven reactor**, so two repo-specific points extend the canonical procedure. -## Reactor version bump (all three poms) +## Reactor version bump (all four poms) -The root `pom.xml` is the parent (`net.ladenthin:llama-parent`); the `llama/` and -`llama-langchain4j/` modules inherit its version **but hardcode it in their ``** -(there is no `${revision}` single-sourcing). A version change — the release strip in Step 1 and the -post-release bump in Step 3 — must touch **all three poms in lockstep**, or the reactor build fails -with `Could not find artifact net.ladenthin:llama-parent:pom:{VERSION}`. Use: +The root `pom.xml` is the parent (`net.ladenthin:llama-parent`); the `llama/`, +`llama-langchain4j/`, and `llama-kotlin/` modules inherit its version **but hardcode it in their +``** (there is no `${revision}` single-sourcing). A version change — the release +strip in Step 1 and the post-release bump in Step 3 — must touch **all four poms in lockstep**, or +the reactor build fails with `Could not find artifact net.ladenthin:llama-parent:pom:{VERSION}`. +(The `llama-android/` Gradle build needs no edit — it parses the root pom's version.) Use: ```bash mvn -q versions:set -DnewVersion={VERSION} -DgenerateBackupPoms=false ``` -from the repo root — it updates the root `` plus both children's `` at +from the repo root — it updates the root `` plus every child's `` at once. See the "Version bump" note in [CLAUDE.md](../CLAUDE.md) for the rationale. ## Extra README dependency snippet -Besides the root `README.md`, the `llama-langchain4j/README.md` `## Dependency` section carries a -**release** dependency snippet that must also be set to `{VERSION}` in Step 1 — it is not covered by -the root-README edits and drifts silently otherwise (the release examples stay at `{VERSION}` on the -Step 3 snapshot bump). - -One reactor `mvn -P release deploy` signs and publishes the parent pom, `llama`, and -`llama-langchain4j` together at the same version. +Besides the root `README.md`, the `llama-langchain4j/README.md` `## Dependency` section, the +`llama-android/README.md` and `llama-kotlin/README.md` Gradle snippets, and the root README's +"Importing in Android" snippets carry **release** dependency versions that must also be set to +`{VERSION}` in Step 1 — they are not covered by the root-README edits and drift silently otherwise +(the release examples stay at `{VERSION}` on the Step 3 snapshot bump). + +One reactor `mvn -P release deploy` signs and publishes the parent pom, `llama`, +`llama-langchain4j`, and `llama-kotlin` together at the same version. The **Android AARs** +(`llama-android`, `llama-android-opencl`) are published by the `publish-release` job's separate +Gradle step (signed Central Portal bundle upload) — no manual action, but they appear as their own +deployment named `llama-android-{VERSION}` in the Central Portal UI. diff --git a/docs/history/llama-cpp-breaking-changes.md b/docs/history/llama-cpp-breaking-changes.md index cf745f86..c207614a 100644 --- a/docs/history/llama-cpp-breaking-changes.md +++ b/docs/history/llama-cpp-breaking-changes.md @@ -423,3 +423,9 @@ Used during `llama.cpp` version bumps: when upgrading, scan this file from the r | b9866–b9867 | upstream verification (sandbox) | All **six** patches (`0001`–`0006`) re-verified against b9867. The b9866→b9867 diff touches **no** patch-target file (`common/arg.*`, `tools/server/server-context.{cpp,h}`, `server-common.cpp`, `server-schema.cpp`, `server-task.h`, `server.cpp`, `test-arg-parser.cpp`, `test-chat.cpp`, the ~34 standalone mains) and **no** OuteTTS generator anchor (`tools/tts/tts.cpp` unchanged) — the only edit is `common/speculative.cpp` — so every patch hunk/offset is byte-identical to b9866. Confirmed end-to-end by a clean `cmake` configure: b9867 fetched and **all six patches applied via the fail-loud `PATCH_COMMAND`** (exit 0; 0005's `is_ckpt_only_rollback` and 0006's `g_llama_server_embedded` markers present), OuteTTS generator anchors held. First bump driven by `.github/scripts/llama-next-version.sh` (b9866→b9867, 2 KiB single-commit final chunk). Full build + `ctest` (target 462/462) to be confirmed by the CI pipeline. | | b9867–b9870 | `common/chat.cpp` + `models/templates/stepfun-ai-Step-3.5-Flash.jinja` (removed) + `tests/test-chat*.cpp` | Internal-only, no API surface. Adds a **StepFun** message-content whitespace workaround (issue #24181): `common_chat_templates_apply_jinja` detects a StepFun template (`src.find("You have access to the following functions in JSONSchema format")`) and, before rendering, trims leading/trailing whitespace from each `common_chat_msg`'s `content`/`reasoning_content` and its `"text"` `content_parts` via a new `static` `workaround::trim_all_content(...)` — otherwise leftover whitespace drove the model into reasoning loops. Uses only existing `common_chat_msg` fields; `common/chat.h` is untouched (no struct/API change). The removed `stepfun-ai-Step-3.5-Flash.jinja` embedded template and the `test-chat*.cpp` additions are **not built here** (`LLAMA_BUILD_TESTS` OFF for the FetchContent subproject). All inside upstream-compiled `common`, flowing through the embedded server / `LlamaModel` chat path automatically. No project source changes required. | | b9867–b9870 | upstream verification (sandbox) | All **six** patches (`0001`–`0006`) re-verified against b9870. The b9867→b9870 diff touches **no** patch-target file (`common/arg.*`, `tools/server/server-context.{cpp,h}`, `server-common.cpp`, `server-schema.cpp`, `server-task.h`, `server.cpp`, `test-arg-parser.cpp`, the ~34 standalone mains) and **no** OuteTTS generator anchor (`tools/tts/tts.cpp` unchanged) — the only source edit is `common/chat.cpp` (a StepFun whitespace workaround), plus `tools/ui/**` (WebUI, auto-followed) and `tests/test-chat*.cpp` (not built) — so every patch hunk/offset is byte-identical to b9867. **Note:** patch `0004` also targets `tests/test-chat.cpp`, which b9870 edits, but `0004`'s hunks add the reasoning-budget cases in a disjoint region (verified clean by the configure below). Confirmed end-to-end by a clean `cmake` configure: b9870 fetched and **all six patches applied via the fail-loud `PATCH_COMMAND`** (exit 0; 0005's `is_ckpt_only_rollback` and 0006's `g_llama_server_embedded` markers present, b9870's `trim_all_content` present), OuteTTS generator anchors held. Full build + `ctest` (target 462/462) to be confirmed by the CI pipeline. | +| b9870–b9873 | `ggml/src/ggml-cpu/ops.cpp` + `src/llama-graph.cpp` + `tests/test-backend-ops.cpp` + `tools/ui/**` | Internal-only, no API surface. Two upstream bugfixes: **(1)** CPU `concat` fixed for **quantized** tensors (`ggml_compute_forward_concat_any` now scales the dim-0 offset and loop bounds by `ggml_blck_size`, with new contiguity/block-multiple asserts for quantized inputs); **(2)** `llm_graph_input_attn_kv{,_iswa}::set_input` guards the K/V rotation inputs with `tensor->buffer` null-checks so an unallocated rotation buffer is skipped instead of dereferenced (upstream #25215). `tests/test-backend-ops.cpp` additions are **not built here** (`LLAMA_BUILD_TESTS` OFF); `tools/ui/**` is the WebUI (auto-followed by `build-webui`). All inside upstream-compiled TUs — no project source changes required. | +| b9870–b9873 | upstream verification (sandbox) | All **eight** patches (`0001`–`0008`) re-verified against b9873: applied in filename order onto a clean b9873 checkout via `git apply --check` + `git apply`, all clean. The b9870→b9873 diff (5 files, ~9.5 KiB) touches **no** patch-target file and **no** OuteTTS generator anchor (`tools/tts/tts.cpp` unchanged), so every patch hunk/offset is byte-identical to b9870. Full build + `ctest` to be confirmed by the CI pipeline. | +| b9873–b9876 | `ggml/src/ggml-backend-meta.cpp` + `ggml/src/ggml-cuda/{concat.cu,ggml-cuda.cu}` | Internal-only, no API surface, ggml-only. **(1)** CUDA `concat` gains the same quantized-tensor block-size handling b9873 added to the CPU op (`concat.cu`); **(2)** tensor-parallel + `-ncmoe` crash fix on MoE models (upstream #25028: `ggml-backend-meta.cpp` + `ggml-cuda.cu` split-buffer handling). Only the CUDA classifiers even compile the `.cu` files; nothing project-side changes. | +| b9873–b9876 | upstream verification (sandbox) | All **eight** patches (`0001`–`0008`) re-verified against b9876: applied in filename order onto a clean b9876 checkout, all clean. The b9873→b9876 diff (3 files, ~9.6 KiB) touches **no** patch-target file and **no** OuteTTS generator anchor (`tools/tts/tts.cpp` unchanged). Full build + `ctest` to be confirmed by the CI pipeline. | +| b9876–b9878 | `ggml/src/ggml-backend-meta.cpp` + `src/llama-model.cpp` | Internal-only, no API surface (2 files, ~1.8 KiB). **(1)** meta backend gains a fail-loud `GGML_ABORT` guard when handed a multi-buffer (upstream #22197); **(2)** `llama_model` now **copies** the borrowed `params.tensor_split` array into an owned vector (`tensor_split_owned`) so tensor-parallel KV-cache split metadata cannot read a dangling caller pointer later. Both inside upstream-compiled TUs; no project source changes required. | +| b9876–b9878 | upstream verification (sandbox) | All **eight** patches (`0001`–`0008`) re-verified against b9878: applied in filename order onto a clean b9878 checkout, all clean. The range touches **no** patch-target file and **no** OuteTTS generator anchor (`tools/tts/tts.cpp` unchanged). Full build + `ctest` to be confirmed by the CI pipeline. | diff --git a/llama-android/README.md b/llama-android/README.md new file mode 100644 index 00000000..2a8743aa --- /dev/null +++ b/llama-android/README.md @@ -0,0 +1,80 @@ + +# llama-android + +Android AAR packaging of [java-llama.cpp](https://github.com/bernardladenthin/java-llama.cpp): +the `net.ladenthin:llama` Java API plus the CI-built `arm64-v8a` native library, consumable from +any Android project as a normal Maven dependency — no git submodule, no NDK build, no manual +ProGuard rules. + +```kotlin +// build.gradle.kts of your app — that's all. +dependencies { + implementation("net.ladenthin:llama-android:5.0.6") + // or, for Qualcomm Adreno GPUs (device must provide an OpenCL ICD): + // implementation("net.ladenthin:llama-android-opencl:5.0.6") +} +``` + +- **minSdk 28** (Android 9.0 Pie) — enforced at build time via the AAR manifest. +- **Multi-ABI**: `arm64-v8a` (devices) + `x86_64` (Android Studio emulator, Chromebooks, + x86-64 Android hardware). App bundles split per ABI, so phones download only arm64. +- **R8/ProGuard safe** — consumer rules ship inside the AAR (`proguard.txt`) and apply + automatically. +- **16 KB page-size compliant** native library (Google Play requirement for Android 15+ targets). +- The Kotlin coroutines/Flow façade lives in the separate, optional + [`llama-kotlin`](../llama-kotlin) artifact. + +Use `LlamaModel` exactly as on the JVM (see the core README). On Android the loader resolves the +native library via `System.loadLibrary("jllama")` from the APK's native-lib directory — where the +AAR's `jni/arm64-v8a/libjllama.so` lands. + +> Do **not** combine this artifact with a `net.ladenthin:llama` JAR dependency in the same app: +> the AAR already contains those classes (and only the Android native library, whereas the JAR +> would drag ~70 MB of desktop natives into your APK as Java resources). + +Models are ordinary GGUF files on device storage; download them at runtime (or bundle small ones +as assets and copy them to files dir) and pass the absolute path to `ModelParameters.setModel`. + +## Two AAR flavors + +| Artifact | Backend | Requirement | +|---|---|---| +| `llama-android` | CPU | any arm64-v8a or x86_64 Android environment, API 28+ | +| `llama-android-opencl` | OpenCL (Adreno-tuned kernels) | device OpenCL ICD (`libOpenCL.so`) — Qualcomm Adreno drivers ship one; devices without an ICD must use the CPU flavor | + +## How this build works + +This directory is a **standalone plain-Gradle build** (no Android Gradle Plugin, no Android SDK +required to build): an AAR is a documented zip, and Gradle's built-in `maven-publish` can publish +it with `aar` — which plain Maven cannot (`android-maven-plugin` is +unmaintained). It is intentionally *not* a Maven reactor module, but it stays version-locked to +the reactor: `build.gradle.kts` parses the version (and the mirrored dependency versions) out of +the Maven poms at configure time, so `mvn versions:set` remains the single bump point. + +The AAR's `classes.jar` repackages the **byte-identical Maven-built core classes** (no +recompilation) minus the bundled desktop native resources and `module-info.class`; the Android +`.so` ships under `jni/arm64-v8a/` instead of as a Java resource. + +### Building locally + +```bash +# 1. Build the core jar (from the repo root) +mvn -pl llama -am -DskipTests package + +# 2. Stage the Android native libraries (CI artifacts, or a dockcross build): +# natives/cpu/arm64-v8a/libjllama.so +# natives/cpu/x86_64/libjllama.so +# natives/opencl/arm64-v8a/libjllama.so + +# 3. Assemble + publish to the local staging repo / mavenLocal +gradle -p llama-android aarCpu aarOpencl +gradle -p llama-android publishToMavenLocal +``` + +CI (`.github/workflows/publish.yml`) assembles both AARs from the freshly built native +artifacts, asserts the AAR structure and the 16 KB LOAD-segment alignment, and compiles a +minimal AGP consumer app against the published AAR as a smoke test. diff --git a/llama-android/build.gradle.kts b/llama-android/build.gradle.kts new file mode 100644 index 00000000..ef2cd266 --- /dev/null +++ b/llama-android/build.gradle.kts @@ -0,0 +1,303 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +// Builds the Android AAR artifacts for java-llama.cpp WITHOUT the Android Gradle +// Plugin and without an Android SDK: +// +// net.ladenthin:llama-android — CPU natives (arm64-v8a) +// net.ladenthin:llama-android-opencl — OpenCL/Adreno natives (arm64-v8a) +// +// An AAR is a documented zip (AndroidManifest.xml + classes.jar + jni// + +// proguard.txt + R.txt). AGP is only required to *consume* it — which the CI +// consumer smoke test does on a runner with the Android SDK. Building it here +// with plain Gradle means: +// 1. classes.jar carries the BYTE-IDENTICAL Maven-built core classes (no +// recompilation, no Lombok/AGP coupling, no drift from the tested jar); +// only the desktop/Android native resources and module-info.class are +// stripped (the .so ships under jni/ instead — LlamaLoader already calls +// System.loadLibrary("jllama") first on Android, see LlamaLoader). +// 2. The published POM says aar, which plain Maven +// cannot produce — this is exactly how AGP-built libraries on Central +// declare themselves, so `implementation("net.ladenthin:llama-android:V")` +// resolves in any Android project without an @aar suffix. +// +// Version lockstep: the version and all mirrored dependency versions are parsed +// from the Maven poms at configure time. `mvn versions:set` (the documented bump +// procedure) is the only version edit point; this build follows automatically. +// +// Inputs expected before running the aar tasks (fail-loud checks below): +// ../llama/target/llama-.jar mvn -pl llama -am -DskipTests package +// natives/cpu/arm64-v8a/libjllama.so CI artifact Linux-Android-aarch64-libraries +// natives/opencl/arm64-v8a/libjllama.so CI artifact android-libraries-opencl + +import org.w3c.dom.Document +import org.w3c.dom.Element +import org.w3c.dom.Node +import javax.xml.parsers.DocumentBuilderFactory + +plugins { + base + `maven-publish` + signing +} + +// --------------------------------------------------------------------------- +// Single-source-of-truth version/metadata parsing from the Maven reactor poms. +// --------------------------------------------------------------------------- + +fun parsePom(path: String): Document { + val factory = DocumentBuilderFactory.newInstance() + factory.isNamespaceAware = false + return factory.newDocumentBuilder().parse(file(path)) +} + +fun directChildText(element: Element, tag: String): String? { + val children = element.childNodes + for (i in 0 until children.length) { + val node = children.item(i) + if (node.nodeType == Node.ELEMENT_NODE && node.nodeName == tag) { + return node.textContent.trim() + } + } + return null +} + +fun directChildElement(element: Element, tag: String): Element? { + val children = element.childNodes + for (i in 0 until children.length) { + val node = children.item(i) + if (node.nodeType == Node.ELEMENT_NODE && node.nodeName == tag) { + return node as Element + } + } + return null +} + +val rootPom = parsePom("../pom.xml") +val corePom = parsePom("../llama/pom.xml") + +val reactorVersion = directChildText(rootPom.documentElement, "version") + ?: error("No found in ../pom.xml — the root reactor pom must declare the version") + +fun coreProperty(name: String): String { + val properties = directChildElement(corePom.documentElement, "properties") + ?: error("No in ../llama/pom.xml") + return directChildText(properties, name) + ?: error("Property <$name> not found in ../llama/pom.xml — keep the AAR pom dependencies in lockstep with the core") +} + +group = "net.ladenthin" +version = reactorVersion + +val coreJarFile = file("../llama/target/llama-$reactorVersion.jar") + +// --------------------------------------------------------------------------- +// AAR content tasks. +// --------------------------------------------------------------------------- + +// The AAR classes.jar: the Maven-built core jar minus (a) every bundled desktop +// native resource tree (they would otherwise be packaged into consumer APKs — +// ~70 MB of dead weight; the Android .so ships under jni/ instead) and +// (b) module-info.class (D8 does not accept JPMS descriptors in classes.jar). +val coreClassesJar = tasks.register("coreClassesJar") { + description = "Repackages the Maven-built core classes as the AAR classes.jar payload." + archiveBaseName.set("classes-payload") + destinationDirectory.set(layout.buildDirectory.dir("intermediates")) + isPreserveFileTimestamps = false + isReproducibleFileOrder = true + doFirst { + require(coreJarFile.isFile) { + "Core jar not found: $coreJarFile — build it first: mvn -pl llama -am -DskipTests package (from the repo root)" + } + } + from(zipTree(coreJarFile)) { + exclude("net/ladenthin/llama/Linux/**") + exclude("net/ladenthin/llama/Linux-Android/**") + exclude("net/ladenthin/llama/Mac/**") + exclude("net/ladenthin/llama/Windows/**") + exclude("module-info.class") + exclude("META-INF/maven/**") + } +} + +// AGP-built AARs always carry an R.txt (empty for resource-less libraries). +val generateRTxt = tasks.register("generateRTxt") { + val rTxt = layout.buildDirectory.file("intermediates/R.txt") + outputs.file(rTxt) + doLast { rTxt.get().asFile.writeText("") } +} + +val sourcesJar = tasks.register("sourcesJar") { + description = "Core Java sources (Central requires a -sources jar per artifact)." + archiveClassifier.set("sources") + isPreserveFileTimestamps = false + isReproducibleFileOrder = true + from("../llama/src/main/java") { + exclude("module-info.java") + } +} + +val javadocJar = tasks.register("javadocJar") { + description = "Javadoc placeholder jar (Central requires a -javadoc jar per artifact; " + + "the full javadoc ships with net.ladenthin:llama, whose classes this AAR repackages)." + archiveClassifier.set("javadoc") + val readme = layout.buildDirectory.file("intermediates/javadoc-readme/README.txt") + doFirst { + val file = readme.get().asFile + file.parentFile.mkdirs() + file.writeText( + "This artifact repackages the net.ladenthin:llama classes for Android.\n" + + "The full javadoc is published with net.ladenthin:llama:$reactorVersion (classifier 'javadoc')\n" + + "and online via javadoc.io.\n" + ) + } + from(readme.map { it.asFile.parentFile }) +} + +fun registerAarTask(taskName: String, artifactBase: String, nativesSubdir: String, requiredAbis: List) = + tasks.register(taskName) { + description = "Assembles $artifactBase-$reactorVersion.aar from the core classes and natives/$nativesSubdir." + archiveBaseName.set(artifactBase) + archiveVersion.set(reactorVersion) + archiveExtension.set("aar") + destinationDirectory.set(layout.buildDirectory.dir("aar")) + isPreserveFileTimestamps = false + isReproducibleFileOrder = true + val nativesDir = file("natives/$nativesSubdir") + doFirst { + // Fail-loud per ABI: a missing staging copy must never silently produce an + // AAR that lacks an advertised ABI (emulator/x86_64 consumers would crash + // at load time instead). + for (abi in requiredAbis) { + val so = File(nativesDir, "$abi/libjllama.so") + require(so.isFile) { + "Missing Android native library: $so — stage the CI-built libjllama.so there " + + "(artifacts 'Linux-Android-aarch64-libraries' / 'Linux-Android-x86_64-libraries' " + + "for cpu, 'android-libraries-opencl' for opencl; the artifact tree is " + + "net/ladenthin/llama/Linux-Android//libjllama.so)" + } + } + } + from("src/main/AndroidManifest.xml") + from(coreClassesJar) { rename { "classes.jar" } } + from("consumer-proguard.txt") { rename { "proguard.txt" } } + from(generateRTxt) + from(nativesDir) { into("jni") } + } + +// CPU AAR is multi-ABI: arm64-v8a for devices, x86_64 for emulators / x86_64 Android +// hardware (Chromebooks etc.). App bundles split per ABI, so phones download only arm64. +// The OpenCL flavor stays arm64-only (Adreno = Qualcomm ARM hardware). +val aarCpu = registerAarTask("aarCpu", "llama-android", "cpu", listOf("arm64-v8a", "x86_64")) +val aarOpencl = registerAarTask("aarOpencl", "llama-android-opencl", "opencl", listOf("arm64-v8a")) + +// --------------------------------------------------------------------------- +// Publishing: POM aar + mirrored core dependencies. +// --------------------------------------------------------------------------- + +// Suppress Gradle Module Metadata: consumers must resolve via the POM +// (aar → .aar artifact), the exact mechanism every +// pre-GMM Android library on Central uses. Ad-hoc-artifact GMM would lack the +// variant attributes AGP expects and could confuse resolution. +tasks.withType().configureEach { enabled = false } + +fun org.gradle.api.publish.maven.MavenPom.commonMetadata(artifactDisplayName: String, backendNote: String) { + name.set(artifactDisplayName) + description.set( + "Android AAR for java-llama.cpp: the net.ladenthin:llama Java API with the $backendNote " + + "arm64-v8a native library packaged under jni/, consumer R8/ProGuard rules, and minSdk 28." + ) + url.set("https://github.com/bernardladenthin/java-llama.cpp") + licenses { + license { + name.set("MIT License") + url.set("https://opensource.org/licenses/MIT") + } + } + developers { + developer { + id.set("bernardladenthin") + name.set("Bernard Ladenthin") + email.set("bernard.ladenthin@gmail.com") + } + } + scm { + connection.set("scm:git:git://github.com/bernardladenthin/java-llama.cpp.git") + developerConnection.set("scm:git:ssh://github.com:bernardladenthin/java-llama.cpp.git") + url.set("https://github.com/bernardladenthin/java-llama.cpp") + } + // Mirror the core's compile-scope dependencies (the AAR repackages those + // classes, so their imports must resolve on the consumer classpath). + // logback-classic is deliberately NOT mirrored: it is the core's JVM-only + // runtime SLF4J binding and does not run on Android — Android consumers + // pick their own binding (e.g. slf4j-android) or run with the no-op one. + withXml { + val dependencies = asNode().appendNode("dependencies") + fun dependency(groupId: String, artifactId: String, versionProperty: String) { + val node = dependencies.appendNode("dependency") + node.appendNode("groupId", groupId) + node.appendNode("artifactId", artifactId) + node.appendNode("version", coreProperty(versionProperty)) + node.appendNode("scope", "compile") + } + dependency("com.fasterxml.jackson.core", "jackson-databind", "jackson.version") + dependency("org.slf4j", "slf4j-api", "slf4j.version") + dependency("org.jspecify", "jspecify", "jspecify.version") + dependency("org.checkerframework", "checker-qual", "checker.version") + } +} + +publishing { + publications { + create("llamaAndroid") { + artifactId = "llama-android" + artifact(aarCpu) + artifact(sourcesJar) + artifact(javadocJar) + pom.packaging = "aar" + pom.commonMetadata("llama-android", "CPU") + } + create("llamaAndroidOpencl") { + artifactId = "llama-android-opencl" + artifact(aarOpencl) + artifact(sourcesJar) + artifact(javadocJar) + pom.packaging = "aar" + pom.commonMetadata("llama-android-opencl", "OpenCL/Adreno") + } + } + repositories { + // Local staging repo in Maven layout — CI zips this into a Central + // Portal bundle for releases, and it doubles as the inspection target + // for the structural AAR checks. + maven { + name = "staging" + url = uri(layout.buildDirectory.dir("staging-repo")) + } + // Central Portal snapshot repository — plain Maven layout, token auth. + // Only wired when CI provides the credentials. + val centralUsername = System.getenv("CENTRAL_USERNAME") + val centralPassword = System.getenv("CENTRAL_PASSWORD") + if (centralUsername != null && centralPassword != null) { + maven { + name = "centralSnapshots" + url = uri("https://central.sonatype.com/repository/maven-snapshots/") + credentials { + username = centralUsername + password = centralPassword + } + } + } + } +} + +// Sign only when CI provides the key (same GPG key the Maven release uses). +val signingKey = System.getenv("MAVEN_GPG_PRIVATE_KEY") +val signingPassphrase = System.getenv("MAVEN_GPG_PASSPHRASE") +if (signingKey != null) { + signing { + useInMemoryPgpKeys(signingKey, signingPassphrase ?: "") + sign(publishing.publications) + } +} diff --git a/llama-android/consumer-proguard.txt b/llama-android/consumer-proguard.txt new file mode 100644 index 00000000..f047395d --- /dev/null +++ b/llama-android/consumer-proguard.txt @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT +# +# Consumer R8/ProGuard rules — shipped as proguard.txt inside the AAR, so AGP +# applies them to every consuming app automatically (no manual rule copying, +# unlike the source-integration flow documented in the core README). + +# The native layer resolves classes, fields, and methods by name: +# JNI_OnLoad calls FindClass/GetMethodID/GetFieldID on net.ladenthin.llama +# types (LlamaModel, exception.LlamaException, value.LogLevel, args.LogFormat, +# callback.LoadProgressCallback, ...), and Jackson reflects over the parameter +# and response POJOs. Keep the whole binding. +-keep class net.ladenthin.llama.** { *; } + +# Native method names must survive for JNI registration. +-keepclasseswithmembernames class * { + native ; +} + +# Jackson databind needs generic signatures and annotations at runtime. +-keepattributes Signature,InnerClasses,EnclosingMethod,*Annotation* +-dontwarn com.fasterxml.jackson.databind.** + +# JVM-only optional integrations referenced from the core classes but absent +# (and unused) on Android. +-dontwarn org.slf4j.** + +# Compile-time-only references R8 cannot resolve on Android. The broad keep above +# retains the whole binding, so R8 verifies every referenced type: +# - com.sun.net.httpserver: the JDK's HTTP server, used only by the JVM-only +# OpenAiCompatServer transport. Those classes stay dexed but are unusable on +# Android by design (loading them throws NoClassDefFoundError; the supported +# Android entry points never touch them). +# - lombok / animal-sniffer: CLASS-retention build-time annotations referenced +# from the bytecode; they exist on no runtime classpath anywhere. +-dontwarn com.sun.net.httpserver.** +-dontwarn lombok.** +-dontwarn org.codehaus.mojo.animal_sniffer.** diff --git a/llama-android/settings.gradle.kts b/llama-android/settings.gradle.kts new file mode 100644 index 00000000..b7af49f2 --- /dev/null +++ b/llama-android/settings.gradle.kts @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +// Standalone Gradle build (NOT a Maven reactor module): Maven cannot produce or +// deploy an artifact with aar (the only android-maven-plugin +// is long dead), while Gradle's built-in maven-publish can. This build stays +// version-locked to the reactor anyway — build.gradle.kts reads the version from +// the root pom.xml, so `mvn versions:set` remains the single bump point. +rootProject.name = "llama-android" diff --git a/llama-android/src/main/AndroidManifest.xml b/llama-android/src/main/AndroidManifest.xml new file mode 100644 index 00000000..31ce33cc --- /dev/null +++ b/llama-android/src/main/AndroidManifest.xml @@ -0,0 +1,16 @@ + + + + + diff --git a/llama-kotlin/README.md b/llama-kotlin/README.md new file mode 100644 index 00000000..49277ac2 --- /dev/null +++ b/llama-kotlin/README.md @@ -0,0 +1,62 @@ + +# llama-kotlin + +Kotlin coroutines façade for [java-llama.cpp](https://github.com/bernardladenthin/java-llama.cpp): +`Flow`-based token streaming and `suspend` wrappers over the `net.ladenthin:llama` JNI API. +Pure Kotlin/JVM — works on desktop JVMs **and** Android. + +```kotlin +dependencies { + implementation("net.ladenthin:llama-kotlin:5.0.6") + // ...plus the binding itself — the façade does NOT drag it in transitively + // (provided scope), so YOU pick the right flavor: + implementation("net.ladenthin:llama:5.0.6") // desktop JVM + // implementation("net.ladenthin:llama-android:5.0.6") // Android (AAR) +} +``` + +## API + +```kotlin +import net.ladenthin.llama.kotlin.* + +// Token streaming as a cold Flow — the native task slot is released on +// completion, error, AND cancellation (take(n), Job.cancel, ...): +model.generateChatFlow(params) + .flowOn(Dispatchers.IO) + .collect { output -> print(output.text) } + +// Suspend wrappers (main-safe; default dispatcher = Dispatchers.IO): +val text: String = model.completeSuspend(params) // coroutine cancel → native cancel +val chat: ChatResponse = model.chatSuspend(request) +val reply: String = model.chatCompleteTextSuspend(params) +val vector: FloatArray = model.embedSuspend("hello") +``` + +`completeSuspend` wires **coroutine cancellation to the binding's cooperative +`CancellationToken`**: cancelling the calling coroutine stops the native generation at the next +token boundary and frees the slot — the missing piece a hand-rolled +`withContext(Dispatchers.IO) { model.complete(params) }` does not give you. + +## Why `provided` scope on the core? + +The desktop `net.ladenthin:llama` JAR bundles native libraries for every desktop platform as Java +resources. If this module depended on it transitively, every Android APK using the façade would +package ~70 MB of dead desktop natives. Declaring the binding yourself keeps the choice explicit: +`llama` on the JVM, the `llama-android` AAR on Android. + +## Build + +Reactor module — built, versioned and released with the core: + +```bash +mvn -pl llama-kotlin -am -DskipTests install # build with the core +mvn -f llama-kotlin/pom.xml test # 6 model-free unit tests +``` + +Requires Kotlin 2.1+ in consuming Kotlin projects (compiled with Kotlin 2.2, metadata readable one +minor back); the bytecode targets Java 8, same as the core. diff --git a/llama-kotlin/pom.xml b/llama-kotlin/pom.xml new file mode 100644 index 00000000..310b4526 --- /dev/null +++ b/llama-kotlin/pom.xml @@ -0,0 +1,202 @@ + + + + 4.0.0 + + + net.ladenthin + llama-parent + 5.0.6-SNAPSHOT + ../pom.xml + + + llama-kotlin + jar + + ${project.groupId}:${project.artifactId} + Kotlin coroutines facade for java-llama.cpp: Flow-based token + streaming and suspend wrappers (with coroutine-cancellation wired to the + binding's cooperative CancellationToken) over the net.ladenthin:llama JNI + API. Pure Kotlin/JVM - works on desktop JVMs and on Android (pair it with + net.ladenthin:llama or net.ladenthin:llama-android respectively). + https://github.com/bernardladenthin/java-llama.cpp + + + + MIT License + https://www.opensource.org/licenses/mit-license.php + repo + + + + + + Bernard Ladenthin + https://github.com/bernardladenthin + + + + + scm:git:https://github.com/bernardladenthin/java-llama.cpp.git + scm:git:https://github.com/bernardladenthin/java-llama.cpp.git + https://github.com/bernardladenthin/java-llama.cpp/tree/main + + + + + central + https://central.sonatype.com/repository/maven-snapshots/ + + + + + UTF-8 + + 2.2.21 + 1.11.0 + 6.1.1 + 3.0 + 3.5.6 + 3.4.0 + 3.4.1 + + + + + + net.ladenthin + llama + ${project.version} + provided + + + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin.version} + + + org.jetbrains.kotlinx + kotlinx-coroutines-core + ${kotlinx.coroutines.version} + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + org.hamcrest + hamcrest + ${hamcrest.version} + test + + + org.jetbrains.kotlinx + kotlinx-coroutines-test + ${kotlinx.coroutines.version} + test + + + + + + ${project.basedir}/src/main/kotlin + ${project.basedir}/src/test/kotlin + + + org.jetbrains.kotlin + kotlin-maven-plugin + ${kotlin.version} + + + 1.8 + + + + compile + + compile + + + + test-compile + + test-compile + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + + + org.apache.maven.plugins + maven-source-plugin + ${source.plugin.version} + + + attach-sources + + jar-no-fork + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${jar.plugin.version} + + + javadoc-placeholder-jar + package + + jar + + + javadoc + ${project.basedir}/src/javadoc-placeholder + + + + + + + + + diff --git a/llama-kotlin/src/javadoc-placeholder/README.txt b/llama-kotlin/src/javadoc-placeholder/README.txt new file mode 100644 index 00000000..4d8181ff --- /dev/null +++ b/llama-kotlin/src/javadoc-placeholder/README.txt @@ -0,0 +1,6 @@ +SPDX-FileCopyrightText: 2026 Bernard Ladenthin +SPDX-License-Identifier: MIT + +net.ladenthin:llama-kotlin is a pure-Kotlin module; its API documentation is the +KDoc in the -sources.jar of this artifact. The underlying Java API it wraps is +documented in the javadoc of net.ladenthin:llama (same version). diff --git a/llama-kotlin/src/main/kotlin/net/ladenthin/llama/kotlin/LlamaFlows.kt b/llama-kotlin/src/main/kotlin/net/ladenthin/llama/kotlin/LlamaFlows.kt new file mode 100644 index 00000000..45f5f142 --- /dev/null +++ b/llama-kotlin/src/main/kotlin/net/ladenthin/llama/kotlin/LlamaFlows.kt @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.kotlin + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import net.ladenthin.llama.LlamaModel +import net.ladenthin.llama.parameters.InferenceParameters +import net.ladenthin.llama.value.LlamaOutput + +/** + * Streams a raw completion as a cold [Flow] of tokens. + * + * Each collection starts a fresh generation via [LlamaModel.generate]. The underlying + * [net.ladenthin.llama.LlamaIterable] is closed when the flow completes **or is cancelled**, so an + * early `take(n)`/cancellation releases the native task slot instead of leaking it. + * + * The token iteration blocks the collecting dispatcher; collect on a background one: + * `model.generateFlow(params).flowOn(Dispatchers.IO)`. + */ +fun LlamaModel.generateFlow(parameters: InferenceParameters): Flow = + closeableIterableFlow { generate(parameters) } + +/** + * Streams an OpenAI-style chat completion as a cold [Flow] of tokens. + * + * Same contract as [generateFlow], backed by [LlamaModel.generateChat] (the model's chat template + * is applied to the `messages` in [parameters]). + */ +fun LlamaModel.generateChatFlow(parameters: InferenceParameters): Flow = + closeableIterableFlow { generateChat(parameters) } + +/** + * Bridges a close-on-abandon iterable (the shape of `LlamaIterable`: `Iterable & AutoCloseable`) + * into a cold [Flow]: [open] runs per collection, items are emitted in order, and the source is + * closed on completion, error, and cancellation alike. + * + * Internal seam so the flow semantics are unit-testable without a loaded model + * (see `CloseableIterableFlowTest`). + */ +internal fun closeableIterableFlow(open: () -> I): Flow where I : Iterable, I : AutoCloseable = + flow { + open().use { source -> + for (item in source) { + emit(item) + } + } + } diff --git a/llama-kotlin/src/main/kotlin/net/ladenthin/llama/kotlin/LlamaSuspend.kt b/llama-kotlin/src/main/kotlin/net/ladenthin/llama/kotlin/LlamaSuspend.kt new file mode 100644 index 00000000..0ff46cfd --- /dev/null +++ b/llama-kotlin/src/main/kotlin/net/ladenthin/llama/kotlin/LlamaSuspend.kt @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.kotlin + +import kotlin.coroutines.CoroutineContext +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.withContext +import net.ladenthin.llama.LlamaModel +import net.ladenthin.llama.callback.CancellationToken +import net.ladenthin.llama.parameters.ChatRequest +import net.ladenthin.llama.parameters.InferenceParameters +import net.ladenthin.llama.value.ChatResponse + +/** + * Runs a blocking completion as a suspending call on [context] (default [Dispatchers.IO]), + * with **coroutine cancellation wired to the binding's cooperative [CancellationToken]**: + * cancelling the calling coroutine cancels the token, the native loop stops at the next token + * boundary (freeing the slot), and the [CancellationException] propagates as usual. + */ +suspend fun LlamaModel.completeSuspend( + parameters: InferenceParameters, + context: CoroutineContext = Dispatchers.IO, +): String = withCancellationToken(context) { token -> complete(parameters, token) } + +/** + * Runs a typed chat completion as a suspending call on [context] (default [Dispatchers.IO]). + * + * Not token-cancellable: [LlamaModel.chat] has no [CancellationToken] overload, so a cancelled + * coroutine resumes only after the native call returns. Use [completeSuspend] or + * [generateChatFlow] when prompt cancellation matters. + */ +suspend fun LlamaModel.chatSuspend( + request: ChatRequest, + context: CoroutineContext = Dispatchers.IO, +): ChatResponse = withContext(context) { chat(request) } + +/** + * Runs an OpenAI-style chat completion and returns only the assistant text, as a suspending + * call on [context] (default [Dispatchers.IO]). Same cancellation caveat as [chatSuspend]. + */ +suspend fun LlamaModel.chatCompleteTextSuspend( + parameters: InferenceParameters, + context: CoroutineContext = Dispatchers.IO, +): String = withContext(context) { chatCompleteText(parameters) } + +/** + * Computes an embedding as a suspending call on [context] (default [Dispatchers.IO]). + */ +suspend fun LlamaModel.embedSuspend( + prompt: String, + context: CoroutineContext = Dispatchers.IO, +): FloatArray = withContext(context) { embed(prompt) } + +/** + * Runs [block] with a fresh [CancellationToken] on [context] and cancels that token as soon as + * the calling coroutine is cancelled, so a cooperative native loop stops at its next check + * instead of running to natural completion. + * + * Structure: the blocking [block] runs in a child [async]; awaiting it is the cancellable + * suspension point. On cancellation the token is cancelled *before* rethrowing, and the + * enclosing [coroutineScope] then waits for [block] to observe the token and return — so the + * function never leaks a still-running generation past its own return. + * + * Internal seam so the wiring is unit-testable without a loaded model (see + * `WithCancellationTokenTest`). + */ +internal suspend fun withCancellationToken( + context: CoroutineContext, + block: (CancellationToken) -> R, +): R = coroutineScope { + val token = CancellationToken() + val work = async(context) { block(token) } + try { + work.await() + } catch (e: CancellationException) { + token.cancel() + throw e + } +} diff --git a/llama-kotlin/src/test/kotlin/net/ladenthin/llama/kotlin/CloseableIterableFlowTest.kt b/llama-kotlin/src/test/kotlin/net/ladenthin/llama/kotlin/CloseableIterableFlowTest.kt new file mode 100644 index 00000000..0a5cb06b --- /dev/null +++ b/llama-kotlin/src/test/kotlin/net/ladenthin/llama/kotlin/CloseableIterableFlowTest.kt @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.kotlin + +import java.util.concurrent.atomic.AtomicInteger +import kotlinx.coroutines.flow.take +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.contains +import org.hamcrest.Matchers.instanceOf +import org.hamcrest.Matchers.`is` +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test + +/** + * Model-free tests for [closeableIterableFlow], the seam behind + * `LlamaModel.generateFlow`/`generateChatFlow`. A fake `Iterable & AutoCloseable` stands in for + * `LlamaIterable` (a final class), pinning the close-on-completion / close-on-cancellation / + * close-on-error contract that keeps native task slots from leaking. + */ +class CloseableIterableFlowTest { + + private class FakeStream( + private val items: List, + private val failAfter: Int = Int.MAX_VALUE, + ) : Iterable, AutoCloseable { + var closed = false + private set + + override fun iterator(): Iterator = object : Iterator { + private var index = 0 + + override fun hasNext(): Boolean = index < items.size + + override fun next(): String { + check(index < failAfter) { "simulated native failure" } + return items[index++] + } + } + + override fun close() { + closed = true + } + } + + @Test + fun emitsAllItemsInOrderAndClosesOnCompletion() = runTest { + val stream = FakeStream(listOf("a", "b", "c")) + + val collected = closeableIterableFlow { stream }.toList() + + assertThat(collected, contains("a", "b", "c")) + assertThat(stream.closed, `is`(true)) + } + + @Test + fun earlyCancellationClosesTheSource() = runTest { + val stream = FakeStream(listOf("a", "b", "c")) + + val collected = closeableIterableFlow { stream }.take(1).toList() + + // take(1) cancels the flow after the first emission; the source must still be + // closed so the native task slot is released. + assertThat(collected, contains("a")) + assertThat(stream.closed, `is`(true)) + } + + @Test + fun iterationFailureClosesTheSourceAndPropagates() = runTest { + val stream = FakeStream(listOf("a", "b"), failAfter = 1) + + val thrown = assertThrows(IllegalStateException::class.java) { + kotlinx.coroutines.runBlocking { + closeableIterableFlow { stream }.toList() + } + } + + assertThat(thrown, instanceOf(IllegalStateException::class.java)) + assertThat(stream.closed, `is`(true)) + } + + @Test + fun coldFlowOpensAFreshSourcePerCollection() = runTest { + val opens = AtomicInteger() + val flow = closeableIterableFlow { + opens.incrementAndGet() + FakeStream(listOf("x")) + } + + flow.toList() + flow.toList() + + assertThat(opens.get(), `is`(2)) + } +} diff --git a/llama-kotlin/src/test/kotlin/net/ladenthin/llama/kotlin/WithCancellationTokenTest.kt b/llama-kotlin/src/test/kotlin/net/ladenthin/llama/kotlin/WithCancellationTokenTest.kt new file mode 100644 index 00000000..7619f048 --- /dev/null +++ b/llama-kotlin/src/test/kotlin/net/ladenthin/llama/kotlin/WithCancellationTokenTest.kt @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.kotlin + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import net.ladenthin.llama.callback.CancellationToken +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.`is` +import org.junit.jupiter.api.Test + +/** + * Model-free tests for [withCancellationToken], the seam behind `LlamaModel.completeSuspend`. + * A blocking block that spins on the token stands in for the native inference loop, pinning + * that coroutine cancellation reaches the cooperative [CancellationToken] (and that normal + * completion does not). + */ +class WithCancellationTokenTest { + + @Test + fun returnsBlockResultAndLeavesTokenUncancelledOnNormalCompletion() = runBlocking { + val seenToken = AtomicReference() + + val result = withCancellationToken(Dispatchers.IO) { token -> + seenToken.set(token) + "done" + } + + assertThat(result, `is`("done")) + assertThat(seenToken.get().isCancelled, `is`(false)) + } + + @Test + fun coroutineCancellationCancelsTheTokenSoTheBlockingLoopStops() = runBlocking { + val blockEntered = CountDownLatch(1) + val blockFinished = CountDownLatch(1) + val seenToken = AtomicReference() + + // launch on Default (not runBlocking's single event-loop thread): the test + // body blocks on latches below, which would otherwise starve the event loop + // before the job is ever dispatched. + val job = launch(Dispatchers.Default) { + withCancellationToken(Dispatchers.IO) { token -> + seenToken.set(token) + blockEntered.countDown() + // Stand-in for the native token loop: spins until the cooperative + // token is cancelled. If cancellation never reaches the token this + // spins forever and the withTimeout below fails the test. + while (!token.isCancelled) { + Thread.sleep(5) + } + blockFinished.countDown() + "partial" + } + } + + assertThat(blockEntered.await(5, TimeUnit.SECONDS), `is`(true)) + withTimeout(5_000) { job.cancelAndJoin() } + + // The block observed the cancel and returned (the scope waited for it), + // proving no still-running generation leaks past the suspend call. + assertThat(blockFinished.await(5, TimeUnit.SECONDS), `is`(true)) + assertThat(seenToken.get().isCancelled, `is`(true)) + assertThat(job.isCancelled, `is`(true)) + } +} diff --git a/llama-langchain4j/README.md b/llama-langchain4j/README.md index e477f5cd..29b5a57d 100644 --- a/llama-langchain4j/README.md +++ b/llama-langchain4j/README.md @@ -18,7 +18,7 @@ pull langchain4j (or its Java 17 floor) transitively. | Class | langchain4j interface | java-llama.cpp call | |-------|-----------------------|---------------------| | `JllamaChatModel` | `ChatModel` | `LlamaModel.chat(...)` | -| `JllamaStreamingChatModel` | `StreamingChatModel` | `LlamaModel.generateChat(...)` (token streaming) | +| `JllamaStreamingChatModel` | `StreamingChatModel` | `LlamaModel.streamChatCompletion(...)` (OAI chunk streaming: text, thinking, tool calls) | | `JllamaEmbeddingModel` | `EmbeddingModel` | `LlamaModel.embed(...)` | | `JllamaScoringModel` | `ScoringModel` (re-ranking) | `LlamaModel.handleRerank(...)` | @@ -95,6 +95,9 @@ mvn test -Dnet.ladenthin.llama.model.path=/abs/path/to/chat.gguf mvn test -Dnet.ladenthin.llama.langchain4j.embedding.model=/abs/path/to/embedding.gguf # re-ranking / scoring (JllamaScoringModelIntegrationTest) mvn test -Dnet.ladenthin.llama.langchain4j.rerank.model=/abs/path/to/reranker.gguf +# tool calling + JSON-schema structured output (JllamaToolCallingIntegrationTest; +# needs a tool-capable instruct model, e.g. Qwen2.5-Instruct) +mvn test -Dnet.ladenthin.llama.langchain4j.tool.model=/abs/path/to/instruct.gguf ``` In CI these reuse the project's existing shared GGUF cache (the chat, nomic-embedding and @@ -102,22 +105,34 @@ jina-reranker models the core test jobs already download) — the `test-java-llama-langchain4j-integration` job restores that cache and the `Linux-x86_64` native library artifact, so no extra model is downloaded. +## Mapped features + +- **Tool calling (blocking).** `ChatRequest.toolSpecifications()` and `toolChoice()` are forwarded to + the native OAI tools path; a response with `tool_calls` comes back as + `AiMessage.toolExecutionRequests()` with finish reason `TOOL_EXECUTION`. Assistant tool-call turns + and `ToolExecutionResultMessage`s in the request history round-trip, so langchain4j `AiServices` + agent loops work against `JllamaChatModel`. `ToolSpecification.parameters()` (the langchain4j + `JsonSchemaElement` tree, including `$defs`/`$ref` recursion) is serialized to standard JSON Schema + by this module. +- **`response_format` (JSON mode).** `ResponseFormat.JSON` maps to the native + `response_format={"type":"json_object"}`; a `ResponseFormat` carrying a `JsonSchema` maps to the + native `json_schema` grammar constraint (structured output). Works on both adapters. +- **Multimodal user input.** `ImageContent` (base64 or URL) and `AudioContent` (inline wav/mp3) map to + the OAI array-form `content` parts routed through the compiled-in mtmd pipeline — the model must be + loaded with a matching `--mmproj` (see the core README's multimodal section). Unsupported media + (URL-only audio, non-wav/mp3 audio, video/PDF) fails loud rather than being silently dropped. +- **Sampling parameters:** `temperature`, `topP`, `topK`, `maxOutputTokens`, `frequencyPenalty`, + `presencePenalty`, `stopSequences`. + +- **Streaming tool calls + thinking events.** `JllamaStreamingChatModel` streams over the + native OpenAI chunk path: `delta.tool_calls` fragments surface as `onPartialToolCall` / + `onCompleteToolCall` events and land on the final response as + `AiMessage.toolExecutionRequests()` (finish reason `TOOL_EXECUTION`); reasoning deltas surface + as `onPartialThinking` and as `AiMessage.thinking()`; both the blocking and the streamed + final response carry the model's real finish reason and token usage. + ## Not mapped yet -- **Tool calling.** `ChatRequest.toolSpecifications()` are not forwarded, so the chat adapters return - assistant *text*, not `AiMessage.toolExecutionRequests()`. (java-llama.cpp itself supports tool - calling via `LlamaModel.chatWithTools` / typed `ToolDefinition`; bridging that to langchain4j - `ToolSpecification` is the planned next step.) -- **Multimodal user input.** A multi-content `UserMessage` is flattened to its text parts; image/audio - content is dropped. -- **Per-token tool-call / thinking stream events.** Streaming forwards plain text via - `onPartialResponse`. -- **`response_format` (JSON mode).** `ChatRequest.responseFormat()` (json_object / json_schema) is not - forwarded; `modelName()` is ignored since one model is bound per adapter. - -Mapped request parameters: `temperature`, `topP`, `topK`, `maxOutputTokens`, `frequencyPenalty`, -`presencePenalty`, `stopSequences`. The non-streaming chat response carries the model's real finish -reason (`stop`/`length`/`tool_calls`) and token usage; the streaming completion carries assembled text -(no per-token usage). +- **`modelName()`** is ignored since one model is bound per adapter. Requires Java 17+ (langchain4j 1.x baseline). Targets `langchain4j-core` 1.17.1. diff --git a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JllamaChatModel.java b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JllamaChatModel.java index dcade59f..62e0ac1b 100644 --- a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JllamaChatModel.java +++ b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JllamaChatModel.java @@ -17,10 +17,14 @@ * {@link LlamaModel} you already own and keep managing that model's lifecycle (try-with-resources or * an explicit {@code close()}). One {@code LlamaModel} can back several adapters at once. * - *

Mapped today: messages (system/user/assistant/tool-result) and the sampling parameters - * {@code temperature}/{@code topP}/{@code topK}/{@code maxOutputTokens}/{@code stopSequences}. - * Tool specifications on the request are not yet forwarded, so this returns assistant text, - * not tool calls — see the module README for the planned tool-calling bridge. + *

Mapped today: messages (system/user/assistant-with-tool-calls/tool-result), multimodal user + * content (text + image/audio parts, routed through the mtmd pipeline when the model was loaded + * with an {@code mmproj}), tool specifications ({@code toolSpecifications()} + {@code toolChoice()} + * — a response with {@code tool_calls} comes back as {@link dev.langchain4j.data.message.AiMessage + * AiMessage}{@code .toolExecutionRequests()} with finish reason {@code TOOL_EXECUTION}), JSON + * {@code responseFormat()} (plain JSON mode and JSON-schema-constrained structured output), and the + * sampling parameters {@code temperature}/{@code topP}/{@code topK}/{@code maxOutputTokens}/{@code + * frequencyPenalty}/{@code presencePenalty}/{@code stopSequences}. */ public final class JllamaChatModel implements ChatModel { diff --git a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JllamaStreamingChatModel.java b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JllamaStreamingChatModel.java index 9bf2124a..8cecaa4e 100644 --- a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JllamaStreamingChatModel.java +++ b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JllamaStreamingChatModel.java @@ -4,26 +4,29 @@ package net.ladenthin.llama.langchain4j; -import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.model.chat.StreamingChatModel; import dev.langchain4j.model.chat.request.ChatRequest; import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; -import dev.langchain4j.model.output.FinishReason; import java.util.Objects; -import net.ladenthin.llama.LlamaIterable; import net.ladenthin.llama.LlamaModel; -import net.ladenthin.llama.value.LlamaOutput; /** * langchain4j {@link StreamingChatModel} backed by an in-process java-llama.cpp model. * - *

Each generated token is forwarded to {@link StreamingChatResponseHandler#onPartialResponse}; a - * final {@link StreamingChatResponseHandler#onCompleteResponse} carries the assembled assistant - * message. Any failure during generation is reported via {@link StreamingChatResponseHandler#onError}. + *

Streams over the native OpenAI {@code chat.completion.chunk} path + * ({@code LlamaModel.streamChatCompletion}), so the full event surface is forwarded: + * assistant text via {@link StreamingChatResponseHandler#onPartialResponse(String)}, reasoning + * deltas via {@link StreamingChatResponseHandler#onPartialThinking}, and streamed tool + * calls — {@code delta.tool_calls} fragments are forwarded as + * {@link StreamingChatResponseHandler#onPartialToolCall} events, completed via + * {@link StreamingChatResponseHandler#onCompleteToolCall}, and carried on the final + * {@link ChatResponse} as {@code AiMessage.toolExecutionRequests()} (finish reason + * {@code TOOL_EXECUTION}). JSON {@code responseFormat()} and multimodal user content are + * forwarded like in the blocking adapter. Any failure during generation is reported via + * {@link StreamingChatResponseHandler#onError}. * - *

The model is borrowed (never closed here) — see {@link JllamaChatModel}. Tool - * specifications are not yet forwarded; this streams plain assistant text. + *

The model is borrowed (never closed here) — see {@link JllamaChatModel}. */ public final class JllamaStreamingChatModel implements StreamingChatModel { @@ -40,20 +43,13 @@ public JllamaStreamingChatModel(LlamaModel model) { @Override public void doChat(ChatRequest chatRequest, StreamingChatResponseHandler handler) { - StringBuilder full = new StringBuilder(); - try (LlamaIterable stream = model.generateChat(LangChain4jMapping.toStreamingParameters(chatRequest))) { - for (LlamaOutput output : stream) { - full.append(output.text); - handler.onPartialResponse(output.text); - } + StreamingChunkAssembler assembler = new StreamingChunkAssembler(handler); + try { + model.streamChatCompletion(LangChain4jMapping.toStreamingParameters(chatRequest), assembler::accept); } catch (Exception e) { handler.onError(e); return; } - handler.onCompleteResponse( - ChatResponse.builder() - .aiMessage(AiMessage.from(full.toString())) - .finishReason(FinishReason.STOP) - .build()); + handler.onCompleteResponse(assembler.complete()); } } diff --git a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JsonSchemaElementSerializer.java b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JsonSchemaElementSerializer.java new file mode 100644 index 00000000..6399310e --- /dev/null +++ b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/JsonSchemaElementSerializer.java @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.langchain4j; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import dev.langchain4j.model.chat.request.json.JsonAnyOfSchema; +import dev.langchain4j.model.chat.request.json.JsonArraySchema; +import dev.langchain4j.model.chat.request.json.JsonBooleanSchema; +import dev.langchain4j.model.chat.request.json.JsonEnumSchema; +import dev.langchain4j.model.chat.request.json.JsonIntegerSchema; +import dev.langchain4j.model.chat.request.json.JsonNullSchema; +import dev.langchain4j.model.chat.request.json.JsonNumberSchema; +import dev.langchain4j.model.chat.request.json.JsonObjectSchema; +import dev.langchain4j.model.chat.request.json.JsonRawSchema; +import dev.langchain4j.model.chat.request.json.JsonReferenceSchema; +import dev.langchain4j.model.chat.request.json.JsonSchemaElement; +import dev.langchain4j.model.chat.request.json.JsonStringSchema; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Serializes a langchain4j {@link JsonSchemaElement} tree to a standard JSON Schema string. + * + *

langchain4j's own serializer lives in its {@code internal} package (not public API), so this + * module carries its own recursive walk over the public element types. The emitted shape follows + * the langchain4j conventions so schemas produced by langchain4j's annotation processing round-trip + * unchanged: object definitions land under {@code $defs}, and a {@link JsonReferenceSchema} becomes + * {@code {"$ref": "#/$defs/<reference>"}}. + * + *

Pure data transform: no JNI, no model state, unit-testable with schema literals (see + * {@code JsonSchemaElementSerializerTest}). + */ +final class JsonSchemaElementSerializer { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private JsonSchemaElementSerializer() {} + + /** + * Serialize a schema element tree to its JSON Schema string form. + * + * @param element the root element (must not be {@code null}) + * @return the JSON Schema as a string + * @throws IllegalArgumentException on an unknown element type or an unparseable + * {@link JsonRawSchema} + */ + static String toJson(JsonSchemaElement element) { + return toJsonNode(element).toString(); + } + + /** + * Serialize a schema element tree to a Jackson node. + * + * @param element the root element (must not be {@code null}) + * @return the JSON Schema as an {@link ObjectNode} + * @throws IllegalArgumentException on an unknown element type or an unparseable + * {@link JsonRawSchema} + */ + static ObjectNode toJsonNode(JsonSchemaElement element) { + if (element instanceof JsonObjectSchema) { + return objectNode((JsonObjectSchema) element); + } + if (element instanceof JsonStringSchema) { + return primitiveNode("string", ((JsonStringSchema) element).description()); + } + if (element instanceof JsonIntegerSchema) { + return primitiveNode("integer", ((JsonIntegerSchema) element).description()); + } + if (element instanceof JsonNumberSchema) { + return primitiveNode("number", ((JsonNumberSchema) element).description()); + } + if (element instanceof JsonBooleanSchema) { + return primitiveNode("boolean", ((JsonBooleanSchema) element).description()); + } + if (element instanceof JsonEnumSchema) { + return enumNode((JsonEnumSchema) element); + } + if (element instanceof JsonArraySchema) { + return arrayNode((JsonArraySchema) element); + } + if (element instanceof JsonReferenceSchema) { + return referenceNode((JsonReferenceSchema) element); + } + if (element instanceof JsonAnyOfSchema) { + return anyOfNode((JsonAnyOfSchema) element); + } + if (element instanceof JsonNullSchema) { + return primitiveNode("null", ((JsonNullSchema) element).description()); + } + if (element instanceof JsonRawSchema) { + return rawNode((JsonRawSchema) element); + } + throw new IllegalArgumentException( + "Unsupported JsonSchemaElement type: " + element.getClass().getName()); + } + + private static ObjectNode objectNode(JsonObjectSchema schema) { + ObjectNode node = MAPPER.createObjectNode(); + node.put("type", "object"); + putDescription(node, schema.description()); + ObjectNode properties = node.putObject("properties"); + Map schemaProperties = schema.properties(); + if (schemaProperties != null) { + for (Map.Entry entry : schemaProperties.entrySet()) { + properties.set(entry.getKey(), toJsonNode(entry.getValue())); + } + } + List required = schema.required(); + if (required != null && !required.isEmpty()) { + ArrayNode requiredNode = node.putArray("required"); + for (String name : required) { + requiredNode.add(name); + } + } + if (schema.additionalProperties() != null) { + node.put("additionalProperties", schema.additionalProperties().booleanValue()); + } + Map definitions = schema.definitions(); + if (definitions != null && !definitions.isEmpty()) { + ObjectNode defs = node.putObject("$defs"); + for (Map.Entry entry : definitions.entrySet()) { + defs.set(entry.getKey(), toJsonNode(entry.getValue())); + } + } + return node; + } + + private static ObjectNode enumNode(JsonEnumSchema schema) { + // langchain4j emits enums as string-typed with an "enum" values list. + ObjectNode node = primitiveNode("string", schema.description()); + ArrayNode values = node.putArray("enum"); + List enumValues = schema.enumValues(); + if (enumValues != null) { + for (String value : enumValues) { + values.add(value); + } + } + return node; + } + + private static ObjectNode arrayNode(JsonArraySchema schema) { + ObjectNode node = MAPPER.createObjectNode(); + node.put("type", "array"); + putDescription(node, schema.description()); + if (schema.items() != null) { + node.set("items", toJsonNode(schema.items())); + } + return node; + } + + private static ObjectNode referenceNode(JsonReferenceSchema schema) { + ObjectNode node = MAPPER.createObjectNode(); + // Mirrors langchain4j's internal convention: definitions live under "$defs". + if (schema.reference() != null) { + node.put("$ref", "#/$defs/" + schema.reference()); + } + return node; + } + + private static ObjectNode anyOfNode(JsonAnyOfSchema schema) { + ObjectNode node = MAPPER.createObjectNode(); + putDescription(node, schema.description()); + ArrayNode anyOf = node.putArray("anyOf"); + List elements = schema.anyOf(); + if (elements != null) { + for (JsonSchemaElement element : elements) { + anyOf.add(toJsonNode(element)); + } + } + return node; + } + + private static ObjectNode rawNode(JsonRawSchema schema) { + try { + return (ObjectNode) MAPPER.readTree(schema.schema()); + } catch (IOException | ClassCastException e) { + throw new IllegalArgumentException("JsonRawSchema does not contain a JSON object: " + schema.schema(), e); + } + } + + private static ObjectNode primitiveNode(String type, String description) { + ObjectNode node = MAPPER.createObjectNode(); + node.put("type", type); + putDescription(node, description); + return node; + } + + private static void putDescription(ObjectNode node, String description) { + if (description != null) { + node.put("description", description); + } + } +} diff --git a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/LangChain4jMapping.java b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/LangChain4jMapping.java index da0ca32b..c137b51b 100644 --- a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/LangChain4jMapping.java +++ b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/LangChain4jMapping.java @@ -5,22 +5,37 @@ package net.ladenthin.llama.langchain4j; import com.fasterxml.jackson.databind.JsonNode; +import dev.langchain4j.agent.tool.ToolExecutionRequest; +import dev.langchain4j.agent.tool.ToolSpecification; +import dev.langchain4j.data.audio.Audio; +import dev.langchain4j.data.image.Image; import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.AudioContent; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.Content; import dev.langchain4j.data.message.ContentType; +import dev.langchain4j.data.message.ImageContent; import dev.langchain4j.data.message.SystemMessage; import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.request.ResponseFormat; +import dev.langchain4j.model.chat.request.ResponseFormatType; +import dev.langchain4j.model.chat.request.ToolChoice; import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.output.FinishReason; import dev.langchain4j.model.output.TokenUsage; import java.io.IOException; +import java.util.ArrayList; +import java.util.Base64; import java.util.List; +import java.util.Locale; import net.ladenthin.llama.json.RerankResponseParser; import net.ladenthin.llama.parameters.InferenceParameters; +import net.ladenthin.llama.value.ContentPart; +import net.ladenthin.llama.value.ToolCall; +import net.ladenthin.llama.value.ToolDefinition; /** * Pure (model-free) translation between langchain4j chat types and java-llama.cpp parameters. @@ -31,12 +46,24 @@ */ final class LangChain4jMapping { + /** + * Parameter schema used when a {@link ToolSpecification} declares no parameters: a + * no-argument tool still needs a syntactically valid JSON Schema object for the OAI + * {@code tools} array. + */ + static final String EMPTY_PARAMETERS_SCHEMA = "{\"type\":\"object\",\"properties\":{}}"; + + /** OAI {@code response_format} payload selecting plain JSON-object mode (no schema). */ + static final String JSON_OBJECT_RESPONSE_FORMAT = "{\"type\":\"json_object\"}"; + private LangChain4jMapping() {} /** * Build a java-llama.cpp typed chat request from a langchain4j chat request. Messages map by - * role; sampling parameters ({@code temperature}/{@code topP}/{@code topK}/{@code - * maxOutputTokens}/{@code stopSequences}) ride along as an inference customizer. + * role (including assistant tool-call turns and multimodal user turns); tool specifications and + * the {@code toolChoice} hint are forwarded; sampling parameters ({@code temperature}/{@code + * topP}/{@code topK}/{@code maxOutputTokens}/{@code stopSequences}) and a JSON {@code + * responseFormat} ride along as an inference customizer. */ static net.ladenthin.llama.parameters.ChatRequest toJllamaRequest(ChatRequest request) { net.ladenthin.llama.parameters.ChatRequest jllama = @@ -44,24 +71,48 @@ static net.ladenthin.llama.parameters.ChatRequest toJllamaRequest(ChatRequest re for (ChatMessage message : request.messages()) { jllama = jllama.appendMessage(toJllamaMessage(message)); } - return jllama.withInferenceCustomizer(params -> applySampling(params, request)); + List tools = request.toolSpecifications(); + if (tools != null) { + for (ToolSpecification tool : tools) { + jllama = jllama.appendTool(toToolDefinition(tool)); + } + } + if (request.toolChoice() != null) { + jllama = jllama.withToolChoice(toToolChoiceString(request.toolChoice())); + } + return jllama.withInferenceCustomizer(params -> applyResponseFormat(applySampling(params, request), request)); } /** - * Build the streaming inference parameters (messages JSON + sampling) for {@code generateChat}. - * Shares {@link #toJllamaRequest(ChatRequest)} so blocking and streaming stay in lockstep. + * Build the streaming inference parameters (messages JSON + tools + sampling) for + * {@code streamChatCompletion}. Shares {@link #toJllamaRequest(ChatRequest)} so blocking and + * streaming stay in lockstep: tool specifications, the {@code toolChoice} hint, JSON + * {@code responseFormat}, and the sampling parameters all ride along — the same wiring + * {@code LlamaModel.chat} applies on the blocking path. */ static InferenceParameters toStreamingParameters(ChatRequest request) { net.ladenthin.llama.parameters.ChatRequest jllama = toJllamaRequest(request); InferenceParameters params = InferenceParameters.empty().withMessagesJson(jllama.buildMessagesJson()); + java.util.Optional toolsJson = jllama.buildToolsJson(); + if (toolsJson.isPresent()) { + params = params.withToolsJson(toolsJson.get()).withUseChatTemplate(true); + java.util.Optional toolChoice = jllama.getToolChoice(); + if (toolChoice.isPresent()) { + params = params.withToolChoice(toolChoice.get()); + } + } return jllama.applyCustomizer(params); } - /** Wrap a java-llama.cpp chat result as a langchain4j {@link ChatResponse}. */ + /** + * Wrap a java-llama.cpp chat result as a langchain4j {@link ChatResponse}. An assistant turn + * carrying {@code tool_calls} becomes an {@link AiMessage} with + * {@link AiMessage#toolExecutionRequests()} populated (and {@code finishReason} mapped to + * {@code TOOL_EXECUTION} by the native {@code "tool_calls"} finish string). + */ static ChatResponse toLangChainResponse(net.ladenthin.llama.value.ChatResponse response) { - ChatResponse.Builder builder = - ChatResponse.builder().aiMessage(AiMessage.from(response.getFirstContent())); + ChatResponse.Builder builder = ChatResponse.builder().aiMessage(toAiMessage(response)); net.ladenthin.llama.value.Usage usage = response.getUsage(); if (usage != null) { builder.tokenUsage( @@ -126,17 +177,59 @@ static double[] parseRerankScores(String json, int count) { return scores; } + /** Convert a langchain4j tool specification to the jllama typed tool definition. */ + static ToolDefinition toToolDefinition(ToolSpecification spec) { + String parametersJson = spec.parameters() == null + ? EMPTY_PARAMETERS_SCHEMA + : JsonSchemaElementSerializer.toJson(spec.parameters()); + return new ToolDefinition( + spec.name(), spec.description() == null ? "" : spec.description(), parametersJson); + } + + /** Map the langchain4j {@link ToolChoice} enum to the OAI {@code tool_choice} string. */ + static String toToolChoiceString(ToolChoice choice) { + switch (choice) { + case REQUIRED: + return "required"; + case NONE: + return "none"; + case AUTO: + default: + return "auto"; + } + } + + private static AiMessage toAiMessage(net.ladenthin.llama.value.ChatResponse response) { + net.ladenthin.llama.value.ChatMessage first = + response.getFirstMessage().orElse(null); + if (first == null || first.getToolCalls().isEmpty()) { + return AiMessage.from(response.getFirstContent()); + } + List requests = new ArrayList<>(first.getToolCalls().size()); + for (ToolCall call : first.getToolCalls()) { + requests.add(ToolExecutionRequest.builder() + .id(call.getId()) + .name(call.getName()) + .arguments(call.getArgumentsJson()) + .build()); + } + AiMessage.Builder builder = AiMessage.builder().toolExecutionRequests(requests); + String text = first.getContent(); + if (text != null && !text.isEmpty()) { + builder.text(text); + } + return builder.build(); + } + private static net.ladenthin.llama.value.ChatMessage toJllamaMessage(ChatMessage message) { switch (message.type()) { case SYSTEM: return new net.ladenthin.llama.value.ChatMessage( "system", ((SystemMessage) message).text()); case USER: - return new net.ladenthin.llama.value.ChatMessage("user", userText((UserMessage) message)); + return toJllamaUserMessage((UserMessage) message); case AI: - String aiText = ((AiMessage) message).text(); - return new net.ladenthin.llama.value.ChatMessage( - "assistant", aiText == null ? "" : aiText); + return toJllamaAssistantMessage((AiMessage) message); case TOOL_EXECUTION_RESULT: ToolExecutionResultMessage tool = (ToolExecutionResultMessage) message; return net.ladenthin.llama.value.ChatMessage.toolResult(tool.id(), tool.text()); @@ -146,18 +239,115 @@ private static net.ladenthin.llama.value.ChatMessage toJllamaMessage(ChatMessage } } - /** Flatten a (possibly multimodal) user message to text; non-text parts (images) are dropped. */ - private static String userText(UserMessage message) { + private static net.ladenthin.llama.value.ChatMessage toJllamaAssistantMessage(AiMessage message) { + String text = message.text(); + if (!message.hasToolExecutionRequests()) { + return new net.ladenthin.llama.value.ChatMessage("assistant", text == null ? "" : text); + } + List calls = new ArrayList<>(message.toolExecutionRequests().size()); + for (ToolExecutionRequest request : message.toolExecutionRequests()) { + calls.add(new ToolCall( + request.id() == null ? "" : request.id(), + request.name(), + request.arguments() == null ? "{}" : request.arguments())); + } + return net.ladenthin.llama.value.ChatMessage.assistantToolCalls(text == null ? "" : text, calls); + } + + private static net.ladenthin.llama.value.ChatMessage toJllamaUserMessage(UserMessage message) { if (message.hasSingleText()) { - return message.singleText(); + return new net.ladenthin.llama.value.ChatMessage("user", message.singleText()); } - StringBuilder text = new StringBuilder(); - for (Content content : message.contents()) { - if (content.type() == ContentType.TEXT) { + if (!hasNonTextContent(message)) { + // Multiple text parts, no media: keep the flat-string form (no array-content needed). + StringBuilder text = new StringBuilder(); + for (Content content : message.contents()) { text.append(((TextContent) content).text()); } + return new net.ladenthin.llama.value.ChatMessage("user", text.toString()); + } + List parts = new ArrayList<>(message.contents().size()); + for (Content content : message.contents()) { + parts.add(toContentPart(content)); + } + return new net.ladenthin.llama.value.ChatMessage("user", parts); + } + + private static boolean hasNonTextContent(UserMessage message) { + for (Content content : message.contents()) { + if (content.type() != ContentType.TEXT) { + return true; + } + } + return false; + } + + /** + * Map one langchain4j content part to the jllama multimodal part. Media requires either an + * inline payload (base64 + MIME type) or, for images, a URL; anything the upstream mtmd + * pipeline cannot consume fails loud instead of being silently dropped. + */ + static ContentPart toContentPart(Content content) { + if (content instanceof TextContent) { + return ContentPart.text(((TextContent) content).text()); + } + if (content instanceof ImageContent) { + return toImagePart(((ImageContent) content).image()); + } + if (content instanceof AudioContent) { + return toAudioPart(((AudioContent) content).audio()); + } + throw new IllegalArgumentException("Unsupported user content type: " + content.type()); + } + + private static ContentPart toImagePart(Image image) { + if (image.base64Data() != null && image.mimeType() != null) { + return ContentPart.imageUrl("data:" + image.mimeType() + ";base64," + image.base64Data()); + } + if (image.url() != null) { + return ContentPart.imageUrl(image.url().toString()); + } + throw new IllegalArgumentException( + "ImageContent carries neither base64 data (with MIME type) nor a URL: " + image); + } + + private static ContentPart toAudioPart(Audio audio) { + String format = toAudioFormat(audio.mimeType()); + byte[] bytes = audio.binaryData(); + if (bytes == null && audio.base64Data() != null) { + bytes = Base64.getDecoder().decode(audio.base64Data()); + } + if (bytes == null) { + throw new IllegalArgumentException( + "AudioContent carries no inline audio data (URL-only audio is not supported): " + audio); + } + return ContentPart.inputAudio(bytes, format); + } + + private static String toAudioFormat(String mimeType) { + if (mimeType != null) { + String normalized = mimeType.toLowerCase(Locale.ROOT); + if (normalized.equals("audio/wav") || normalized.equals("audio/x-wav") || normalized.equals("audio/wave")) { + return "wav"; + } + if (normalized.equals("audio/mpeg") || normalized.equals("audio/mp3")) { + return "mp3"; + } + } + throw new IllegalArgumentException( + "Unsupported audio MIME type (only wav/mp3 reach the mtmd pipeline): " + mimeType); + } + + private static InferenceParameters applyResponseFormat(InferenceParameters params, ChatRequest request) { + ResponseFormat format = request.responseFormat(); + if (format == null || format.type() != ResponseFormatType.JSON) { + return params; + } + if (format.jsonSchema() != null && format.jsonSchema().rootElement() != null) { + return params.withJsonSchema( + JsonSchemaElementSerializer.toJson(format.jsonSchema().rootElement())); } - return text.toString(); + return params.withResponseFormat(JSON_OBJECT_RESPONSE_FORMAT); } private static InferenceParameters applySampling(InferenceParameters params, ChatRequest request) { diff --git a/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/StreamingChunkAssembler.java b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/StreamingChunkAssembler.java new file mode 100644 index 00000000..b224a28f --- /dev/null +++ b/llama-langchain4j/src/main/java/net/ladenthin/llama/langchain4j/StreamingChunkAssembler.java @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.langchain4j; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.langchain4j.agent.tool.ToolExecutionRequest; +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.model.chat.response.ChatResponse; +import dev.langchain4j.model.chat.response.CompleteToolCall; +import dev.langchain4j.model.chat.response.PartialThinking; +import dev.langchain4j.model.chat.response.PartialToolCall; +import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; +import dev.langchain4j.model.output.TokenUsage; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Assembles the native OpenAI {@code chat.completion.chunk} stream + * ({@code LlamaModel.streamChatCompletion}) into langchain4j streaming events and the final + * {@link ChatResponse}: + * + *

    + *
  • {@code delta.content} → {@link StreamingChatResponseHandler#onPartialResponse(String)} + *
  • {@code delta.reasoning_content} → + * {@link StreamingChatResponseHandler#onPartialThinking(PartialThinking)} + *
  • {@code delta.tool_calls[*]} fragments → accumulated per OpenAI {@code index} (id and + * name arrive on the first fragment, arguments split across fragments), each forwarded as + * {@link StreamingChatResponseHandler#onPartialToolCall(PartialToolCall)} and completed as + * {@link StreamingChatResponseHandler#onCompleteToolCall(CompleteToolCall)} + *
  • {@code finish_reason} / trailing {@code usage} chunk → finish reason and + * {@link TokenUsage} on the final response + *
+ * + *

Pure data transform over chunk JSON strings — no JNI, no model — so the whole state + * machine is unit-testable with canned chunks (see {@code StreamingChunkAssemblerTest}). + * Not thread-safe; one instance per streamed request (chunks arrive in order on one thread).

+ */ +final class StreamingChunkAssembler { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** Per-index accumulation state for one streamed tool call. */ + private static final class PartialTool { + String id = ""; + String name = ""; + final StringBuilder arguments = new StringBuilder(); + } + + private final StreamingChatResponseHandler handler; + private final StringBuilder text = new StringBuilder(); + private final StringBuilder thinking = new StringBuilder(); + private final Map toolsByIndex = new TreeMap<>(); + private String finishReason = ""; + private TokenUsage tokenUsage; + + /** + * Creates an assembler forwarding intermediate events to {@code handler}. + * + * @param handler the langchain4j streaming handler to notify per chunk + */ + StreamingChunkAssembler(StreamingChatResponseHandler handler) { + this.handler = handler; + } + + /** + * Consume one {@code chat.completion.chunk} JSON string, forwarding the matching + * streaming events. + * + * @param chunkJson the chunk as emitted by {@code streamChatCompletion} + * @throws UncheckedIOException if the chunk is not parseable JSON (native contract + * violation — surfaces via {@code onError} in the adapter) + */ + void accept(String chunkJson) { + JsonNode chunk; + try { + chunk = MAPPER.readTree(chunkJson); + } catch (IOException e) { + throw new UncheckedIOException("Unparseable chat.completion.chunk: " + chunkJson, e); + } + JsonNode usage = chunk.path("usage"); + if (usage.isObject()) { + tokenUsage = new TokenUsage( + usage.path("prompt_tokens").asInt(0), usage.path("completion_tokens").asInt(0)); + } + JsonNode choice = chunk.path("choices").path(0); + if (choice.path("finish_reason").isTextual()) { + finishReason = choice.path("finish_reason").asText(); + } + JsonNode delta = choice.path("delta"); + JsonNode content = delta.path("content"); + if (content.isTextual() && !content.asText().isEmpty()) { + text.append(content.asText()); + handler.onPartialResponse(content.asText()); + } + JsonNode reasoning = delta.path("reasoning_content"); + if (reasoning.isTextual() && !reasoning.asText().isEmpty()) { + thinking.append(reasoning.asText()); + handler.onPartialThinking(new PartialThinking(reasoning.asText())); + } + JsonNode toolCalls = delta.path("tool_calls"); + if (toolCalls.isArray()) { + for (JsonNode fragment : toolCalls) { + acceptToolCallFragment(fragment); + } + } + } + + private void acceptToolCallFragment(JsonNode fragment) { + int index = fragment.path("index").asInt(0); + PartialTool tool = toolsByIndex.get(index); + if (tool == null) { + tool = new PartialTool(); + toolsByIndex.put(index, tool); + } + if (fragment.path("id").isTextual() && !fragment.path("id").asText().isEmpty()) { + tool.id = fragment.path("id").asText(); + } + JsonNode function = fragment.path("function"); + if (function.path("name").isTextual() && !function.path("name").asText().isEmpty()) { + tool.name = function.path("name").asText(); + } + String argumentsFragment = + function.path("arguments").isTextual() ? function.path("arguments").asText() : ""; + tool.arguments.append(argumentsFragment); + handler.onPartialToolCall(PartialToolCall.builder() + .index(index) + .id(tool.id) + .name(tool.name) + .partialArguments(argumentsFragment) + .build()); + } + + /** + * Finalize the stream: emit {@code onCompleteToolCall} for every accumulated call and + * build the final {@link ChatResponse} (text and/or tool execution requests, thinking, + * finish reason, token usage when the stream carried a usage chunk). + * + * @return the assembled final response for {@code onCompleteResponse} + */ + ChatResponse complete() { + AiMessage.Builder message = AiMessage.builder(); + if (text.length() > 0) { + message.text(text.toString()); + } + if (thinking.length() > 0) { + message.thinking(thinking.toString()); + } + if (!toolsByIndex.isEmpty()) { + List requests = new ArrayList<>(toolsByIndex.size()); + for (Map.Entry entry : toolsByIndex.entrySet()) { + PartialTool tool = entry.getValue(); + ToolExecutionRequest request = ToolExecutionRequest.builder() + .id(tool.id) + .name(tool.name) + .arguments(tool.arguments.toString()) + .build(); + handler.onCompleteToolCall(new CompleteToolCall(entry.getKey(), request)); + requests.add(request); + } + message.toolExecutionRequests(requests); + } + ChatResponse.Builder response = ChatResponse.builder() + .aiMessage(message.build()) + .finishReason(LangChain4jMapping.toFinishReason(finishReason.isEmpty() ? null : finishReason)); + if (tokenUsage != null) { + response.tokenUsage(tokenUsage); + } + return response.build(); + } +} diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaToolCallingIntegrationTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaToolCallingIntegrationTest.java new file mode 100644 index 00000000..2f24169c --- /dev/null +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JllamaToolCallingIntegrationTest.java @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.langchain4j; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.langchain4j.agent.tool.ToolExecutionRequest; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.request.ResponseFormat; +import dev.langchain4j.model.chat.request.ResponseFormatType; +import dev.langchain4j.model.chat.request.ToolChoice; +import dev.langchain4j.model.chat.request.json.JsonBooleanSchema; +import dev.langchain4j.model.chat.request.json.JsonObjectSchema; +import dev.langchain4j.model.chat.request.json.JsonSchema; +import dev.langchain4j.model.chat.request.json.JsonStringSchema; +import dev.langchain4j.model.chat.response.ChatResponse; +import dev.langchain4j.model.output.FinishReason; +import java.nio.file.Files; +import java.nio.file.Paths; +import net.ladenthin.llama.LlamaModel; +import net.ladenthin.llama.parameters.ModelParameters; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Real-model coverage for the langchain4j tool-calling and JSON-mode bridges. Self-skips unless a + * tool-capable GGUF is provided via {@code -Dnet.ladenthin.llama.langchain4j.tool.model} (the CI + * job passes the same Qwen2.5-Instruct model the core {@code ToolCallingIntegrationTest} uses), + * mirroring {@code JllamaChatModelIntegrationTest}'s self-skip pattern. The model is loaded with + * jinja enabled — required for the upstream tool-call chat-template path. + */ +class JllamaToolCallingIntegrationTest { + + /** System property naming the tool-capable GGUF this test loads. */ + static final String PROP_TOOL_MODEL = "net.ladenthin.llama.langchain4j.tool.model"; + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static LlamaModel model; + + @BeforeAll + static void loadModel() { + String path = System.getProperty(PROP_TOOL_MODEL); + Assumptions.assumeTrue(path != null && !path.isEmpty(), "tool model path property not set"); + Assumptions.assumeTrue(Files.exists(Paths.get(path)), "model file not present: " + path); + model = new LlamaModel(new ModelParameters() + .setModel(path) + .setCtxSize(8192) + .setFit(false) + .enableJinja()); + } + + @AfterAll + static void closeModel() { + if (model != null) { + model.close(); + } + } + + @Test + void requiredToolCallComesBackAsToolExecutionRequest() throws Exception { + JllamaChatModel chat = new JllamaChatModel(model); + + ChatResponse response = chat.chat(ChatRequest.builder() + .messages(UserMessage.from("Report success using the test tool.")) + .toolSpecifications(dev.langchain4j.agent.tool.ToolSpecification.builder() + .name("test") + .description("Report success") + .parameters(JsonObjectSchema.builder() + .addProperty("success", new JsonBooleanSchema()) + .required("success") + .build()) + .build()) + .toolChoice(ToolChoice.REQUIRED) + .maxOutputTokens(512) + .temperature(0.0) + .build()); + + assertThat(response.aiMessage().hasToolExecutionRequests(), is(true)); + ToolExecutionRequest request = + response.aiMessage().toolExecutionRequests().get(0); + assertThat(request.name(), is("test")); + // The grammar enforces the required field, so the arguments must be JSON carrying it. + JsonNode arguments = MAPPER.readTree(request.arguments()); + assertThat(arguments.has("success"), is(true)); + assertThat(response.finishReason(), is(FinishReason.TOOL_EXECUTION)); + } + + @Test + void streamedRequiredToolCallComesBackAsToolExecutionRequest() throws Exception { + JllamaStreamingChatModel streaming = new JllamaStreamingChatModel(model); + java.util.concurrent.CompletableFuture done = new java.util.concurrent.CompletableFuture<>(); + + streaming.chat( + ChatRequest.builder() + .messages(UserMessage.from("Report success using the test tool.")) + .toolSpecifications(dev.langchain4j.agent.tool.ToolSpecification.builder() + .name("test") + .description("Report success") + .parameters(JsonObjectSchema.builder() + .addProperty("success", new JsonBooleanSchema()) + .required("success") + .build()) + .build()) + .toolChoice(ToolChoice.REQUIRED) + .maxOutputTokens(512) + .temperature(0.0) + .build(), + new dev.langchain4j.model.chat.response.StreamingChatResponseHandler() { + @Override + public void onPartialResponse(String partialResponse) {} + + @Override + public void onCompleteResponse(ChatResponse completeResponse) { + done.complete(completeResponse); + } + + @Override + public void onError(Throwable error) { + done.completeExceptionally(error); + } + }); + + ChatResponse response = done.get(120, java.util.concurrent.TimeUnit.SECONDS); + assertThat(response.aiMessage().hasToolExecutionRequests(), is(true)); + ToolExecutionRequest request = + response.aiMessage().toolExecutionRequests().get(0); + assertThat(request.name(), is("test")); + assertThat(MAPPER.readTree(request.arguments()).has("success"), is(true)); + assertThat(response.finishReason(), is(FinishReason.TOOL_EXECUTION)); + } + + @Test + void jsonSchemaResponseFormatYieldsConformingJson() throws Exception { + JllamaChatModel chat = new JllamaChatModel(model); + + ChatResponse response = chat.chat(ChatRequest.builder() + .messages(UserMessage.from("The person is called Alice. Extract the person.")) + .responseFormat(ResponseFormat.builder() + .type(ResponseFormatType.JSON) + .jsonSchema(JsonSchema.builder() + .name("Person") + .rootElement(JsonObjectSchema.builder() + .addProperty("name", new JsonStringSchema()) + .required("name") + .build()) + .build()) + .build()) + .maxOutputTokens(256) + .temperature(0.0) + .build()); + + // The native json_schema grammar constraint forces conforming output. + JsonNode parsed = MAPPER.readTree(response.aiMessage().text()); + assertThat(parsed.path("name").isTextual(), is(true)); + assertThat(response.aiMessage(), is(notNullValue())); + } +} diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JsonSchemaElementSerializerTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JsonSchemaElementSerializerTest.java new file mode 100644 index 00000000..70a9afef --- /dev/null +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/JsonSchemaElementSerializerTest.java @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.langchain4j; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.langchain4j.model.chat.request.json.JsonAnyOfSchema; +import dev.langchain4j.model.chat.request.json.JsonArraySchema; +import dev.langchain4j.model.chat.request.json.JsonBooleanSchema; +import dev.langchain4j.model.chat.request.json.JsonEnumSchema; +import dev.langchain4j.model.chat.request.json.JsonIntegerSchema; +import dev.langchain4j.model.chat.request.json.JsonNullSchema; +import dev.langchain4j.model.chat.request.json.JsonNumberSchema; +import dev.langchain4j.model.chat.request.json.JsonObjectSchema; +import dev.langchain4j.model.chat.request.json.JsonRawSchema; +import dev.langchain4j.model.chat.request.json.JsonReferenceSchema; +import dev.langchain4j.model.chat.request.json.JsonSchemaElement; +import dev.langchain4j.model.chat.request.json.JsonStringSchema; +import java.io.IOException; +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +/** Model-free tests for the langchain4j {@code JsonSchemaElement} → JSON Schema serializer. */ +class JsonSchemaElementSerializerTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private static JsonNode serialize(JsonSchemaElement element) throws IOException { + return MAPPER.readTree(JsonSchemaElementSerializer.toJson(element)); + } + + @Test + void serializesObjectWithPrimitivesRequiredAndAdditionalProperties() throws IOException { + JsonObjectSchema schema = JsonObjectSchema.builder() + .description("a person") + .addProperty("name", JsonStringSchema.builder().description("full name").build()) + .addProperty("age", new JsonIntegerSchema()) + .addProperty("height", new JsonNumberSchema()) + .addProperty("active", new JsonBooleanSchema()) + .required("name", "age") + .additionalProperties(false) + .build(); + + JsonNode node = serialize(schema); + + assertThat(node.path("type").asText(), is("object")); + assertThat(node.path("description").asText(), is("a person")); + assertThat(node.path("properties").path("name").path("type").asText(), is("string")); + assertThat(node.path("properties").path("name").path("description").asText(), is("full name")); + assertThat(node.path("properties").path("age").path("type").asText(), is("integer")); + assertThat(node.path("properties").path("height").path("type").asText(), is("number")); + assertThat(node.path("properties").path("active").path("type").asText(), is("boolean")); + assertThat(node.path("required").get(0).asText(), is("name")); + assertThat(node.path("required").get(1).asText(), is("age")); + assertThat(node.path("additionalProperties").asBoolean(true), is(false)); + } + + @Test + void serializesEnumAsStringTypeWithValues() throws IOException { + JsonEnumSchema schema = JsonEnumSchema.builder() + .description("unit") + .enumValues("CELSIUS", "FAHRENHEIT") + .build(); + + JsonNode node = serialize(schema); + + // langchain4j convention: enums are string-typed with an "enum" list. + assertThat(node.path("type").asText(), is("string")); + assertThat(node.path("enum").get(0).asText(), is("CELSIUS")); + assertThat(node.path("enum").get(1).asText(), is("FAHRENHEIT")); + } + + @Test + void serializesArrayWithItems() throws IOException { + JsonArraySchema schema = JsonArraySchema.builder() + .description("tags") + .items(new JsonStringSchema()) + .build(); + + JsonNode node = serialize(schema); + + assertThat(node.path("type").asText(), is("array")); + assertThat(node.path("items").path("type").asText(), is("string")); + } + + @Test + void serializesReferenceAndDefinitionsInLangchainConvention() throws IOException { + // Recursive shape: a node whose children reference the node definition itself. + JsonObjectSchema schema = JsonObjectSchema.builder() + .addProperty( + "root", + JsonReferenceSchema.builder().reference("TreeNode").build()) + .definitions(java.util.Collections.singletonMap( + "TreeNode", + JsonObjectSchema.builder() + .addProperty("value", new JsonStringSchema()) + .build())) + .build(); + + JsonNode node = serialize(schema); + + // Must match langchain4j's internal convention so schema round-trips are stable. + assertThat(node.path("properties").path("root").path("$ref").asText(), is("#/$defs/TreeNode")); + assertThat( + node.path("$defs").path("TreeNode").path("properties").path("value").path("type").asText(), + is("string")); + } + + @Test + void serializesAnyOfIncludingNull() throws IOException { + JsonAnyOfSchema schema = JsonAnyOfSchema.builder() + .anyOf(Arrays.asList(new JsonStringSchema(), new JsonNullSchema())) + .build(); + + JsonNode node = serialize(schema); + + assertThat(node.path("anyOf").get(0).path("type").asText(), is("string")); + assertThat(node.path("anyOf").get(1).path("type").asText(), is("null")); + } + + @Test + void passesRawSchemaThroughVerbatim() throws IOException { + JsonRawSchema schema = JsonRawSchema.from("{\"type\":\"object\",\"properties\":{\"x\":{\"type\":\"integer\"}}}"); + + JsonNode node = serialize(schema); + + assertThat(node.path("properties").path("x").path("type").asText(), is("integer")); + } + + @Test + void rejectsUnparseableRawSchema() { + JsonRawSchema broken = JsonRawSchema.from("not json"); + + assertThrows(IllegalArgumentException.class, () -> JsonSchemaElementSerializer.toJson(broken)); + } + + @Test + void objectWithoutPropertiesStillCarriesEmptyPropertiesObject() throws IOException { + JsonNode node = serialize(JsonObjectSchema.builder().build()); + + // The OAI tools array expects "parameters" to be a full (if empty) object schema. + assertThat(node.path("type").asText(), is("object")); + assertThat(node.path("properties").isObject(), is(true)); + assertThat(node.path("properties").size(), is(0)); + } +} diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/LangChain4jMappingTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/LangChain4jMappingTest.java index 745213c0..40cea27c 100644 --- a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/LangChain4jMappingTest.java +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/LangChain4jMappingTest.java @@ -8,19 +8,37 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.assertThrows; +import dev.langchain4j.agent.tool.ToolExecutionRequest; +import dev.langchain4j.agent.tool.ToolSpecification; +import dev.langchain4j.data.audio.Audio; import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.AudioContent; +import dev.langchain4j.data.message.ImageContent; import dev.langchain4j.data.message.SystemMessage; import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.ToolExecutionResultMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.request.ResponseFormat; +import dev.langchain4j.model.chat.request.ResponseFormatType; +import dev.langchain4j.model.chat.request.ToolChoice; +import dev.langchain4j.model.chat.request.json.JsonObjectSchema; +import dev.langchain4j.model.chat.request.json.JsonSchema; +import dev.langchain4j.model.chat.request.json.JsonStringSchema; import dev.langchain4j.model.output.FinishReason; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import net.ladenthin.llama.parameters.InferenceParameters; +import net.ladenthin.llama.value.ChatChoice; import net.ladenthin.llama.value.ChatMessage; +import net.ladenthin.llama.value.ContentPart; +import net.ladenthin.llama.value.Timings; +import net.ladenthin.llama.value.ToolCall; +import net.ladenthin.llama.value.ToolDefinition; +import net.ladenthin.llama.value.Usage; import org.junit.jupiter.api.Test; /** Model-free tests for the pure langchain4j<->java-llama.cpp transforms. */ @@ -50,7 +68,7 @@ void mapsEveryRoleAndContent() { } @Test - void flattensMultimodalUserMessageToText() { + void flattensMultiTextUserMessageToText() { ChatRequest request = ChatRequest.builder() .messages(UserMessage.from(TextContent.from("Hello "), TextContent.from("world"))) @@ -60,6 +78,8 @@ void flattensMultimodalUserMessageToText() { assertThat(mapped.getRole(), is("user")); assertThat(mapped.getContent(), is("Hello world")); + // Text-only content needs no multimodal array form. + assertThat(mapped.hasParts(), is(false)); } @Test @@ -132,4 +152,241 @@ void rerankScoresFallBackToArrayOrderWhenIndexAbsent() { assertThat(scores[0], is(0.7)); assertThat(scores[1], is(0.2)); } + + // ------------------------------------------------------------------ + // Tool calling + // ------------------------------------------------------------------ + + @Test + void forwardsToolSpecificationsAndToolChoice() { + ToolSpecification weather = ToolSpecification.builder() + .name("get_weather") + .description("Current weather for a city") + .parameters(JsonObjectSchema.builder() + .addProperty("city", JsonStringSchema.builder() + .description("city name") + .build()) + .required("city") + .build()) + .build(); + ChatRequest request = ChatRequest.builder() + .messages(UserMessage.from("weather in Berlin?")) + .toolSpecifications(weather) + .toolChoice(ToolChoice.REQUIRED) + .build(); + + net.ladenthin.llama.parameters.ChatRequest jllama = LangChain4jMapping.toJllamaRequest(request); + + assertThat(jllama.getTools().size(), is(1)); + ToolDefinition tool = jllama.getTools().get(0); + assertThat(tool.getName(), is("get_weather")); + assertThat(tool.getDescription(), is("Current weather for a city")); + assertThat(tool.getParametersSchemaJson(), containsString("\"city\"")); + assertThat(tool.getParametersSchemaJson(), containsString("\"required\"")); + assertThat(jllama.getToolChoice().orElse(""), is("required")); + // The OAI tools array must materialize for LlamaModel.chat to forward it natively. + assertThat(jllama.buildToolsJson().orElse(""), containsString("\"function\"")); + } + + @Test + void toolWithoutParametersGetsEmptyObjectSchema() { + ToolSpecification noArgs = + ToolSpecification.builder().name("get_time").build(); + + ToolDefinition tool = LangChain4jMapping.toToolDefinition(noArgs); + + assertThat(tool.getParametersSchemaJson(), is(LangChain4jMapping.EMPTY_PARAMETERS_SCHEMA)); + } + + @Test + void streamingParametersCarryToolsAndToolChoice() { + ChatRequest request = ChatRequest.builder() + .messages(UserMessage.from("weather?")) + .toolSpecifications(ToolSpecification.builder() + .name("get_weather") + .parameters(JsonObjectSchema.builder() + .addProperty("city", new JsonStringSchema()) + .build()) + .build()) + .toolChoice(ToolChoice.REQUIRED) + .build(); + + String json = LangChain4jMapping.toStreamingParameters(request).toString(); + + // The streaming blob must carry the same tools wiring the blocking path applies. + assertThat(json, containsString("\"tools\"")); + assertThat(json, containsString("get_weather")); + assertThat(json, containsString("\"tool_choice\"")); + assertThat(json, containsString("required")); + } + + @Test + void mapsToolChoiceEnumToOaiStrings() { + assertThat(LangChain4jMapping.toToolChoiceString(ToolChoice.AUTO), is("auto")); + assertThat(LangChain4jMapping.toToolChoiceString(ToolChoice.REQUIRED), is("required")); + assertThat(LangChain4jMapping.toToolChoiceString(ToolChoice.NONE), is("none")); + } + + @Test + void mapsAssistantToolCallTurnIntoHistory() { + // Multi-turn tool history: the assistant's earlier tool-call turn must survive verbatim + // so the model sees the full call/result exchange. + AiMessage toolCallTurn = AiMessage.from(ToolExecutionRequest.builder() + .id("call_7") + .name("get_weather") + .arguments("{\"city\":\"Berlin\"}") + .build()); + ChatRequest request = ChatRequest.builder() + .messages( + UserMessage.from("weather in Berlin?"), + toolCallTurn, + ToolExecutionResultMessage.from("call_7", "get_weather", "22C")) + .build(); + + List messages = LangChain4jMapping.toJllamaRequest(request).getMessages(); + + ChatMessage assistant = messages.get(1); + assertThat(assistant.getRole(), is("assistant")); + assertThat(assistant.getToolCalls().size(), is(1)); + ToolCall call = assistant.getToolCalls().get(0); + assertThat(call.getId(), is("call_7")); + assertThat(call.getName(), is("get_weather")); + assertThat(call.getArgumentsJson(), is("{\"city\":\"Berlin\"}")); + ChatMessage result = messages.get(2); + assertThat(result.getRole(), is("tool")); + assertThat(result.getToolCallId().orElse(""), is("call_7")); + } + + @Test + void toolCallResponseBecomesAiMessageWithToolExecutionRequests() { + ChatMessage assistant = ChatMessage.assistantToolCalls( + "", Arrays.asList(new ToolCall("call_1", "get_weather", "{\"city\":\"Berlin\"}"))); + net.ladenthin.llama.value.ChatResponse response = new net.ladenthin.llama.value.ChatResponse( + "id-1", + Arrays.asList(new ChatChoice(0, assistant, "tool_calls")), + new Usage(10, 5), + new Timings(0, 0, 0, 0, 0, 0, 0, 0, 0), + "{}"); + + dev.langchain4j.model.chat.response.ChatResponse mapped = + LangChain4jMapping.toLangChainResponse(response); + + assertThat(mapped.aiMessage().hasToolExecutionRequests(), is(true)); + ToolExecutionRequest request = mapped.aiMessage().toolExecutionRequests().get(0); + assertThat(request.id(), is("call_1")); + assertThat(request.name(), is("get_weather")); + assertThat(request.arguments(), is("{\"city\":\"Berlin\"}")); + assertThat(mapped.finishReason(), is(FinishReason.TOOL_EXECUTION)); + } + + // ------------------------------------------------------------------ + // response_format / JSON mode + // ------------------------------------------------------------------ + + @Test + void jsonResponseFormatWithoutSchemaMapsToJsonObjectMode() { + ChatRequest request = ChatRequest.builder() + .messages(UserMessage.from("emit json")) + .responseFormat(ResponseFormat.JSON) + .build(); + + String json = LangChain4jMapping.toStreamingParameters(request).toString(); + + assertThat(json, containsString("\"response_format\"")); + assertThat(json, containsString("json_object")); + } + + @Test + void jsonResponseFormatWithSchemaMapsToJsonSchemaConstraint() { + ResponseFormat format = ResponseFormat.builder() + .type(ResponseFormatType.JSON) + .jsonSchema(JsonSchema.builder() + .name("Person") + .rootElement(JsonObjectSchema.builder() + .addProperty("name", new JsonStringSchema()) + .required("name") + .build()) + .build()) + .build(); + ChatRequest request = ChatRequest.builder() + .messages(UserMessage.from("extract the person")) + .responseFormat(format) + .build(); + + String json = LangChain4jMapping.toStreamingParameters(request).toString(); + + assertThat(json, containsString("\"json_schema\"")); + assertThat(json, containsString("\"name\"")); + } + + @Test + void textResponseFormatAddsNoConstraint() { + ChatRequest request = ChatRequest.builder() + .messages(UserMessage.from("hi")) + .responseFormat(ResponseFormat.TEXT) + .build(); + + String json = LangChain4jMapping.toStreamingParameters(request).toString(); + + assertThat(json, not(containsString("\"response_format\""))); + assertThat(json, not(containsString("\"json_schema\""))); + } + + // ------------------------------------------------------------------ + // Multimodal user content + // ------------------------------------------------------------------ + + @Test + void multimodalUserMessageMapsToContentParts() { + // 1x1 transparent PNG, base64. + String base64Png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk" + + "YPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + ChatRequest request = ChatRequest.builder() + .messages(UserMessage.from( + TextContent.from("What is in this image?"), ImageContent.from(base64Png, "image/png"))) + .build(); + + ChatMessage mapped = LangChain4jMapping.toJllamaRequest(request).getMessages().get(0); + + assertThat(mapped.hasParts(), is(true)); + List parts = mapped.getParts().orElse(java.util.Collections.emptyList()); + assertThat(parts.size(), is(2)); + assertThat(parts.get(0).getType(), is(ContentPart.Type.TEXT)); + assertThat(parts.get(0).getText(), is("What is in this image?")); + assertThat(parts.get(1).getType(), is(ContentPart.Type.IMAGE_URL)); + assertThat(parts.get(1).getImageUrl(), is("data:image/png;base64," + base64Png)); + } + + @Test + void audioContentMapsToInputAudioPart() { + byte[] fakeWav = new byte[] {'R', 'I', 'F', 'F'}; + AudioContent audio = AudioContent.from(Audio.builder() + .base64Data(java.util.Base64.getEncoder().encodeToString(fakeWav)) + .mimeType("audio/wav") + .build()); + + ContentPart part = LangChain4jMapping.toContentPart(audio); + + assertThat(part.getType(), is(ContentPart.Type.INPUT_AUDIO)); + assertThat(part.getAudioFormat(), is("wav")); + assertThat(part.getAudioData(), is(java.util.Base64.getEncoder().encodeToString(fakeWav))); + } + + @Test + void unsupportedAudioMimeTypeFailsLoud() { + AudioContent audio = AudioContent.from(Audio.builder() + .base64Data("AAAA") + .mimeType("audio/ogg") + .build()); + + assertThrows(IllegalArgumentException.class, () -> LangChain4jMapping.toContentPart(audio)); + } + + @Test + void imageWithoutDataOrUrlFailsLoud() { + ImageContent image = ImageContent.from( + dev.langchain4j.data.image.Image.builder().build()); + + assertThrows(IllegalArgumentException.class, () -> LangChain4jMapping.toContentPart(image)); + } } diff --git a/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/StreamingChunkAssemblerTest.java b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/StreamingChunkAssemblerTest.java new file mode 100644 index 00000000..1f2a9b2c --- /dev/null +++ b/llama-langchain4j/src/test/java/net/ladenthin/llama/langchain4j/StreamingChunkAssemblerTest.java @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.langchain4j; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import dev.langchain4j.model.chat.response.ChatResponse; +import dev.langchain4j.model.chat.response.CompleteToolCall; +import dev.langchain4j.model.chat.response.PartialThinking; +import dev.langchain4j.model.chat.response.PartialToolCall; +import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; +import dev.langchain4j.model.output.FinishReason; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Model-free tests for the chunk-stream state machine behind {@code JllamaStreamingChatModel}: + * canned OpenAI {@code chat.completion.chunk} JSON drives text/thinking/tool-call event + * forwarding and final-response assembly. + */ +class StreamingChunkAssemblerTest { + + private static final class RecordingHandler implements StreamingChatResponseHandler { + final List partials = new ArrayList<>(); + final List thinking = new ArrayList<>(); + final List partialToolCalls = new ArrayList<>(); + final List completeToolCalls = new ArrayList<>(); + + @Override + public void onPartialResponse(String partialResponse) { + partials.add(partialResponse); + } + + @Override + public void onPartialThinking(PartialThinking partialThinking) { + thinking.add(partialThinking.text()); + } + + @Override + public void onPartialToolCall(PartialToolCall partialToolCall) { + partialToolCalls.add(partialToolCall); + } + + @Override + public void onCompleteToolCall(CompleteToolCall completeToolCall) { + completeToolCalls.add(completeToolCall); + } + + @Override + public void onCompleteResponse(ChatResponse completeResponse) { + throw new AssertionError("assembler never calls onCompleteResponse itself"); + } + + @Override + public void onError(Throwable error) { + throw new AssertionError("assembler never calls onError itself", error); + } + } + + private static String contentChunk(String content) { + return "{\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0," + + "\"delta\":{\"content\":\"" + content + "\"},\"finish_reason\":null}]}"; + } + + @Test + void assemblesPlainTextStream() { + RecordingHandler handler = new RecordingHandler(); + StreamingChunkAssembler assembler = new StreamingChunkAssembler(handler); + + assembler.accept(contentChunk("Hel")); + assembler.accept(contentChunk("lo")); + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}"); + ChatResponse response = assembler.complete(); + + assertThat(handler.partials, contains("Hel", "lo")); + assertThat(response.aiMessage().text(), is("Hello")); + assertThat(response.finishReason(), is(FinishReason.STOP)); + assertThat(response.aiMessage().hasToolExecutionRequests(), is(false)); + } + + @Test + void assemblesStreamedToolCallAcrossFragments() { + RecordingHandler handler = new RecordingHandler(); + StreamingChunkAssembler assembler = new StreamingChunkAssembler(handler); + + // Fragment 1: id + name + start of arguments. Fragment 2: rest of arguments. + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0," + + "\"id\":\"call_1\",\"type\":\"function\"," + + "\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"ci\"}}]},\"finish_reason\":null}]}"); + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0," + + "\"function\":{\"arguments\":\"ty\\\":\\\"Berlin\\\"}\"}}]},\"finish_reason\":null}]}"); + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]," + + "\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":5}}"); + ChatResponse response = assembler.complete(); + + assertThat(handler.partialToolCalls.size(), is(2)); + assertThat(handler.partialToolCalls.get(0).name(), is("get_weather")); + assertThat(handler.partialToolCalls.get(0).partialArguments(), is("{\"ci")); + assertThat(handler.partialToolCalls.get(1).partialArguments(), is("ty\":\"Berlin\"}")); + assertThat(handler.completeToolCalls.size(), is(1)); + assertThat( + handler.completeToolCalls.get(0).toolExecutionRequest().arguments(), + is("{\"city\":\"Berlin\"}")); + assertThat(response.aiMessage().hasToolExecutionRequests(), is(true)); + assertThat(response.aiMessage().toolExecutionRequests().get(0).id(), is("call_1")); + assertThat(response.aiMessage().toolExecutionRequests().get(0).name(), is("get_weather")); + assertThat(response.finishReason(), is(FinishReason.TOOL_EXECUTION)); + assertThat(response.tokenUsage().inputTokenCount(), is(10)); + assertThat(response.tokenUsage().outputTokenCount(), is(5)); + } + + @Test + void assemblesParallelToolCallsByIndex() { + RecordingHandler handler = new RecordingHandler(); + StreamingChunkAssembler assembler = new StreamingChunkAssembler(handler); + + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[" + + "{\"index\":0,\"id\":\"a\",\"function\":{\"name\":\"first\",\"arguments\":\"{}\"}}," + + "{\"index\":1,\"id\":\"b\",\"function\":{\"name\":\"second\",\"arguments\":\"{}\"}}" + + "]},\"finish_reason\":\"tool_calls\"}]}"); + ChatResponse response = assembler.complete(); + + assertThat(handler.completeToolCalls.size(), is(2)); + assertThat(response.aiMessage().toolExecutionRequests().get(0).name(), is("first")); + assertThat(response.aiMessage().toolExecutionRequests().get(1).name(), is("second")); + } + + @Test + void forwardsThinkingDeltasAndKeepsThinkingOnFinalMessage() { + RecordingHandler handler = new RecordingHandler(); + StreamingChunkAssembler assembler = new StreamingChunkAssembler(handler); + + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"let me \"}," + + "\"finish_reason\":null}]}"); + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{\"reasoning_content\":\"think\"}," + + "\"finish_reason\":null}]}"); + assembler.accept(contentChunk("42")); + assembler.accept("{\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}"); + ChatResponse response = assembler.complete(); + + assertThat(handler.thinking, contains("let me ", "think")); + assertThat(response.aiMessage().thinking(), is("let me think")); + assertThat(response.aiMessage().text(), is("42")); + } + + @Test + void noUsageChunkMeansNoTokenUsage() { + RecordingHandler handler = new RecordingHandler(); + StreamingChunkAssembler assembler = new StreamingChunkAssembler(handler); + + assembler.accept(contentChunk("x")); + ChatResponse response = assembler.complete(); + + assertThat(response.tokenUsage(), is(nullValue())); + } + + @Test + void unparseableChunkFailsLoud() { + StreamingChunkAssembler assembler = new StreamingChunkAssembler(new RecordingHandler()); + + assertThrows(UncheckedIOException.class, () -> assembler.accept("not json")); + } +} diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index 523d4b3b..73412d84 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -85,6 +85,30 @@ endif() # toolchain keeps working. if(ANDROID OR ANDROID_ABI OR OS_NAME MATCHES "Android" OR CMAKE_CXX_COMPILER MATCHES "android") add_compile_definitions(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__) + # 16 KB page-size compliance (Google Play requirement for apps targeting + # Android 15+): every LOAD segment of the shipped .so must be aligned to a + # multiple of 16384. The dockcross cross-linker currently emits 0x4000 + # alignment by default, but that is a toolchain default, not a guarantee — + # pin it explicitly so a toolchain image bump cannot silently regress Play + # compatibility. CI asserts the resulting alignment with readelf. + add_link_options(-Wl,-z,max-page-size=16384) + # The shipped .so must dlopen with ONLY bionic system libraries present — + # an app that consumes the AAR bundles no other native libs. The dockcross + # cross-clang, left alone, emits two DT_NEEDED entries that do not exist on + # any Android device and made System.loadLibrary fail with + # UnsatisfiedLinkError (caught on the API-30 emulator CI job; readelf on the + # released 5.0.5 arm64 lib shows the same latent defect): + # - libomp.so: ggml's OpenMP backend links LLVM's libomp, which Android + # does not ship. Turn OpenMP off; ggml falls back to its own std::thread + # pool (same trade the Windows-arm64 clang-cl job makes with + # -DGGML_OPENMP=OFF, and what the Google NDK toolchain does by default). + # - libc++_shared.so: the NDK's shared C++ runtime, only present when an + # app packages it itself. Link libc++ statically instead (upstream + # llama.cpp's Android builds do the same); bionic system libs are the + # only remaining DT_NEEDED. CI enforces this with a whitelist check on + # the AAR's .so files. + set(GGML_OPENMP OFF CACHE BOOL "Android: no libomp.so on devices; use ggml's std::thread pool" FORCE) + add_link_options(-static-libstdc++) endif() set(LLAMA_BUILD_COMMON ON) @@ -150,7 +174,7 @@ set(LLAMA_BUILD_APP OFF CACHE BOOL "" FORCE) FetchContent_Declare( llama.cpp GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git - GIT_TAG b9870 + GIT_TAG b9878 PATCH_COMMAND ${CMAKE_COMMAND} -DPATCH_DIR=${CMAKE_CURRENT_SOURCE_DIR}/patches -DLLAMA_SRC= @@ -173,7 +197,7 @@ execute_process( COMMAND ${CMAKE_COMMAND} -DTTS_SRC=${llama.cpp_SOURCE_DIR}/tools/tts/tts.cpp -DOUT_CPP=${JLLAMA_TTS_GEN_CPP} - -DLLAMA_TAG=b9870 + -DLLAMA_TAG=b9878 -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/generate-tts-upstream.cmake RESULT_VARIABLE JLLAMA_TTS_GEN_RESULT ) diff --git a/llama/patches/0007-server-attach-http-frontend.patch b/llama/patches/0007-server-attach-http-frontend.patch new file mode 100644 index 00000000..908ba2fa --- /dev/null +++ b/llama/patches/0007-server-attach-http-frontend.patch @@ -0,0 +1,288 @@ +--- a/tools/server/server.cpp ++++ b/tools/server/server.cpp +@@ -94,8 +94,111 @@ + }; + } + ++// [jllama] Route table shared by the standalone single-model server, the router, and the ++// embedded attach mode (llama_server_attach below). Extracted verbatim from llama_server() so the ++// entry points cannot drift. The resumable-streaming routes are NOT registered here: their ++// handlers differ between router and non-router mode, so each entry point wires its own. Returns ++// false when the experimental built-in tools were requested but failed to set up. ++[[nodiscard]] static bool llama_server_register_common_routes( ++ server_http_context & ctx_http, ++ server_routes & routes, ++ const common_params & params, ++ server_tools & tools) { ++ ctx_http.get ("/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check) ++ ctx_http.get ("/v1/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check) ++ ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics)); ++ ctx_http.get ("/props", ex_wrapper(routes.get_props)); ++ ctx_http.post("/props", ex_wrapper(routes.post_props)); ++ ctx_http.get ("/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) ++ ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) ++ ctx_http.post("/completion", ex_wrapper(routes.post_completions)); // legacy ++ ctx_http.post("/completions", ex_wrapper(routes.post_completions)); ++ ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai)); ++ ctx_http.post("/chat/completions", ex_wrapper(routes.post_chat_completions)); ++ ctx_http.post("/v1/chat/completions", ex_wrapper(routes.post_chat_completions)); ++ ctx_http.post("/v1/chat/completions/control", ex_wrapper(routes.post_control)); ++ ctx_http.post("/v1/responses", ex_wrapper(routes.post_responses_oai)); ++ ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai)); ++ ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); ++ ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); ++ ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API ++ ctx_http.post("/infill", ex_wrapper(routes.post_infill)); ++ ctx_http.post("/embedding", ex_wrapper(routes.post_embeddings)); // legacy ++ ctx_http.post("/embeddings", ex_wrapper(routes.post_embeddings)); ++ ctx_http.post("/v1/embeddings", ex_wrapper(routes.post_embeddings_oai)); ++ ctx_http.post("/rerank", ex_wrapper(routes.post_rerank)); ++ ctx_http.post("/reranking", ex_wrapper(routes.post_rerank)); ++ ctx_http.post("/v1/rerank", ex_wrapper(routes.post_rerank)); ++ ctx_http.post("/v1/reranking", ex_wrapper(routes.post_rerank)); ++ ctx_http.post("/tokenize", ex_wrapper(routes.post_tokenize)); ++ ctx_http.post("/detokenize", ex_wrapper(routes.post_detokenize)); ++ ctx_http.post("/apply-template", ex_wrapper(routes.post_apply_template)); ++ // token counting ++ ctx_http.post("/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok)); ++ ctx_http.post("/v1/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok)); ++ ctx_http.post("/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai)); ++ ctx_http.post("/v1/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai)); ++ ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting ++ // LoRA adapters hotswap ++ ctx_http.get ("/lora-adapters", ex_wrapper(routes.get_lora_adapters)); ++ ctx_http.post("/lora-adapters", ex_wrapper(routes.post_lora_adapters)); ++ // Save & load slots ++ ctx_http.get ("/slots", ex_wrapper(routes.get_slots)); ++ ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots)); ++ ++ // Google Cloud Platform (Vertex AI) compat ++ ctx_http.register_gcp_compat(); ++ ++ // return 403 for disabled features ++ server_http_context::handler_t res_403 = [](const server_http_req &) { ++ auto res = std::make_unique(); ++ res->status = 403; ++ res->data = safe_json_to_str({ ++ {"error", { ++ {"message", "this feature is disabled"}, ++ {"type", "feature_disabled"}, ++ }} ++ }); ++ return res; ++ }; ++ ++ // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP) ++ if (params.ui_mcp_proxy) { ++ SRV_WRN("%s", "-----------------\n"); ++ SRV_WRN("%s", "CORS proxy is enabled, do not expose server to untrusted environments\n"); ++ SRV_WRN("%s", "This feature is EXPERIMENTAL and may be removed or changed in future versions\n"); ++ SRV_WRN("%s", "-----------------\n"); ++ ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get)); ++ ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post)); ++ } else { ++ ctx_http.get ("/cors-proxy", ex_wrapper(res_403)); ++ ctx_http.post("/cors-proxy", ex_wrapper(res_403)); ++ } ++ ++ // EXPERIMENTAL built-in tools ++ if (!params.server_tools.empty()) { ++ try { ++ tools.setup(params.server_tools); ++ } catch (const std::exception & e) { ++ SRV_ERR("tools setup failed: %s\n", e.what()); ++ return false; ++ } ++ SRV_WRN("%s", "-----------------\n"); ++ SRV_WRN("%s", "Built-in tools are enabled, do not expose server to untrusted environments\n"); ++ SRV_WRN("%s", "This feature is EXPERIMENTAL and may be changed in the future\n"); ++ SRV_WRN("%s", "-----------------\n"); ++ ctx_http.get ("/tools", ex_wrapper(tools.handle_get)); ++ ctx_http.post("/tools", ex_wrapper(tools.handle_post)); ++ } else { ++ ctx_http.get ("/tools", ex_wrapper(res_403)); ++ ctx_http.post("/tools", ex_wrapper(res_403)); ++ } ++ return true; ++} ++ + // satisfies -Wmissing-declarations + int llama_server(int argc, char ** argv); ++int llama_server_attach(int argc, char ** argv, server_context & ctx_server); + + int llama_server(int argc, char ** argv) { + std::setlocale(LC_NUMERIC, "C"); +@@ -230,47 +333,9 @@ + ctx_http.del ("/models", ex_wrapper(models_routes->del_router_models)); + } + +- ctx_http.get ("/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check) +- ctx_http.get ("/v1/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check) +- ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics)); +- ctx_http.get ("/props", ex_wrapper(routes.get_props)); +- ctx_http.post("/props", ex_wrapper(routes.post_props)); +- ctx_http.get ("/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) +- ctx_http.get ("/v1/models", ex_wrapper(routes.get_models)); // public endpoint (no API key check) +- ctx_http.post("/completion", ex_wrapper(routes.post_completions)); // legacy +- ctx_http.post("/completions", ex_wrapper(routes.post_completions)); +- ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai)); +- ctx_http.post("/chat/completions", ex_wrapper(routes.post_chat_completions)); +- ctx_http.post("/v1/chat/completions", ex_wrapper(routes.post_chat_completions)); +- ctx_http.post("/v1/chat/completions/control", ex_wrapper(routes.post_control)); +- ctx_http.post("/v1/responses", ex_wrapper(routes.post_responses_oai)); +- ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai)); +- ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); +- ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); +- ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API +- ctx_http.post("/infill", ex_wrapper(routes.post_infill)); +- ctx_http.post("/embedding", ex_wrapper(routes.post_embeddings)); // legacy +- ctx_http.post("/embeddings", ex_wrapper(routes.post_embeddings)); +- ctx_http.post("/v1/embeddings", ex_wrapper(routes.post_embeddings_oai)); +- ctx_http.post("/rerank", ex_wrapper(routes.post_rerank)); +- ctx_http.post("/reranking", ex_wrapper(routes.post_rerank)); +- ctx_http.post("/v1/rerank", ex_wrapper(routes.post_rerank)); +- ctx_http.post("/v1/reranking", ex_wrapper(routes.post_rerank)); +- ctx_http.post("/tokenize", ex_wrapper(routes.post_tokenize)); +- ctx_http.post("/detokenize", ex_wrapper(routes.post_detokenize)); +- ctx_http.post("/apply-template", ex_wrapper(routes.post_apply_template)); +- // token counting +- ctx_http.post("/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok)); +- ctx_http.post("/v1/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok)); +- ctx_http.post("/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai)); +- ctx_http.post("/v1/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai)); +- ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting +- // LoRA adapters hotswap +- ctx_http.get ("/lora-adapters", ex_wrapper(routes.get_lora_adapters)); +- ctx_http.post("/lora-adapters", ex_wrapper(routes.post_lora_adapters)); +- // Save & load slots +- ctx_http.get ("/slots", ex_wrapper(routes.get_slots)); +- ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots)); ++ if (!llama_server_register_common_routes(ctx_http, routes, params, tools)) { ++ return 1; ++ } + + // resumable streaming, the conversation_id is the session identity end to end. router and + // child wire different handlers under the same paths: a child binds the local g_stream_sessions +@@ -295,53 +360,6 @@ + ctx_http.post("/v1/streams/lookup", ex_wrapper(streams_lookup_h)); + ctx_http.del ("/v1/stream/:conv_id", ex_wrapper(stream_delete_h)); + +- // Google Cloud Platform (Vertex AI) compat +- ctx_http.register_gcp_compat(); +- +- // return 403 for disabled features +- server_http_context::handler_t res_403 = [](const server_http_req &) { +- auto res = std::make_unique(); +- res->status = 403; +- res->data = safe_json_to_str({ +- {"error", { +- {"message", "this feature is disabled"}, +- {"type", "feature_disabled"}, +- }} +- }); +- return res; +- }; +- +- // CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP) +- if (params.ui_mcp_proxy) { +- SRV_WRN("%s", "-----------------\n"); +- SRV_WRN("%s", "CORS proxy is enabled, do not expose server to untrusted environments\n"); +- SRV_WRN("%s", "This feature is EXPERIMENTAL and may be removed or changed in future versions\n"); +- SRV_WRN("%s", "-----------------\n"); +- ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get)); +- ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post)); +- } else { +- ctx_http.get ("/cors-proxy", ex_wrapper(res_403)); +- ctx_http.post("/cors-proxy", ex_wrapper(res_403)); +- } +- +- // EXPERIMENTAL built-in tools +- if (!params.server_tools.empty()) { +- try { +- tools.setup(params.server_tools); +- } catch (const std::exception & e) { +- SRV_ERR("tools setup failed: %s\n", e.what()); +- return 1; +- } +- SRV_WRN("%s", "-----------------\n"); +- SRV_WRN("%s", "Built-in tools are enabled, do not expose server to untrusted environments\n"); +- SRV_WRN("%s", "This feature is EXPERIMENTAL and may be changed in the future\n"); +- SRV_WRN("%s", "-----------------\n"); +- ctx_http.get ("/tools", ex_wrapper(tools.handle_get)); +- ctx_http.post("/tools", ex_wrapper(tools.handle_post)); +- } else { +- ctx_http.get ("/tools", ex_wrapper(res_403)); +- ctx_http.post("/tools", ex_wrapper(res_403)); +- } + + // + // Handle downloading model +@@ -503,3 +521,68 @@ + + return 0; + } ++ ++// [jllama] Attach the upstream HTTP frontend — full route table, WebUI assets, resumable ++// streaming — to an ALREADY-LOADED server_context owned by an embedding caller (a LlamaModel ++// driven over JNI). Unlike llama_server(): no common_init(), no backend/NUMA init, no model ++// load and no start_loop() — the caller's worker thread keeps driving the context, the HTTP ++// routes only post tasks to its queue (the queue is the synchronization point). Parses only the ++// HTTP-relevant argv (--host/--port/--api-key/...; no -m). Blocks until ++// llama_server_request_shutdown(); never terminates the shared server_context or frees the ++// backend — the embedding caller owns both. Shares shutdown_handler with llama_server(), so the ++// single-instance-per-process rule covers both entry points. ++int llama_server_attach(int argc, char ** argv, server_context & ctx_server) { ++ std::setlocale(LC_NUMERIC, "C"); ++ ++ common_params params; ++ // embedded callers always forward a clean UTF-8 argv (see llama_server's embedded parse note) ++ if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) { ++ return 1; ++ } ++ ++ // stream session GC: lifecycle is symmetric with the HTTP frontend (same as llama_server) ++ g_stream_sessions.start_gc(); ++ ++ server_http_context ctx_http; ++ if (!ctx_http.init(params)) { ++ SRV_ERR("%s", "failed to initialize HTTP server\n"); ++ g_stream_sessions.stop_gc(); ++ return 1; ++ } ++ ++ server_routes routes(params, ctx_server); ++ server_tools tools; ++ if (!llama_server_register_common_routes(ctx_http, routes, params, tools)) { ++ g_stream_sessions.stop_gc(); ++ return 1; ++ } ++ ++ // single-model (non-router) resumable-streaming handlers ++ ctx_http.get ("/v1/stream/:conv_id", ex_wrapper(make_stream_get_handler())); ++ ctx_http.post("/v1/streams/lookup", ex_wrapper(make_streams_lookup_handler())); ++ ctx_http.del ("/v1/stream/:conv_id", ex_wrapper(make_stream_delete_handler())); ++ ++ if (!ctx_http.start()) { ++ SRV_ERR("%s", "exiting due to HTTP server error\n"); ++ g_stream_sessions.stop_gc(); ++ return 1; ++ } ++ ++ // the caller's model is already loaded, so the frontend is ready immediately ++ routes.update_meta(ctx_server); ++ ctx_http.is_ready.store(true); ++ ++ shutdown_handler = [&](int) { ++ ctx_http.stop(); ++ }; ++ ++ SRV_INF("attached to existing server context, listening on %s\n", ctx_http.listening_address.c_str()); ++ ++ // block until llama_server_request_shutdown() stops the HTTP thread ++ if (ctx_http.thread.joinable()) { ++ ctx_http.thread.join(); ++ } ++ ++ g_stream_sessions.stop_gc(); ++ return 0; ++} diff --git a/llama/patches/0008-server-models-worker-cmd-override.patch b/llama/patches/0008-server-models-worker-cmd-override.patch new file mode 100644 index 00000000..b8f2a325 --- /dev/null +++ b/llama/patches/0008-server-models-worker-cmd-override.patch @@ -0,0 +1,32 @@ +--- a/tools/server/server-models.cpp ++++ b/tools/server/server-models.cpp +@@ -215,6 +215,29 @@ + if (app_cmd != nullptr && app_cmd[0] != '\0' && !bin_path.empty()) { + args.insert(args.begin() + 1, app_cmd); + } ++ ++ // [jllama] Allow overriding the worker command entirely via LLAMA_SERVER_WORKER_CMD ++ // (whitespace-split; individual tokens cannot contain spaces). The router spawns each model ++ // instance by re-executing its own binary (get_server_exec_path() == /proc/self/exe and ++ // friends) — but when llama_server runs EMBEDDED in a host process (e.g. a JVM driving it ++ // through a JNI library), that binary is the host (java), not a llama-server. The override ++ // lets such hosts relaunch workers through their own bootstrap, e.g. ++ // LLAMA_SERVER_WORKER_CMD="/usr/bin/java -cp app.jar net.ladenthin.llama.server.NativeServer". ++ // Only the leading binary-path token of the rendered args is replaced; everything the router ++ // computed (host, port, alias, model args) is preserved. ++ const char * worker_cmd = std::getenv("LLAMA_SERVER_WORKER_CMD"); ++ if (worker_cmd != nullptr && worker_cmd[0] != '\0' && !args.empty()) { ++ std::vector cmd_tokens; ++ std::istringstream cmd_stream((std::string(worker_cmd))); ++ std::string token; ++ while (cmd_stream >> token) { ++ cmd_tokens.push_back(token); ++ } ++ if (!cmd_tokens.empty()) { ++ args.erase(args.begin()); ++ args.insert(args.begin(), cmd_tokens.begin(), cmd_tokens.end()); ++ } ++ } + } + + void server_model_meta::update_caps() { diff --git a/llama/spotbugs-exclude.xml b/llama/spotbugs-exclude.xml index eb2fdcfa..2fc0dffa 100644 --- a/llama/spotbugs-exclude.xml +++ b/llama/spotbugs-exclude.xml @@ -202,6 +202,19 @@ SPDX-License-Identifier: MIT + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/llama/src/main/cpp/jllama.cpp b/llama/src/main/cpp/jllama.cpp index faa8fadd..ffd3c846 100644 --- a/llama/src/main/cpp/jllama.cpp +++ b/llama/src/main/cpp/jllama.cpp @@ -71,6 +71,10 @@ jclass c_error_oom = nullptr; jmethodID cc_hash_map = nullptr; jmethodID cc_integer = nullptr; jmethodID cc_float = nullptr; +// String(byte[], String charsetName) — the standard-UTF-8-safe way to build a +// Java String from native bytes (NewStringUTF expects Modified UTF-8 and is +// spec-invalid for supplementary-plane characters such as 4-byte emoji). +jmethodID cc_string_bytes_charset = nullptr; // methods jmethodID m_get_bytes = nullptr; @@ -145,6 +149,31 @@ static const static_object_binding g_static_object_bindings[] = { return get_jllama_context_impl(env, obj, f_model_pointer); } +/** + * Builds a Java String from raw standard-UTF-8 bytes via the cached + * String(byte[], "UTF-8") constructor (see utf8_to_jstring_impl for why + * NewStringUTF must not be used for payload text). + */ +[[nodiscard]] static jstring utf8_to_jstring(JNIEnv *env, const std::string &s) { + return utf8_to_jstring_impl(env, s, c_string, cc_string_bytes_charset, o_utf_8); +} + +/** + * Serialises a json value to a Java String (UTF-8-safe on both the dump and + * the JNI crossing; see json_to_jstring_impl). + */ +[[nodiscard]] static jstring json_to_jstring(JNIEnv *env, const json &j) { + return json_to_jstring_impl(env, j, c_string, cc_string_bytes_charset, o_utf_8); +} + +/** + * Serialises a vector of task results to a Java String (see + * results_to_jstring_impl). + */ +[[nodiscard]] static jstring results_to_jstring(JNIEnv *env, const std::vector &results) { + return results_to_jstring_impl(env, results, c_string, cc_string_bytes_charset, o_utf_8); +} + /** * Formats e as a JSON invalid-request error and throws it via JNI. */ @@ -260,7 +289,7 @@ static void populate_completion_task(server_task &task, jllama_context *jctx, in auto br = rd.wait_for_all([] { return false; }); if (!batch_ok_or_throw(env, br)) return nullptr; - return results_to_jstring_impl(env, br.results); + return results_to_jstring(env, br.results); } /** @@ -328,7 +357,7 @@ std::string parse_jstring(JNIEnv *env, jstring java_string) { auto result = rd.next([] { return false; }); if (!result_ok_or_throw(env, result)) return nullptr; - return json_to_jstring_impl(env, result->to_json()); + return json_to_jstring(env, result->to_json()); } // Post a single slot file task (SAVE or RESTORE), wait for its result, and @@ -518,8 +547,9 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) { cc_hash_map = env->GetMethodID(c_hash_map, "", "()V"); cc_integer = env->GetMethodID(c_integer, "", "(I)V"); cc_float = env->GetMethodID(c_float, "", "(F)V"); + cc_string_bytes_charset = env->GetMethodID(c_string, "", "([BLjava/lang/String;)V"); - if (!(cc_hash_map && cc_integer && cc_float)) { + if (!(cc_hash_map && cc_integer && cc_float && cc_string_bytes_charset)) { goto error; } @@ -783,7 +813,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_getModelMetaJson(J {"n_vocab", llama_vocab_n_tokens(jctx->vocab)}, {"special_tokens", special_tokens_json(jctx->vocab)}, }; - return json_to_jstring_impl(env, meta); + return json_to_jstring(env, meta); } auto m = ctx_server->get_meta(); // Read general.architecture from GGUF metadata via the llama C API. @@ -824,7 +854,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_getModelMetaJson(J } j["metadata"] = std::move(meta_map); } - return json_to_jstring_impl(env, j); + return json_to_jstring(env, j); } JNIEXPORT jint JNICALL Java_net_ladenthin_llama_LlamaModel_requestCompletion(JNIEnv *env, jobject obj, @@ -887,7 +917,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_receiveCompletionJ break; } - return json_to_jstring_impl(env, response); + return json_to_jstring(env, response); } // Streaming OpenAI chat: poll one step of a chat.completion.chunk stream. @@ -935,7 +965,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_receiveChatComplet break; } - return json_to_jstring_impl(env, wrap_stream_chunk(std::move(payload), stop)); + return json_to_jstring(env, wrap_stream_chunk(std::move(payload), stop)); } JNIEXPORT jfloatArray JNICALL Java_net_ladenthin_llama_LlamaModel_embed(JNIEnv *env, jobject obj, jstring jprompt) { @@ -1012,7 +1042,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleRerank(JNIEn auto br = rd.wait_for_all([] { return false; }); if (!batch_ok_or_throw(env, br)) return nullptr; - return json_to_jstring_impl(env, rerank_results_to_json(br.results, document_vector)); + return json_to_jstring(env, rerank_results_to_json(br.results, document_vector)); } JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_applyTemplate(JNIEnv *env, jobject obj, jstring jparams) { @@ -1033,7 +1063,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_applyTemplate(JNIE return nullptr; } std::string tok_str = templateData.at("prompt"); - return env->NewStringUTF(tok_str.c_str()); + return utf8_to_jstring(env, tok_str); } JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleChatCompletions(JNIEnv *env, jobject obj, @@ -1180,7 +1210,13 @@ JNIEXPORT void JNICALL Java_net_ladenthin_llama_LlamaModel_setLogger(JNIEnv *env if (env == nullptr || text == nullptr) { return; } - jstring message = env->NewStringUTF(text); + // Log lines can embed payload text (prompts, model metadata), so the + // message must cross as standard UTF-8, not Modified UTF-8. + jstring message = utf8_to_jstring(env, text); + if (message == nullptr) { + env->ExceptionClear(); // allocation failed; drop this log line + return; + } jobject log_level = log_level_to_jobject(level); env->CallVoidMethod(o_log_callback, m_biconsumer_accept, log_level, message); env->DeleteLocalRef(message); @@ -1349,7 +1385,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleEmbeddings(J ? format_embeddings_response_oaicompat( body, json_value(body, "model", std::string(DEFAULT_OAICOMPAT_MODEL)), responses, use_base64) : responses; - return json_to_jstring_impl(env, out); + return json_to_jstring(env, out); } JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleTokenize(JNIEnv *env, jobject obj, jstring jcontent, @@ -1388,7 +1424,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleTokenize(JNI tokens_response = tokens; } - return json_to_jstring_impl(env, format_tokenizer_response(tokens_response)); + return json_to_jstring(env, format_tokenizer_response(tokens_response)); } JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleDetokenize(JNIEnv *env, jobject obj, @@ -1396,7 +1432,7 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleDetokenize(J REQUIRE_SERVER_CONTEXT(nullptr); const auto tokens = jint_array_to_tokens_impl(env, jtokens); - return json_to_jstring_impl(env, format_detokenized_response(detokenize(jctx, tokens))); + return json_to_jstring(env, format_detokenized_response(detokenize(jctx, tokens))); } JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleSlotAction(JNIEnv *env, jobject obj, jint action, @@ -1423,6 +1459,55 @@ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_handleSlotAction(J } } +JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_getLoraAdaptersJson(JNIEnv *env, jobject obj) { + REQUIRE_SERVER_CONTEXT(nullptr); + + return dispatch_one_shot_task(env, ctx_server, server_task(SERVER_TASK_TYPE_GET_LORA)); +} + +JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_setLoraAdaptersJson(JNIEnv *env, jobject obj, + jstring jadapters) { + REQUIRE_SERVER_CONTEXT(nullptr); + + json data; + if (!parse_json_params(env, jadapters, data)) { + return nullptr; + } + if (!data.is_array()) { + // Same contract as the upstream POST /lora-adapters route body. + env->ThrowNew(c_llama_error, "LoRA adapter list must be a JSON array of {id, scale} objects"); + return nullptr; + } + server_task task(SERVER_TASK_TYPE_SET_LORA); + task.set_lora = parse_lora_request(data); + return dispatch_one_shot_task(env, ctx_server, std::move(task)); +} + +JNIEXPORT void JNICALL Java_net_ladenthin_llama_LlamaQuantizer_quantizeNative(JNIEnv *env, jclass, jstring jinput, + jstring joutput, jint ftype, jint nthread, + jboolean allowRequantize) { + try { + const std::string input_path = parse_jstring(env, jinput); + const std::string output_path = parse_jstring(env, joutput); + // Idempotent; intentionally never paired with llama_backend_free here — a LlamaModel + // loaded in the same JVM shares the backend and must not have it freed underneath it. + llama_backend_init(); + llama_model_quantize_params qparams = llama_model_quantize_default_params(); + qparams.ftype = static_cast(ftype); + qparams.nthread = nthread; + qparams.allow_requantize = (allowRequantize == JNI_TRUE); + const uint32_t rc = llama_model_quantize(input_path.c_str(), output_path.c_str(), &qparams); + if (rc != 0) { + const std::string msg = "Quantization of '" + input_path + "' failed with code " + std::to_string(rc); + env->ThrowNew(c_llama_error, msg.c_str()); + } + } catch (const std::exception &e) { + env->ThrowNew(c_llama_error, e.what()); + } catch (...) { + env->ThrowNew(c_llama_error, "Unknown C++ exception during quantization"); + } +} + JNIEXPORT jboolean JNICALL Java_net_ladenthin_llama_LlamaModel_configureParallelInference(JNIEnv *env, jobject obj, jstring jconfig) { REQUIRE_SERVER_CONTEXT(JNI_FALSE); diff --git a/llama/src/main/cpp/jllama.h b/llama/src/main/cpp/jllama.h index ff93bd94..b4090fe5 100644 --- a/llama/src/main/cpp/jllama.h +++ b/llama/src/main/cpp/jllama.h @@ -153,6 +153,20 @@ JNIEXPORT jboolean JNICALL Java_net_ladenthin_llama_LlamaModel_configureParallel */ JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_getModelMetaJson(JNIEnv *, jobject); +/* + * Class: net_ladenthin_llama_LlamaModel + * Method: getLoraAdaptersJson + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_getLoraAdaptersJson(JNIEnv *, jobject); + +/* + * Class: net_ladenthin_llama_LlamaModel + * Method: setLoraAdaptersJson + * Signature: (Ljava/lang/String;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_net_ladenthin_llama_LlamaModel_setLoraAdaptersJson(JNIEnv *, jobject, jstring); + #ifdef __cplusplus } #endif diff --git a/llama/src/main/cpp/jni_helpers.hpp b/llama/src/main/cpp/jni_helpers.hpp index 79003190..5118d69f 100644 --- a/llama/src/main/cpp/jni_helpers.hpp +++ b/llama/src/main/cpp/jni_helpers.hpp @@ -15,7 +15,7 @@ // // Layer B — JNI + server orchestration: // configure_multimodal_task_impl, configure_task_slot_impl, -// json_to_jstring_impl, results_to_jstring_impl, +// utf8_to_jstring_impl, json_to_jstring_impl, results_to_jstring_impl, // embedding_to_jfloat_array_impl, tokens_to_jint_array_impl // // Pure JSON transforms (no JNI, no llama state) live in json_helpers.hpp, @@ -185,14 +185,49 @@ inline void configure_task_slot_impl(server_task &task, const json &data) { task.id_slot = json_value(data, "id_slot", -1); } +// --------------------------------------------------------------------------- +// utf8_to_jstring_impl +// +// Builds a java.lang.String from raw standard-UTF-8 bytes via the +// String(byte[], String charsetName) constructor. NewStringUTF must NOT be +// used for payload text: JNI specifies *Modified* UTF-8 input, where +// supplementary-plane characters (e.g. every 4-byte emoji sequence) are +// encoded as CESU-8 surrogate pairs — standard UTF-8 payloads containing +// them are spec-invalid (Android CheckJNI aborts, HotSpot mangles). Routing +// through byte[] + charset keeps the bytes standard UTF-8 end to end and is +// the mirror of parse_jstring's String.getBytes("UTF-8") input path. +// +// string_init_bytes_charset is the cached method id of +// String(byte[], String) and charset_name a cached "UTF-8" jstring. +// Returns nullptr with a pending OOM exception if the byte array cannot be +// allocated. +// --------------------------------------------------------------------------- +[[nodiscard]] inline jstring utf8_to_jstring_impl(JNIEnv *env, const std::string &s, jclass string_class, + jmethodID string_init_bytes_charset, jobject charset_name) { + const jsize length = static_cast(s.size()); + jbyteArray bytes = env->NewByteArray(length); + if (bytes == nullptr) { + return nullptr; // OOM exception already pending + } + env->SetByteArrayRegion(bytes, 0, length, reinterpret_cast(s.data())); + auto result = (jstring)env->NewObject(string_class, string_init_bytes_charset, bytes, charset_name); + env->DeleteLocalRef(bytes); + return result; +} + // --------------------------------------------------------------------------- // json_to_jstring_impl // -// Serialises any json value to a JNI string via dump() + NewStringUTF. +// Serialises any json value to a JNI string. Serialisation goes through +// upstream safe_json_to_str (dump with error_handler_t::replace) so a +// payload string that ends in an incomplete UTF-8 sequence — possible on the +// non-stream path when generation stops mid-codepoint at the token limit — +// is replaced with U+FFFD instead of throwing json::type_error 316. The +// resulting UTF-8 bytes cross into Java via utf8_to_jstring_impl. // --------------------------------------------------------------------------- -[[nodiscard]] inline jstring json_to_jstring_impl(JNIEnv *env, const json &j) { - std::string s = j.dump(); - return env->NewStringUTF(s.c_str()); +[[nodiscard]] inline jstring json_to_jstring_impl(JNIEnv *env, const json &j, jclass string_class, + jmethodID string_init_bytes_charset, jobject charset_name) { + return utf8_to_jstring_impl(env, safe_json_to_str(j), string_class, string_init_bytes_charset, charset_name); } // --------------------------------------------------------------------------- @@ -202,8 +237,10 @@ inline void configure_task_slot_impl(server_task &task, const json &data) { // construction to results_to_json (json_helpers.hpp) and serialisation to // json_to_jstring_impl. // --------------------------------------------------------------------------- -[[nodiscard]] inline jstring results_to_jstring_impl(JNIEnv *env, const std::vector &results) { - return json_to_jstring_impl(env, results_to_json(results)); +[[nodiscard]] inline jstring results_to_jstring_impl(JNIEnv *env, const std::vector &results, + jclass string_class, jmethodID string_init_bytes_charset, + jobject charset_name) { + return json_to_jstring_impl(env, results_to_json(results), string_class, string_init_bytes_charset, charset_name); } // --------------------------------------------------------------------------- diff --git a/llama/src/main/cpp/native_server.cpp b/llama/src/main/cpp/native_server.cpp index d9cfa527..1234a3ff 100644 --- a/llama/src/main/cpp/native_server.cpp +++ b/llama/src/main/cpp/native_server.cpp @@ -12,12 +12,23 @@ // is_terminating state in file-scope globals, so a second concurrent llama_server() would clobber // them. NativeServer enforces this on the Java side. +// Upstream server headers must precede jni_helpers.hpp (include order rule, see CLAUDE.md); +// they provide server_context so the attach path can reach a LlamaModel's jllama_context. +#include "server-context.h" +#include "server-queue.h" +#include "server-task.h" +#include "server-common.h" +#include "server-chat.h" +#include "utils.hpp" +#include "jni_helpers.hpp" + #include "native_server_bridge.h" #include #include #include +#include #include #include #include @@ -35,14 +46,10 @@ struct native_server { int exit_code = -1; }; -} // namespace - -extern "C" { - -JNIEXPORT jlong JNICALL Java_net_ladenthin_llama_server_NativeServer_startNativeServer(JNIEnv *env, jclass, - jobjectArray jargs) { - auto *srv = new native_server(); - +// Copies the forwarded Java argv into srv->args/argv with a synthetic argv[0]. The argv pointers +// reference the std::string storage in `args`, which is filled once (with reserve) and never +// mutated afterwards, so the pointers stay valid for the worker's lifetime. +void fill_native_server_args(JNIEnv *env, jobjectArray jargs, native_server *srv) { const jsize n = (jargs != nullptr) ? env->GetArrayLength(jargs) : 0; srv->args.reserve(static_cast(n) + 1); srv->args.emplace_back("llama-server"); // argv[0] @@ -64,6 +71,25 @@ JNIEXPORT jlong JNICALL Java_net_ladenthin_llama_server_NativeServer_startNative for (auto &arg : srv->args) { srv->argv.push_back(const_cast(arg.c_str())); } +} + +// Throws net.ladenthin.llama.exception.LlamaException with the given message (best-effort: if the +// class cannot be resolved the pending NoClassDefFoundError is surfaced instead). +void throw_llama_exception(JNIEnv *env, const char *message) { + jclass exception_class = env->FindClass("net/ladenthin/llama/exception/LlamaException"); + if (exception_class != nullptr) { + env->ThrowNew(exception_class, message); + } +} + +} // namespace + +extern "C" { + +JNIEXPORT jlong JNICALL Java_net_ladenthin_llama_server_NativeServer_startNativeServer(JNIEnv *env, jclass, + jobjectArray jargs) { + auto *srv = new native_server(); + fill_native_server_args(env, jargs, srv); // Embedded mode: no process signal handlers, honor the forwarded argv (see patches/0006). llama_server_set_embedded(true); @@ -104,4 +130,64 @@ JNIEXPORT jboolean JNICALL Java_net_ladenthin_llama_server_NativeServer_isRunnin return (srv != nullptr && !srv->finished.load()) ? JNI_TRUE : JNI_FALSE; } +JNIEXPORT jlong JNICALL Java_net_ladenthin_llama_server_NativeServer_startAttachedNativeServer(JNIEnv *env, jclass, + jobject jmodel, + jobjectArray jargs) { + if (jmodel == nullptr) { + throw_llama_exception(env, "model must not be null"); + return 0; + } + // Read the LlamaModel's native context handle directly (the same "ctx" long field jllama.cpp + // caches in JNI_OnLoad; this TU resolves it itself to stay decoupled from those globals). + jclass model_class = env->GetObjectClass(jmodel); + jfieldID ctx_field = env->GetFieldID(model_class, "ctx", "J"); + if (ctx_field == nullptr) { + return 0; // NoSuchFieldError already pending + } + const jlong model_handle = env->GetLongField(jmodel, ctx_field); + if (model_handle == 0) { + throw_llama_exception(env, "model is not loaded (or already closed)"); + return 0; + } + auto *jctx = reinterpret_cast(model_handle); // NOLINT(*-no-int-to-ptr) + + auto *srv = new native_server(); + fill_native_server_args(env, jargs, srv); + + // The attach entry always parses the forwarded argv; set the embedded flag anyway so any + // shared embedded-mode behavior in server.cpp stays consistent with startNativeServer. + llama_server_set_embedded(true); + + server_context *ctx_server = &jctx->server; + srv->worker = std::thread([srv, ctx_server]() { + srv->exit_code = llama_server_attach(static_cast(srv->argv.size()), srv->argv.data(), *ctx_server); + srv->finished.store(true); + }); + + return reinterpret_cast(srv); +} + +JNIEXPORT void JNICALL Java_net_ladenthin_llama_server_NativeServer_setWorkerCommandNative(JNIEnv *env, jclass, + jstring jcommand) { + // Sets/clears LLAMA_SERVER_WORKER_CMD in the process environment, which the router-mode + // model manager (server-models.cpp, patches/0008) reads when spawning worker instances. + std::string value; + if (jcommand != nullptr) { + const char *chars = env->GetStringUTFChars(jcommand, nullptr); + if (chars != nullptr) { + value = chars; + env->ReleaseStringUTFChars(jcommand, chars); + } + } +#if defined(_WIN32) + _putenv_s("LLAMA_SERVER_WORKER_CMD", value.c_str()); // empty value removes the variable +#else + if (value.empty()) { + unsetenv("LLAMA_SERVER_WORKER_CMD"); + } else { + setenv("LLAMA_SERVER_WORKER_CMD", value.c_str(), 1); + } +#endif +} + } // extern "C" diff --git a/llama/src/main/cpp/native_server_bridge.h b/llama/src/main/cpp/native_server_bridge.h index 1a40c766..b93ea904 100644 --- a/llama/src/main/cpp/native_server_bridge.h +++ b/llama/src/main/cpp/native_server_bridge.h @@ -17,6 +17,16 @@ // re-deriving it from the process command line) and can be stopped out-of-band (the SIGTERM // path) since its server_context is local to llama_server(). -int llama_server(int argc, char ** argv); +// - llama_server_attach: added by patches/0007-server-attach-http-frontend.patch. Attaches the +// upstream HTTP frontend (route table + WebUI + resumable streaming) to an ALREADY-LOADED +// server_context owned by a LlamaModel — no second model load, no start_loop; the LlamaModel's +// worker keeps driving the context and the HTTP routes post tasks to its queue. Blocks until +// llama_server_request_shutdown() (shared shutdown path with llama_server, so the +// single-instance-per-process rule covers both entry points). + +struct server_context; + +int llama_server(int argc, char **argv); +int llama_server_attach(int argc, char **argv, server_context &ctx_server); void llama_server_set_embedded(bool embedded); void llama_server_request_shutdown(); diff --git a/llama/src/main/java/net/ladenthin/llama/GgufInspector.java b/llama/src/main/java/net/ladenthin/llama/GgufInspector.java new file mode 100644 index 00000000..eda2bb69 --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/GgufInspector.java @@ -0,0 +1,234 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import net.ladenthin.llama.value.GgufMetadata; + +/** + * Pure-Java GGUF header/metadata reader — inspects a model file without loading it: + * no native library, no tensor data read, no RAM committed beyond the key/value table + * (typically a few MB even for models whose tensor payload is many GB). Useful for model + * pickers, download validators, and tooling that must describe a GGUF before deciding to + * load it (e.g. checking {@code general.architecture} or the trained context length). + * + *

Supports GGUF container versions 2 and 3, in both little-endian (the common case) and + * big-endian byte order (s390x-converted files; the byte order is auto-detected from the + * version field). GGUF v1 (pre-2023, 32-bit lengths) is rejected with a clear error. + * Only the header and key/value table are read; parsing stops before the tensor-info + * section, so inspection cost is independent of model size.

+ * + *

For metadata of an already-loaded model use + * {@link LlamaModel#getModelMeta()} instead.

+ * + * 
{@code
+ * GgufMetadata meta = GgufInspector.read(Paths.get("models/Qwen3-0.6B-Q4_K_M.gguf"));
+ * String arch = meta.getArchitecture().orElse("unknown");
+ * long ctx    = meta.getContextLength().orElse(0);
+ * }
+ */ +public final class GgufInspector { + + /** The 4-byte GGUF magic, {@code 'G' 'G' 'U' 'F'}. */ + private static final byte[] MAGIC = {'G', 'G', 'U', 'F'}; + + /** Sanity cap on the declared key/value count (a real table has dozens of entries). */ + private static final long MAX_KV_COUNT = 1L << 20; + + /** Sanity cap on a single string length (largest real strings are chat templates / tokens). */ + private static final long MAX_STRING_BYTES = 1L << 27; + + /** Sanity cap on a single array element count (largest real arrays are ~150k-token vocabs). */ + private static final long MAX_ARRAY_ELEMENTS = 1L << 26; + + // GGUF metadata value type ids (gguf.h: enum gguf_type). + private static final int TYPE_UINT8 = 0; + private static final int TYPE_INT8 = 1; + private static final int TYPE_UINT16 = 2; + private static final int TYPE_INT16 = 3; + private static final int TYPE_UINT32 = 4; + private static final int TYPE_INT32 = 5; + private static final int TYPE_FLOAT32 = 6; + private static final int TYPE_BOOL = 7; + private static final int TYPE_STRING = 8; + private static final int TYPE_ARRAY = 9; + private static final int TYPE_UINT64 = 10; + private static final int TYPE_INT64 = 11; + private static final int TYPE_FLOAT64 = 12; + + private GgufInspector() {} + + /** + * Read the GGUF header and key/value table from a file. + * + * @param ggufFile path to the GGUF file + * @return the decoded metadata + * @throws IOException if the file cannot be read, is not a GGUF file, uses the + * unsupported v1 container, or is structurally invalid/truncated + */ + public static GgufMetadata read(Path ggufFile) throws IOException { + try (InputStream in = new BufferedInputStream(Files.newInputStream(ggufFile))) { + return read(in); + } + } + + /** + * Read the GGUF header and key/value table from a stream. Reads exactly the header + + * key/value section; the stream is not consumed past it (tensor infos and tensor data + * stay unread). + * + * @param in the stream positioned at the start of the GGUF file + * @return the decoded metadata + * @throws IOException if the stream is not a GGUF file, uses the unsupported v1 + * container, or is structurally invalid/truncated + */ + public static GgufMetadata read(InputStream in) throws IOException { + DataInputStream data = new DataInputStream(in); + byte[] magic = new byte[MAGIC.length]; + data.readFully(magic); + for (int i = 0; i < MAGIC.length; i++) { + if (magic[i] != MAGIC[i]) { + throw new IOException("Not a GGUF file (magic mismatch: read 0x" + + String.format("%02X%02X%02X%02X", magic[0], magic[1], magic[2], magic[3]) + + ", expected 'GGUF')"); + } + } + + // The version field doubles as the byte-order detector: read little-endian first; + // a big-endian (s390x-converted) file then shows the version in the high bytes. + Reader reader = new Reader(data, ByteOrder.LITTLE_ENDIAN); + long versionLe = reader.u32(); + long version = versionLe; + if (versionLe > 0xFFFFL && Long.reverseBytes(versionLe << 32) <= 0xFFFFL) { + version = Long.reverseBytes(versionLe << 32); + reader = new Reader(data, ByteOrder.BIG_ENDIAN); + } + if (version == 1) { + throw new IOException( + "GGUF v" + version + " is not supported (pre-2023 32-bit container); re-convert the model"); + } + if (version != 2 && version != 3) { + throw new IOException("Unsupported GGUF version: " + version); + } + + long tensorCount = reader.u64(); + long kvCount = reader.u64(); + if (kvCount < 0 || kvCount > MAX_KV_COUNT) { + throw new IOException("Implausible GGUF key/value count: " + kvCount); + } + + Map entries = new LinkedHashMap(); + for (long i = 0; i < kvCount; i++) { + String key = reader.string(); + int type = (int) reader.u32(); + entries.put(key, readValue(reader, type)); + } + return new GgufMetadata((int) version, tensorCount, entries); + } + + private static Object readValue(Reader reader, int type) throws IOException { + switch (type) { + case TYPE_UINT8: + return Long.valueOf(reader.u8()); + case TYPE_INT8: + return Long.valueOf((byte) reader.u8()); + case TYPE_UINT16: + return Long.valueOf(reader.u16()); + case TYPE_INT16: + return Long.valueOf((short) reader.u16()); + case TYPE_UINT32: + return Long.valueOf(reader.u32()); + case TYPE_INT32: + return Long.valueOf(reader.i32()); + case TYPE_FLOAT32: + return Double.valueOf(Float.intBitsToFloat(reader.i32())); + case TYPE_BOOL: + return Boolean.valueOf(reader.u8() != 0); + case TYPE_STRING: + return reader.string(); + case TYPE_ARRAY: + return readArray(reader); + case TYPE_UINT64: + case TYPE_INT64: + // u64 values above Long.MAX_VALUE wrap negative; real metadata never gets there. + return Long.valueOf(reader.u64()); + case TYPE_FLOAT64: + return Double.valueOf(Double.longBitsToDouble(reader.u64())); + default: + throw new IOException("Unknown GGUF metadata value type id: " + type); + } + } + + private static Object readArray(Reader reader) throws IOException { + int elementType = (int) reader.u32(); + long count = reader.u64(); + if (count < 0 || count > MAX_ARRAY_ELEMENTS) { + throw new IOException("Implausible GGUF array element count: " + count); + } + List values = new ArrayList((int) Math.min(count, 1 << 16)); + for (long i = 0; i < count; i++) { + values.add(readValue(reader, elementType)); + } + return Collections.unmodifiableList(values); + } + + /** Byte-order-aware primitive reader over the underlying stream. */ + private static final class Reader { + private final DataInputStream in; + private final ByteOrder order; + private final byte[] scratch = new byte[8]; + + Reader(DataInputStream in, ByteOrder order) { + this.in = in; + this.order = order; + } + + int u8() throws IOException { + return in.readUnsignedByte(); + } + + int u16() throws IOException { + in.readFully(scratch, 0, 2); + return ByteBuffer.wrap(scratch, 0, 2).order(order).getShort() & 0xFFFF; + } + + long u32() throws IOException { + return i32() & 0xFFFFFFFFL; + } + + int i32() throws IOException { + in.readFully(scratch, 0, 4); + return ByteBuffer.wrap(scratch, 0, 4).order(order).getInt(); + } + + long u64() throws IOException { + in.readFully(scratch, 0, 8); + return ByteBuffer.wrap(scratch, 0, 8).order(order).getLong(); + } + + String string() throws IOException { + long length = u64(); + if (length < 0 || length > MAX_STRING_BYTES) { + throw new IOException("Implausible GGUF string length: " + length); + } + byte[] bytes = new byte[(int) length]; + in.readFully(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/LlamaModel.java b/llama/src/main/java/net/ladenthin/llama/LlamaModel.java index 6dbe19d0..8d13b5d6 100644 --- a/llama/src/main/java/net/ladenthin/llama/LlamaModel.java +++ b/llama/src/main/java/net/ladenthin/llama/LlamaModel.java @@ -64,6 +64,11 @@ public class LlamaModel implements AutoCloseable { private static final com.fasterxml.jackson.databind.ObjectMapper OBJECT_MAPPER = new com.fasterxml.jackson.databind.ObjectMapper(); + private static final net.ladenthin.llama.json.EmbeddingResponseParser EMBEDDING_RESPONSE_PARSER = + new net.ladenthin.llama.json.EmbeddingResponseParser(); + private static final net.ladenthin.llama.json.LoraAdapterResponseParser LORA_ADAPTER_RESPONSE_PARSER = + new net.ladenthin.llama.json.LoraAdapterResponseParser(); + static { LlamaLoader.initialize(); } @@ -377,6 +382,32 @@ public LlamaIterable generate(InferenceParameters parameters) { */ public native float[] embed(String prompt); + /** + * Get the embeddings of several strings in one native batch. Prompts are dispatched to the + * native scheduler together (continuous batching across the configured parallel slots), so + * this is the preferred form for embedding many inputs — e.g. indexing documents for a + * retrieval pipeline. As with {@link #embed(String)}, the prompts are not preprocessed in + * any way. + * + * @param prompts the strings to embed; may be empty (returns an empty list) + * @return one embedding vector per prompt, in the same order as {@code prompts} + * @throws net.ladenthin.llama.exception.LlamaException if embedding mode was not activated (see + * {@link net.ladenthin.llama.parameters.ModelParameters#enableEmbedding()}), a prompt is empty, or the + * native response cannot be parsed + */ + public List embed(java.util.Collection prompts) { + if (prompts.isEmpty()) { + return new java.util.ArrayList(); + } + String response = handleEmbeddings(EMBEDDING_RESPONSE_PARSER.toBatchRequestJson(prompts), true); + List embeddings = EMBEDDING_RESPONSE_PARSER.parse(response); + if (embeddings.size() != prompts.size()) { + throw new LlamaException("Expected " + prompts.size() + " embeddings but the native response contained " + + embeddings.size()); + } + return embeddings; + } + /** * Tokenize a prompt given the native tokenizer * @@ -902,6 +933,68 @@ public String restoreSlot(int slotId, String filepath) { return handleSlotAction(2, slotId, filepath); } + // ------------------------------------------------------------------ + // Runtime LoRA adapter control + // ------------------------------------------------------------------ + + /** + * List the LoRA adapters registered on the loaded model with their current scales. + * Adapters are loaded at model-load time via + * {@link net.ladenthin.llama.parameters.ModelParameters#addLoraAdapter(String)} / + * {@link net.ladenthin.llama.parameters.ModelParameters#addLoraScaledAdapter(String, float)} + * (optionally with + * {@link net.ladenthin.llama.parameters.ModelParameters#setLoraInitWithoutApply()} to load + * them disabled); this call returns an empty list when the model was loaded without + * adapters. Typed counterpart of the upstream {@code GET /lora-adapters} endpoint. + * + * @return the registered adapters with their current scales; empty when none were loaded + * @throws net.ladenthin.llama.exception.LlamaException if the native call fails + */ + public List getLoraAdapters() { + return LORA_ADAPTER_RESPONSE_PARSER.parse(getLoraAdaptersJson()); + } + + /** + * Set the scales of the loaded LoRA adapters at runtime, without reloading the model. + * Mirrors the upstream {@code POST /lora-adapters} contract: adapters present in + * {@code scales} get the given scale, all other adapters are set to {@code 0} + * (disabled). Ids not corresponding to a loaded adapter are ignored. The native side + * clears affected KV caches when the effective adapter set changes. + * + * @param scales adapter id (see {@link net.ladenthin.llama.value.LoraAdapter#getId()}) to + * new scale; an empty map disables every adapter + * @throws IllegalArgumentException if a scale is NaN or infinite + * @throws net.ladenthin.llama.exception.LlamaException if the native call fails + */ + public void setLoraAdapters(Map scales) { + setLoraAdaptersJson(LORA_ADAPTER_RESPONSE_PARSER.toRequestJson(scales)); + } + + /** + * Set the scale of a single LoRA adapter at runtime. Convenience form of + * {@link #setLoraAdapters(Map)} — note that, per the upstream contract, every + * other adapter is set to {@code 0} (disabled) by this call. + * + * @param adapterId the adapter id (see {@link net.ladenthin.llama.value.LoraAdapter#getId()}) + * @param scale the new scale; {@code 0} disables the adapter + * @throws IllegalArgumentException if {@code scale} is NaN or infinite + * @throws net.ladenthin.llama.exception.LlamaException if the native call fails + */ + public void setLoraAdapter(int adapterId, float scale) { + setLoraAdapters(java.util.Collections.singletonMap(adapterId, scale)); + } + + /** + * List the LoRA adapters as the raw native JSON array (the upstream + * {@code GET /lora-adapters} response shape). Prefer {@link #getLoraAdapters()} for typed + * access. + * + * @return JSON array of {@code {id, path, scale, task_name, prompt_prefix}} objects + */ + public native String getLoraAdaptersJson(); + + private native String setLoraAdaptersJson(String adaptersJson); + /** * Configure runtime inference parameters. * Accepts a JSON string with optional keys: diff --git a/llama/src/main/java/net/ladenthin/llama/LlamaQuantizer.java b/llama/src/main/java/net/ladenthin/llama/LlamaQuantizer.java new file mode 100644 index 00000000..8fca432e --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/LlamaQuantizer.java @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama; + +import java.util.Objects; +import net.ladenthin.llama.args.QuantizationType; +import net.ladenthin.llama.loader.LlamaLoader; + +/** + * In-JVM GGUF model quantization over llama.cpp's {@code llama_model_quantize} — the Java + * counterpart of the {@code llama-quantize} CLI tool. Converts a GGUF file to another + * {@linkplain QuantizationType quantization scheme} without leaving the JVM or shelling out. + * + * 
{@code
+ * LlamaQuantizer.quantize("model-f16.gguf", "model-q4_k_m.gguf", QuantizationType.Q4_K_M);
+ * }
+ * + *

Quantizing an already-quantized input (re-quantization) is refused by llama.cpp by + * default because it degrades quality; opt in explicitly via + * {@link #quantize(String, String, QuantizationType, int, boolean)} with + * {@code allowRequantize = true}.

+ * + *

Quantization is CPU-bound and can take minutes for large models; it initializes the shared + * llama backend (a no-op if a {@link LlamaModel} is already loaded) and may safely run while + * models are loaded in the same JVM.

+ */ +public final class LlamaQuantizer { + + static { + LlamaLoader.initialize(); + } + + private LlamaQuantizer() {} + + /** + * Quantize {@code inputPath} to {@code outputPath} with default settings (all available + * hardware threads, no re-quantization of already-quantized inputs). + * + * @param inputPath the source GGUF (typically F32/F16/BF16) + * @param outputPath the destination GGUF to write + * @param type the target quantization scheme + * @throws net.ladenthin.llama.exception.LlamaException if quantization fails (missing input, + * unwritable output, re-quantization without opt-in, unsupported tensor layout, …) + */ + public static void quantize(String inputPath, String outputPath, QuantizationType type) { + quantize(inputPath, outputPath, type, 0, false); + } + + /** + * Quantize {@code inputPath} to {@code outputPath} with explicit thread count and + * re-quantization opt-in. + * + * @param inputPath the source GGUF + * @param outputPath the destination GGUF to write + * @param type the target quantization scheme + * @param threads quantization threads; {@code <= 0} uses all hardware threads + * @param allowRequantize permit quantizing tensors that are not F32/F16 (re-quantizing an + * already-quantized model — lossy on top of lossy, quality degrades) + * @throws net.ladenthin.llama.exception.LlamaException if quantization fails + */ + public static void quantize( + String inputPath, String outputPath, QuantizationType type, int threads, boolean allowRequantize) { + Objects.requireNonNull(inputPath, "inputPath"); + Objects.requireNonNull(outputPath, "outputPath"); + Objects.requireNonNull(type, "type"); + quantizeNative(inputPath, outputPath, type.getFtypeValue(), threads, allowRequantize); + } + + private static native void quantizeNative( + String inputPath, String outputPath, int ftype, int threads, boolean allowRequantize); +} diff --git a/llama/src/main/java/net/ladenthin/llama/Session.java b/llama/src/main/java/net/ladenthin/llama/Session.java index 56acbd32..7979c5c0 100644 --- a/llama/src/main/java/net/ladenthin/llama/Session.java +++ b/llama/src/main/java/net/ladenthin/llama/Session.java @@ -10,6 +10,7 @@ import net.ladenthin.llama.parameters.InferenceParameters; import net.ladenthin.llama.value.ChatMessage; import net.ladenthin.llama.value.Pair; +import net.ladenthin.llama.value.SessionCheckpoint; import org.jspecify.annotations.Nullable; /** @@ -187,6 +188,79 @@ public List getMessages() { return state.snapshot(); } + /** + * Capture a point-in-time snapshot of this conversation: the slot's KV-cache state is + * saved to {@code filepath} (native slot-save) and paired with the transcript turns + * committed so far. The returned checkpoint can later be passed to + * {@link #rewind(SessionCheckpoint)} — undoing every turn after this point — or used as + * the branch point for {@link #fork(int, String)}. + *

+ * The checkpoint file's lifecycle is caller-managed (same as {@link #save(String)}); + * KV dumps grow with context usage. Rejected while a stream is in progress. + *

+ * + * @param filepath destination file for the slot KV state + * @return the checkpoint pairing that file with the transcript snapshot + * @throws IllegalStateException if a stream is in progress + */ + public SessionCheckpoint checkpoint(String filepath) { + return state.runWhenNotStreaming("checkpoint", () -> { + model.saveSlot(slotId, filepath); + return new SessionCheckpoint(filepath, state.turnsSnapshot()); + }); + } + + /** + * Rewind this conversation to a {@link #checkpoint(String)}: the slot's KV cache is + * restored from the checkpoint file and the in-memory transcript is reset to the + * checkpointed turns, atomically — native state and transcript cannot drift apart. + * Turns sent after the checkpoint are discarded; the conversation continues from the + * checkpointed point (e.g. to retry a turn with different wording or sampling). + * + * @param checkpoint a checkpoint previously taken on a session of the same model + * (typically this one) + * @throws IllegalStateException if a stream is in progress + */ + public void rewind(SessionCheckpoint checkpoint) { + state.runWhenNotStreaming("rewind", () -> { + model.restoreSlot(slotId, checkpoint.getFilepath()); + state.restoreTurns(checkpoint.getTurns()); + return null; + }); + } + + /** + * Fork this conversation into a new {@link Session} on another slot: the current slot + * state is checkpointed to {@code filepath} and restored into {@code newSlotId}, and the + * new session starts with a copy of this session's transcript (and the same system + * message and parameter customizer). Both sessions then continue independently — + * A/B-answering the same conversation, speculative side-branches, etc. + *

+ * {@code newSlotId} must be a different slot served by the same model — load the model + * with {@link net.ladenthin.llama.parameters.ModelParameters#setParallel(int)} ≥ 2 to have one available. The + * checkpoint file's lifecycle is caller-managed and may be deleted once the fork returns. + *

+ * + * @param newSlotId the slot the forked session is bound to (must differ from this session's) + * @param filepath scratch file used to transfer the slot KV state + * @return the forked session, positioned at this session's current state + * @throws IllegalArgumentException if {@code newSlotId} equals this session's slot id + * @throws IllegalStateException if a stream is in progress + */ + public Session fork(int newSlotId, String filepath) { + if (newSlotId == slotId) { + throw new IllegalArgumentException( + "fork target slot must differ from the session's own slot " + slotId + ", was " + newSlotId); + } + return state.runWhenNotStreaming("fork", () -> { + model.saveSlot(slotId, filepath); + model.restoreSlot(newSlotId, filepath); + Session forked = new Session(model, newSlotId, state.getSystemMessage(), paramsCustomizer); + forked.state.restoreTurns(state.turnsSnapshot()); + return forked; + }); + } + /** Erase the bound slot's KV cache. Does not modify the in-memory transcript. */ @Override public void close() { diff --git a/llama/src/main/java/net/ladenthin/llama/SessionState.java b/llama/src/main/java/net/ladenthin/llama/SessionState.java index 99168ae4..86ffcc39 100644 --- a/llama/src/main/java/net/ladenthin/llama/SessionState.java +++ b/llama/src/main/java/net/ladenthin/llama/SessionState.java @@ -189,6 +189,44 @@ public List snapshot() { } } + /** + * Return a fresh copy of the committed (role, text) turns for checkpointing. Rejected + * while a stream is in progress: the pending user turn is already committed at that + * point, so a mid-stream snapshot would capture a dangling half-round. + * + * @return a fresh copy of the committed turns, in order + * @throws IllegalStateException if a stream is in progress + */ + public List> turnsSnapshot() { + synchronized (lock) { + requireNotStreaming("checkpoint"); + return transcript.turnsSnapshot(); + } + } + + /** + * Replace the transcript's committed turns with a checkpointed snapshot, for + * rewinding. Rejected while a stream is in progress. + * + * @param turns the (role, text) turns to restore, in order + * @throws IllegalStateException if a stream is in progress + */ + public void restoreTurns(List> turns) { + synchronized (lock) { + requireNotStreaming("rewind"); + transcript.resetTurns(turns); + } + } + + /** + * System-message accessor (fixed at construction). + * + * @return the system prompt, or {@code null} when none was configured + */ + public @Nullable String getSystemMessage() { + return transcript.getSystemMessage(); + } + private void requireNotStreaming(String operation) { if (streamingActive) { throw new IllegalStateException("stream in progress on slot " + slotId diff --git a/llama/src/main/java/net/ladenthin/llama/args/QuantizationType.java b/llama/src/main/java/net/ladenthin/llama/args/QuantizationType.java new file mode 100644 index 00000000..062c31b6 --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/args/QuantizationType.java @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.args; + +/** + * Target file type (quantization scheme) for {@link net.ladenthin.llama.LlamaQuantizer}. + * + *

Each constant maps 1-to-1 to a {@code llama_ftype} enumerator in {@code include/llama.h} + * (llama.cpp b9870); the stored integer is the exact native enum value passed to + * {@code llama_model_quantize}. Enumerators that upstream removed or commented out are not + * represented here. + */ +public enum QuantizationType { + + /** All tensors kept as F32 — {@code LLAMA_FTYPE_ALL_F32 = 0}. */ + ALL_F32(0), + /** Mostly F16, except 1d tensors — {@code LLAMA_FTYPE_MOSTLY_F16 = 1}. */ + F16(1), + /** Mostly Q4_0 — {@code LLAMA_FTYPE_MOSTLY_Q4_0 = 2}. */ + Q4_0(2), + /** Mostly Q4_1 — {@code LLAMA_FTYPE_MOSTLY_Q4_1 = 3}. */ + Q4_1(3), + /** Mostly Q8_0 — {@code LLAMA_FTYPE_MOSTLY_Q8_0 = 7}. */ + Q8_0(7), + /** Mostly Q5_0 — {@code LLAMA_FTYPE_MOSTLY_Q5_0 = 8}. */ + Q5_0(8), + /** Mostly Q5_1 — {@code LLAMA_FTYPE_MOSTLY_Q5_1 = 9}. */ + Q5_1(9), + /** Mostly Q2_K — {@code LLAMA_FTYPE_MOSTLY_Q2_K = 10}. */ + Q2_K(10), + /** Mostly Q3_K, small — {@code LLAMA_FTYPE_MOSTLY_Q3_K_S = 11}. */ + Q3_K_S(11), + /** Mostly Q3_K, medium — {@code LLAMA_FTYPE_MOSTLY_Q3_K_M = 12}. */ + Q3_K_M(12), + /** Mostly Q3_K, large — {@code LLAMA_FTYPE_MOSTLY_Q3_K_L = 13}. */ + Q3_K_L(13), + /** Mostly Q4_K, small — {@code LLAMA_FTYPE_MOSTLY_Q4_K_S = 14}. */ + Q4_K_S(14), + /** Mostly Q4_K, medium — {@code LLAMA_FTYPE_MOSTLY_Q4_K_M = 15}. */ + Q4_K_M(15), + /** Mostly Q5_K, small — {@code LLAMA_FTYPE_MOSTLY_Q5_K_S = 16}. */ + Q5_K_S(16), + /** Mostly Q5_K, medium — {@code LLAMA_FTYPE_MOSTLY_Q5_K_M = 17}. */ + Q5_K_M(17), + /** Mostly Q6_K — {@code LLAMA_FTYPE_MOSTLY_Q6_K = 18}. */ + Q6_K(18), + /** Mostly IQ2_XXS — {@code LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19}. */ + IQ2_XXS(19), + /** Mostly IQ2_XS — {@code LLAMA_FTYPE_MOSTLY_IQ2_XS = 20}. */ + IQ2_XS(20), + /** Mostly Q2_K, small — {@code LLAMA_FTYPE_MOSTLY_Q2_K_S = 21}. */ + Q2_K_S(21), + /** Mostly IQ3_XS — {@code LLAMA_FTYPE_MOSTLY_IQ3_XS = 22}. */ + IQ3_XS(22), + /** Mostly IQ3_XXS — {@code LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23}. */ + IQ3_XXS(23), + /** Mostly IQ1_S — {@code LLAMA_FTYPE_MOSTLY_IQ1_S = 24}. */ + IQ1_S(24), + /** Mostly IQ4_NL — {@code LLAMA_FTYPE_MOSTLY_IQ4_NL = 25}. */ + IQ4_NL(25), + /** Mostly IQ3_S — {@code LLAMA_FTYPE_MOSTLY_IQ3_S = 26}. */ + IQ3_S(26), + /** Mostly IQ3_M — {@code LLAMA_FTYPE_MOSTLY_IQ3_M = 27}. */ + IQ3_M(27), + /** Mostly IQ2_S — {@code LLAMA_FTYPE_MOSTLY_IQ2_S = 28}. */ + IQ2_S(28), + /** Mostly IQ2_M — {@code LLAMA_FTYPE_MOSTLY_IQ2_M = 29}. */ + IQ2_M(29), + /** Mostly IQ4_XS — {@code LLAMA_FTYPE_MOSTLY_IQ4_XS = 30}. */ + IQ4_XS(30), + /** Mostly IQ1_M — {@code LLAMA_FTYPE_MOSTLY_IQ1_M = 31}. */ + IQ1_M(31), + /** Mostly BF16 — {@code LLAMA_FTYPE_MOSTLY_BF16 = 32}. */ + BF16(32), + /** Mostly TQ1_0 — {@code LLAMA_FTYPE_MOSTLY_TQ1_0 = 36}. */ + TQ1_0(36), + /** Mostly TQ2_0 — {@code LLAMA_FTYPE_MOSTLY_TQ2_0 = 37}. */ + TQ2_0(37), + /** Mostly MXFP4 (MoE) — {@code LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38}. */ + MXFP4_MOE(38), + /** Mostly NVFP4 — {@code LLAMA_FTYPE_MOSTLY_NVFP4 = 39}. */ + NVFP4(39), + /** Mostly Q1_0 — {@code LLAMA_FTYPE_MOSTLY_Q1_0 = 40}. */ + Q1_0(40); + + private final int ftypeValue; + + QuantizationType(int ftypeValue) { + this.ftypeValue = ftypeValue; + } + + /** + * The native {@code llama_ftype} enum value this constant maps to. + * + * @return the integer passed to {@code llama_model_quantize} + */ + public int getFtypeValue() { + return ftypeValue; + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/json/EmbeddingResponseParser.java b/llama/src/main/java/net/ladenthin/llama/json/EmbeddingResponseParser.java new file mode 100644 index 00000000..6f2a458c --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/json/EmbeddingResponseParser.java @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Pure JSON transforms for the OpenAI-compatible embeddings wire format used by the typed + * batch-embedding API ({@code net.ladenthin.llama.LlamaModel#embed(java.util.Collection)}). + * + *

All methods are stateless and have zero dependency on JNI, native libraries, or llama + * model state — they can be tested with JSON string literals alone (see + * {@code EmbeddingResponseParserTest}). + * + *

The native OAI embeddings response has the shape: + * 

{@code
+ * {
+ *   "object": "list",
+ *   "data": [
+ *     {"object": "embedding", "embedding": [0.1, 0.2, ...], "index": 0},
+ *     {"object": "embedding", "embedding": [0.3, 0.4, ...], "index": 1}
+ *   ]
+ * }
+ * }
+ */ +public class EmbeddingResponseParser { + + /** Creates a new {@link EmbeddingResponseParser}. */ + public EmbeddingResponseParser() {} + + /** Shared Jackson mapper; thread-safe and reused across all instances. */ + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + /** + * Parse the embedding vectors from a raw OAI-format JSON response string. Delegates to + * {@link #parse(JsonNode)} after a single {@code readTree} call. + * + * @param json raw OAI embeddings response JSON + * @return one float vector per input, ordered by the response's {@code index} field; + * empty list on parse failure + */ + public List parse(String json) { + try { + return parse(OBJECT_MAPPER.readTree(json)); + } catch (IOException e) { + return new ArrayList<>(); + } + } + + /** + * Parse the embedding vectors from a pre-parsed OAI-format response node. Entries are + * ordered by their {@code "index"} field (falling back to array position when absent), so + * the result lines up with the request's input order. Entries whose {@code "embedding"} + * is not a numeric array are skipped. + * + * @param node pre-parsed OAI embeddings response + * @return one float vector per parsed entry, ordered by index; empty list when + * {@code "data"} is absent or not an array + */ + public List parse(JsonNode node) { + List vectors = new ArrayList(); + JsonNode data = node.path("data"); + if (!data.isArray()) { + return new ArrayList(); + } + int position = 0; + for (JsonNode entry : data) { + JsonNode embedding = entry.path("embedding"); + if (!embedding.isArray()) { + position++; + continue; + } + float[] vector = new float[embedding.size()]; + for (int i = 0; i < vector.length; i++) { + vector[i] = (float) embedding.get(i).asDouble(0.0); + } + vectors.add(new IndexedVector(entry.path("index").asInt(position), vector)); + position++; + } + // Stable sort by the response's index field so the vectors line up with the + // request's input order even if the native side reordered completions. + vectors.sort((a, b) -> Integer.compare(a.index, b.index)); + List ordered = new ArrayList(vectors.size()); + for (IndexedVector entry : vectors) { + ordered.add(entry.vector); + } + return ordered; + } + + /** One parsed embedding paired with the response's {@code index} field for reordering. */ + private static final class IndexedVector { + final int index; + final float[] vector; + + IndexedVector(int index, float[] vector) { + this.index = index; + this.vector = vector; + } + } + + /** + * Build the OAI batch-embedding request body {@code {"input": [prompt, ...]}} for + * the native embeddings endpoint. + * + * @param prompts the strings to embed, in order; must not be empty + * @return the request JSON string + */ + public String toBatchRequestJson(Collection prompts) { + ObjectNode root = OBJECT_MAPPER.createObjectNode(); + ArrayNode input = root.putArray("input"); + for (String prompt : prompts) { + input.add(prompt); + } + return root.toString(); + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/json/LoraAdapterResponseParser.java b/llama/src/main/java/net/ladenthin/llama/json/LoraAdapterResponseParser.java new file mode 100644 index 00000000..6a637ed9 --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/json/LoraAdapterResponseParser.java @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import net.ladenthin.llama.value.LoraAdapter; + +/** + * Pure JSON transforms for the native LoRA-adapter list and scale-update wire formats. + * + *

All methods are stateless and have zero dependency on JNI, native libraries, or llama + * model state — they can be tested with JSON string literals alone (see + * {@code LoraAdapterResponseParserTest}). + * + *

The native {@code GET /lora-adapters}-equivalent task produces a JSON array: + * 

{@code
+ * [
+ *   {"id": 0, "path": "adapter.gguf", "scale": 0.5,
+ *    "task_name": "classification", "prompt_prefix": ""}
+ * ]
+ * }
+ * + *

The scale-update request (the upstream {@code POST /lora-adapters} body) is a JSON array + * of {@code {"id": , "scale": }} objects, built by + * {@link #toRequestJson(Map)}. + */ +public class LoraAdapterResponseParser { + + /** Creates a new {@link LoraAdapterResponseParser}. */ + public LoraAdapterResponseParser() {} + + /** Shared Jackson mapper; thread-safe and reused across all instances. */ + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + /** + * Parse the adapter list from a raw JSON array string. Delegates to {@link #parse(JsonNode)} + * after a single {@code readTree} call. + * + * @param json raw JSON array string from the native LoRA-adapter list response + * @return list of adapters; empty list on parse failure or empty array + */ + public List parse(String json) { + try { + return parse(OBJECT_MAPPER.readTree(json)); + } catch (IOException e) { + return new ArrayList<>(); + } + } + + /** + * Parse the adapter list from a pre-parsed {@link JsonNode} array. Each element carries + * {@code "id"} (int), {@code "path"} (string), {@code "scale"} (number), {@code "task_name"} + * (string) and {@code "prompt_prefix"} (string); absent fields fall back to {@code -1}, + * {@code ""}, {@code 0}, {@code ""} and {@code ""} respectively. Returns an empty list when + * the node is not an array or is empty. + * + * @param arr pre-parsed {@link JsonNode} array of adapter objects + * @return list of adapters; empty list if the node is not an array or is empty + */ + public List parse(JsonNode arr) { + List adapters = new ArrayList(); + if (!arr.isArray()) { + return adapters; + } + for (JsonNode entry : arr) { + adapters.add(new LoraAdapter( + entry.path("id").asInt(-1), + entry.path("path").asText(""), + (float) entry.path("scale").asDouble(0.0), + entry.path("task_name").asText(""), + entry.path("prompt_prefix").asText(""))); + } + return adapters; + } + + /** + * Build the scale-update request body — a JSON array of {@code {"id", "scale"}} objects in + * the map's iteration order — matching the upstream {@code POST /lora-adapters} contract: + * adapters listed in the map get the given scale, all other adapters are set to {@code 0} + * (disabled) by the native side. + * + * @param scales adapter id to new scale; may be empty (disables every adapter) + * @return the JSON array request string + * @throws IllegalArgumentException if a scale is NaN or infinite (would not serialize to + * valid JSON) + */ + public String toRequestJson(Map scales) { + com.fasterxml.jackson.databind.node.ArrayNode array = OBJECT_MAPPER.createArrayNode(); + for (Map.Entry entry : scales.entrySet()) { + float scale = entry.getValue(); + if (Float.isNaN(scale) || Float.isInfinite(scale)) { + throw new IllegalArgumentException( + "LoRA scale for adapter " + entry.getKey() + " must be finite, got: " + scale); + } + array.addObject().put("id", entry.getKey().intValue()).put("scale", scale); + } + return array.toString(); + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/json/RouterModelsResponseParser.java b/llama/src/main/java/net/ladenthin/llama/json/RouterModelsResponseParser.java new file mode 100644 index 00000000..a148cf0e --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/json/RouterModelsResponseParser.java @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import net.ladenthin.llama.value.RouterModel; + +/** + * Pure JSON transform for the router-mode model registry wire format (upstream + * {@code GET /models} in router mode). + * + *

All methods are stateless and have zero dependency on JNI, native libraries, or llama + * model state — they can be tested with JSON string literals alone (see + * {@code RouterModelsResponseParserTest}). + * + *

The router response has the shape (see {@code get_router_models} in upstream + * {@code tools/server/server-models.cpp}): + * 

{@code
+ * {
+ *   "object": "list",
+ *   "data": [
+ *     {"id": "Qwen3-0.6B-Q4_K_M",
+ *      "status": {"value": "loaded", "args": [...]},
+ *      "source": "models_dir", ...},
+ *     {"id": "broken-model",
+ *      "status": {"value": "unloaded", "failed": true, "exit_code": 1}, ...}
+ *   ]
+ * }
+ * }
+ */ +public class RouterModelsResponseParser { + + /** Creates a new {@link RouterModelsResponseParser}. */ + public RouterModelsResponseParser() {} + + /** Shared Jackson mapper; thread-safe and reused across all instances. */ + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + /** + * Parse the model list from a raw {@code GET /models} JSON response string. Delegates to + * {@link #parse(JsonNode)} after a single {@code readTree} call. + * + * @param json raw router {@code GET /models} response JSON + * @return list of models; empty list on parse failure + */ + public List parse(String json) { + try { + return parse(OBJECT_MAPPER.readTree(json)); + } catch (IOException e) { + return new ArrayList<>(); + } + } + + /** + * Parse the model list from a pre-parsed response node. Entries are read from the + * {@code "data"} array (falling back to {@code "models"} for older/alternate shapes); the + * identifier is read from {@code "id"} (falling back to {@code "name"}). The lifecycle + * status comes from {@code status.value}; a missing status maps to + * {@link RouterModel.Status#UNKNOWN} with an empty raw value. The failure marker is read + * from {@code status.failed} / {@code status.exit_code}. + * + * @param root pre-parsed router {@code GET /models} response + * @return list of models; empty list when no entry array is present + */ + public List parse(JsonNode root) { + List models = new ArrayList(); + JsonNode data = root.path("data"); + if (!data.isArray()) { + data = root.path("models"); + } + if (!data.isArray()) { + return models; + } + for (JsonNode entry : data) { + String id = entry.path("id").asText(entry.path("name").asText("")); + JsonNode status = entry.path("status"); + String statusValue = status.path("value").asText(""); + models.add(new RouterModel( + id, + RouterModel.Status.fromValue(statusValue), + statusValue, + status.path("failed").asBoolean(false), + status.path("exit_code").asInt(0))); + } + return models; + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/loader/LlamaLoader.java b/llama/src/main/java/net/ladenthin/llama/loader/LlamaLoader.java index d19ead4c..96121e19 100644 --- a/llama/src/main/java/net/ladenthin/llama/loader/LlamaLoader.java +++ b/llama/src/main/java/net/ladenthin/llama/loader/LlamaLoader.java @@ -142,7 +142,11 @@ private static void loadNativeLibrary(String name) { System.loadLibrary(name); return; } catch (UnsatisfiedLinkError e) { - triedPaths.add("Directly from .apk/lib"); + // Carry the dlopen reason into the final error: "library not in the APK" + // and "library present but a DT_NEEDED dependency is missing" are + // indistinguishable without it (the latter shipped once — the Android .so + // linked libomp.so/libc++_shared.so, which no device has). + triedPaths.add("Directly from .apk/lib (" + e.getMessage() + ")"); } } diff --git a/llama/src/main/java/net/ladenthin/llama/server/NativeServer.java b/llama/src/main/java/net/ladenthin/llama/server/NativeServer.java index 65caf6c8..d85628cf 100644 --- a/llama/src/main/java/net/ladenthin/llama/server/NativeServer.java +++ b/llama/src/main/java/net/ladenthin/llama/server/NativeServer.java @@ -9,7 +9,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import lombok.ToString; +import net.ladenthin.llama.LlamaModel; import net.ladenthin.llama.loader.LlamaLoader; +import org.jspecify.annotations.Nullable; /** * Runs the full upstream llama.cpp HTTP server — including its embedded @@ -24,12 +26,28 @@ * {@code --port}, {@code --ui}/{@code --no-ui}, …). Unlike {@link OpenAiCompatServer}, no per-flag * Java mapping is involved.

* - *

Independent lifecycle. {@code NativeServer} loads its own model from - * the forwarded arguments — exactly like running {@code llama-server.exe} — and is unrelated to any - * {@code net.ladenthin.llama.LlamaModel} you may also have open. Reusing an already-loaded - * {@code LlamaModel}'s context instead of loading a second copy is a possible future enhancement - * (see {@code TODO.md}). While the native server runs it owns the process-wide llama backend and - * routes llama.cpp logging to stderr/file (llama-server's own logging), not the JNI log callback.

+ *

Two lifecycles. The classic constructor ({@link #NativeServer(String...)}) + * loads its own model from the forwarded arguments — exactly like running + * {@code llama-server.exe} — and is unrelated to any {@link LlamaModel} you may also have open; + * while it runs it owns the process-wide llama backend and routes llama.cpp logging to + * stderr/file (llama-server's own logging), not the JNI log callback. The attach + * constructor ({@link #NativeServer(LlamaModel, String...)}) instead serves an + * already-loaded {@code LlamaModel} — no second copy of the weights, no second + * model load: the model's own worker thread keeps driving inference and the HTTP routes post + * tasks to its queue. In attach mode the arguments carry only the HTTP-side flags + * ({@code --host}, {@code --port}, {@code --api-key}, …; no {@code -m}), and the caller keeps + * full ownership of the model: do not {@code close()} the model while the server is + * attached — stop the server first (server before model, like unwinding + * try-with-resources).

+ * + *

Router mode. Starting without any model argument puts the upstream server + * in router mode ({@code --models-dir}, {@code GET/POST /models}, per-request model selection). + * The router serves each model from a worker subprocess that upstream spawns by + * re-executing its own binary — which, inside a JVM, is {@code java}, not a llama-server. Set + * {@link #setWorkerCommand(String...)} before starting a router so workers relaunch through this + * library instead, e.g. {@code setWorkerCommand(javaBin, "-cp", classpath, + * "net.ladenthin.llama.server.NativeServer")} — each worker is then a fresh JVM running the + * classic single-model {@code NativeServer}.

* *

Single instance per process. The upstream server keeps its shutdown state in * file-scope globals, so only one {@code NativeServer} may run at a time; {@link #start()} throws if @@ -65,6 +83,14 @@ public final class NativeServer implements AutoCloseable { /** The llama-server argument vector, forwarded verbatim to the native entry point. */ private final String[] args; + /** + * The already-loaded model this server attaches to, or {@code null} for the classic + * standalone lifecycle (the server loads its own model from {@link #args}). Excluded from + * {@code toString} — rendering it would dump the model's own identity block on every log line. + */ + @ToString.Exclude + private final @Nullable LlamaModel attachedModel; + /** Native handle (pointer) while running, or {@code 0} when not started / stopped. */ private volatile long handle; @@ -84,6 +110,37 @@ public NativeServer(String... args) { Objects.requireNonNull(arg, "args element"); } this.args = args.clone(); + this.attachedModel = null; + } + + /** + * Creates a native-server bridge that attaches the full upstream HTTP frontend — + * route table, WebUI, resumable streaming — to an already-loaded {@link LlamaModel}, instead + * of loading a second copy of the weights. + * + *

The arguments carry only the HTTP-side llama-server flags ({@code --host}, + * {@code --port}, {@code --api-key}, {@code --slots}, …); no model argument is needed or + * used. Because the model is already loaded, the server reports ready on {@code GET /health} + * as soon as the socket is up.

+ * + *

Lifecycle contract: the caller keeps full ownership of {@code model}. + * The model must stay open for as long as the server runs — close the server first, then the + * model. Closing the model while attached leaves the HTTP routes pointing at freed native + * state.

+ * + * @param model the loaded model whose native server context this server should serve; must + * not be {@code null} and must not be closed while the server runs + * @param args the HTTP-side llama-server arguments (e.g. {@code "--host", "127.0.0.1", + * "--port", "8080"}); must not be {@code null} or contain {@code null} elements + */ + public NativeServer(LlamaModel model, String... args) { + Objects.requireNonNull(model, "model"); + Objects.requireNonNull(args, "args"); + for (final String arg : args) { + Objects.requireNonNull(arg, "args element"); + } + this.args = args.clone(); + this.attachedModel = model; } /** @@ -107,7 +164,7 @@ public NativeServer start() { // Load libjllama lazily here (not in a static initializer) so construction, argument // parsing and close() stay usable — and unit-testable — without the native library. LlamaLoader.initialize(); - handle = startNativeServer(args); + handle = attachedModel != null ? startAttachedNativeServer(attachedModel, args) : startNativeServer(args); } catch (final RuntimeException | Error e) { RUNNING.set(false); throw e; @@ -231,12 +288,64 @@ public static void main(String[] args) throws InterruptedException { } } + /** + * Sets (or clears) the router-mode worker command for this process — the command + * line prefix used to spawn each model-worker subprocess when this server runs in router + * mode. By default the upstream router re-executes its own binary, which inside a JVM is + * {@code java} itself and cannot serve a model; point it at this library's bootstrap instead: + * + * 
{@code
+     * String javaBin = System.getProperty("java.home") + File.separator + "bin" + File.separator + "java";
+     * NativeServer.setWorkerCommand(javaBin, "-cp", System.getProperty("java.class.path"),
+     *         "net.ladenthin.llama.server.NativeServer");
+     * }
+ * + *

The tokens are stored in the process environment ({@code LLAMA_SERVER_WORKER_CMD}, + * whitespace-joined) and read by the native router when it spawns a worker; the router's + * computed worker arguments ({@code --host}, {@code --port}, alias, model flags) are appended + * after them. Because the variable is whitespace-split natively, no token may contain + * whitespace (e.g. a classpath with spaces is unsupported).

+ * + *

Calling with no tokens clears the override (workers re-exec the process binary again, + * the upstream default).

+ * + * @param command the worker command tokens, e.g. {@code "java", "-cp", "app.jar", + * "net.ladenthin.llama.server.NativeServer"}; empty clears the override + * @throws IllegalArgumentException if a token is null, empty, or contains whitespace + */ + public static void setWorkerCommand(String... command) { + Objects.requireNonNull(command, "command"); + StringBuilder joined = new StringBuilder(); + for (final String token : command) { + if (token == null || token.isEmpty() || token.matches(".*\\s.*")) { + throw new IllegalArgumentException( + "worker command tokens must be non-empty and must not contain whitespace, got: " + token); + } + if (joined.length() > 0) { + joined.append(' '); + } + joined.append(token); + } + LlamaLoader.initialize(); + setWorkerCommandNative(joined.length() == 0 ? null : joined.toString()); + } + /** * Starts the native server on a worker thread and returns an opaque handle. The argv is * forwarded verbatim (with a synthetic {@code argv[0]}). */ private static native long startNativeServer(String[] args); + /** + * Starts the attach-mode server (HTTP frontend over the given model's native server context) + * on a worker thread and returns an opaque handle compatible with + * {@link #stopNativeServer(long)} / {@link #isRunningNative(long)}. + */ + private static native long startAttachedNativeServer(LlamaModel model, String[] args); + + /** Sets/clears the {@code LLAMA_SERVER_WORKER_CMD} process environment variable. */ + private static native void setWorkerCommandNative(@Nullable String command); + /** Signals shutdown, joins the worker thread, and frees the handle. */ private static native void stopNativeServer(long handle); diff --git a/llama/src/main/java/net/ladenthin/llama/server/RouterClient.java b/llama/src/main/java/net/ladenthin/llama/server/RouterClient.java new file mode 100644 index 00000000..f20129d7 --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/server/RouterClient.java @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.server; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import lombok.EqualsAndHashCode; +import net.ladenthin.llama.json.RouterModelsResponseParser; +import net.ladenthin.llama.value.RouterModel; +import org.jspecify.annotations.Nullable; + +/** + * Typed client for the upstream router mode model-management endpoints + * ({@code GET /models}, {@code POST /models/load}, {@code POST /models/unload}) — the API a + * {@link NativeServer} exposes when started with {@code --models-dir} instead of a model + * (see the README "Router mode" section and {@code patches/0008}). + * + *

Replaces the raw HTTP+JSON boilerplate callers previously had to write themselves + * (hand-building request bodies, knowing that a model's lifecycle state lives at + * {@code status.value}, polling until {@code "loaded"}). Works against the in-JVM + * {@link NativeServer} or any external {@code llama-server} router alike — it is plain HTTP, + * no JNI. + * + * 

{@code
+ * NativeServer.setWorkerCommand(javaBin, "-cp", classpath, "net.ladenthin.llama.server.NativeServer");
+ * try (NativeServer router = new NativeServer("--models-dir", dir, "--port", "8080").start()) {
+ *     RouterClient client = new RouterClient(8080);
+ *     List models = client.listModels();
+ *     client.loadModel(models.get(0).getId());
+ *     client.awaitModelLoaded(models.get(0).getId(), 240_000L);
+ *     // ... POST /v1/chat/completions with {"model": models.get(0).getId(), ...}
+ * }
+ * }
+ * + *

Instances are immutable and safe to share across threads; every call opens a short-lived + * {@link HttpURLConnection}.

+ * + *

{@code equals}/{@code hashCode} are generated by Lombok over the {@code host}/{@code port} + * fields (two clients pointing at the same router compare equal). {@code toString} is + * intentionally handwritten (not Lombok-generated) so the client renders as its target URL, + * "{@code RouterClient(http://host:port)}", in log traces.

+ */ +@EqualsAndHashCode +public final class RouterClient { + + /** Poll interval used by {@link #awaitModelLoaded(String, long)}. */ + public static final long POLL_INTERVAL_MILLIS = 500L; + + /** Per-request connect/read timeout for the plain model-management calls. */ + private static final int REQUEST_TIMEOUT_MILLIS = 30_000; + + private static final RouterModelsResponseParser PARSER = new RouterModelsResponseParser(); + + private final String host; + private final int port; + + /** + * Client for a router on {@code 127.0.0.1}. + * + * @param port the router's HTTP port + */ + public RouterClient(int port) { + this("127.0.0.1", port); + } + + /** + * Client for a router on an arbitrary host. + * + * @param host the router's host name or address + * @param port the router's HTTP port + */ + public RouterClient(String host, int port) { + this.host = host; + this.port = port; + } + + /** + * List every model the router knows about ({@code GET /models}), with its lifecycle + * status and failure marker. + * + * @return the model entries, in server order + * @throws IOException if the request fails or the router answers non-2xx + */ + public List listModels() throws IOException { + return PARSER.parse(request("GET", "/models", null)); + } + + /** + * Look up a single model by identifier. + * + * @param modelId the model identifier (as listed by {@link #listModels()}) + * @return the entry, or {@link Optional#empty()} when the router does not list it + * @throws IOException if the request fails or the router answers non-2xx + */ + public Optional findModel(String modelId) throws IOException { + for (RouterModel model : listModels()) { + if (model.getId().equals(modelId)) { + return Optional.of(model); + } + } + return Optional.empty(); + } + + /** + * Ask the router to start a worker for {@code modelId} ({@code POST /models/load}). + * Non-blocking on the server side: the call returns as soon as the load is initiated — + * use {@link #awaitModelLoaded(String, long)} to wait for readiness. + * + * @param modelId the model identifier to load + * @throws IOException if the request fails or the router rejects it (unknown model, + * already running, ...) — the router's error body is included in the message + */ + public void loadModel(String modelId) throws IOException { + request("POST", "/models/load", modelBody(modelId)); + } + + /** + * Ask the router to stop the worker for {@code modelId} ({@code POST /models/unload}). + * + * @param modelId the model identifier to unload + * @throws IOException if the request fails or the router rejects it (unknown model, not + * running) — the router's error body is included in the message + */ + public void unloadModel(String modelId) throws IOException { + request("POST", "/models/unload", modelBody(modelId)); + } + + /** + * Poll {@link #listModels()} until {@code modelId} reaches + * {@link RouterModel.Status#LOADED}, its worker is flagged as failed, or the timeout + * elapses. Fails fast (instead of running out the timeout) when the router stops listing + * the model or marks it failed. + * + * @param modelId the model identifier to wait for + * @param timeoutMillis how long to keep polling + * @return the entry in its {@code LOADED} state + * @throws IOException if a poll request fails + * @throws InterruptedException if the calling thread is interrupted while waiting + * @throws IllegalStateException if the model is not listed by the router, its worker + * failed (exit code included), or the timeout elapses + */ + public RouterModel awaitModelLoaded(String modelId, long timeoutMillis) throws IOException, InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + RouterModel last = null; + while (System.currentTimeMillis() < deadline) { + Optional model = findModel(modelId); + if (!model.isPresent()) { + throw new IllegalStateException( + "Router does not list model '" + modelId + "' — check --models-dir and the identifier"); + } + last = model.get(); + if (last.isFailed()) { + throw new IllegalStateException( + "Router worker for model '" + modelId + "' failed with exit code " + last.getExitCode()); + } + if (last.getStatus() == RouterModel.Status.LOADED) { + return last; + } + // Interruptible pause between polls; a latch that is never counted down is the + // project's Thread.sleep-free wait primitive (see LlamaArchitectureTest#noThreadSleep). + if (new CountDownLatch(1).await(POLL_INTERVAL_MILLIS, TimeUnit.MILLISECONDS)) { + throw new IllegalStateException( + "unreachable while polling model '" + modelId + "': the poll latch is never counted down"); + } + } + throw new IllegalStateException("Model '" + modelId + "' did not reach LOADED within " + timeoutMillis + + " ms (last status: " + last + ")"); + } + + private static String modelBody(String modelId) { + ObjectNode body = RouterModelsResponseParser.OBJECT_MAPPER.createObjectNode(); + body.put("model", modelId); + return body.toString(); + } + + /** + * Execute one HTTP request and return the response body. Non-2xx responses raise an + * {@link IOException} carrying the status code and the router's error body (upstream + * answers with a JSON {@code {"error": ...}} object), so callers see the actual reason. + */ + private String request(String method, String path, @Nullable String body) throws IOException { + // URI.create(...).toURL() instead of the URL(String) constructor: the latter is + // deprecated (no validation/encoding) — flagged by CodeQL; this form is Java-8-safe. + URL url = URI.create("http://" + host + ":" + port + path).toURL(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + try { + connection.setRequestMethod(method); + connection.setConnectTimeout(REQUEST_TIMEOUT_MILLIS); + connection.setReadTimeout(REQUEST_TIMEOUT_MILLIS); + if (body != null) { + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", "application/json"); + try (OutputStream out = connection.getOutputStream()) { + out.write(body.getBytes(StandardCharsets.UTF_8)); + } + } + int code = connection.getResponseCode(); + InputStream stream = code >= 200 && code < 300 ? connection.getInputStream() : connection.getErrorStream(); + String response = readFully(stream); + if (code < 200 || code >= 300) { + throw new IOException("Router " + method + " " + path + " answered HTTP " + code + ": " + response); + } + return response; + } finally { + connection.disconnect(); + } + } + + @Override + public String toString() { + return "RouterClient(http://" + host + ":" + port + ")"; + } + + private static String readFully(@Nullable InputStream stream) throws IOException { + if (stream == null) { + return ""; + } + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] chunk = new byte[8192]; + int read; + while ((read = stream.read(chunk)) != -1) { + buffer.write(chunk, 0, read); + } + return new String(buffer.toByteArray(), StandardCharsets.UTF_8); + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/value/ChatTranscript.java b/llama/src/main/java/net/ladenthin/llama/value/ChatTranscript.java index 806815e0..cf96a799 100644 --- a/llama/src/main/java/net/ladenthin/llama/value/ChatTranscript.java +++ b/llama/src/main/java/net/ladenthin/llama/value/ChatTranscript.java @@ -173,4 +173,26 @@ public List snapshot() { public int size() { return turns.size(); } + + /** + * Return a fresh copy of the committed turns, for checkpointing. Mutating the + * returned list never affects this transcript. + * + * @return a fresh copy of the committed (role, text) turns, in order + */ + public List> turnsSnapshot() { + return new ArrayList>(turns); + } + + /** + * Replace the committed turns with a checkpointed snapshot, for rewinding. The + * input is copied, so later mutation of {@code newTurns} never affects this + * transcript. The system message is fixed at construction and unaffected. + * + * @param newTurns the (role, text) turns to restore, in order + */ + public void resetTurns(List> newTurns) { + turns.clear(); + turns.addAll(newTurns); + } } diff --git a/llama/src/main/java/net/ladenthin/llama/value/GgufMetadata.java b/llama/src/main/java/net/ladenthin/llama/value/GgufMetadata.java new file mode 100644 index 00000000..6261a64d --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/value/GgufMetadata.java @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.value; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import lombok.EqualsAndHashCode; + +/** + * Metadata read from a GGUF file's header and key/value table by + * {@link net.ladenthin.llama.GgufInspector} — without loading the model (no native + * library, no tensor data, no RAM commitment). Complements {@link ModelMeta}, which describes an + * already-loaded model via the native layer. + * + *

The full key/value table is exposed via {@link #getEntries()} (GGUF keys such as + * {@code general.architecture} mapped to their decoded values), with typed convenience accessors + * for the fields a model picker typically needs. Integer-typed GGUF values (u8…u64/i8…i64) decode + * to {@link Long}, floats to {@link Double}, booleans to {@link Boolean}, strings to + * {@link String}, and arrays to unmodifiable {@code List}.

+ * + *

{@code equals}/{@code hashCode} are generated by Lombok over all fields. {@code toString} + * is intentionally handwritten (not Lombok-generated): the entry table can hold six-figure + * token arrays, so it renders a compact one-line summary instead of dumping the map.

+ */ +@EqualsAndHashCode +public final class GgufMetadata { + + /** GGUF key holding the model architecture (e.g. {@code "llama"}, {@code "qwen3"}). */ + public static final String KEY_ARCHITECTURE = "general.architecture"; + + /** GGUF key holding the human-readable model name. */ + public static final String KEY_NAME = "general.name"; + + /** GGUF key holding the total parameter count. */ + public static final String KEY_PARAMETER_COUNT = "general.parameter_count"; + + /** GGUF key holding the numeric {@code llama_ftype} quantization id. */ + public static final String KEY_FILE_TYPE = "general.file_type"; + + /** GGUF key holding the Jinja chat template. */ + public static final String KEY_CHAT_TEMPLATE = "tokenizer.chat_template"; + + /** + * Suffix of the per-architecture context-length key + * ({@code ".context_length"}). + */ + public static final String KEY_SUFFIX_CONTEXT_LENGTH = ".context_length"; + + private final int version; + private final long tensorCount; + private final Map entries; + + /** + * Construct a metadata snapshot. + * + * @param version the GGUF container version (2 or 3) + * @param tensorCount the number of tensors declared in the header + * @param entries the decoded key/value table, in file order + */ + public GgufMetadata(int version, long tensorCount, Map entries) { + this.version = version; + this.tensorCount = tensorCount; + this.entries = Collections.unmodifiableMap(new LinkedHashMap(entries)); + } + + /** + * GGUF container version accessor. + * @return the container version (2 or 3) + */ + public int getVersion() { + return version; + } + + /** + * Tensor count accessor. + * @return the number of tensors declared in the header + */ + public long getTensorCount() { + return tensorCount; + } + + /** + * Full key/value table accessor. + * @return an unmodifiable view of the decoded key/value table, in file order + */ + public Map getEntries() { + return entries; + } + + /** + * Raw value lookup. + * + * @param key the GGUF key + * @return the decoded value, or {@link Optional#empty()} when the key is absent + */ + public Optional getValue(String key) { + return Optional.ofNullable(entries.get(key)); + } + + /** + * String value lookup. + * + * @param key the GGUF key + * @return the string value, or {@link Optional#empty()} when absent or not a string + */ + public Optional getString(String key) { + Object value = entries.get(key); + return value instanceof String ? Optional.of((String) value) : Optional.empty(); + } + + /** + * Integer value lookup (any numeric GGUF type, widened to {@code long}). + * + * @param key the GGUF key + * @return the numeric value, or {@link OptionalLong#empty()} when absent or not numeric + */ + public OptionalLong getLong(String key) { + Object value = entries.get(key); + return value instanceof Number ? OptionalLong.of(((Number) value).longValue()) : OptionalLong.empty(); + } + + /** + * Model architecture accessor ({@value #KEY_ARCHITECTURE}). + * @return the architecture (e.g. {@code "llama"}), or {@link Optional#empty()} when absent + */ + public Optional getArchitecture() { + return getString(KEY_ARCHITECTURE); + } + + /** + * Model name accessor ({@value #KEY_NAME}). + * @return the human-readable model name, or {@link Optional#empty()} when absent + */ + public Optional getModelName() { + return getString(KEY_NAME); + } + + /** + * Parameter count accessor ({@value #KEY_PARAMETER_COUNT}). + * @return the total parameter count, or {@link OptionalLong#empty()} when absent + */ + public OptionalLong getParameterCount() { + return getLong(KEY_PARAMETER_COUNT); + } + + /** + * Quantization id accessor ({@value #KEY_FILE_TYPE}): the numeric {@code llama_ftype} + * (matches the values behind {@link net.ladenthin.llama.args.QuantizationType}). + * @return the {@code llama_ftype} id, or {@link OptionalLong#empty()} when absent + */ + public OptionalLong getFileType() { + return getLong(KEY_FILE_TYPE); + } + + /** + * Chat template accessor ({@value #KEY_CHAT_TEMPLATE}). + * @return the Jinja chat template, or {@link Optional#empty()} when absent + */ + public Optional getChatTemplate() { + return getString(KEY_CHAT_TEMPLATE); + } + + /** + * Trained context length accessor: reads {@code ".context_length"}, so it + * requires {@link #getArchitecture()} to be present. + * @return the trained context length, or {@link OptionalLong#empty()} when the + * architecture or the length key is absent + */ + public OptionalLong getContextLength() { + Optional architecture = getArchitecture(); + if (!architecture.isPresent()) { + return OptionalLong.empty(); + } + return getLong(architecture.get() + KEY_SUFFIX_CONTEXT_LENGTH); + } + + @Override + public String toString() { + return "GGUF v" + version + " (" + tensorCount + " tensors, " + entries.size() + " keys, arch=" + + getArchitecture().orElse("?") + ")"; + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/value/LoraAdapter.java b/llama/src/main/java/net/ladenthin/llama/value/LoraAdapter.java new file mode 100644 index 00000000..0351765c --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/value/LoraAdapter.java @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.value; + +import lombok.EqualsAndHashCode; +import lombok.ToString; + +/** + * A LoRA adapter registered on the loaded model, as reported by the native + * {@code GET /lora-adapters}-equivalent task (see + * {@code net.ladenthin.llama.LlamaModel#getLoraAdapters()}). + *

+ * Adapters are loaded at model-load time via + * {@link net.ladenthin.llama.parameters.ModelParameters#addLoraAdapter(String)} / + * {@link net.ladenthin.llama.parameters.ModelParameters#addLoraScaledAdapter(String, float)}; their + * {@linkplain #getScale() scale} can then be changed at runtime without reloading the model via + * {@code net.ladenthin.llama.LlamaModel#setLoraAdapters(java.util.Map)}. A scale of {@code 0} + * disables the adapter. + *

+ * + *

{@code toString} and {@code equals}/{@code hashCode} are generated by Lombok over all + * fields.

+ */ +@ToString +@EqualsAndHashCode +public final class LoraAdapter { + + private final int id; + private final String path; + private final float scale; + private final String taskName; + private final String promptPrefix; + + /** + * Construct a LoRA adapter view. + * + * @param id the adapter id (its position in the load-time adapter list) + * @param path the GGUF file path the adapter was loaded from + * @param scale the currently applied scale; {@code 0} means disabled + * @param taskName the adapter's task name from its GGUF metadata; may be empty + * @param promptPrefix the adapter's prompt prefix from its GGUF metadata; may be empty + */ + public LoraAdapter(int id, String path, float scale, String taskName, String promptPrefix) { + this.id = id; + this.path = path; + this.scale = scale; + this.taskName = taskName; + this.promptPrefix = promptPrefix; + } + + /** + * Adapter id used to address this adapter in scale updates. + * + * @return the adapter id (its position in the load-time adapter list) + */ + public int getId() { + return id; + } + + /** + * File path of the adapter. + * + * @return the GGUF file path the adapter was loaded from + */ + public String getPath() { + return path; + } + + /** + * Currently applied adapter scale. + * + * @return the scale; {@code 0} means the adapter is disabled + */ + public float getScale() { + return scale; + } + + /** + * Task name declared in the adapter's GGUF metadata. + * + * @return the task name, or an empty string when the adapter declares none + */ + public String getTaskName() { + return taskName; + } + + /** + * Prompt prefix declared in the adapter's GGUF metadata. + * + * @return the prompt prefix, or an empty string when the adapter declares none + */ + public String getPromptPrefix() { + return promptPrefix; + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/value/RouterModel.java b/llama/src/main/java/net/ladenthin/llama/value/RouterModel.java new file mode 100644 index 00000000..6ee8c00a --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/value/RouterModel.java @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.value; + +import lombok.EqualsAndHashCode; + +/** + * One model entry from the router-mode model registry (the upstream {@code GET /models} + * response served by {@link net.ladenthin.llama.server.NativeServer} when started with + * {@code --models-dir}). Carries the model identifier, its lifecycle {@link Status}, the raw + * status string as emitted by the server, and the failure marker the router attaches when a + * worker exited abnormally. + * + *

{@code equals}/{@code hashCode} are generated by Lombok over all fields. + * {@code toString} is intentionally handwritten (not Lombok-generated) so that router traces + * in logs render as "{@code id [status]}" or "{@code id [status, failed exit=N]}" instead of + * a verbose field dump.

+ */ +@EqualsAndHashCode +public final class RouterModel { + + /** + * Router-side model lifecycle state. Mirrors the upstream {@code server_model_status} + * enum (serialized by {@code server_model_status_to_string} in {@code server-models.h}). + */ + public enum Status { + /** The model file is being downloaded. */ + DOWNLOADING, + /** The model file is downloaded but no worker has been started. */ + DOWNLOADED, + /** No worker is running for this model. */ + UNLOADED, + /** A worker is starting / loading the model. */ + LOADING, + /** The worker is up and serving requests. */ + LOADED, + /** The worker is sleeping (idle-unloaded, resumable). */ + SLEEPING, + /** Any status string this binding does not recognize. */ + UNKNOWN; + + /** + * Map the upstream status string (e.g. {@code "loaded"}) to the enum. Matching is + * exact: upstream emits exactly these lowercase strings + * ({@code server_model_status_to_string}), so no case folding is applied (which + * would also trip findsecbugs' IMPROPER_UNICODE on transformation-based matching). + * + * @param value the raw {@code status.value} string; may be {@code null} + * @return the matching status, or {@link #UNKNOWN} for {@code null}/unrecognized values + */ + public static Status fromValue(String value) { + if (value == null) { + return UNKNOWN; + } + switch (value) { + case "downloading": + return DOWNLOADING; + case "downloaded": + return DOWNLOADED; + case "unloaded": + return UNLOADED; + case "loading": + return LOADING; + case "loaded": + return LOADED; + case "sleeping": + return SLEEPING; + default: + return UNKNOWN; + } + } + } + + private final String id; + private final Status status; + private final String statusValue; + private final boolean failed; + private final int exitCode; + + /** + * Construct a router model entry. + * + * @param id the model identifier used in {@code /models/load} and per-request + * {@code "model"} selection + * @param status the parsed lifecycle status + * @param statusValue the raw {@code status.value} string as emitted by the server (kept so + * an {@link Status#UNKNOWN} mapping still exposes what the server said) + * @param failed whether the router flagged the model's worker as failed + * ({@code status.failed} in the wire format) + * @param exitCode the failed worker's exit code ({@code status.exit_code}); {@code 0} + * when not failed + */ + public RouterModel(String id, Status status, String statusValue, boolean failed, int exitCode) { + this.id = id; + this.status = status; + this.statusValue = statusValue; + this.failed = failed; + this.exitCode = exitCode; + } + + /** + * Model identifier accessor. + * @return the model identifier + */ + public String getId() { + return id; + } + + /** + * Lifecycle status accessor. + * @return the parsed lifecycle status + */ + public Status getStatus() { + return status; + } + + /** + * Raw status string accessor. + * @return the raw {@code status.value} string as emitted by the server + */ + public String getStatusValue() { + return statusValue; + } + + /** + * Failure marker accessor. + * @return {@code true} when the router flagged the model's worker as failed + */ + public boolean isFailed() { + return failed; + } + + /** + * Failed worker exit code accessor. + * @return the worker's exit code; {@code 0} when not failed + */ + public int getExitCode() { + return exitCode; + } + + @Override + public String toString() { + if (failed) { + return id + " [" + statusValue + ", failed exit=" + exitCode + "]"; + } + return id + " [" + statusValue + "]"; + } +} diff --git a/llama/src/main/java/net/ladenthin/llama/value/SessionCheckpoint.java b/llama/src/main/java/net/ladenthin/llama/value/SessionCheckpoint.java new file mode 100644 index 00000000..15a58115 --- /dev/null +++ b/llama/src/main/java/net/ladenthin/llama/value/SessionCheckpoint.java @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.value; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; + +/** + * A point-in-time snapshot of a {@link net.ladenthin.llama.Session}: the file that holds the + * slot's saved KV-cache state (written by the native slot-save action) paired with the + * transcript turns committed at checkpoint time. Produced by + * {@link net.ladenthin.llama.Session#checkpoint(String)} and consumed by + * {@link net.ladenthin.llama.Session#rewind(SessionCheckpoint)} — the KV file restores the + * native state, the turn list restores the matching in-memory transcript, so the two can + * never drift apart. + * + *

The checkpoint file's lifecycle is caller-managed (same philosophy as + * {@code Session.save(filepath)}): delete it when no checkpoint referencing it is needed + * anymore. KV dumps grow with context usage and can reach hundreds of MB for long + * conversations.

+ * + *

{@code equals}/{@code hashCode} are generated by Lombok over all fields. + * {@code toString} is intentionally handwritten (not Lombok-generated) so a checkpoint + * renders compactly as "{@code filepath (N turns)}" instead of dumping the transcript.

+ */ +@EqualsAndHashCode +public final class SessionCheckpoint { + + private final String filepath; + private final List> turns; + + /** + * Construct a checkpoint. + * + * @param filepath the file the slot KV state was saved to + * @param turns the transcript turns committed at checkpoint time, in order (copied) + */ + public SessionCheckpoint(String filepath, List> turns) { + this.filepath = filepath; + this.turns = Collections.unmodifiableList(new ArrayList>(turns)); + } + + /** + * Checkpoint file accessor. + * @return the file holding the saved slot KV state + */ + public String getFilepath() { + return filepath; + } + + /** + * Transcript snapshot accessor. + * @return an unmodifiable view of the (role, text) turns committed at checkpoint time + */ + public List> getTurns() { + return turns; + } + + @Override + public String toString() { + return filepath + " (" + turns.size() + " turns)"; + } +} diff --git a/llama/src/test/cpp/test_jni_helpers.cpp b/llama/src/test/cpp/test_jni_helpers.cpp index cafc924d..86b4be5a 100644 --- a/llama/src/test/cpp/test_jni_helpers.cpp +++ b/llama/src/test/cpp/test_jni_helpers.cpp @@ -15,7 +15,7 @@ // // Layer B tests (need upstream server headers + mock JNIEnv): // configure_multimodal_task_impl, configure_task_slot_impl, -// json_to_jstring_impl, results_to_jstring_impl, +// utf8_to_jstring_impl, json_to_jstring_impl, results_to_jstring_impl, // embedding_to_jfloat_array_impl, tokens_to_jint_array_impl // // JNIEnv is mocked via a zero-filled JNINativeInterface_ table with only the @@ -64,7 +64,22 @@ static std::string g_throw_message; static std::string g_new_string_utf_value; static jlong g_mock_handle = 0; +// String-construction stubs (utf8_to_jstring_impl path): the produced Java +// string is built via NewByteArray + SetByteArrayRegion + NewObject(String, +// ([BLjava/lang/String;)V, bytes, charsetName), so the mock captures +// the raw byte payload and the constructor arguments. +static std::string g_string_bytes; +static jsize g_byte_alloc_len = -1; +static bool g_fail_new_byte_array = false; +static bool g_new_object_called = false; +static jmethodID g_ctor_method = nullptr; +static jobject g_ctor_bytes_arg = nullptr; +static jobject g_ctor_charset_arg = nullptr; +static int g_delete_local_ref_count = 0; + static jstring g_new_string_utf_sentinel = reinterpret_cast(0xBEEF); +static jbyteArray g_byte_array_sentinel = reinterpret_cast(0xB17E); +static jstring g_ctor_string_sentinel = reinterpret_cast(0x57A); static jint JNICALL stub_ThrowNew(JNIEnv *, jclass, const char *msg) { g_throw_called = true; @@ -76,13 +91,36 @@ static jstring JNICALL stub_NewStringUTF(JNIEnv *, const char *utf) { g_new_string_utf_value = utf ? utf : ""; return g_new_string_utf_sentinel; } +static jbyteArray JNICALL stub_NewByteArray(JNIEnv *, jsize len) { + if (g_fail_new_byte_array) { + return nullptr; + } + g_byte_alloc_len = len; + return g_byte_array_sentinel; +} +static void JNICALL stub_SetByteArrayRegion(JNIEnv *, jbyteArray, jsize, jsize len, const jbyte *buf) { + g_string_bytes.assign(reinterpret_cast(buf), static_cast(len)); +} +// JNIEnv_::NewObject forwards its varargs to functions->NewObjectV. +static jobject JNICALL stub_NewObjectV(JNIEnv *, jclass, jmethodID mid, va_list args) { + g_new_object_called = true; + g_ctor_method = mid; + g_ctor_bytes_arg = va_arg(args, jobject); + g_ctor_charset_arg = va_arg(args, jobject); + return (jobject)g_ctor_string_sentinel; +} +static void JNICALL stub_DeleteLocalRef(JNIEnv *, jobject) { g_delete_local_ref_count++; } -// Minimal env: ThrowNew + GetLongField + NewStringUTF. +// Minimal env: ThrowNew + GetLongField + string-construction stubs. JNIEnv *make_mock_env(JNINativeInterface_ &table, JNIEnv_ &env_obj) { std::memset(&table, 0, sizeof(table)); table.ThrowNew = stub_ThrowNew; table.GetLongField = stub_GetLongField; table.NewStringUTF = stub_NewStringUTF; + table.NewByteArray = stub_NewByteArray; + table.SetByteArrayRegion = stub_SetByteArrayRegion; + table.NewObjectV = stub_NewObjectV; + table.DeleteLocalRef = stub_DeleteLocalRef; env_obj.functions = &table; return &env_obj; } @@ -94,6 +132,10 @@ struct MockJniFixture : ::testing::Test { JNIEnv *env = nullptr; jfieldID dummy_field = reinterpret_cast(0x1); jclass dummy_class = reinterpret_cast(0x2); + // Sentinels passed to the parameterized string-conversion helpers. + jclass string_class = reinterpret_cast(0x53); + jmethodID string_ctor = reinterpret_cast(0x54); + jobject charset_name = reinterpret_cast(0x55); void SetUp() override { env = make_mock_env(table, env_obj); @@ -101,6 +143,14 @@ struct MockJniFixture : ::testing::Test { g_throw_called = false; g_throw_message.clear(); g_new_string_utf_value.clear(); + g_string_bytes.clear(); + g_byte_alloc_len = -1; + g_fail_new_byte_array = false; + g_new_object_called = false; + g_ctor_method = nullptr; + g_ctor_bytes_arg = nullptr; + g_ctor_charset_arg = nullptr; + g_delete_local_ref_count = 0; } }; @@ -382,15 +432,59 @@ TEST_F(ArrayFixture, JintArrayToTokens_ReleasesWithAbortFlag) { EXPECT_EQ(g_release_mode, JNI_ABORT); } +// ============================================================ +// utf8_to_jstring_impl +// ============================================================ + +TEST_F(MockJniFixture, Utf8ToJstring_Ascii_BytesAndCtorArgsCaptured) { + jstring js = utf8_to_jstring_impl(env, "hello", string_class, string_ctor, charset_name); + EXPECT_EQ(js, g_ctor_string_sentinel); + EXPECT_EQ(g_string_bytes, "hello"); + EXPECT_EQ(g_byte_alloc_len, 5); + EXPECT_EQ(g_ctor_method, string_ctor); + EXPECT_EQ(g_ctor_bytes_arg, (jobject)g_byte_array_sentinel); + EXPECT_EQ(g_ctor_charset_arg, charset_name); +} + +// The whole point of the byte[]-based path: a supplementary-plane character +// (4-byte standard UTF-8) must cross the JNI boundary byte-identical. +// NewStringUTF would require Modified UTF-8 (CESU-8 surrogate encoding) here. +TEST_F(MockJniFixture, Utf8ToJstring_FourByteEmoji_PreservedAsStandardUtf8) { + const std::string emoji = "\xF0\x9F\x98\x80"; // U+1F600 + jstring js = utf8_to_jstring_impl(env, emoji, string_class, string_ctor, charset_name); + EXPECT_EQ(js, g_ctor_string_sentinel); + EXPECT_EQ(g_string_bytes, emoji); + EXPECT_EQ(g_byte_alloc_len, 4); +} + +TEST_F(MockJniFixture, Utf8ToJstring_EmptyString_AllocatesZeroLength) { + jstring js = utf8_to_jstring_impl(env, "", string_class, string_ctor, charset_name); + EXPECT_EQ(js, g_ctor_string_sentinel); + EXPECT_EQ(g_byte_alloc_len, 0); + EXPECT_TRUE(g_string_bytes.empty()); +} + +TEST_F(MockJniFixture, Utf8ToJstring_ReleasesByteArrayLocalRef) { + (void)utf8_to_jstring_impl(env, "x", string_class, string_ctor, charset_name); + EXPECT_EQ(g_delete_local_ref_count, 1); +} + +TEST_F(MockJniFixture, Utf8ToJstring_ByteArrayAllocFails_ReturnsNullWithoutCtor) { + g_fail_new_byte_array = true; + jstring js = utf8_to_jstring_impl(env, "x", string_class, string_ctor, charset_name); + EXPECT_EQ(js, nullptr); + EXPECT_FALSE(g_new_object_called); +} + // ============================================================ // json_to_jstring_impl // ============================================================ TEST_F(MockJniFixture, JsonToJstring_Object_RoundTrips) { json j = {{"key", "value"}, {"n", 42}}; - jstring js = json_to_jstring_impl(env, j); + jstring js = json_to_jstring_impl(env, j, string_class, string_ctor, charset_name); EXPECT_NE(js, nullptr); - json parsed = json::parse(g_new_string_utf_value); + json parsed = json::parse(g_string_bytes); EXPECT_TRUE(parsed.is_object()); EXPECT_EQ(parsed.value("key", ""), "value"); EXPECT_EQ(parsed.value("n", 0), 42); @@ -398,22 +492,45 @@ TEST_F(MockJniFixture, JsonToJstring_Object_RoundTrips) { TEST_F(MockJniFixture, JsonToJstring_Array_RoundTrips) { json j = json::array({1, 2, 3}); - jstring js = json_to_jstring_impl(env, j); + jstring js = json_to_jstring_impl(env, j, string_class, string_ctor, charset_name); EXPECT_NE(js, nullptr); - json parsed = json::parse(g_new_string_utf_value); + json parsed = json::parse(g_string_bytes); EXPECT_TRUE(parsed.is_array()); ASSERT_EQ(parsed.size(), 3u); } -TEST_F(MockJniFixture, JsonToJstring_ReturnsSentinel) { - jstring js = json_to_jstring_impl(env, {{"ok", true}}); - EXPECT_EQ(js, reinterpret_cast(0xBEEF)); +TEST_F(MockJniFixture, JsonToJstring_ReturnsCtorResult) { + jstring js = json_to_jstring_impl(env, {{"ok", true}}, string_class, string_ctor, charset_name); + EXPECT_EQ(js, g_ctor_string_sentinel); } TEST_F(MockJniFixture, JsonToJstring_NullJson_SerializesToNullString) { - jstring js = json_to_jstring_impl(env, json(nullptr)); + jstring js = json_to_jstring_impl(env, json(nullptr), string_class, string_ctor, charset_name); + EXPECT_NE(js, nullptr); + EXPECT_EQ(g_string_bytes, "null"); +} + +// A payload string ending in an incomplete UTF-8 sequence (possible on the +// non-stream path when generation stops mid-codepoint at the token limit) +// must serialise via error_handler_t::replace instead of throwing +// json::type_error 316. +TEST_F(MockJniFixture, JsonToJstring_TruncatedUtf8Content_ReplacesInsteadOfThrows) { + json j; + j["content"] = std::string("hi \xF0\x9F\x98", 6); // emoji cut after 3 of 4 bytes + jstring js = nullptr; + EXPECT_NO_THROW(js = json_to_jstring_impl(env, j, string_class, string_ctor, charset_name)); EXPECT_NE(js, nullptr); - EXPECT_EQ(g_new_string_utf_value, "null"); + // The produced JSON is valid UTF-8 with U+FFFD in place of the fragment. + json parsed = json::parse(g_string_bytes); + const std::string content = parsed.value("content", ""); + EXPECT_NE(content.find("hi "), std::string::npos); + EXPECT_NE(content.find("\xEF\xBF\xBD"), std::string::npos); +} + +TEST_F(MockJniFixture, JsonToJstring_EmojiContent_KeptAsFourByteUtf8) { + json j = {{"content", "\xF0\x9F\x98\x80"}}; + (void)json_to_jstring_impl(env, j, string_class, string_ctor, charset_name); + EXPECT_NE(g_string_bytes.find("\xF0\x9F\x98\x80"), std::string::npos); } // ============================================================ @@ -424,10 +541,10 @@ TEST_F(MockJniFixture, ResultsToJstring_SingleResult_ReturnsBareObject) { std::vector results; results.push_back(make_ok(1, "hello")); - jstring js = results_to_jstring_impl(env, results); + jstring js = results_to_jstring_impl(env, results, string_class, string_ctor, charset_name); EXPECT_NE(js, nullptr); - json parsed = json::parse(g_new_string_utf_value); + json parsed = json::parse(g_string_bytes); EXPECT_TRUE(parsed.is_object()); EXPECT_EQ(parsed.value("content", ""), "hello"); } @@ -437,10 +554,10 @@ TEST_F(MockJniFixture, ResultsToJstring_MultipleResults_ReturnsArray) { results.push_back(make_ok(2, "first")); results.push_back(make_ok(3, "second")); - jstring js = results_to_jstring_impl(env, results); + jstring js = results_to_jstring_impl(env, results, string_class, string_ctor, charset_name); EXPECT_NE(js, nullptr); - json parsed = json::parse(g_new_string_utf_value); + json parsed = json::parse(g_string_bytes); EXPECT_TRUE(parsed.is_array()); ASSERT_EQ(parsed.size(), 2u); EXPECT_EQ(parsed[0].value("content", ""), "first"); @@ -449,9 +566,9 @@ TEST_F(MockJniFixture, ResultsToJstring_MultipleResults_ReturnsArray) { TEST_F(MockJniFixture, ResultsToJstring_EmptyVector_ReturnsEmptyArray) { std::vector results; - jstring js = results_to_jstring_impl(env, results); + jstring js = results_to_jstring_impl(env, results, string_class, string_ctor, charset_name); EXPECT_NE(js, nullptr); - json parsed = json::parse(g_new_string_utf_value); + json parsed = json::parse(g_string_bytes); EXPECT_TRUE(parsed.is_array()); EXPECT_TRUE(parsed.empty()); } diff --git a/llama/src/test/cpp/test_server.cpp b/llama/src/test/cpp/test_server.cpp index 5ff9d894..404c7add 100644 --- a/llama/src/test/cpp/test_server.cpp +++ b/llama/src/test/cpp/test_server.cpp @@ -848,6 +848,72 @@ TEST(ServerTaskResultApplyLora, ToJson_SuccessTrue) { EXPECT_TRUE(j.at("success").get()); } +// ============================================================ +// server_task_result_get_lora::to_json +// The GET /lora-adapters shape parsed by the Java +// LoraAdapterResponseParser (LlamaModel.getLoraAdapters). +// ============================================================ + +TEST(ServerTaskResultGetLora, ToJson_NoAdapters_EmptyArray) { + server_task_result_get_lora r; + const json j = r.to_json(); + ASSERT_TRUE(j.is_array()); + EXPECT_TRUE(j.empty()); +} + +TEST(ServerTaskResultGetLora, ToJson_EntryCarriesIdPathScaleTaskNamePromptPrefix) { + server_task_result_get_lora r; + common_adapter_lora_info info; + info.path = "adapter.gguf"; + info.scale = 0.5f; + info.task_name = "classification"; + info.prompt_prefix = "prefix"; + r.loras.push_back(server_task_result_get_lora::lora{info, "", {}}); + + const json j = r.to_json(); + ASSERT_TRUE(j.is_array()); + ASSERT_EQ(j.size(), 1u); + EXPECT_EQ(j[0].at("id").get(), 0); + EXPECT_EQ(j[0].at("path").get(), "adapter.gguf"); + EXPECT_FLOAT_EQ(j[0].at("scale").get(), 0.5f); + EXPECT_EQ(j[0].at("task_name").get(), "classification"); + EXPECT_EQ(j[0].at("prompt_prefix").get(), "prefix"); + // alora fields are only present when invocation tokens exist + EXPECT_FALSE(j[0].contains("alora_invocation_string")); + EXPECT_FALSE(j[0].contains("alora_invocation_tokens")); +} + +TEST(ServerTaskResultGetLora, ToJson_AloraTokens_PresentWhenNonEmpty) { + server_task_result_get_lora r; + common_adapter_lora_info info; + info.path = "alora.gguf"; + info.scale = 1.0f; + r.loras.push_back(server_task_result_get_lora::lora{info, "", {7, 8}}); + + const json j = r.to_json(); + ASSERT_EQ(j.size(), 1u); + EXPECT_EQ(j[0].at("alora_invocation_string").get(), ""); + const auto toks = j[0].at("alora_invocation_tokens").get>(); + ASSERT_EQ(toks.size(), 2u); + EXPECT_EQ(toks[0], 7); + EXPECT_EQ(toks[1], 8); +} + +TEST(ServerTaskResultGetLora, ToJson_IdIsArrayIndex) { + server_task_result_get_lora r; + common_adapter_lora_info a; + a.path = "a.gguf"; + common_adapter_lora_info b; + b.path = "b.gguf"; + r.loras.push_back(server_task_result_get_lora::lora{a, "", {}}); + r.loras.push_back(server_task_result_get_lora::lora{b, "", {}}); + + const json j = r.to_json(); + ASSERT_EQ(j.size(), 2u); + EXPECT_EQ(j[0].at("id").get(), 0); + EXPECT_EQ(j[1].at("id").get(), 1); +} + // ============================================================ // server_task_result_error::to_json // jllama.cpp calls is_error() then get_result_error_message() diff --git a/llama/src/test/cpp/test_utils.cpp b/llama/src/test/cpp/test_utils.cpp index a2382387..cbba7a40 100644 --- a/llama/src/test/cpp/test_utils.cpp +++ b/llama/src/test/cpp/test_utils.cpp @@ -1139,6 +1139,54 @@ TEST(AreLoraEqual, PathDifference_Ignored) { EXPECT_TRUE(are_lora_equal({a}, {b})); } +// ============================================================ +// parse_lora_request +// Parses the POST /lora-adapters body shape [{id, scale}, ...] +// into the id -> scale map consumed by SERVER_TASK_TYPE_SET_LORA +// (also the wire format of LlamaModel.setLoraAdapters). +// ============================================================ + +TEST(ParseLoraRequest, EmptyArray_ReturnsEmptyMap) { + const auto result = parse_lora_request(json::array()); + EXPECT_TRUE(result.empty()); +} + +TEST(ParseLoraRequest, SingleEntry_MapsIdToScale) { + const json body = json::array({{{"id", 0}, {"scale", 0.5f}}}); + const auto result = parse_lora_request(body); + ASSERT_EQ(result.size(), 1u); + EXPECT_FLOAT_EQ(result.at(0), 0.5f); +} + +TEST(ParseLoraRequest, MultipleEntries_AllMapped) { + const json body = json::array({{{"id", 0}, {"scale", 1.0f}}, {{"id", 2}, {"scale", 0.25f}}}); + const auto result = parse_lora_request(body); + ASSERT_EQ(result.size(), 2u); + EXPECT_FLOAT_EQ(result.at(0), 1.0f); + EXPECT_FLOAT_EQ(result.at(2), 0.25f); +} + +TEST(ParseLoraRequest, MissingId_DefaultsToMinusOne) { + const json body = json::array({{{"scale", 0.75f}}}); + const auto result = parse_lora_request(body); + ASSERT_EQ(result.size(), 1u); + EXPECT_FLOAT_EQ(result.at(-1), 0.75f); +} + +TEST(ParseLoraRequest, MissingScale_DefaultsToZero) { + const json body = json::array({{{"id", 1}}}); + const auto result = parse_lora_request(body); + ASSERT_EQ(result.size(), 1u); + EXPECT_FLOAT_EQ(result.at(1), 0.0f); +} + +TEST(ParseLoraRequest, DuplicateId_LastWins) { + const json body = json::array({{{"id", 0}, {"scale", 0.1f}}, {{"id", 0}, {"scale", 0.9f}}}); + const auto result = parse_lora_request(body); + ASSERT_EQ(result.size(), 1u); + EXPECT_FLOAT_EQ(result.at(0), 0.9f); +} + // ============================================================ // StripFlagFromArgv // Helper used by loadModel to remove --vocab-only from argv diff --git a/llama/src/test/java/net/ladenthin/llama/AudioInputIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/AudioInputIntegrationTest.java index 5176c5c1..4ec34dfb 100644 --- a/llama/src/test/java/net/ladenthin/llama/AudioInputIntegrationTest.java +++ b/llama/src/test/java/net/ladenthin/llama/AudioInputIntegrationTest.java @@ -34,10 +34,12 @@ * compiled-in {@code mtmd} audio pipeline. * * - *

Self-skips when any of the three system properties - * ({@link TestConstants#PROP_AUDIO_MODEL_PATH} / {@link TestConstants#PROP_AUDIO_MMPROJ_PATH} / - * {@link TestConstants#PROP_AUDIO_PATH}) is unset or its file is missing, so it runs only in CI or on a - * dev machine where the (large) audio model and a clip have been staged. + *

The audio prompt defaults to the committed clip + * {@link TestConstants#DEFAULT_AUDIO_INPUT_PATH} (override via + * {@link TestConstants#PROP_AUDIO_PATH}). Self-skips when the model/mmproj properties + * ({@link TestConstants#PROP_AUDIO_MODEL_PATH} / {@link TestConstants#PROP_AUDIO_MMPROJ_PATH}) are + * unset or any referenced file is missing, so it runs only where the (large) audio model has been + * staged. */ public class AudioInputIntegrationTest { @@ -48,7 +50,7 @@ public class AudioInputIntegrationTest { public static void setup() { String modelPath = System.getProperty(TestConstants.PROP_AUDIO_MODEL_PATH); String mmprojPath = System.getProperty(TestConstants.PROP_AUDIO_MMPROJ_PATH); - audioPath = System.getProperty(TestConstants.PROP_AUDIO_PATH); + audioPath = System.getProperty(TestConstants.PROP_AUDIO_PATH, TestConstants.DEFAULT_AUDIO_INPUT_PATH); Assumptions.assumeTrue( modelPath != null && !modelPath.isEmpty(), @@ -56,9 +58,6 @@ public static void setup() { Assumptions.assumeTrue( mmprojPath != null && !mmprojPath.isEmpty(), "Audio mmproj path not set (-D" + TestConstants.PROP_AUDIO_MMPROJ_PATH + "=...)"); - Assumptions.assumeTrue( - audioPath != null && !audioPath.isEmpty(), - "Audio clip path not set (-D" + TestConstants.PROP_AUDIO_PATH + "=...)"); Assumptions.assumeTrue(new File(modelPath).exists(), "Audio model file missing: " + modelPath); Assumptions.assumeTrue(new File(mmprojPath).exists(), "Audio mmproj file missing: " + mmprojPath); Assumptions.assumeTrue(new File(audioPath).exists(), "Audio clip missing: " + audioPath); diff --git a/llama/src/test/java/net/ladenthin/llama/GgufInspectorTest.java b/llama/src/test/java/net/ladenthin/llama/GgufInspectorTest.java new file mode 100644 index 00000000..bad79f74 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/GgufInspectorTest.java @@ -0,0 +1,251 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import net.ladenthin.llama.value.GgufMetadata; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +@ClaudeGenerated( + purpose = "Verify the pure-Java GGUF header/metadata reader against in-memory generated " + + "fixtures (no committed binaries, no native library): every metadata value " + + "type, v2/v3 containers, big-endian auto-detection, the fail-loud paths " + + "(magic mismatch, v1, unknown version/type, truncation, implausible lengths), " + + "and that parsing never touches the tensor payload.") +public class GgufInspectorTest { + + @TempDir + Path tempDir; + + /** Minimal GGUF byte-stream writer mirroring the container layout (header + KV table). */ + private static final class GgufWriter { + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private final ByteOrder order; + + GgufWriter(ByteOrder order) { + this.order = order; + } + + GgufWriter magic() { + out.write('G'); + out.write('G'); + out.write('U'); + out.write('F'); + return this; + } + + GgufWriter u32(long value) { + out.write(ByteBuffer.allocate(4).order(order).putInt((int) value).array(), 0, 4); + return this; + } + + GgufWriter u64(long value) { + out.write(ByteBuffer.allocate(8).order(order).putLong(value).array(), 0, 8); + return this; + } + + GgufWriter u8(int value) { + out.write(value); + return this; + } + + GgufWriter str(String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + u64(bytes.length); + out.write(bytes, 0, bytes.length); + return this; + } + + GgufWriter raw(byte[] bytes) { + out.write(bytes, 0, bytes.length); + return this; + } + + byte[] bytes() { + return out.toByteArray(); + } + } + + /** Builds a representative v3 file: header + a KV table covering the common value types. */ + private static byte[] sampleGguf(ByteOrder order, long version) { + GgufWriter writer = new GgufWriter(order); + writer.magic().u32(version); + writer.u64(291); // tensor count + writer.u64(8); // kv count + writer.str("general.architecture").u32(8).str("llama"); + writer.str("general.name").u32(8).str("Test Model"); + writer.str("general.parameter_count").u32(10).u64(751_632_384L); // u64 + writer.str("general.file_type").u32(4).u32(15); // u32 (Q4_K_M) + writer.str("llama.context_length").u32(4).u32(40_960); + writer.str("llama.rope.freq_base").u32(6).u32(Float.floatToIntBits(1_000_000.0f)); // f32 + writer.str("tokenizer.chat_template").u32(8).str("{{ messages }}"); + // array of i32 + writer.str("tokenizer.ggml.token_type") + .u32(9) + .u32(5) + .u64(3) + .u32(1) + .u32(1) + .u32(2); + // Tensor-info section would follow; parsing must stop before it, so arbitrary + // trailing bytes must not disturb the result. + writer.raw(new byte[] {(byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF}); + return writer.bytes(); + } + + @Test + public void readsHeaderAndAllValueTypes() throws IOException { + GgufMetadata meta = GgufInspector.read(new ByteArrayInputStream(sampleGguf(ByteOrder.LITTLE_ENDIAN, 3))); + + assertThat(meta.getVersion(), is(3)); + assertThat(meta.getTensorCount(), is(291L)); + assertThat(meta.getEntries().size(), is(8)); + assertThat(meta.getArchitecture().orElse(""), is("llama")); + assertThat(meta.getModelName().orElse(""), is("Test Model")); + assertThat(meta.getParameterCount().orElse(0), is(751_632_384L)); + assertThat(meta.getFileType().orElse(0), is(15L)); + assertThat(meta.getContextLength().orElse(0), is(40_960L)); + assertThat(meta.getChatTemplate().orElse(""), is("{{ messages }}")); + assertThat(meta.getValue("llama.rope.freq_base").orElse(null), is((Object) Double.valueOf(1_000_000.0))); + assertThat(meta.getValue("tokenizer.ggml.token_type").orElse(null), is((Object) + Arrays.asList(1L, 1L, 2L))); + } + + @Test + public void readsVersion2Container() throws IOException { + GgufMetadata meta = GgufInspector.read(new ByteArrayInputStream(sampleGguf(ByteOrder.LITTLE_ENDIAN, 2))); + + assertThat(meta.getVersion(), is(2)); + assertThat(meta.getArchitecture().orElse(""), is("llama")); + } + + @Test + public void autoDetectsBigEndianContainer() throws IOException { + GgufMetadata meta = GgufInspector.read(new ByteArrayInputStream(sampleGguf(ByteOrder.BIG_ENDIAN, 3))); + + assertThat(meta.getVersion(), is(3)); + assertThat(meta.getTensorCount(), is(291L)); + assertThat(meta.getArchitecture().orElse(""), is("llama")); + assertThat(meta.getContextLength().orElse(0), is(40_960L)); + } + + @Test + public void readsFromPath(@TempDir Path dir) throws IOException { + Path file = dir.resolve("sample.gguf"); + Files.write(file, sampleGguf(ByteOrder.LITTLE_ENDIAN, 3)); + + GgufMetadata meta = GgufInspector.read(file); + + assertThat(meta.getModelName().orElse(""), is("Test Model")); + } + + @Test + public void rejectsNonGgufMagic() { + byte[] bytes = new GgufWriter(ByteOrder.LITTLE_ENDIAN) + .raw("GGML".getBytes(StandardCharsets.UTF_8)) + .u32(3) + .bytes(); + + IOException thrown = assertThrows(IOException.class, () -> GgufInspector.read(new ByteArrayInputStream(bytes))); + + assertThat(thrown.getMessage(), containsString("magic")); + } + + @Test + public void rejectsVersion1() { + byte[] bytes = new GgufWriter(ByteOrder.LITTLE_ENDIAN).magic().u32(1).bytes(); + + IOException thrown = assertThrows(IOException.class, () -> GgufInspector.read(new ByteArrayInputStream(bytes))); + + assertThat(thrown.getMessage(), containsString("v1")); + } + + @Test + public void rejectsUnknownVersion() { + byte[] bytes = new GgufWriter(ByteOrder.LITTLE_ENDIAN).magic().u32(4).bytes(); + + IOException thrown = assertThrows(IOException.class, () -> GgufInspector.read(new ByteArrayInputStream(bytes))); + + assertThat(thrown.getMessage(), containsString("version")); + } + + @Test + public void rejectsUnknownValueTypeId() { + byte[] bytes = new GgufWriter(ByteOrder.LITTLE_ENDIAN) + .magic() + .u32(3) + .u64(0) + .u64(1) + .str("key") + .u32(99) + .bytes(); + + IOException thrown = assertThrows(IOException.class, () -> GgufInspector.read(new ByteArrayInputStream(bytes))); + + assertThat(thrown.getMessage(), containsString("type id")); + } + + @Test + public void rejectsImplausibleStringLength() { + byte[] bytes = new GgufWriter(ByteOrder.LITTLE_ENDIAN) + .magic() + .u32(3) + .u64(0) + .u64(1) + .u64(Long.MAX_VALUE) // key length + .bytes(); + + IOException thrown = assertThrows(IOException.class, () -> GgufInspector.read(new ByteArrayInputStream(bytes))); + + assertThat(thrown.getMessage(), containsString("string length")); + } + + @Test + public void failsLoudOnTruncatedFile() { + byte[] full = sampleGguf(ByteOrder.LITTLE_ENDIAN, 3); + byte[] truncated = Arrays.copyOf(full, 40); + + assertThrows(IOException.class, () -> GgufInspector.read(new ByteArrayInputStream(truncated))); + } + + @Test + public void readsRealModelFileWhenPresent() throws IOException { + // Real-file sanity (CI: the shared GGUF cache; locally self-skips without models/). + Path model = java.nio.file.Paths.get(TestConstants.REASONING_MODEL_PATH); + org.junit.jupiter.api.Assumptions.assumeTrue(Files.exists(model), "reasoning model not present"); + + GgufMetadata meta = GgufInspector.read(model); + + assertThat(meta.getVersion() >= 2, is(true)); + assertThat(meta.getTensorCount() > 0, is(true)); + assertThat(meta.getArchitecture().isPresent(), is(true)); + assertThat(meta.getContextLength().orElse(0) > 0, is(true)); + } + + @Test + public void arrayValuesAreUnmodifiable() throws IOException { + GgufMetadata meta = GgufInspector.read(new ByteArrayInputStream(sampleGguf(ByteOrder.LITTLE_ENDIAN, 3))); + @SuppressWarnings("unchecked") + List tokenTypes = + (List) meta.getValue("tokenizer.ggml.token_type").orElseThrow(AssertionError::new); + + assertThrows(UnsupportedOperationException.class, () -> tokenTypes.add(0L)); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/LlamaArchitectureTest.java b/llama/src/test/java/net/ladenthin/llama/LlamaArchitectureTest.java index a3100e57..d0e302db 100644 --- a/llama/src/test/java/net/ladenthin/llama/LlamaArchitectureTest.java +++ b/llama/src/test/java/net/ladenthin/llama/LlamaArchitectureTest.java @@ -124,8 +124,10 @@ public class LlamaArchitectureTest { .mayOnlyBeAccessedByLayers("Server") .whereLayer("Loader") .mayOnlyBeAccessedByLayers("Api", "Server") + // Server: RouterClient parses the router GET /models wire format via + // json.RouterModelsResponseParser (a pure transform, tested model-free). .whereLayer("Json") - .mayOnlyBeAccessedByLayers("Api") + .mayOnlyBeAccessedByLayers("Api", "Server") .whereLayer("Parameters") .mayOnlyBeAccessedByLayers("Api", "Loader", "Server") .whereLayer("Value") diff --git a/llama/src/test/java/net/ladenthin/llama/LlamaEmbeddingsTest.java b/llama/src/test/java/net/ladenthin/llama/LlamaEmbeddingsTest.java index 69c52b73..a2f52b85 100644 --- a/llama/src/test/java/net/ladenthin/llama/LlamaEmbeddingsTest.java +++ b/llama/src/test/java/net/ladenthin/llama/LlamaEmbeddingsTest.java @@ -140,6 +140,45 @@ public void testMeanAndLastPoolingDiffer() { assertTrue(differ, "MEAN and LAST pooling must produce different vectors for multi-token input"); } + // ------------------------------------------------------------------------- + // Batch embeddings — embed(Collection) + // ------------------------------------------------------------------------- + + /** + * The batch form must return one correctly-sized vector per prompt in request order: + * identical prompts map to (near-)identical vectors, different prompts to different ones. + */ + @Test + public void testEmbedBatchReturnsOneVectorPerPromptInOrder() { + model = openModel(PoolingType.MEAN); + java.util.List batch = + model.embed(java.util.Arrays.asList(TEXT, "A completely different sentence.", TEXT)); + + assertEquals(3, batch.size()); + for (float[] vector : batch) { + assertEquals(EXPECTED_DIM, vector.length); + assertEmbeddingValid(vector, PoolingType.MEAN); + } + // Same prompt at positions 0 and 2 → same vector (same model state, same params). + org.junit.jupiter.api.Assertions.assertArrayEquals(batch.get(0), batch.get(2), 1e-3f); + // Different prompt at position 1 → measurably different vector. + boolean differ = false; + for (int i = 0; i < EXPECTED_DIM; i++) { + if (Math.abs(batch.get(0)[i] - batch.get(1)[i]) > 1e-4f) { + differ = true; + break; + } + } + assertTrue(differ, "Different prompts must not produce identical embeddings"); + } + + /** An empty prompt collection short-circuits to an empty result without a native call. */ + @Test + public void testEmbedBatchEmptyCollection() { + model = openModel(PoolingType.MEAN); + assertTrue(model.embed(java.util.Collections.emptyList()).isEmpty()); + } + // ------------------------------------------------------------------------- // Private helpers // ------------------------------------------------------------------------- diff --git a/llama/src/test/java/net/ladenthin/llama/QuantizerIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/QuantizerIntegrationTest.java new file mode 100644 index 00000000..e7bea93c --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/QuantizerIntegrationTest.java @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import net.ladenthin.llama.args.QuantizationType; +import net.ladenthin.llama.exception.LlamaException; +import net.ladenthin.llama.parameters.InferenceParameters; +import net.ladenthin.llama.parameters.ModelParameters; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Integration tests for {@link LlamaQuantizer} against a real GGUF. Uses the small draft model + * (AMD-Llama-135m, Q2_K) so the re-quantization round trip stays fast; the produced Q4_0 file is + * then loaded and asked for a short completion, proving the output is a valid, loadable model. + */ +@ClaudeGenerated( + purpose = "Exercise llama_model_quantize end to end over the JNI surface: successful " + + "re-quantization producing a loadable GGUF, the default refusal to requantize " + + "an already-quantized input, and the missing-input error path.") +public class QuantizerIntegrationTest { + + @TempDir + static Path tempDir; + + private static void assumeDraftModel() { + Assumptions.assumeTrue( + new File(TestConstants.DRAFT_MODEL_PATH).exists(), + "Draft model not found, skipping QuantizerIntegrationTest"); + } + + @Test + public void quantize_producesLoadableModel() throws Exception { + assumeDraftModel(); + Path output = tempDir.resolve("draft-q4_0.gguf"); + + LlamaQuantizer.quantize(TestConstants.DRAFT_MODEL_PATH, output.toString(), QuantizationType.Q4_0, 0, true); + + assertTrue(Files.exists(output), "quantized output must exist"); + assertTrue(Files.size(output) > 10_000_000L, "quantized 135M model should be well above 10 MB"); + + int gpuLayers = Integer.getInteger(TestConstants.PROP_TEST_NGL, TestConstants.DEFAULT_TEST_NGL); + try (LlamaModel model = new LlamaModel(new ModelParameters() + .setModel(output.toString()) + .setCtxSize(128) + .setGpuLayers(gpuLayers) + .setFit(false))) { + String completion = model.complete( + new InferenceParameters("def main():").withNPredict(4).withTemperature(0.0f)); + assertNotNull(completion, "quantized model must be able to complete"); + } + } + + /** Re-quantizing an already-quantized GGUF without the explicit opt-in must fail loudly. */ + @Test + public void quantize_requantizeWithoutOptIn_throws() { + assumeDraftModel(); + Path output = tempDir.resolve("draft-requant-refused.gguf"); + assertThrows( + LlamaException.class, + () -> LlamaQuantizer.quantize( + TestConstants.DRAFT_MODEL_PATH, output.toString(), QuantizationType.Q4_0)); + } + + @Test + public void quantize_missingInput_throws() { + assumeDraftModel(); // gates on the native lib being present in this environment + Path output = tempDir.resolve("never-written.gguf"); + assertThrows( + LlamaException.class, + () -> LlamaQuantizer.quantize( + "models/does-not-exist.gguf", output.toString(), QuantizationType.Q8_0, 0, true)); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/RuntimeLoraIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/RuntimeLoraIntegrationTest.java new file mode 100644 index 00000000..8a2176b2 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/RuntimeLoraIntegrationTest.java @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +import java.io.File; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import net.ladenthin.llama.parameters.ModelParameters; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for the runtime LoRA adapter control API + * ({@link LlamaModel#getLoraAdapters()} / {@link LlamaModel#setLoraAdapters(Map)}), the typed + * counterpart of the upstream {@code GET/POST /lora-adapters} endpoints. + * + *

CI ships no LoRA adapter GGUF, so these tests pin the adapter-less contract end to end: + * the list round-trips as empty, and scale updates are accepted (upstream ignores ids that do + * not correspond to a loaded adapter and rebuilds the — empty — adapter list). The + * adapter-carrying paths of the wire format are covered model-free by + * {@code LoraAdapterResponseParserTest} and the native C++ tests + * ({@code ServerTaskResultGetLora}/{@code ParseLoraRequest}). + */ +@ClaudeGenerated( + purpose = "Exercise the new getLoraAdapters/setLoraAdapters JNI round-trip against a real " + + "model (adapter-less contract: empty list, accepted scale updates, stable " + + "native state across repeated calls).") +public class RuntimeLoraIntegrationTest { + + private static LlamaModel model; + + @BeforeAll + public static void setup() { + Assumptions.assumeTrue( + new File(TestConstants.REASONING_MODEL_PATH).exists(), + "Reasoning model not found, skipping RuntimeLoraIntegrationTest"); + int gpuLayers = Integer.getInteger(TestConstants.PROP_TEST_NGL, TestConstants.DEFAULT_TEST_NGL); + model = new LlamaModel(new ModelParameters() + .setModel(TestConstants.REASONING_MODEL_PATH) + .setCtxSize(512) + .setGpuLayers(gpuLayers) + .setFit(false)); + } + + @AfterAll + public static void tearDown() { + if (model != null) { + model.close(); + } + } + + @Test + public void getLoraAdapters_withoutAdapters_returnsEmptyList() { + assertThat(model.getLoraAdapters(), is(empty())); + } + + @Test + public void getLoraAdaptersJson_withoutAdapters_isEmptyJsonArray() { + assertThat(model.getLoraAdaptersJson(), is("[]")); + } + + @Test + public void setLoraAdapters_unknownIdIsIgnored_listStaysEmpty() { + // Upstream construct_lora_list only looks up ids of loaded adapters, so an unknown id + // must be accepted silently rather than erroring. + Map scales = new HashMap<>(); + scales.put(0, 0.5f); + assertDoesNotThrow(() -> model.setLoraAdapters(scales)); + assertThat(model.getLoraAdapters(), is(empty())); + } + + @Test + public void setLoraAdapters_emptyMap_isAccepted() { + assertDoesNotThrow(() -> model.setLoraAdapters(Collections.emptyMap())); + assertThat(model.getLoraAdapters(), is(empty())); + } + + @Test + public void setLoraAdapter_singleConvenienceForm_isAccepted() { + assertDoesNotThrow(() -> model.setLoraAdapter(0, 0.0f)); + assertThat(model.getLoraAdapters(), is(empty())); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/SessionForkRewindIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/SessionForkRewindIntegrationTest.java new file mode 100644 index 00000000..b3e24a02 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/SessionForkRewindIntegrationTest.java @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +import java.io.File; +import java.nio.file.Path; +import java.util.List; +import net.ladenthin.llama.parameters.ModelParameters; +import net.ladenthin.llama.value.ChatMessage; +import net.ladenthin.llama.value.SessionCheckpoint; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +@ClaudeGenerated( + purpose = "Real-model coverage for the Session fork/rewind API: checkpoint + rewind " + + "restores both the KV slot state and the transcript atomically and the " + + "conversation continues from the branch point; fork produces an independent " + + "session on a second slot carrying the same transcript; guard rails " + + "(fork onto the own slot) fail fast.") +public class SessionForkRewindIntegrationTest { + + private static LlamaModel model; + + @TempDir + static Path tempDir; + + @BeforeAll + public static void loadModel() { + String modelPath = System.getProperty("net.ladenthin.llama.model.path", TestConstants.REASONING_MODEL_PATH); + Assumptions.assumeTrue(new File(modelPath).exists(), "Model missing: " + modelPath); + int gpuLayers = Integer.getInteger(TestConstants.PROP_TEST_NGL, TestConstants.DEFAULT_TEST_NGL); + model = new LlamaModel(new ModelParameters() + .setModel(modelPath) + .setCtxSize(2048) + .setGpuLayers(gpuLayers) + // Two slots: slot 0 hosts the primary session, slot 1 receives the fork. + .setParallel(2)); + } + + @AfterAll + public static void closeModel() { + if (model != null) { + model.close(); + } + } + + private static Session newSession(int slotId) { + return new Session( + model, + slotId, + "You are terse.", + p -> p.withNPredict(24).withTemperature(0.0f).withSeed(42)); + } + + @Test + public void rewindRestoresTranscriptAndConversationContinues() { + try (Session session = newSession(0)) { + session.send("Say OK."); + List atCheckpoint = session.getMessages(); + SessionCheckpoint checkpoint = + session.checkpoint(tempDir.resolve("rewind.bin").toString()); + + session.send("Say MORE."); + assertThat(session.getMessages().size(), is(atCheckpoint.size() + 2)); + + session.rewind(checkpoint); + + // Transcript is back at the branch point... + assertThat(session.getMessages(), is(atCheckpoint)); + // ...and the session is fully usable from there (KV state restored with it). + String retried = session.send("Say YES."); + assertThat(retried.isEmpty(), is(false)); + assertThat(session.getMessages().size(), is(atCheckpoint.size() + 2)); + } + } + + @Test + public void forkCreatesIndependentSessionWithSameTranscript() { + try (Session original = newSession(0)) { + original.send("Say OK."); + + try (Session forked = original.fork(1, tempDir.resolve("fork.bin").toString())) { + // The fork starts as an exact transcript copy... + assertThat(forked.getMessages(), is(original.getMessages())); + + // ...and both continue independently from the branch point. + String forkedReply = forked.send("Say A."); + String originalReply = original.send("Say B."); + assertThat(forkedReply.isEmpty(), is(false)); + assertThat(originalReply.isEmpty(), is(false)); + assertThat(forked.getMessages(), is(not(original.getMessages()))); + assertThat( + forked.getMessages().size(), is(original.getMessages().size())); + } + } + } + + @Test + public void forkOntoOwnSlotFailsFast() { + try (Session session = newSession(0)) { + org.junit.jupiter.api.Assertions.assertThrows( + IllegalArgumentException.class, + () -> session.fork(0, tempDir.resolve("self.bin").toString())); + } + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/SessionStateTest.java b/llama/src/test/java/net/ladenthin/llama/SessionStateTest.java index d829497e..ab80c401 100644 --- a/llama/src/test/java/net/ladenthin/llama/SessionStateTest.java +++ b/llama/src/test/java/net/ladenthin/llama/SessionStateTest.java @@ -132,4 +132,50 @@ public void runUnderLock_runsActionRegardlessOfStreamingState() { assertThat(ran[0], is(true)); } + + @Test + public void turnsSnapshot_andRestoreTurns_roundTrip() { + SessionState state = new SessionState(0, "sys"); + state.send("a", (systemMessage, wireMessages) -> "b"); + java.util.List> checkpoint = state.turnsSnapshot(); + state.send("c", (systemMessage, wireMessages) -> "d"); + assertThat(state.snapshot().size(), is(5)); // system + 2 rounds + + state.restoreTurns(checkpoint); + + assertThat(roles(state.snapshot()), contains("system", "user", "assistant")); + assertThat(state.snapshot().get(1).getContent(), is("a")); + } + + @Test + public void turnsSnapshot_rejectedWhileStreaming() { + SessionState state = new SessionState(0, null); + state.beginStream("q", (systemMessage, wireMessages) -> "HANDLE"); + + IllegalStateException thrown = + org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class, state::turnsSnapshot); + + assertThat(thrown.getMessage().contains("checkpoint"), is(true)); + state.commitStreamedReply("r"); + assertThat(state.turnsSnapshot().size(), is(2)); // usable again after commit + } + + @Test + public void restoreTurns_rejectedWhileStreaming() { + SessionState state = new SessionState(0, null); + state.beginStream("q", (systemMessage, wireMessages) -> "HANDLE"); + + IllegalStateException thrown = org.junit.jupiter.api.Assertions.assertThrows( + IllegalStateException.class, + () -> state.restoreTurns( + java.util.Collections.>emptyList())); + + assertThat(thrown.getMessage().contains("rewind"), is(true)); + } + + @Test + public void getSystemMessage_exposesConstructionValue() { + assertThat(new SessionState(0, "sys").getSystemMessage(), is("sys")); + assertThat(new SessionState(0, null).getSystemMessage(), is((String) null)); + } } diff --git a/llama/src/test/java/net/ladenthin/llama/TestConstants.java b/llama/src/test/java/net/ladenthin/llama/TestConstants.java index 883d91c4..261b396f 100644 --- a/llama/src/test/java/net/ladenthin/llama/TestConstants.java +++ b/llama/src/test/java/net/ladenthin/llama/TestConstants.java @@ -84,11 +84,20 @@ public class TestConstants { /** * System property holding a path to a {@code .wav} or {@code .mp3} clip used as the audio prompt in - * {@code AudioInputIntegrationTest}. The matching extension drives format detection in + * {@code AudioInputIntegrationTest}. When unset the test falls back to + * {@link #DEFAULT_AUDIO_INPUT_PATH}, which points at a small clip committed under + * {@code src/test/resources/audios/}. The matching extension drives format detection in * {@code ContentPart.audioFile(Path)}. */ public static final String PROP_AUDIO_PATH = LlamaSystemProperties.PREFIX + ".audio.input"; + /** + * Path used by {@code AudioInputIntegrationTest} when {@link #PROP_AUDIO_PATH} is unset. Points at + * the committed test resource so only the (large) audio model + mmproj have to be staged + * out-of-band. + */ + public static final String DEFAULT_AUDIO_INPUT_PATH = "src/test/resources/audios/sample.wav"; + /** * System property holding a path to the text-to-codes (OuteTTS) GGUF used by * {@code TtsIntegrationTest}. The test self-skips when this or the vocoder is unset/missing. diff --git a/llama/src/test/java/net/ladenthin/llama/Utf8RoundTripIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/Utf8RoundTripIntegrationTest.java new file mode 100644 index 00000000..eacc00e8 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/Utf8RoundTripIntegrationTest.java @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import net.ladenthin.llama.parameters.InferenceParameters; +import net.ladenthin.llama.parameters.ModelParameters; +import net.ladenthin.llama.value.LlamaOutput; +import net.ladenthin.llama.value.Pair; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Integration tests pinning that text payloads cross the JNI boundary as standard + * UTF-8 in both directions. + * + *

Two failure modes are guarded: + *

    + *
  • {@code NewStringUTF} expects Modified UTF-8, where supplementary-plane characters + * (every 4-byte emoji sequence) are CESU-8 surrogate pairs — passing standard UTF-8 + * through it mangles emoji on HotSpot and aborts under Android CheckJNI. The native + * layer therefore builds response strings via {@code String(byte[], "UTF-8")} + * ({@code utf8_to_jstring_impl}); {@link #applyTemplate_supplementaryPlane_roundTrips()} + * exercises that path deterministically (no generation involved).
    • + *
  • Streamed chunks must never split a multi-byte codepoint. The upstream server core + * holds back incomplete UTF-8 at the end of the generated text + * ({@code server_context::process_token}); the streaming tests assert no chunk carries + * a replacement character or a lone surrogate, whatever the model generates.
    • + *
+ */ +@ClaudeGenerated( + purpose = "Pin standard-UTF-8 round-trip correctness across the JNI boundary: " + + "deterministic applyTemplate echo of emoji/CJK input and well-formed " + + "(no replacement char, no lone surrogate) streamed chunks.") +public class Utf8RoundTripIntegrationTest { + + /** BMP + supplementary-plane sample: CJK (3-byte UTF-8) and emoji (4-byte UTF-8). */ + private static final String UNICODE_SAMPLE = "你好 😀🚀 grüße"; + + private static LlamaModel model; + + @BeforeAll + public static void setup() { + Assumptions.assumeTrue( + new File(TestConstants.REASONING_MODEL_PATH).exists(), + "Reasoning model not found, skipping Utf8RoundTripIntegrationTest"); + int gpuLayers = Integer.getInteger(TestConstants.PROP_TEST_NGL, TestConstants.DEFAULT_TEST_NGL); + model = new LlamaModel(new ModelParameters() + .setModel(TestConstants.REASONING_MODEL_PATH) + .setCtxSize(512) + .setGpuLayers(gpuLayers) + .setFit(false)); + } + + @AfterAll + public static void tearDown() { + if (model != null) { + model.close(); + } + } + + /** Asserts {@code text} is well-formed UTF-16: no lone surrogate, no replacement char. */ + private static void assertWellFormed(String text, String context) { + assertFalse(text.indexOf('�') >= 0, context + " contains a U+FFFD replacement character: " + text); + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (Character.isHighSurrogate(c)) { + assertTrue( + i + 1 < text.length() && Character.isLowSurrogate(text.charAt(i + 1)), + context + " contains a lone high surrogate at index " + i + ": " + text); + i++; + } else { + assertFalse( + Character.isLowSurrogate(c), + context + " contains a lone low surrogate at index " + i + ": " + text); + } + } + } + + /** + * The rendered chat template must echo emoji (supplementary plane) and CJK input + * byte-correct through the native jstring construction — deterministic, no sampling. + */ + @Test + public void applyTemplate_supplementaryPlane_roundTrips() { + List> messages = new ArrayList<>(); + messages.add(new Pair<>("user", UNICODE_SAMPLE)); + InferenceParameters params = new InferenceParameters("").withMessages(null, messages); + + String prompt = model.applyTemplate(params); + assertTrue( + prompt.contains(UNICODE_SAMPLE), + "applyTemplate must round-trip supplementary-plane characters, got: " + prompt); + assertWellFormed(prompt, "applyTemplate result"); + } + + /** + * Every streamed chunk must be well-formed regardless of what the model generates: the + * native side may only flush completed codepoints to the iterator. + */ + @Test + public void streaming_chunksAreAlwaysWellFormedUtf8() { + InferenceParameters params = new InferenceParameters("Repeat this exactly: " + UNICODE_SAMPLE + "\n") + .withNPredict(48) + .withSeed(42) + .withTemperature(0.0f); + StringBuilder all = new StringBuilder(); + for (LlamaOutput output : model.generate(params)) { + assertWellFormed(output.text, "streamed chunk"); + all.append(output.text); + } + assertWellFormed(all.toString(), "concatenated stream"); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/args/QuantizationTypeTest.java b/llama/src/test/java/net/ladenthin/llama/args/QuantizationTypeTest.java new file mode 100644 index 00000000..e5522712 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/args/QuantizationTypeTest.java @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.args; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Stream; +import net.ladenthin.llama.ClaudeGenerated; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +@ClaudeGenerated( + purpose = "Pin every QuantizationType constant to its exact native llama_ftype enum value " + + "(llama.cpp b9870 include/llama.h) — a wrong value would silently quantize to a " + + "different scheme.") +public class QuantizationTypeTest { + + static Stream expectedFtypeValues() { + return Stream.of( + Arguments.of(QuantizationType.ALL_F32, 0), + Arguments.of(QuantizationType.F16, 1), + Arguments.of(QuantizationType.Q4_0, 2), + Arguments.of(QuantizationType.Q4_1, 3), + Arguments.of(QuantizationType.Q8_0, 7), + Arguments.of(QuantizationType.Q5_0, 8), + Arguments.of(QuantizationType.Q5_1, 9), + Arguments.of(QuantizationType.Q2_K, 10), + Arguments.of(QuantizationType.Q3_K_S, 11), + Arguments.of(QuantizationType.Q3_K_M, 12), + Arguments.of(QuantizationType.Q3_K_L, 13), + Arguments.of(QuantizationType.Q4_K_S, 14), + Arguments.of(QuantizationType.Q4_K_M, 15), + Arguments.of(QuantizationType.Q5_K_S, 16), + Arguments.of(QuantizationType.Q5_K_M, 17), + Arguments.of(QuantizationType.Q6_K, 18), + Arguments.of(QuantizationType.IQ2_XXS, 19), + Arguments.of(QuantizationType.IQ2_XS, 20), + Arguments.of(QuantizationType.Q2_K_S, 21), + Arguments.of(QuantizationType.IQ3_XS, 22), + Arguments.of(QuantizationType.IQ3_XXS, 23), + Arguments.of(QuantizationType.IQ1_S, 24), + Arguments.of(QuantizationType.IQ4_NL, 25), + Arguments.of(QuantizationType.IQ3_S, 26), + Arguments.of(QuantizationType.IQ3_M, 27), + Arguments.of(QuantizationType.IQ2_S, 28), + Arguments.of(QuantizationType.IQ2_M, 29), + Arguments.of(QuantizationType.IQ4_XS, 30), + Arguments.of(QuantizationType.IQ1_M, 31), + Arguments.of(QuantizationType.BF16, 32), + Arguments.of(QuantizationType.TQ1_0, 36), + Arguments.of(QuantizationType.TQ2_0, 37), + Arguments.of(QuantizationType.MXFP4_MOE, 38), + Arguments.of(QuantizationType.NVFP4, 39), + Arguments.of(QuantizationType.Q1_0, 40)); + } + + @ParameterizedTest + @MethodSource("expectedFtypeValues") + public void ftypeValueMatchesNativeEnum(QuantizationType type, int expected) { + assertThat(type.getFtypeValue(), is(expected)); + } + + /** Every constant must be covered by the mapping table above. */ + @Test + public void mappingTableCoversAllConstants() { + assertThat((int) expectedFtypeValues().count(), is(QuantizationType.values().length)); + } + + /** llama_ftype values are unique; two constants sharing a value would be a copy-paste bug. */ + @Test + public void ftypeValuesAreUnique() { + Set seen = new HashSet<>(); + for (QuantizationType type : QuantizationType.values()) { + assertThat("duplicate ftype value " + type.getFtypeValue(), seen.add(type.getFtypeValue()), is(true)); + } + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/json/EmbeddingResponseParserTest.java b/llama/src/test/java/net/ladenthin/llama/json/EmbeddingResponseParserTest.java new file mode 100644 index 00000000..a31cc84a --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/json/EmbeddingResponseParserTest.java @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.json; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import net.ladenthin.llama.ClaudeGenerated; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link EmbeddingResponseParser}. + * No native library or model file needed — JSON string literals only. + */ +@ClaudeGenerated( + purpose = "Verify EmbeddingResponseParser parses the OAI embeddings response shape " + + "(index-ordered, skipping non-array embeddings) and builds the batch " + + "{\"input\": [...]} request with correct JSON escaping.") +public class EmbeddingResponseParserTest { + + private final EmbeddingResponseParser parser = new EmbeddingResponseParser(); + + // ------------------------------------------------------------------ + // parse(String) + // ------------------------------------------------------------------ + + @Test + public void testParse_singleEmbedding() { + String json = "{\"object\":\"list\",\"data\":[" + + "{\"object\":\"embedding\",\"embedding\":[0.1,0.2,0.3],\"index\":0}]}"; + List result = parser.parse(json); + assertThat(result, hasSize(1)); + assertArrayEquals(new float[] {0.1f, 0.2f, 0.3f}, result.get(0), 0.0001f); + } + + @Test + public void testParse_orderedByIndexField() { + // The native scheduler may complete prompts out of order; the parser must + // restore the request order via each entry's index field. + String json = "{\"data\":[" + + "{\"embedding\":[2.0],\"index\":1}," + + "{\"embedding\":[3.0],\"index\":2}," + + "{\"embedding\":[1.0],\"index\":0}]}"; + List result = parser.parse(json); + assertThat(result, hasSize(3)); + assertArrayEquals(new float[] {1.0f}, result.get(0), 0.0f); + assertArrayEquals(new float[] {2.0f}, result.get(1), 0.0f); + assertArrayEquals(new float[] {3.0f}, result.get(2), 0.0f); + } + + @Test + public void testParse_missingIndexFallsBackToPosition() { + String json = "{\"data\":[{\"embedding\":[1.0]},{\"embedding\":[2.0]}]}"; + List result = parser.parse(json); + assertThat(result, hasSize(2)); + assertArrayEquals(new float[] {1.0f}, result.get(0), 0.0f); + assertArrayEquals(new float[] {2.0f}, result.get(1), 0.0f); + } + + @Test + public void testParse_nonArrayEmbeddingSkipped() { + // encoding_format=base64 renders the embedding as a string; the float parser skips it. + String json = "{\"data\":[{\"embedding\":\"AAAA\",\"index\":0},{\"embedding\":[1.0],\"index\":1}]}"; + List result = parser.parse(json); + assertThat(result, hasSize(1)); + assertArrayEquals(new float[] {1.0f}, result.get(0), 0.0f); + } + + @Test + public void testParse_emptyData() { + assertThat(parser.parse("{\"data\":[]}"), is(empty())); + } + + @Test + public void testParse_missingData() { + assertThat(parser.parse("{\"object\":\"list\"}"), is(empty())); + } + + @Test + public void testParse_malformedJson() { + assertThat(parser.parse("not json"), is(empty())); + } + + // ------------------------------------------------------------------ + // toBatchRequestJson(Collection) + // ------------------------------------------------------------------ + + @Test + public void testToBatchRequestJson_simple() { + String json = parser.toBatchRequestJson(Arrays.asList("first", "second")); + assertThat(json, is("{\"input\":[\"first\",\"second\"]}")); + } + + @Test + public void testToBatchRequestJson_escapesSpecialCharacters() { + String json = parser.toBatchRequestJson(Collections.singletonList("say \"hi\"\nplease")); + assertThat(json, is("{\"input\":[\"say \\\"hi\\\"\\nplease\"]}")); + } + + @Test + public void testToBatchRequestJson_keepsUnicodeText() throws java.io.IOException { + String prompt = "emoji 😀 and umlaut ä"; + String json = parser.toBatchRequestJson(Collections.singletonList(prompt)); + // Round-trip through the shared mapper to prove lossless encoding. + com.fasterxml.jackson.databind.JsonNode root = EmbeddingResponseParser.OBJECT_MAPPER.readTree(json); + assertThat(root.path("input").get(0).asText(), is(prompt)); + } + + @Test + public void testToBatchRequestJson_emptyCollection() { + assertThat(parser.toBatchRequestJson(Collections.emptyList()), is("{\"input\":[]}")); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/json/LoraAdapterResponseParserTest.java b/llama/src/test/java/net/ladenthin/llama/json/LoraAdapterResponseParserTest.java new file mode 100644 index 00000000..e860cb05 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/json/LoraAdapterResponseParserTest.java @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.json; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import net.ladenthin.llama.ClaudeGenerated; +import net.ladenthin.llama.value.LoraAdapter; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link LoraAdapterResponseParser}. + * No native library or model file needed — JSON string literals only. + */ +@ClaudeGenerated( + purpose = "Verify LoraAdapterResponseParser parses the native GET /lora-adapters array " + + "shape (incl. field defaults and tolerated alora extras) and builds the " + + "POST /lora-adapters request body with finite-scale validation.") +public class LoraAdapterResponseParserTest { + + private final LoraAdapterResponseParser parser = new LoraAdapterResponseParser(); + + // ------------------------------------------------------------------ + // parse(String) + // ------------------------------------------------------------------ + + @Test + public void testParse_fullEntry() { + String json = "[{\"id\":0,\"path\":\"adapter.gguf\",\"scale\":0.5," + + "\"task_name\":\"classification\",\"prompt_prefix\":\"prefix\"}]"; + List result = parser.parse(json); + assertThat(result, hasSize(1)); + LoraAdapter adapter = result.get(0); + assertThat(adapter.getId(), is(0)); + assertThat(adapter.getPath(), is("adapter.gguf")); + assertEquals(0.5f, adapter.getScale(), 0.0001f); + assertThat(adapter.getTaskName(), is("classification")); + assertThat(adapter.getPromptPrefix(), is("prefix")); + } + + @Test + public void testParse_missingFieldsFallBackToDefaults() { + List result = parser.parse("[{}]"); + assertThat(result, hasSize(1)); + LoraAdapter adapter = result.get(0); + assertThat(adapter.getId(), is(-1)); + assertThat(adapter.getPath(), is("")); + assertEquals(0.0f, adapter.getScale(), 0.0f); + assertThat(adapter.getTaskName(), is("")); + assertThat(adapter.getPromptPrefix(), is("")); + } + + @Test + public void testParse_aloraExtrasAreTolerated() { + // aLoRA adapters additionally carry invocation fields; the parser must not trip on them. + String json = "[{\"id\":1,\"path\":\"alora.gguf\",\"scale\":1.0,\"task_name\":\"\"," + + "\"prompt_prefix\":\"\",\"alora_invocation_string\":\"\"," + + "\"alora_invocation_tokens\":[7,8]}]"; + List result = parser.parse(json); + assertThat(result, hasSize(1)); + assertThat(result.get(0).getPath(), is("alora.gguf")); + } + + @Test + public void testParse_multipleEntriesPreserveOrder() { + String json = + "[{\"id\":0,\"path\":\"a.gguf\",\"scale\":1.0}," + "{\"id\":1,\"path\":\"b.gguf\",\"scale\":0.0}]"; + List result = parser.parse(json); + assertThat(result, hasSize(2)); + assertThat(result.get(0).getPath(), is("a.gguf")); + assertThat(result.get(1).getPath(), is("b.gguf")); + } + + @Test + public void testParse_emptyArray() { + assertThat(parser.parse("[]"), is(empty())); + } + + @Test + public void testParse_nonArray() { + assertThat(parser.parse("{\"success\":true}"), is(empty())); + } + + @Test + public void testParse_malformedJson() { + assertThat(parser.parse("not json"), is(empty())); + } + + // ------------------------------------------------------------------ + // toRequestJson(Map) + // ------------------------------------------------------------------ + + @Test + public void testToRequestJson_singleEntry() { + String json = parser.toRequestJson(Collections.singletonMap(0, 0.5f)); + assertThat(json, is("[{\"id\":0,\"scale\":0.5}]")); + } + + @Test + public void testToRequestJson_multipleEntriesInIterationOrder() { + Map scales = new LinkedHashMap<>(); + scales.put(2, 0.25f); + scales.put(0, 1.0f); + String json = parser.toRequestJson(scales); + assertThat(json, is("[{\"id\":2,\"scale\":0.25},{\"id\":0,\"scale\":1.0}]")); + } + + @Test + public void testToRequestJson_emptyMap() { + assertThat(parser.toRequestJson(Collections.emptyMap()), is("[]")); + } + + @Test + public void testToRequestJson_nanScaleRejected() { + assertThrows( + IllegalArgumentException.class, () -> parser.toRequestJson(Collections.singletonMap(0, Float.NaN))); + } + + @Test + public void testToRequestJson_infiniteScaleRejected() { + assertThrows( + IllegalArgumentException.class, + () -> parser.toRequestJson(Collections.singletonMap(0, Float.POSITIVE_INFINITY))); + } + + // ------------------------------------------------------------------ + // Round trip: request built here parses back on the C++ side contract + // ------------------------------------------------------------------ + + @Test + public void testRequestJson_isValidAdapterArrayShape() { + // The request shape is a subset of the response shape, so the parser reads it back. + String json = parser.toRequestJson(Collections.singletonMap(3, 0.75f)); + List parsed = parser.parse(json); + assertThat(parsed, hasSize(1)); + assertThat(parsed.get(0).getId(), is(3)); + assertEquals(0.75f, parsed.get(0).getScale(), 0.0001f); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/json/RouterModelsResponseParserTest.java b/llama/src/test/java/net/ladenthin/llama/json/RouterModelsResponseParserTest.java new file mode 100644 index 00000000..a9395ea1 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/json/RouterModelsResponseParserTest.java @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.json; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +import java.util.List; +import net.ladenthin.llama.ClaudeGenerated; +import net.ladenthin.llama.value.RouterModel; +import org.junit.jupiter.api.Test; + +@ClaudeGenerated( + purpose = "Verify the router GET /models wire-format parser against the upstream " + + "get_router_models JSON shape: data/models array fallback, id/name fallback, " + + "status.value mapping, the failed/exit_code marker, and tolerance for " + + "missing-status and unparseable input.") +public class RouterModelsResponseParserTest { + + private final RouterModelsResponseParser parser = new RouterModelsResponseParser(); + + @Test + public void parsesUpstreamShape() { + String json = "{\"object\":\"list\",\"data\":[" + + "{\"id\":\"qwen\",\"status\":{\"value\":\"loaded\",\"args\":[\"-c\",\"512\"]},\"source\":\"models_dir\"}," + + "{\"id\":\"llama\",\"status\":{\"value\":\"unloaded\"}}" + + "]}"; + + List models = parser.parse(json); + + assertThat(models.size(), is(2)); + assertThat(models.get(0).getId(), is("qwen")); + assertThat(models.get(0).getStatus(), is(RouterModel.Status.LOADED)); + assertThat(models.get(0).getStatusValue(), is("loaded")); + assertThat(models.get(0).isFailed(), is(false)); + assertThat(models.get(1).getId(), is("llama")); + assertThat(models.get(1).getStatus(), is(RouterModel.Status.UNLOADED)); + } + + @Test + public void parsesFailedWorkerMarker() { + String json = "{\"data\":[{\"id\":\"broken\"," + + "\"status\":{\"value\":\"unloaded\",\"failed\":true,\"exit_code\":1}}]}"; + + RouterModel model = parser.parse(json).get(0); + + assertThat(model.isFailed(), is(true)); + assertThat(model.getExitCode(), is(1)); + assertThat(model.getStatus(), is(RouterModel.Status.UNLOADED)); + } + + @Test + public void fallsBackToModelsArrayAndNameField() { + // Alternate shape tolerance: "models" array with "name" identifiers. + String json = "{\"models\":[{\"name\":\"by-name\",\"status\":{\"value\":\"loading\"}}]}"; + + List models = parser.parse(json); + + assertThat(models.size(), is(1)); + assertThat(models.get(0).getId(), is("by-name")); + assertThat(models.get(0).getStatus(), is(RouterModel.Status.LOADING)); + } + + @Test + public void missingStatusMapsToUnknownWithEmptyRawValue() { + String json = "{\"data\":[{\"id\":\"bare\"}]}"; + + RouterModel model = parser.parse(json).get(0); + + assertThat(model.getStatus(), is(RouterModel.Status.UNKNOWN)); + assertThat(model.getStatusValue(), is("")); + assertThat(model.isFailed(), is(false)); + assertThat(model.getExitCode(), is(0)); + } + + @Test + public void unrecognizedStatusKeepsRawValue() { + String json = "{\"data\":[{\"id\":\"m\",\"status\":{\"value\":\"hibernating\"}}]}"; + + RouterModel model = parser.parse(json).get(0); + + assertThat(model.getStatus(), is(RouterModel.Status.UNKNOWN)); + assertThat(model.getStatusValue(), is("hibernating")); + } + + @Test + public void emptyDataYieldsEmptyList() { + assertThat(parser.parse("{\"data\":[],\"object\":\"list\"}").isEmpty(), is(true)); + } + + @Test + public void missingArraysYieldEmptyList() { + assertThat(parser.parse("{\"object\":\"list\"}").isEmpty(), is(true)); + } + + @Test + public void unparseableInputYieldsEmptyList() { + assertThat(parser.parse("not json").isEmpty(), is(true)); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/server/NativeServerAttachIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/server/NativeServerAttachIntegrationTest.java new file mode 100644 index 00000000..eb2a891c --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/server/NativeServerAttachIntegrationTest.java @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.server; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.net.ServerSocket; +import net.ladenthin.llama.ClaudeGenerated; +import net.ladenthin.llama.LlamaModel; +import net.ladenthin.llama.TestConstants; +import net.ladenthin.llama.parameters.InferenceParameters; +import net.ladenthin.llama.parameters.ModelParameters; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Integration test for {@link NativeServer}'s attach mode + * ({@link NativeServer#NativeServer(net.ladenthin.llama.LlamaModel, String...)}): the full + * upstream HTTP frontend serving an already-loaded {@link LlamaModel} — one copy of the weights, + * shared between direct JNI calls and HTTP requests. + */ +@ClaudeGenerated( + purpose = "Prove NativeServer attach mode end to end: the upstream HTTP frontend serves " + + "an already-loaded LlamaModel (health/props/completion/chat over HTTP) while " + + "direct JNI calls on the same model keep working — one copy of the weights.") +public class NativeServerAttachIntegrationTest extends OpenAiServerTestSupport { + + private static LlamaModel model; + private static NativeServer server; + private static int port; + + @BeforeAll + public static void setup() throws Exception { + Assumptions.assumeTrue( + new File(TestConstants.REASONING_MODEL_PATH).exists(), + "Reasoning model not found, skipping NativeServerAttachIntegrationTest"); + int gpuLayers = Integer.getInteger(TestConstants.PROP_TEST_NGL, TestConstants.DEFAULT_TEST_NGL); + model = new LlamaModel(new ModelParameters() + .setModel(TestConstants.REASONING_MODEL_PATH) + .setCtxSize(1024) + .setGpuLayers(gpuLayers) + .setFit(false)); + port = findFreePort(); + server = new NativeServer(model, "--host", "127.0.0.1", "--port", Integer.toString(port)).start(); + awaitHealthy(); + } + + @AfterAll + public static void tearDown() { + // Server before model: the attached routes point into the model's native server context. + if (server != null) { + server.close(); + } + if (model != null) { + model.close(); + } + } + + private static int findFreePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + private static void awaitHealthy() throws Exception { + long deadline = System.currentTimeMillis() + 30_000L; + IOException last = null; + while (System.currentTimeMillis() < deadline) { + try { + Response health = new NativeServerAttachIntegrationTest().get(port, "/health", ""); + if (health.code == 200) { + return; + } + } catch (IOException e) { + last = e; + } + Thread.sleep(200L); + } + fail("attached server did not become healthy within 30s" + (last != null ? ": " + last : "")); + } + + @Test + public void health_isOk() throws IOException { + assertThat(get(port, "/health", "").code, is(200)); + } + + /** The attached frontend reads model metadata straight from the shared, loaded context. */ + @Test + public void props_reportModelContext() throws IOException { + Response props = get(port, "/props", ""); + assertThat(props.code, is(200)); + assertThat(props.body, containsString("default_generation_settings")); + } + + @Test + public void completion_overHttp_served() throws IOException { + Response response = post(port, "/completion", "{\"prompt\":\"Hello\",\"n_predict\":4,\"temperature\":0}", ""); + assertThat(response.body, response.code, is(200)); + assertThat(response.body, containsString("\"content\"")); + } + + @Test + public void chatCompletion_overHttp_served() throws IOException { + Response response = post( + port, + "/v1/chat/completions", + "{\"messages\":[{\"role\":\"user\",\"content\":\"Say one word.\"}],\"max_tokens\":8}", + ""); + assertThat(response.body, response.code, is(200)); + assertThat(response.body, containsString("\"choices\"")); + } + + /** The model stays fully usable for direct JNI calls while the HTTP frontend is attached. */ + @Test + public void directJniCalls_stillWork_whileAttached() { + String completion = + model.complete(new InferenceParameters("2+2=").withNPredict(4).withTemperature(0.0f)); + assertNotNull(completion); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/server/NativeServerSmokeTest.java b/llama/src/test/java/net/ladenthin/llama/server/NativeServerSmokeTest.java index 389136f9..892da39e 100644 --- a/llama/src/test/java/net/ladenthin/llama/server/NativeServerSmokeTest.java +++ b/llama/src/test/java/net/ladenthin/llama/server/NativeServerSmokeTest.java @@ -62,4 +62,11 @@ public void nullArgsRejected() { public void nullArgElementRejected() { assertThrows(NullPointerException.class, () -> new NativeServer("-m", null)); } + + @Test + public void attachMode_nullModelRejected() { + assertThrows( + NullPointerException.class, + () -> new NativeServer((net.ladenthin.llama.LlamaModel) null, "--port", "8080")); + } } diff --git a/llama/src/test/java/net/ladenthin/llama/server/NativeServerWorkerCommandTest.java b/llama/src/test/java/net/ladenthin/llama/server/NativeServerWorkerCommandTest.java new file mode 100644 index 00000000..113000d8 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/server/NativeServerWorkerCommandTest.java @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.server; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import net.ladenthin.llama.ClaudeGenerated; +import org.junit.jupiter.api.Test; + +/** + * Model-free unit tests for {@link NativeServer#setWorkerCommand(String...)} argument validation. + * Only the rejection paths are tested here — they fire before the native library is + * loaded, so this class runs on a pure-Java checkout. The accepted path (env round trip into the + * router's worker spawn) is exercised by {@code RouterModeIntegrationTest}. + */ +@ClaudeGenerated( + purpose = "Pin setWorkerCommand's fail-fast validation: tokens with whitespace, empty " + + "tokens, null tokens and a null array must be rejected before any native call, " + + "because the env variable is whitespace-split natively.") +public class NativeServerWorkerCommandTest { + + @Test + public void tokenWithSpaceRejected() { + assertThrows(IllegalArgumentException.class, () -> NativeServer.setWorkerCommand("java", "-cp", "a b.jar")); + } + + @Test + public void tokenWithTabRejected() { + assertThrows(IllegalArgumentException.class, () -> NativeServer.setWorkerCommand("java\t-cp")); + } + + @Test + public void emptyTokenRejected() { + assertThrows(IllegalArgumentException.class, () -> NativeServer.setWorkerCommand("java", "")); + } + + @Test + public void nullTokenRejected() { + assertThrows(IllegalArgumentException.class, () -> NativeServer.setWorkerCommand("java", (String) null)); + } + + @Test + public void nullArrayRejected() { + assertThrows(NullPointerException.class, () -> NativeServer.setWorkerCommand((String[]) null)); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/server/RouterClientTest.java b/llama/src/test/java/net/ladenthin/llama/server/RouterClientTest.java new file mode 100644 index 00000000..aaa5a504 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/server/RouterClientTest.java @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.server; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import net.ladenthin.llama.ClaudeGenerated; +import net.ladenthin.llama.value.RouterModel; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +@ClaudeGenerated( + purpose = "Model-free verification of RouterClient against a stub HTTP server speaking " + + "the upstream router wire format: list/find parsing, load/unload request " + + "bodies, error surfacing with the router's error body, and the " + + "awaitModelLoaded state machine (poll-until-loaded, fail-fast on failed " + + "worker or unknown model, timeout).") +public class RouterClientTest { + + private HttpServer server; + private RouterClient client; + + /** Body served for GET /models; test cases swap it (or a supplier-driven sequence). */ + private final AtomicReference modelsBody = new AtomicReference<>("{\"data\":[],\"object\":\"list\"}"); + + /** Counts GET /models calls so await tests can serve a status sequence. */ + private final AtomicInteger modelsCalls = new AtomicInteger(); + + private final AtomicReference lastLoadBody = new AtomicReference<>(""); + private final AtomicReference lastUnloadBody = new AtomicReference<>(""); + + @BeforeEach + public void startStub() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/models", exchange -> { + String path = exchange.getRequestURI().getPath(); + if ("/models/load".equals(path)) { + lastLoadBody.set(readBody(exchange)); + respond(exchange, 200, "{\"success\":true}"); + } else if ("/models/unload".equals(path)) { + lastUnloadBody.set(readBody(exchange)); + respond(exchange, 200, "{\"success\":true}"); + } else { + modelsCalls.incrementAndGet(); + respond(exchange, 200, modelsBody.get()); + } + }); + server.start(); + client = new RouterClient("127.0.0.1", server.getAddress().getPort()); + } + + @AfterEach + public void stopStub() { + if (server != null) { + server.stop(0); + } + } + + private static String readBody(HttpExchange exchange) throws IOException { + try (InputStream in = exchange.getRequestBody()) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] chunk = new byte[4096]; + int read; + while ((read = in.read(chunk)) != -1) { + buffer.write(chunk, 0, read); + } + return new String(buffer.toByteArray(), StandardCharsets.UTF_8); + } + } + + private static void respond(HttpExchange exchange, int code, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(code, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + exchange.close(); + } + + private static String entry(String id, String status) { + return "{\"id\":\"" + id + "\",\"status\":{\"value\":\"" + status + "\"}}"; + } + + @Test + public void listModels_parsesRouterResponse() throws IOException { + modelsBody.set( + "{\"object\":\"list\",\"data\":[" + entry("qwen", "loaded") + "," + entry("llama", "unloaded") + "]}"); + + List models = client.listModels(); + + assertThat(models.size(), is(2)); + assertThat(models.get(0).getId(), is("qwen")); + assertThat(models.get(0).getStatus(), is(RouterModel.Status.LOADED)); + assertThat(models.get(1).getStatus(), is(RouterModel.Status.UNLOADED)); + } + + @Test + public void findModel_matchesById() throws IOException { + modelsBody.set("{\"data\":[" + entry("qwen", "loading") + "]}"); + + assertThat(client.findModel("qwen").isPresent(), is(true)); + assertThat(client.findModel("qwen").get().getStatus(), is(RouterModel.Status.LOADING)); + assertThat(client.findModel("absent").isPresent(), is(false)); + } + + @Test + public void loadModel_postsModelFieldToLoadEndpoint() throws IOException { + client.loadModel("qwen"); + + assertThat(lastLoadBody.get(), is("{\"model\":\"qwen\"}")); + } + + @Test + public void unloadModel_postsModelFieldToUnloadEndpoint() throws IOException { + client.unloadModel("qwen"); + + assertThat(lastUnloadBody.get(), is("{\"model\":\"qwen\"}")); + } + + @Test + public void non2xxResponseSurfacesRouterErrorBody() throws IOException { + // Dedicated stub that always rejects, mirroring upstream's error JSON shape. + HttpServer errorServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + errorServer.createContext( + "/models", + exchange -> respond(exchange, 400, "{\"error\":{\"message\":\"model is already running\"}}")); + errorServer.start(); + try { + RouterClient errorClient = + new RouterClient("127.0.0.1", errorServer.getAddress().getPort()); + + IOException thrown = assertThrows(IOException.class, () -> errorClient.loadModel("qwen")); + + assertThat(thrown.getMessage(), containsString("HTTP 400")); + assertThat(thrown.getMessage(), containsString("model is already running")); + } finally { + errorServer.stop(0); + } + } + + @Test + public void awaitModelLoaded_pollsUntilLoaded() throws Exception { + modelsCalls.set(0); + // Serve "loading" for the first two polls, then "loaded" — via a body that depends on + // the call counter (swapped inside the handler through modelsBody on each poll). + modelsBody.set("{\"data\":[" + entry("qwen", "loading") + "]}"); + Thread flipper = new Thread(() -> { + try { + while (modelsCalls.get() < 2) { + Thread.sleep(20L); + } + modelsBody.set("{\"data\":[" + entry("qwen", "loaded") + "]}"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + flipper.start(); + + RouterModel loaded = client.awaitModelLoaded("qwen", 30_000L); + + flipper.join(5_000L); + assertThat(loaded.getStatus(), is(RouterModel.Status.LOADED)); + } + + @Test + public void awaitModelLoaded_failsFastOnFailedWorker() { + modelsBody.set("{\"data\":[{\"id\":\"qwen\"," + + "\"status\":{\"value\":\"unloaded\",\"failed\":true,\"exit_code\":137}}]}"); + + IllegalStateException thrown = + assertThrows(IllegalStateException.class, () -> client.awaitModelLoaded("qwen", 30_000L)); + + assertThat(thrown.getMessage(), containsString("exit code 137")); + } + + @Test + public void awaitModelLoaded_failsFastOnUnknownModel() { + modelsBody.set("{\"data\":[" + entry("other", "loaded") + "]}"); + + IllegalStateException thrown = + assertThrows(IllegalStateException.class, () -> client.awaitModelLoaded("qwen", 30_000L)); + + assertThat(thrown.getMessage(), containsString("does not list model 'qwen'")); + } + + @Test + public void awaitModelLoaded_timesOutWithLastStatusInMessage() { + modelsBody.set("{\"data\":[" + entry("qwen", "loading") + "]}"); + + IllegalStateException thrown = + assertThrows(IllegalStateException.class, () -> client.awaitModelLoaded("qwen", 600L)); + + assertThat(thrown.getMessage(), containsString("did not reach LOADED")); + assertThat(thrown.getMessage(), containsString("loading")); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/server/RouterModeIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/server/RouterModeIntegrationTest.java new file mode 100644 index 00000000..fc8b2a85 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/server/RouterModeIntegrationTest.java @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.server; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Locale; +import net.ladenthin.llama.ClaudeGenerated; +import net.ladenthin.llama.TestConstants; +import net.ladenthin.llama.value.RouterModel; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Integration test for upstream router mode running inside the JVM via + * {@link NativeServer}: started without a model argument, the server manages models from + * {@code --models-dir} and serves each from a worker subprocess. In-JVM the upstream worker-spawn + * (re-exec of the process binary) would relaunch {@code java} with llama-server arguments, so + * {@link NativeServer#setWorkerCommand(String...)} redirects workers to a fresh JVM running the + * classic single-model {@code NativeServer} (patch {@code 0008}). + * + *

Linux-only: worker relaunch and the symlinked models dir are exercised on one platform; + * router mode itself is upstream functionality, this test pins the embedded wiring.

+ */ +@ClaudeGenerated( + purpose = "Prove in-JVM router mode end to end: model listing from --models-dir, " + + "explicit /models/load spawning a worker JVM through setWorkerCommand, and a " + + "chat completion proxied to that worker.") +public class RouterModeIntegrationTest extends OpenAiServerTestSupport { + + /** Generous ceiling for worker-JVM spawn + model load on a cold CI runner. */ + private static final long MODEL_READY_TIMEOUT_MILLIS = 240_000L; + + @TempDir + static Path modelsDir; + + private static NativeServer server; + private static int port; + private static String modelName; + + /** + * Set only after {@link NativeServer#setWorkerCommand(String...)} succeeded in setup. + * tearDown must clear the override ONLY in that case: setWorkerCommand loads the native + * library, and when the class self-skipped via an {@code @BeforeAll} assumption (no model / + * non-Linux / lib-less analysis runner such as the Sonar job), {@code @AfterAll} still runs — + * an unconditional call would then die with UnsatisfiedLinkError instead of skipping cleanly. + */ + private static boolean workerCommandSet; + + @BeforeAll + public static void setup() throws Exception { + Assumptions.assumeTrue( + System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("linux"), + "Router worker-relaunch test runs on Linux only"); + File reasoningModel = new File(TestConstants.REASONING_MODEL_PATH); + Assumptions.assumeTrue(reasoningModel.exists(), "Reasoning model not found, skipping router test"); + String classpath = System.getProperty("java.class.path", ""); + Assumptions.assumeTrue( + !classpath.isEmpty() && !classpath.matches(".*\\s.*"), + "Classpath contains whitespace; worker command cannot carry it"); + + // The models dir must contain ONLY the model under test (the CI models/ dir also holds + // mmproj/vocoder files the router would list); a symlink avoids copying ~400 MB. + Files.createSymbolicLink( + modelsDir.resolve(reasoningModel.getName()), + reasoningModel.getAbsoluteFile().toPath()); + + // Workers must relaunch through this library, not through the JVM binary itself. + String javaBin = + Paths.get(System.getProperty("java.home"), "bin", "java").toString(); + NativeServer.setWorkerCommand(javaBin, "-cp", classpath, "net.ladenthin.llama.server.NativeServer"); + workerCommandSet = true; + + int gpuLayers = Integer.getInteger(TestConstants.PROP_TEST_NGL, TestConstants.DEFAULT_TEST_NGL); + port = findFreePort(); + // The router forwards its own base arguments (context size, GPU layers) to each worker. + server = new NativeServer( + "--host", + "127.0.0.1", + "--port", + Integer.toString(port), + "--models-dir", + modelsDir.toString(), + "--models-max", + "1", + "-c", + "512", + "-ngl", + Integer.toString(gpuLayers)) + .start(); + awaitHttp("/health", 30_000L); + + // Model discovery + load + readiness go through the typed RouterClient, so this + // integration test exercises it end to end against a real router (the wire-shape + // details are unit-tested model-free in RouterClientTest against a stub server). + RouterClient client = new RouterClient(port); + modelName = discoverModelName(client); + client.loadModel(modelName); + RouterModel loaded = client.awaitModelLoaded(modelName, MODEL_READY_TIMEOUT_MILLIS); + assertThat(loaded.getStatus(), is(RouterModel.Status.LOADED)); + } + + @AfterAll + public static void tearDown() { + if (server != null) { + server.close(); // router clean-up unloads (terminates) all worker instances + } + if (workerCommandSet) { + NativeServer.setWorkerCommand(); // clear the process-wide override + } + } + + private static int findFreePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + private static void awaitHttp(String path, long timeoutMillis) throws Exception { + long deadline = System.currentTimeMillis() + timeoutMillis; + IOException last = null; + while (System.currentTimeMillis() < deadline) { + try { + if (new RouterModeIntegrationTest().get(port, path, "").code == 200) { + return; + } + } catch (IOException e) { + last = e; + } + Thread.sleep(200L); + } + fail("router did not answer " + path + " within " + timeoutMillis + "ms" + (last != null ? ": " + last : "")); + } + + /** Finds the entry for the symlinked GGUF in the typed model list and returns its identifier. */ + private static String discoverModelName(RouterClient client) throws Exception { + List models = client.listModels(); + for (RouterModel model : models) { + if (model.getId().contains("Qwen3")) { + return model.getId(); + } + } + fail("GET /models did not list the symlinked model: " + models); + return ""; // unreachable + } + + @Test + public void chatCompletion_isProxiedToWorker() throws IOException { + Response response = post( + port, + "/v1/chat/completions", + "{\"model\":\"" + modelName + "\",\"messages\":[{\"role\":\"user\",\"content\":\"Say one word.\"}]," + + "\"max_tokens\":8}", + ""); + assertThat(response.body, response.code, is(200)); + assertThat(response.body, containsString("\"choices\"")); + } + + @Test + public void models_listContainsLoadedModel() throws IOException { + Response models = get(port, "/models", ""); + assertThat(models.code, is(200)); + assertThat(models.body, containsString(modelName)); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/value/ChatTranscriptTest.java b/llama/src/test/java/net/ladenthin/llama/value/ChatTranscriptTest.java index c064fddb..93adc498 100644 --- a/llama/src/test/java/net/ladenthin/llama/value/ChatTranscriptTest.java +++ b/llama/src/test/java/net/ladenthin/llama/value/ChatTranscriptTest.java @@ -296,4 +296,53 @@ void streamShape() { assertThat(t.snapshot().get(1).getRole(), is("assistant")); } } + + @Nested + @DisplayName("checkpoint snapshot / rewind reset") + class CheckpointShape { + + @Test + @DisplayName("turnsSnapshot() is a detached copy") + void turnsSnapshotIsDetachedCopy() { + ChatTranscript t = new ChatTranscript(null); + t.appendRound("hi", "hello"); + + List> snapshot = t.turnsSnapshot(); + snapshot.clear(); // mutating the copy must not affect the transcript + + assertThat(t.size(), is(2)); + assertThat(t.turnsSnapshot(), hasSize(2)); + } + + @Test + @DisplayName("resetTurns() replaces the committed turns and copies the input") + void resetTurnsReplacesAndCopies() { + ChatTranscript t = new ChatTranscript("sys"); + t.appendRound("a", "b"); + t.appendRound("c", "d"); + + List> checkpoint = new java.util.ArrayList<>( + java.util.Arrays.asList(new Pair<>("user", "a"), new Pair<>("assistant", "b"))); + t.resetTurns(checkpoint); + checkpoint.clear(); // later mutation of the input must not leak in + + assertThat(t.size(), is(2)); + List snap = t.snapshot(); + // system message survives the reset (fixed at construction) + assertThat(snap.get(0).getRole(), is("system")); + assertThat(snap.get(1).getContent(), is("a")); + assertThat(snap.get(2).getContent(), is("b")); + } + + @Test + @DisplayName("resetTurns() to empty rewinds to the very start") + void resetTurnsToEmpty() { + ChatTranscript t = new ChatTranscript(null); + t.appendRound("a", "b"); + + t.resetTurns(java.util.Collections.>emptyList()); + + assertThat(t.size(), is(0)); + } + } } diff --git a/llama/src/test/java/net/ladenthin/llama/value/GgufMetadataTest.java b/llama/src/test/java/net/ladenthin/llama/value/GgufMetadataTest.java new file mode 100644 index 00000000..0d29f72c --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/value/GgufMetadataTest.java @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.value; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import net.ladenthin.llama.ClaudeGenerated; +import org.junit.jupiter.api.Test; + +@ClaudeGenerated( + purpose = "Verify the GgufMetadata value type: raw/typed lookups (present, absent, " + + "wrong-type), every convenience accessor incl. the architecture-dependent " + + "context-length lookup, immutability of the entry view, the Lombok " + + "equals/hashCode contract, and the compact handwritten toString.") +public class GgufMetadataTest { + + private static GgufMetadata sample() { + Map entries = new LinkedHashMap<>(); + entries.put(GgufMetadata.KEY_ARCHITECTURE, "llama"); + entries.put(GgufMetadata.KEY_NAME, "Test Model"); + entries.put(GgufMetadata.KEY_PARAMETER_COUNT, 751_632_384L); + entries.put(GgufMetadata.KEY_FILE_TYPE, 15L); + entries.put("llama.context_length", 40_960L); + entries.put(GgufMetadata.KEY_CHAT_TEMPLATE, "{{ messages }}"); + entries.put("general.some_flag", Boolean.TRUE); + return new GgufMetadata(3, 291L, entries); + } + + @Test + public void headerGettersRoundTrip() { + GgufMetadata meta = sample(); + assertThat(meta.getVersion(), is(3)); + assertThat(meta.getTensorCount(), is(291L)); + assertThat(meta.getEntries().size(), is(7)); + } + + @Test + public void rawAndTypedLookups() { + GgufMetadata meta = sample(); + assertThat(meta.getValue("general.some_flag").orElse(null), is((Object) Boolean.TRUE)); + assertThat(meta.getValue("absent").isPresent(), is(false)); + assertThat(meta.getString(GgufMetadata.KEY_NAME).orElse(""), is("Test Model")); + // Wrong-type lookups answer empty rather than throwing. + assertThat(meta.getString(GgufMetadata.KEY_FILE_TYPE).isPresent(), is(false)); + assertThat(meta.getLong(GgufMetadata.KEY_FILE_TYPE).orElse(0), is(15L)); + assertThat(meta.getLong(GgufMetadata.KEY_NAME).isPresent(), is(false)); + assertThat(meta.getLong("absent").isPresent(), is(false)); + } + + @Test + public void convenienceAccessorsPresent() { + GgufMetadata meta = sample(); + assertThat(meta.getArchitecture().orElse(""), is("llama")); + assertThat(meta.getModelName().orElse(""), is("Test Model")); + assertThat(meta.getParameterCount().orElse(0), is(751_632_384L)); + assertThat(meta.getFileType().orElse(0), is(15L)); + assertThat(meta.getChatTemplate().orElse(""), is("{{ messages }}")); + assertThat(meta.getContextLength().orElse(0), is(40_960L)); + } + + @Test + public void convenienceAccessorsAbsent() { + GgufMetadata empty = new GgufMetadata(2, 0L, Collections.emptyMap()); + assertThat(empty.getArchitecture().isPresent(), is(false)); + assertThat(empty.getModelName().isPresent(), is(false)); + assertThat(empty.getParameterCount().isPresent(), is(false)); + assertThat(empty.getFileType().isPresent(), is(false)); + assertThat(empty.getChatTemplate().isPresent(), is(false)); + // No architecture -> the per-architecture context-length key cannot be derived. + assertThat(empty.getContextLength().isPresent(), is(false)); + } + + @Test + public void contextLengthRequiresBothArchitectureAndLengthKey() { + GgufMetadata archOnly = new GgufMetadata( + 3, 1L, Collections.singletonMap(GgufMetadata.KEY_ARCHITECTURE, "llama")); + assertThat(archOnly.getContextLength().isPresent(), is(false)); + } + + @Test + public void entriesViewIsUnmodifiableAndDetachedFromInput() { + Map input = new LinkedHashMap<>(); + input.put("k", 1L); + GgufMetadata meta = new GgufMetadata(3, 1L, input); + input.put("later", 2L); // mutation after construction must not leak in + + assertThat(meta.getEntries().size(), is(1)); + assertThrows( + UnsupportedOperationException.class, () -> meta.getEntries().put("x", 1L)); + } + + @Test + public void equalsAndHashCode_sameValues() { + assertEquals(sample(), sample()); + assertEquals(sample().hashCode(), sample().hashCode()); + } + + @Test + public void equals_differsPerField() { + GgufMetadata base = sample(); + Map entries = new LinkedHashMap<>(base.getEntries()); + assertNotEquals(base, new GgufMetadata(2, 291L, entries)); + assertNotEquals(base, new GgufMetadata(3, 290L, entries)); + Map fewer = new LinkedHashMap<>(entries); + fewer.remove(GgufMetadata.KEY_NAME); + assertNotEquals(base, new GgufMetadata(3, 291L, fewer)); + } + + @Test + public void toString_isCompactSummary() { + assertThat(sample().toString(), is("GGUF v3 (291 tensors, 7 keys, arch=llama)")); + GgufMetadata empty = new GgufMetadata(2, 0L, Collections.emptyMap()); + assertThat(empty.toString(), is("GGUF v2 (0 tensors, 0 keys, arch=?)")); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/value/LoraAdapterTest.java b/llama/src/test/java/net/ladenthin/llama/value/LoraAdapterTest.java new file mode 100644 index 00000000..503b8936 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/value/LoraAdapterTest.java @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.value; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import net.ladenthin.llama.ClaudeGenerated; +import org.junit.jupiter.api.Test; + +@ClaudeGenerated( + purpose = "Verify the LoraAdapter value type: constructor/getter round-trip for every " + + "field, and the Lombok-generated equals/hashCode/toString contracts used when " + + "adapters are compared or logged.") +public class LoraAdapterTest { + + private static LoraAdapter sample() { + return new LoraAdapter(1, "adapter.gguf", 0.5f, "classification", "prefix: "); + } + + @Test + public void gettersRoundTrip() { + LoraAdapter adapter = sample(); + assertThat(adapter.getId(), is(1)); + assertThat(adapter.getPath(), is("adapter.gguf")); + assertEquals(0.5f, adapter.getScale(), 0.0f); + assertThat(adapter.getTaskName(), is("classification")); + assertThat(adapter.getPromptPrefix(), is("prefix: ")); + } + + @Test + public void equalsAndHashCode_sameValues() { + assertEquals(sample(), sample()); + assertEquals(sample().hashCode(), sample().hashCode()); + } + + @Test + public void equals_differsPerField() { + LoraAdapter base = sample(); + assertNotEquals(base, new LoraAdapter(2, "adapter.gguf", 0.5f, "classification", "prefix: ")); + assertNotEquals(base, new LoraAdapter(1, "other.gguf", 0.5f, "classification", "prefix: ")); + assertNotEquals(base, new LoraAdapter(1, "adapter.gguf", 1.0f, "classification", "prefix: ")); + assertNotEquals(base, new LoraAdapter(1, "adapter.gguf", 0.5f, "other", "prefix: ")); + assertNotEquals(base, new LoraAdapter(1, "adapter.gguf", 0.5f, "classification", "other")); + } + + @Test + public void toStringContainsFields() { + String rendered = sample().toString(); + assertThat(rendered.contains("adapter.gguf"), is(true)); + assertThat(rendered.contains("classification"), is(true)); + assertThat(rendered, is(not(""))); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/value/RouterModelTest.java b/llama/src/test/java/net/ladenthin/llama/value/RouterModelTest.java new file mode 100644 index 00000000..73ab8a52 --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/value/RouterModelTest.java @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.value; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import net.ladenthin.llama.ClaudeGenerated; +import org.junit.jupiter.api.Test; + +@ClaudeGenerated( + purpose = "Verify the RouterModel value type: constructor/getter round-trip, the " + + "Status.fromValue mapping for every upstream server_model_status string " + + "(plus null/unknown tolerance), the Lombok equals/hashCode contract, and " + + "the handwritten toString shapes used in router log traces.") +public class RouterModelTest { + + private static RouterModel sample() { + return new RouterModel("Qwen3-0.6B-Q4_K_M", RouterModel.Status.LOADED, "loaded", false, 0); + } + + @Test + public void gettersRoundTrip() { + RouterModel model = sample(); + assertThat(model.getId(), is("Qwen3-0.6B-Q4_K_M")); + assertThat(model.getStatus(), is(RouterModel.Status.LOADED)); + assertThat(model.getStatusValue(), is("loaded")); + assertThat(model.isFailed(), is(false)); + assertThat(model.getExitCode(), is(0)); + } + + @Test + public void statusFromValue_mapsEveryUpstreamString() { + // The exact strings emitted by upstream server_model_status_to_string (server-models.h). + assertThat(RouterModel.Status.fromValue("downloading"), is(RouterModel.Status.DOWNLOADING)); + assertThat(RouterModel.Status.fromValue("downloaded"), is(RouterModel.Status.DOWNLOADED)); + assertThat(RouterModel.Status.fromValue("unloaded"), is(RouterModel.Status.UNLOADED)); + assertThat(RouterModel.Status.fromValue("loading"), is(RouterModel.Status.LOADING)); + assertThat(RouterModel.Status.fromValue("loaded"), is(RouterModel.Status.LOADED)); + assertThat(RouterModel.Status.fromValue("sleeping"), is(RouterModel.Status.SLEEPING)); + } + + @Test + public void statusFromValue_matchesExactlyNotCaseFolded() { + // Upstream emits exactly lowercase strings; matching is deliberately exact + // (no Unicode case transformation — findsecbugs IMPROPER_UNICODE). + assertThat(RouterModel.Status.fromValue("LOADED"), is(RouterModel.Status.UNKNOWN)); + } + + @Test + public void statusFromValue_toleratesNullAndUnrecognized() { + assertThat(RouterModel.Status.fromValue(null), is(RouterModel.Status.UNKNOWN)); + assertThat(RouterModel.Status.fromValue(""), is(RouterModel.Status.UNKNOWN)); + assertThat(RouterModel.Status.fromValue("hibernating"), is(RouterModel.Status.UNKNOWN)); + } + + @Test + public void equalsAndHashCode_sameValues() { + assertEquals(sample(), sample()); + assertEquals(sample().hashCode(), sample().hashCode()); + } + + @Test + public void equals_differsPerField() { + RouterModel base = sample(); + assertNotEquals(base, new RouterModel("other", RouterModel.Status.LOADED, "loaded", false, 0)); + assertNotEquals(base, new RouterModel("Qwen3-0.6B-Q4_K_M", RouterModel.Status.LOADING, "loaded", false, 0)); + assertNotEquals(base, new RouterModel("Qwen3-0.6B-Q4_K_M", RouterModel.Status.LOADED, "loading", false, 0)); + assertNotEquals(base, new RouterModel("Qwen3-0.6B-Q4_K_M", RouterModel.Status.LOADED, "loaded", true, 0)); + assertNotEquals(base, new RouterModel("Qwen3-0.6B-Q4_K_M", RouterModel.Status.LOADED, "loaded", false, 1)); + } + + @Test + public void toString_healthyShape() { + assertThat(sample().toString(), is("Qwen3-0.6B-Q4_K_M [loaded]")); + } + + @Test + public void toString_failedShapeIncludesExitCode() { + RouterModel failed = new RouterModel("broken", RouterModel.Status.UNLOADED, "unloaded", true, 137); + assertThat(failed.toString(), is("broken [unloaded, failed exit=137]")); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/value/SessionCheckpointTest.java b/llama/src/test/java/net/ladenthin/llama/value/SessionCheckpointTest.java new file mode 100644 index 00000000..1aba74fc --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/value/SessionCheckpointTest.java @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.value; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import net.ladenthin.llama.ClaudeGenerated; +import org.junit.jupiter.api.Test; + +@ClaudeGenerated( + purpose = "Verify the SessionCheckpoint value type: getter round-trip, defensive copy " + + "and unmodifiability of the turn snapshot, the Lombok equals/hashCode " + + "contract, and the compact handwritten toString.") +public class SessionCheckpointTest { + + private static List> turns() { + return Arrays.asList(new Pair<>("user", "hi"), new Pair<>("assistant", "hello")); + } + + private static SessionCheckpoint sample() { + return new SessionCheckpoint("/tmp/cp.bin", turns()); + } + + @Test + public void gettersRoundTrip() { + SessionCheckpoint checkpoint = sample(); + assertThat(checkpoint.getFilepath(), is("/tmp/cp.bin")); + assertThat(checkpoint.getTurns(), is(turns())); + } + + @Test + public void turnsAreCopiedAndUnmodifiable() { + List> input = new ArrayList<>(turns()); + SessionCheckpoint checkpoint = new SessionCheckpoint("/tmp/cp.bin", input); + input.clear(); // later mutation of the input must not leak in + + assertThat(checkpoint.getTurns().size(), is(2)); + assertThrows( + UnsupportedOperationException.class, () -> checkpoint.getTurns().add(new Pair<>("user", "x"))); + } + + @Test + public void equalsAndHashCode_sameValues() { + assertEquals(sample(), sample()); + assertEquals(sample().hashCode(), sample().hashCode()); + } + + @Test + public void equals_differsPerField() { + SessionCheckpoint base = sample(); + assertNotEquals(base, new SessionCheckpoint("/tmp/other.bin", turns())); + assertNotEquals(base, new SessionCheckpoint("/tmp/cp.bin", Collections.>emptyList())); + } + + @Test + public void toString_isCompactSummary() { + assertThat(sample().toString(), is("/tmp/cp.bin (2 turns)")); + } +} diff --git a/llama/src/test/resources/audios/README.md b/llama/src/test/resources/audios/README.md new file mode 100644 index 00000000..7bf098c4 --- /dev/null +++ b/llama/src/test/resources/audios/README.md @@ -0,0 +1,29 @@ +# Test resource audio clips + +## `sample.wav` + +A ~2.6 s stereo 16-bit 48 kHz WAV (~488 KB) used by +`AudioInputIntegrationTest` as the committed default audio prompt, so the +audio-input path can be exercised end-to-end without staging a clip +out-of-band (the audio *model* + mmproj still have to be supplied via +system properties — see below). + +### Provenance + +Recorded and provided by Bernard Ladenthin (project copyright holder) +specifically for use as a test fixture in this project. + +### License + +The author grants this file for use in this project under the project's +MIT license. The `SPDX-FileCopyrightText: 2026 Bernard Ladenthin +` annotation in `REUSE.toml` records that +grant (WAV carries no comment/metadata channel for an in-file header, so +the annotation lives in `REUSE.toml`, same as `images/test-image.jpg`). + +### Override + +To point the test at a different clip without overwriting this one, set +the `net.ladenthin.llama.audio.input` system property on the `mvn test` +command line. `.wav` and `.mp3` work; the file extension drives format +detection in `ContentPart.audioFile(Path)`. diff --git a/llama/src/test/resources/audios/sample.wav b/llama/src/test/resources/audios/sample.wav new file mode 100644 index 00000000..f3f41d17 Binary files /dev/null and b/llama/src/test/resources/audios/sample.wav differ diff --git a/pom.xml b/pom.xml index 032c9789..2e1eb308 100644 --- a/pom.xml +++ b/pom.xml @@ -58,6 +58,7 @@ SPDX-License-Identifier: MIT llama llama-langchain4j + llama-kotlin