From afd3248b8270bf4aa66f5f3fb0e2c9cbc64337a1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 9 Sep 2026 13:19:14 +0000 Subject: [PATCH 01/22] server: preemption notices, asynchronous parks and exact concurrency, on b10869 The tree of unslothai/llama.cpp#197 at cb30d969c, its branch merged with the upstream release tag b10869, as one commit on the tag so the nightly's pin merge has a single merge base. --- .github/actions/prebuilt-alert/action.yml | 119 + .github/workflows/bench.yml.disabled | 6 +- .github/workflows/unsloth-pin-preflight.yml | 276 +++ .github/workflows/unsloth-pr-set-lint.yml | 164 ++ .github/workflows/unsloth-prebuilt-cpu.yml | 376 +++ .../unsloth-prebuilt-cuda-windows.yml | 310 +++ .github/workflows/unsloth-prebuilt-cuda.yml | 290 +++ .../workflows/unsloth-prebuilt-deadman.yml | 81 + .github/workflows/unsloth-prebuilt-macos.yml | 211 ++ .github/workflows/unsloth-prebuilt-retry.yml | 119 + .github/workflows/unsloth-prebuilt-rocm.yml | 878 +++++++ .github/workflows/unsloth-prebuilt-vulkan.yml | 409 ++++ .github/workflows/unsloth-prebuilt.yml | 1317 ++++++++++ .github/workflows/unsloth-repin-bot.yml | 209 ++ .../workflows/unsloth-upstream-sync-guard.yml | 79 + common/arg.cpp | 24 + common/common.cpp | 160 ++ common/common.h | 19 + common/speculative.cpp | 11 +- ggml/include/ggml-backend.h | 6 + ggml/include/ggml-cuda.h | 3 + ggml/src/ggml-backend-impl.h | 5 +- ggml/src/ggml-backend-meta.cpp | 1 + ggml/src/ggml-backend.cpp | 21 + ggml/src/ggml-blas/ggml-blas.cpp | 1 + ggml/src/ggml-cann/ggml-cann.cpp | 5 + ggml/src/ggml-cpu/ggml-cpu.cpp | 2 + ggml/src/ggml-cuda/common.cuh | 5 + ggml/src/ggml-cuda/fattn-common.cuh | 13 +- ggml/src/ggml-cuda/fattn-vec.cuh | 17 +- ggml/src/ggml-cuda/fattn.cu | 51 + ggml/src/ggml-cuda/ggml-cuda.cu | 368 ++- ggml/src/ggml-cuda/mmvq.cu | 36 +- ggml/src/ggml-cuda/mmvq.cuh | 3 + ggml/src/ggml-cuda/vendors/hip.h | 2 + ggml/src/ggml-cuda/vendors/musa.h | 2 + ggml/src/ggml-et/ggml-et.cpp | 6 + ggml/src/ggml-hexagon/ggml-hexagon.cpp | 4 +- ggml/src/ggml-metal/ggml-metal-device.m | 4 + ggml/src/ggml-metal/ggml-metal.cpp | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 5 + ggml/src/ggml-openvino/ggml-openvino.cpp | 5 + ggml/src/ggml-rpc/ggml-rpc.cpp | 6 +- ggml/src/ggml-sycl/ggml-sycl.cpp | 4 +- ggml/src/ggml-virtgpu/ggml-backend-device.cpp | 1 + ggml/src/ggml-vulkan/ggml-vulkan.cpp | 5 + ggml/src/ggml-webgpu/ggml-webgpu.cpp | 7 + ggml/src/ggml-zdnn/ggml-zdnn.cpp | 1 + ggml/src/ggml-zendnn/ggml-zendnn.cpp | 1 + include/llama.h | 61 + scripts/unsloth/additive_merge.py | 265 ++ scripts/unsloth/assemble_metadata.py | 533 ++++ scripts/unsloth/assert_macho_minos.sh | 55 + scripts/unsloth/carry_vintage.py | 160 ++ scripts/unsloth/check_workflow_scalars.py | 97 + scripts/unsloth/feature-checks.json | 104 + scripts/unsloth/feature_matrix.py | 200 ++ scripts/unsloth/merge_checks.py | 314 +++ scripts/unsloth/package_bundle.py | 381 +++ scripts/unsloth/pin_contract.py | 361 +++ scripts/unsloth/pin_merge.py | 220 ++ scripts/unsloth/pr-set.json | 35 + scripts/unsloth/repin.py | 272 +++ scripts/unsloth/sync_deletes.py | 142 ++ scripts/unsloth/test_additive_merge.py | 206 ++ scripts/unsloth/test_carry_vintage.py | 378 +++ scripts/unsloth/test_feature_matrix.py | 153 ++ scripts/unsloth/test_merge_checks.py | 370 +++ scripts/unsloth/test_pin_contract.py | 187 ++ scripts/unsloth/test_pin_merge.py | 274 +++ scripts/unsloth/test_sync_deletes.py | 150 ++ scripts/unsloth/test_upload_release_assets.sh | 111 + scripts/unsloth/test_verify_upstream_sync.py | 158 ++ scripts/unsloth/upload_release_assets.sh | 165 ++ scripts/unsloth/upstream-sync.json | 26 + scripts/unsloth/verify_upstream_sync.py | 345 +++ src/llama-adapter.cpp | 15 + src/llama-batch.cpp | 84 +- src/llama-batch.h | 11 +- src/llama-context.cpp | 878 ++++++- src/llama-context.h | 22 + src/llama-graph.cpp | 30 +- src/llama-graph.h | 4 +- src/llama-impl.cpp | 163 ++ src/llama-impl.h | 15 + src/llama-kv-cache.cpp | 399 ++- src/llama-kv-cache.h | 39 + src/llama-memory-hybrid.cpp | 36 +- src/llama-memory-hybrid.h | 2 + src/llama-memory-recurrent.cpp | 5 +- src/llama-memory.h | 3 + tests/CMakeLists.txt | 20 + tests/test-backend-ops.cpp | 52 + tests/test-exact-buft.cpp | 60 + tests/test-exact-geometry.cpp | 55 + tests/test-exact-pages.cpp | 152 ++ tests/test-server-tokens.cpp | 184 ++ tests/test-state-restore-fragmented.cpp | 14 + tests/test-state-seq-copy.cpp | 140 ++ tools/server/README.md | 21 + tools/server/server-common.cpp | 6 + tools/server/server-common.h | 9 + tools/server/server-context.cpp | 2141 +++++++++++++++-- tools/server/server-queue.cpp | 5 +- tools/server/server-task.cpp | 49 +- tools/server/server-task.h | 20 + tools/server/tests/unit/test_preempt.py | 887 +++++++ .../server/tests/unit/test_preempt_notify.py | 267 ++ 108 files changed, 16949 insertions(+), 211 deletions(-) create mode 100644 .github/actions/prebuilt-alert/action.yml create mode 100644 .github/workflows/unsloth-pin-preflight.yml create mode 100644 .github/workflows/unsloth-pr-set-lint.yml create mode 100644 .github/workflows/unsloth-prebuilt-cpu.yml create mode 100644 .github/workflows/unsloth-prebuilt-cuda-windows.yml create mode 100644 .github/workflows/unsloth-prebuilt-cuda.yml create mode 100644 .github/workflows/unsloth-prebuilt-deadman.yml create mode 100644 .github/workflows/unsloth-prebuilt-macos.yml create mode 100644 .github/workflows/unsloth-prebuilt-retry.yml create mode 100644 .github/workflows/unsloth-prebuilt-rocm.yml create mode 100644 .github/workflows/unsloth-prebuilt-vulkan.yml create mode 100644 .github/workflows/unsloth-prebuilt.yml create mode 100644 .github/workflows/unsloth-repin-bot.yml create mode 100644 .github/workflows/unsloth-upstream-sync-guard.yml create mode 100644 scripts/unsloth/additive_merge.py create mode 100644 scripts/unsloth/assemble_metadata.py create mode 100755 scripts/unsloth/assert_macho_minos.sh create mode 100755 scripts/unsloth/carry_vintage.py create mode 100644 scripts/unsloth/check_workflow_scalars.py create mode 100644 scripts/unsloth/feature-checks.json create mode 100644 scripts/unsloth/feature_matrix.py create mode 100755 scripts/unsloth/merge_checks.py create mode 100644 scripts/unsloth/package_bundle.py create mode 100644 scripts/unsloth/pin_contract.py create mode 100755 scripts/unsloth/pin_merge.py create mode 100644 scripts/unsloth/pr-set.json create mode 100644 scripts/unsloth/repin.py create mode 100755 scripts/unsloth/sync_deletes.py create mode 100644 scripts/unsloth/test_additive_merge.py create mode 100644 scripts/unsloth/test_carry_vintage.py create mode 100644 scripts/unsloth/test_feature_matrix.py create mode 100644 scripts/unsloth/test_merge_checks.py create mode 100644 scripts/unsloth/test_pin_contract.py create mode 100644 scripts/unsloth/test_pin_merge.py create mode 100644 scripts/unsloth/test_sync_deletes.py create mode 100755 scripts/unsloth/test_upload_release_assets.sh create mode 100644 scripts/unsloth/test_verify_upstream_sync.py create mode 100755 scripts/unsloth/upload_release_assets.sh create mode 100644 scripts/unsloth/upstream-sync.json create mode 100755 scripts/unsloth/verify_upstream_sync.py create mode 100644 tests/test-exact-buft.cpp create mode 100644 tests/test-exact-geometry.cpp create mode 100644 tests/test-exact-pages.cpp create mode 100644 tests/test-server-tokens.cpp create mode 100644 tests/test-state-seq-copy.cpp create mode 100644 tools/server/tests/unit/test_preempt.py create mode 100644 tools/server/tests/unit/test_preempt_notify.py diff --git a/.github/actions/prebuilt-alert/action.yml b/.github/actions/prebuilt-alert/action.yml new file mode 100644 index 000000000000..f7fc2b4be039 --- /dev/null +++ b/.github/actions/prebuilt-alert/action.yml @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: Prebuilt failure alert +description: Open, update or close a deduplicated issue tracking prebuilt pipeline health. + +inputs: + status: + description: 'failure or success' + required: true + key: + description: 'Dedup key; all alerts sharing it collapse onto one issue' + required: true + title: + description: 'Issue title' + required: true + details: + description: 'Markdown describing what broke' + required: false + default: '' + label: + description: 'Label applied to the issue' + required: false + default: 'prebuilt-failure' + token: + description: 'Token with issues:write' + required: true + +runs: + using: composite + steps: + - name: Open, update or close the tracking issue + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + STATUS: ${{ inputs.status }} + KEY: ${{ inputs.key }} + TITLE: ${{ inputs.title }} + DETAILS: ${{ inputs.details }} + LABEL: ${{ inputs.label }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + REPO: ${{ github.repository }} + run: | + # GitHub runs `shell: bash` with -e, which `set -uo pipefail` does not + # undo. Alerting must never fail the run it reports on, so turn it off + # explicitly; every gh call below is already individually tolerated. + set +e + set -uo pipefail + + case "$STATUS" in + failure|success) ;; + *) echo "::warning::prebuilt-alert: unknown status '$STATUS'"; exit 0 ;; + esac + + MARKER="" + + # Scan open labelled issues rather than the search index, which lags by + # minutes and would double-open on back-to-back runs. + EXISTING="" + while read -r num; do + [ -n "$num" ] || continue + if gh issue view "$num" --repo "$REPO" --json body --jq .body 2>/dev/null | grep -qF "$MARKER"; then + EXISTING="$num"; break + fi + done < <(gh issue list --repo "$REPO" --label "$LABEL" --state open \ + --limit 100 --json number --jq '.[].number' 2>/dev/null || true) + + if [ "$STATUS" = "success" ]; then + if [ -n "$EXISTING" ]; then + gh issue comment "$EXISTING" --repo "$REPO" \ + --body "Recovered: [\`${GITHUB_WORKFLOW}\` run](${RUN_URL}) succeeded. Closing." >/dev/null 2>&1 || true + gh issue close "$EXISTING" --repo "$REPO" >/dev/null 2>&1 \ + || echo "::warning::prebuilt-alert: could not close #${EXISTING}" + echo "closed #${EXISTING} ($KEY)" + fi + exit 0 + fi + + BODY="$(printf '%s\n\n%s\n\n**Run:** %s\n\n%s\n' \ + "$MARKER" \ + "\`${GITHUB_WORKFLOW}\` failed on \`${GITHUB_EVENT_NAME}\`." \ + "$RUN_URL" \ + "$DETAILS")" + + # The run summary always works: no token, no permission, no repo + # setting. Issues can be disabled (they are on a fresh fork), and an + # alert nobody can read is the failure this whole pipeline keeps + # having, so write the details somewhere visible before trying. + { + echo "## ${TITLE}" + echo + echo "$DETAILS" + } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" + + if [ -n "$EXISTING" ]; then + # Comment rather than open a second issue, so an outage is one thread. + gh issue comment "$EXISTING" --repo "$REPO" --body "$BODY" >/dev/null 2>&1 \ + || echo "::warning::prebuilt-alert: could not comment on #${EXISTING}" + echo "updated #${EXISTING} ($KEY)" + else + # gh refuses to create with a label that does not exist yet. + gh label create "$LABEL" --repo "$REPO" --color B60205 \ + --description "Prebuilt release pipeline is failing" >/dev/null 2>&1 || true + NEW="$(gh issue create --repo "$REPO" --title "$TITLE" --label "$LABEL" \ + --body "$BODY" 2>&1 | tail -1)" + case "$NEW" in + https://*) echo "opened $NEW ($KEY)" ;; + *) + # The one case worth shouting about: a real failure went + # unreported. Name the usual cause so it is actionable. + if [ "$(gh api "repos/${REPO}" --jq .has_issues 2>/dev/null)" = "false" ]; then + echo "::error::prebuilt-alert: issues are disabled on ${REPO}, so ${KEY} cannot be tracked; see the run summary for details" + else + echo "::error::prebuilt-alert: could not open an issue for ${KEY} (${NEW}); the failure at ${RUN_URL} is unreported" + fi + ;; + esac + fi + exit 0 diff --git a/.github/workflows/bench.yml.disabled b/.github/workflows/bench.yml.disabled index f2d7e16e981a..5829359ad10e 100644 --- a/.github/workflows/bench.yml.disabled +++ b/.github/workflows/bench.yml.disabled @@ -162,7 +162,7 @@ jobs: tools/server/bench/*.log - name: Commit status - uses: Sibz/github-status-action@v1 + uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1 with: authToken: ${{secrets.GITHUB_TOKEN}} sha: ${{ inputs.sha || github.event.pull_request.head.sha || github.sha }} @@ -172,7 +172,7 @@ jobs: state: 'success' - name: Upload benchmark images - uses: devicons/public-upload-to-imgur@v2.2.2 + uses: devicons/public-upload-to-imgur@352cf5f2805c692539a96cfe49a09669e6fca88e # v2.2.2 continue-on-error: true # Important as it looks unstable: 503 id: imgur_step with: @@ -221,7 +221,7 @@ jobs: echo "IMAGE_3=${{ fromJSON(steps.imgur_step.outputs.imgur_urls)[3] }}" >> $GITHUB_ENV - name: Comment PR - uses: mshick/add-pr-comment@v2 + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 id: comment_pr if: ${{ github.event.pull_request != '' && matrix.pr_comment_enabled == 'true' }} with: diff --git a/.github/workflows/unsloth-pin-preflight.yml b/.github/workflows/unsloth-pin-preflight.yml new file mode 100644 index 000000000000..f390f381e6fc --- /dev/null +++ b/.github/workflows/unsloth-pin-preflight.yml @@ -0,0 +1,276 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: Unsloth pin preflight + +# Runs the nightly's merge a few hours early, on the same base tag it will +# pick, and files the conflict before the schedule burns 39 build jobs on it. +# +# unsloth-pr-set-lint.yml checks that pins are well formed and belong to their +# PR, which catches a bad edit. It cannot catch the failure that actually +# recurs: a pin that was fine yesterday and stops merging today because the +# base tag moved under it. That is what killed 08-02, and four nights in a +# week between the two causes. + +on: + schedule: + - cron: '47 16 * * *' + push: + paths: + - scripts/unsloth/pr-set.json + workflow_dispatch: + +permissions: + # write, not read: the mirror step pushes refs/pins/. With read it + # failed every time with "Permission to unslothai/llama.cpp.git denied to + # github-actions[bot]" (403), and the only refs that existed were ones + # pushed by hand. + contents: write + issues: write + +# Two runs of the same ref probe the same pins against the same base, so the +# second adds nothing and just competes for runners. On 08-04 a dispatch and the +# schedule sat queued together for an hour. Newest wins: it sees the newest +# pr-set.json. +# +# Per ref, though, not globally. This file also runs on any push that touches +# pr-set.json, so with one shared group a push to a second branch cancelled the +# first branch's run: observed on 09-03, where the run that would have said +# whether a repin fixed the nightly was cancelled by an unrelated branch, and +# the PR was left showing the failure from before the fix. +concurrency: + group: unsloth-pin-preflight-${{ github.ref }} + cancel-in-progress: true + +jobs: + preflight: + name: Dry-run the pin merges + runs-on: ubuntu-24.04 + env: + GH_TOKEN: ${{ github.token }} + REPIN_TOKEN: ${{ secrets.REPIN_TOKEN }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - name: Resolve base and dry-run the merges + id: p + run: | + set -uo pipefail + # Everything below reports through `status`/`details`, so a death + # anywhere else leaves both empty and the alert blank: a red X on a + # scheduled run nobody opens. Report the abort through the same + # channel as a finding, so the repin bot sees a failure either way. + trap 'rc=$?; if [ "$rc" != 0 ]; then { + echo "status=failure" + echo "details<> "$GITHUB_OUTPUT"; fi' EXIT + AGE_H="${UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS:-6}" + CUTOFF="$(date -u -d "-${AGE_H} hours" +%s)" + # Same base tag the nightly resolves: newest aged b#### build. + # Upstream marks those prerelease since 08-21, so match the tag shape. + BASE="$(gh api 'repos/ggml-org/llama.cpp/releases?per_page=100' \ + | jq -r --argjson cutoff "$CUTOFF" '[.[] | select(.draft==false) | select(.tag_name|test("^b[0-9]+$")) | select((.published_at|fromdateiso8601) <= $cutoff)] | max_by(.published_at|fromdateiso8601) | .tag_name')" + if [ -z "$BASE" ] || [ "$BASE" = "null" ]; then + echo "::warning::no aged upstream release found; skipping" + echo "status=skip" >> "$GITHUB_OUTPUT"; exit 0 + fi + echo "base $BASE" + + git clone -q --filter=blob:none https://github.com/ggml-org/llama.cpp.git scratch + cd scratch + # GITHUB_TOKEN mirrors most pins, but GitHub refuses any ref that ADDS + # a workflow file the repo does not already have. Observed on 08-03, + # three pins mirrored and the fourth rejected: + # ! [remote rejected] ... -> refs/pins/c3fb9724... + # (refusing to allow a GitHub App to create or update workflow + # `.github/workflows/build-self-hosted.yml` without `workflows` + # permission) + # The check applies to any ref, not just branches. REPIN_TOKEN has + # workflow scope and covers those; without it we still mirror what we + # can rather than nothing, and warn about the rest. + MIRROR_TOKEN="${REPIN_TOKEN:-$GH_TOKEN}" + MIRROR="https://x-access-token:${MIRROR_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git fetch -q --no-tags origin "refs/tags/${BASE}:refs/tags/${BASE}" + git checkout -q --detach "refs/tags/${BASE}" + + # Mirror every pin to refs/pins/ before probing anything. The + # 07-31 outage was a reviewed commit pruned out of its PR by a force + # push, which no amount of checking can recover from after the fact. + # A ref of our own keeps the object alive; resolve falls back to it. + # All these repos are in one fork network, so this transfers nothing. + # Done first, and past failures, so a conflict in an early pin does + # not leave the later ones unmirrored. + while read -r url; do + SRC="$(sed -E 's|https://github.com/([^/]+)/llama.cpp/pull/.*|\1|' <<<"$url")/llama.cpp" + SHA="$(sed -E 's|.*/commits/([0-9a-f]{40})/?$|\1|' <<<"$url")" + if git ls-remote --exit-code "$MIRROR" "refs/pins/${SHA}" >/dev/null 2>&1; then + echo "mirrored already: ${SHA:0:10}" + continue + fi + if ! git fetch -q --no-tags "https://github.com/${SRC}.git" "$SHA" 2>/dev/null; then + echo "::warning::cannot mirror ${SHA:0:10}; it is already unfetchable from ${SRC}" + continue + fi + # Report git's own error. Guessing the cause hid a 403 behind a + # workflow-scope message for a week. + if ERR="$(git push -q "$MIRROR" "${SHA}:refs/pins/${SHA}" 2>&1)"; then + echo "mirrored ${SHA:0:10}" + else + echo "::warning::could not mirror refs/pins/${SHA:0:10}: $(sed "s|${MIRROR_TOKEN}|***|g" <<<"$ERR" | tr '\n' ' '). A ref adding a workflow file needs REPIN_TOKEN (workflow scope); that pin stays deletable by a force-push." + fi + done < <(jq -r '.prs[] | if type == "string" then . else .url end' \ + ../scripts/unsloth/pr-set.json) + + PROBLEMS="" + MERGED="" + while read -r url REQUIRED; do + SRC="$(sed -E 's|https://github.com/([^/]+)/llama.cpp/pull/.*|\1|' <<<"$url")/llama.cpp" + NUM="$(sed -E 's|.*/pull/([0-9]+)/commits/.*|\1|' <<<"$url")" + SHA="$(sed -E 's|.*/commits/([0-9a-f]{40})/?$|\1|' <<<"$url")" + + STATE="$(gh api "repos/${SRC}/pulls/${NUM}" --jq .state 2>/dev/null || echo unknown)" + if [ "$STATE" != "open" ]; then + # Non-open required pins are still merged by the nightly, so keep + # probing them here rather than reporting them as a problem; an + # optional one is skipped there, so skip it here too. + if [ "$REQUIRED" = "false" ]; then + continue + fi + echo "note: ${SRC}#${NUM} is ${STATE}; still probing because required pins are merged regardless of state" + fi + if ! git fetch -q --no-tags "https://github.com/${SRC}.git" "$SHA" 2>/dev/null; then + PROBLEMS="${PROBLEMS}- \`${SRC}#${NUM}\` pinned commit \`${SHA:0:10}\` cannot be fetched; it was probably force-pushed away.\n" + continue + fi + if git -c user.name=preflight -c user.email=preflight@local \ + -c merge.conflictStyle=diff3 \ + merge --no-ff --no-edit -m "probe ${SRC}#${NUM}" "$SHA" >/dev/null 2>&1; then + echo "ok ${SRC}#${NUM}" + MERGED=1 + continue + fi + # Mirror resolve: a pure add/add is what the nightly will merge + # automatically, so reporting it as a conflict here is a false + # alarm. Anything additive_merge.py refuses is still a conflict. + if python3 ../scripts/unsloth/additive_merge.py >/dev/null 2>&1 \ + && [ -z "$(git diff --name-only --diff-filter=U)" ]; then + git -c user.name=preflight -c user.email=preflight@local commit -q --no-edit + echo "ok ${SRC}#${NUM} (additive resolve)" + MERGED=1 + continue + fi + FILES="$(git diff --name-only --diff-filter=U | sed 's/^/ /')" + # `|| true` is load-bearing, not tidying. GitHub runs a `run:` block + # under `bash -e` whatever this script's own `set` line says, `head` + # closes the pipe after 20 lines, and pipefail then makes the whole + # assignment fail. So on 09-03 the step died right here, on the first + # real conflict, with `grep: write error: Broken pipe` and no alert: + # the one path this job exists to report was the one it could not + # survive. It only fires when the conflict diff is bigger than the + # 64 KiB pipe buffer, since a smaller one is written before `head` + # ever closes it, which is why most conflicts got reported fine. + HUNKS="$(git diff --diff-filter=U -U0 2>/dev/null | grep -E '^\+|^-' | grep -vE '^(\+\+\+|---)' | head -20 || true)" + git merge --abort 2>/dev/null + PROBLEMS="${PROBLEMS}- \`${SRC}#${NUM}\` (\`${SHA:0:10}\`) does not merge onto \`${BASE}\` + the pins before it.\n\n Conflicting files:\n\n\`\`\`\n${FILES}\n\`\`\`\n\n
conflict hunks\n\n\`\`\`diff\n${HUNKS}\n\`\`\`\n\n
\n" + # Stop here, like resolve does. Probing later pins against a tree + # missing this one reports conflicts that are consequences of it. + PROBLEMS="${PROBLEMS}\nLater pins were not probed; fix this one first.\n" + break + done < <(jq -r '.prs[] | if type == "string" then {url: ., required: true} else . end + | "\(.url)\t\(if .required == null then true else .required end)"' \ + ../scripts/unsloth/pr-set.json | tr '\t' ' ') + + # Only when a pin actually merged. With no pins, or only optional closed ones, this tree is pristine upstream, and a finding there is not a pin problem to alert on. The nightly gates the same check on MERGED_PINS. + if [ -z "$PROBLEMS" ] && [ -n "$MERGED" ]; then + # Merging cleanly is not the same as merging correctly. Two mistakes + # made on 08-27 compiled fine and would have shipped: a tensor-map key + # defined twice, which Python resolves silently by keeping the last, + # and an arch arm made unreachable by the same arch appearing in an + # earlier fallthrough condition. Both are checked here, on the tree the + # pins just produced, because this is the first point it exists. + if ! python3 ../scripts/unsloth/merge_checks.py --root . ; then + PROBLEMS="${PROBLEMS}- the merged tree builds, but \`scripts/unsloth/merge_checks.py\` found a resolution that is silently wrong. See the run log for file and line.\n" + fi + + # The other half of that question. merge_checks.py asks whether the + # tree contains something wrong; this asks whether it still contains + # what each pin carries. A pin that has rotted into a no-op, or an + # arch registration a resolution quietly dropped, is invisible to + # every other check here and to the compiler. + if ! python3 ../scripts/unsloth/pin_contract.py --root . --base "$BASE" \ + --pr-set ../scripts/unsloth/pr-set.json --report "${RUNNER_TEMP}/pin_contract.json" ; then + PROBLEMS="${PROBLEMS}- the merged tree is missing code a pin carries. See the run log for the pin and file.\n" + fi + NOTES="$(jq -r '.notices[]?' "${RUNNER_TEMP}/pin_contract.json" 2>/dev/null || true)" + if [ -n "$NOTES" ]; then + PROBLEMS="${PROBLEMS}- pins upstream has taken over, safe to delete from \`pr-set.json\`:\n\n\`\`\`\n${NOTES}\n\`\`\`\n" + fi + + # A clean merge is not a compiling tree. On 09-03 ggml-org#27754 + # merged with no conflicts at all and did not compile: upstream had + # added a parameter to build_attn_mha and the pin's new + # build_attn_sparse still called the old signature. Nothing above + # can see that. CPU only and the `llama` target only, which is where + # that translation unit lives; 59s cold at -j4 with no ccache. + GATE_OK=1 + if ! cmake -B "${RUNNER_TEMP}/gate" -DCMAKE_BUILD_TYPE=Release \ + -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=ON -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_CURL=OFF > /dev/null \ + || ! cmake --build "${RUNNER_TEMP}/gate" -j "$(nproc)" \ + --target llama test-llama-archs test-backend-ops test-mtmd-impl ; then + GATE_OK= + PROBLEMS="${PROBLEMS}- the pins merge cleanly and the merged tree does not compile. See the run log for the file and line; this is the failure that only shows up in the CUDA leg once the nightly has fanned out.\n" + fi + + # The last question, and the only one that needs a binary: does each + # feature we ship still work. Everything above is about the source. + # CPU only, because no runner in this pipeline has a GPU -- see the + # note in feature_matrix.py about what that does and does not prove. + if [ -n "$GATE_OK" ]; then + if ! python3 ../scripts/unsloth/feature_matrix.py \ + --build-dir "${RUNNER_TEMP}/gate" \ + --feature-checks ../scripts/unsloth/feature-checks.json \ + --report "${RUNNER_TEMP}/feature_matrix.json" ; then + PROBLEMS="${PROBLEMS}- the merged tree compiles and a feature we ship could not be shown to work. See the run log for which feature and which probe.\n" + fi + fi + fi + + if [ -z "$PROBLEMS" ]; then + echo "all pins merge cleanly onto ${BASE}" + echo "status=success" >> "$GITHUB_OUTPUT" + echo "details=" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "status=failure" >> "$GITHUB_OUTPUT" + { + echo 'details<> "$GITHUB_OUTPUT" + + - name: Alert + if: ${{ steps.p.outputs.status != 'skip' }} + uses: ./.github/actions/prebuilt-alert + with: + status: ${{ steps.p.outputs.status }} + key: llama-pin-preflight + title: 'Pinned PRs no longer merge onto the current base tag' + details: ${{ steps.p.outputs.details }} + token: ${{ github.token }} + + # The probe reports through its output, so without this the run still + # ends green and unsloth-repin-bot.yml, which waits for a failed + # workflow_run, never fires. On 08-05 the Inkling pin stopped merging + # onto b10280, the alert said so, the run said success, and the bot + # skipped. Fails last so the alert is always posted first. + - name: Fail the run when a pin does not merge + if: ${{ steps.p.outputs.status == 'failure' }} + run: | + echo "::error::pins do not merge onto the current base tag; see the alert above" + exit 1 diff --git a/.github/workflows/unsloth-pr-set-lint.yml b/.github/workflows/unsloth-pr-set-lint.yml new file mode 100644 index 000000000000..8a898539f022 --- /dev/null +++ b/.github/workflows/unsloth-pr-set-lint.yml @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: "Unsloth: lint pr-set.json" + +# Tripwire for edits to the mix-build PR set: validates the file the moment a +# push or PR touches it (the team usually edits it by pushing straight to +# master, so the push trigger is the main hook). A bad entry can never build -- +# the nightly's resolve job re-runs the same checks, including that each +# pinned commit actually belongs to the PR it is listed under -- but a red +# lint does not stop the schedule; this just surfaces the mistake on the +# commit within seconds instead of failing the 3 AM build. + +on: + push: + paths: + - scripts/unsloth/pr-set.json + - scripts/unsloth/additive_merge.py + - scripts/unsloth/pin_merge.py + - scripts/unsloth/merge_checks.py + - scripts/unsloth/carry_vintage.py + - scripts/unsloth/sync_deletes.py + - scripts/unsloth/test_*.py + - scripts/unsloth/check_workflow_scalars.py + # Every workflow, not just this one: the size guard only guards a file if editing it runs the guard. + - .github/workflows/*.yml + - .github/workflows/*.yaml + pull_request: + paths: + - scripts/unsloth/pr-set.json + - scripts/unsloth/additive_merge.py + - scripts/unsloth/pin_merge.py + - scripts/unsloth/merge_checks.py + - scripts/unsloth/carry_vintage.py + - scripts/unsloth/sync_deletes.py + - scripts/unsloth/test_*.py + - scripts/unsloth/check_workflow_scalars.py + # Every workflow, not just this one: the size guard only guards a file if editing it runs the guard. + - .github/workflows/*.yml + - .github/workflows/*.yaml + +permissions: + contents: read + +jobs: + lint: + name: Validate pr-set.json + runs-on: ubuntu-24.04 + env: + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - run: | + set -euo pipefail + FILE=scripts/unsloth/pr-set.json + # Mirrors the resolve step's schema: a bare url string (required), + # or {"url": ..., "required": false} for a pin the release may ship + # without. + jq -e '.prs | type == "array" and all(.[]; + type == "string" + or (type == "object" and (.url | type == "string") + and ((if .required == null then true else .required end) | type == "boolean")))' "$FILE" >/dev/null \ + || { echo "::error file=$FILE::.prs must be an array of PR url strings, or {url, required} objects" >&2; exit 1; } + URL_RE='^https://github\.com/(ggml-org|unslothai)/llama\.cpp/pull/([0-9]+)/commits/([0-9a-f]{40})/?$' + fail=0 + while read -r url REQUIRED; do + if ! [[ "$url" =~ $URL_RE ]]; then + echo "::error file=$FILE::malformed entry '$url' (expected https://github.com/{ggml-org,unslothai}/llama.cpp/pull//commits/<40-hex-sha>)" + fail=1 + continue + fi + SRC="${BASH_REMATCH[1]}/llama.cpp"; NUM="${BASH_REMATCH[2]}"; PIN="${BASH_REMATCH[3]}" + # Don't let a 404 under set -e kill the collect-all-errors loop. + if ! PR_JSON="$(gh api "repos/${SRC}/pulls/${NUM}" --jq '{state: .state, commits: .commits, head: .head.sha}')"; then + echo "::error file=$FILE::could not fetch ${SRC}#${NUM} (nonexistent PR number in '$url', or a transient API failure)" + fail=1 + continue + fi + STATE="$(jq -r .state <<<"$PR_JSON")" + COMMITS="$(jq -r .commits <<<"$PR_JSON")" + HEAD="$(jq -r .head <<<"$PR_JSON")" + if [ "$STATE" != "open" ]; then + if [ "$REQUIRED" != "false" ]; then + echo "::warning file=$FILE::${SRC}#${NUM} is ${STATE}; the nightly will keep merging its pinned commit (a no-op once the base tag contains it), so drop the entry when you no longer want that code" + else + echo "::warning file=$FILE::${SRC}#${NUM} is ${STATE}; the nightly will skip this optional entry" + fi + fi + # The commits listing is capped at 250 by the API; past that the + # membership check cannot be trusted, so skip it rather than + # false-fail a legitimate giant PR. + if [ "$COMMITS" -gt 250 ]; then + echo "::notice::${SRC}#${NUM} has ${COMMITS} commits (over the API listing cap); skipping pin membership check" + elif ! gh api "repos/${SRC}/pulls/${NUM}/commits" --paginate --jq '.[].sha' | grep -qx "$PIN"; then + echo "::error file=$FILE::pinned commit ${PIN} is not a commit of ${SRC}#${NUM}" + fail=1 + continue + fi + [ "$PIN" = "$HEAD" ] || echo "::notice::${SRC}#${NUM} pin ${PIN} is behind its head ${HEAD}" + echo "OK: ${SRC}#${NUM} @ ${PIN} (${STATE}, required=${REQUIRED})" + done < <(jq -r '.prs[] | if type == "string" then {url: ., required: true} else . end + | "\(.url)\t\(if .required == null then true else .required end)"' "$FILE" | tr '\t' ' ') + exit "$fail" + + resolver-tests: + # These tests existed for additive_merge.py and ran nowhere, so a change to a merge resolver was covered by nothing at all. + # Every resolver and check under scripts/unsloth/ runs here. + name: Resolver and merge-check tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: { fetch-depth: 1 } + + - name: Run the resolver tests + run: | + set -euo pipefail + fail=0 + for t in scripts/unsloth/test_additive_merge.py \ + scripts/unsloth/test_pin_merge.py \ + scripts/unsloth/test_merge_checks.py \ + scripts/unsloth/test_sync_deletes.py \ + scripts/unsloth/test_carry_vintage.py \ + scripts/unsloth/test_pin_contract.py \ + scripts/unsloth/test_feature_matrix.py; do + echo "::group::$t" + python3 "$t" || fail=1 + echo "::endgroup::" + done + exit "$fail" + + # A pin nobody decided about is the failure this whole file exists to stop. + # Being in `unchecked` with a reason is a fine answer; being in neither map + # is how DiffusionGemma went five weeks with no coverage and no record of it. + - name: Every pin is either checked or knowingly unchecked + run: | + set -euo pipefail + python3 - <<'PY' + import json, re, sys + pins = json.load(open("scripts/unsloth/pr-set.json"))["prs"] + doc = json.load(open("scripts/unsloth/feature-checks.json")) + owned = {f["owner"] for f in doc["features"].values() if f.get("owner")} + known = owned | set(doc.get("unchecked", {})) + fail = 0 + for entry in pins: + url = entry if isinstance(entry, str) else entry["url"] + m = re.match(r"https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/", url) + pin = f"{m.group(1)}#{m.group(2)}" + if pin not in known: + print(f"::error file=scripts/unsloth/feature-checks.json::{pin} is pinned " + "and appears in neither `features` nor `unchecked`; say which it is") + fail = 1 + for pin in sorted(owned & set(doc.get("unchecked", {}))): + print(f"::error file=scripts/unsloth/feature-checks.json::{pin} is in both " + "`features` and `unchecked`") + fail = 1 + print(f"{len(pins)} pin(s), {len(owned)} with a feature check, " + f"{len(doc.get('unchecked', {}))} knowingly unchecked") + sys.exit(fail) + PY + + # An over-limit run: script makes the whole file uncompilable, and nothing else sees it: yaml, actionlint and GitHub's own parser all pass it. See check_workflow_scalars.py. + - name: Check no workflow string is near GitHub's size limit + run: python3 scripts/unsloth/check_workflow_scalars.py --root . + diff --git a/.github/workflows/unsloth-prebuilt-cpu.yml b/.github/workflows/unsloth-prebuilt-cpu.yml new file mode 100644 index 000000000000..45e609ae2d0f --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-cpu.yml @@ -0,0 +1,376 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: "Unsloth prebuilt: CPU" + +# Reusable child of unsloth-prebuilt.yml. Builds the CPU-only bundles for +# Linux + Windows, each on x64 and arm64 (one matrix entry per arch). Every +# matrix entry uploads a single app-*.{tar.gz|zip} artifact for the parent's +# assemble step to pick up. +# +# The Linux build mirrors oobabooga/llama-cpp-binaries' build-wheels-cpu.yml +# (GGML_BACKEND_DL + GGML_CPU_ALL_VARIANTS + GGML_RPC, no GPU backend). The +# Windows build instead matches ggml-org/llama.cpp release.yml's windows-cpu +# job (clang/LLVM toolchain file + "Ninja Multi-Config" + OpenMP + BoringSSL, +# arm64 cross-compiled from x64) -- clang is upstream's proven CPU path, so we +# follow it rather than llama-cpp-binaries' MSVC recipe; MSVC stays only where +# it is mandatory (CUDA/nvcc). Adapted to this repo's conventions: +# app----cpu archives packaged straight from build/bin like +# the ROCm/macOS children (no embedded UNSLOTH_PREBUILT_INFO.json -- +# assemble_metadata.py derives the manifest entry from the filename), $ORIGIN +# RPATH on Linux. arm64 extends llama-cpp-binaries (x64-only) so the release +# covers the arm64 CPU hosts that previously fell back to ggml-org upstream. +# Both Linux legs build on ubuntu-22.04 so the bundles keep a glibc 2.35 / +# GLIBCXX <= 3.4.30 floor and load on Ubuntu 22.04 and Debian 12 hosts. + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + repo: + description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' + required: false + default: 'ggml-org/llama.cpp' + type: string + source_artifact: + description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' + required: false + default: '' + type: string + +permissions: + contents: read + +jobs: + build-linux: + name: linux/${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { arch: x64, runner: ubuntu-22.04 } + - { arch: arm64, runner: ubuntu-22.04-arm } + steps: + # The parent's resolve job built the source tree (upstream base + any mix + # PRs, with the build number/commit and Unsloth fingerprint baked + # into cmake/build-info.cmake) and uploaded it as an artifact; extract it + # instead of cloning -- no .git needed, the build number is already baked. + - name: Download source @ ${{ inputs.tag }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: ${{ inputs.source_artifact }} + path: srcpkg + - name: Extract source + shell: bash + run: | + set -eux + mkdir -p src + tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 + + - name: Install build dependencies + run: | + set -eux + sudo apt-get update + sudo apt-get install -y build-essential libssl-dev ninja-build + + # arm64 builds on ubuntu-22.04-arm so the bundle keeps the x64 leg's + # loader floor (a 24.04 build needs GLIBC_2.38 and fails to load on + # Ubuntu 22.04 / Debian 12). Jammy's gcc can't target armv9.2-a+sme and + # a PPA gcc would raise the libstdc++ floor back, so use clang, which + # links against the system libstdc++. + - name: Toolchain (clang 19 on arm64) + if: matrix.arch == 'arm64' + run: | + set -eux + wget -q https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 19 + sudo apt-get install -y libomp-19-dev + { + echo "CC=clang-19" + echo "CXX=clang++-19" + } >> "$GITHUB_ENV" + + - name: ccache + uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 + with: + key: cpu-linux-${{ matrix.arch }}-${{ inputs.tag }} + restore-keys: | + cpu-linux-${{ matrix.arch }} + append-timestamp: false + variant: ccache + max-size: 2G + save: false + + - name: Configure + working-directory: src + run: | + set -eux + # Build recipe mirrors llama-cpp-binaries' CPU wheel (backend-DL + + # all CPU variants + RPC, no GPU backend). RPATH=$ORIGIN so the + # bundle's sibling .so files resolve from the binary's own directory. + # LLAMA_FATAL_WARNINGS below is -Werror. The arm64 image compiles with + # clang-19 against GCC 12's libstdc++, where std::stable_sort still + # reaches the deprecated get_temporary_buffer; GCC buries that in a + # system header, clang reports it at our instantiation. That failed + # this leg on 08-27 over a deprecation in code we do not own, so that + # one diagnostic is off. Every other warning stays fatal. + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_NATIVE=OFF \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_RPC=ON \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_SERVER=ON \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + + - name: Build + working-directory: src + run: | + set -eux + # Build the full tool + server set (no --target), matching the macOS + # bins. Backend modules (CPU variants, RPC) build as ggml deps. + cmake --build build --config Release -j "$(nproc)" + strip build/bin/llama-* || true + + - name: Bundle OpenMP runtime (arm64) + if: matrix.arch == 'arm64' + run: cp /usr/lib/llvm-19/lib/libomp.so.5 src/build/bin/ + + # DiffusionGemma binaries (example targets present only in #24423 mix + # builds): best-effort, never fail the job. See the CUDA child for the + # rationale. The bundle tars all of build/bin, so anything produced here + # is shipped automatically. + - name: Build DiffusionGemma binaries (best-effort; mix builds only) + working-directory: src + run: | + set -u + if [ ! -d examples/diffusion-gemma-server ]; then + echo "no DiffusionGemma sources in this tree; skipping" + exit 0 + fi + cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON \ + || { echo "reconfigure for examples failed; skipping DiffusionGemma binaries"; exit 0; } + if cmake --build build --config Release -j "$(nproc)" \ + --target llama-diffusion-gemma-visual-server llama-diffusion-cli; then + strip build/bin/llama-diffusion-gemma-visual-server build/bin/llama-diffusion-cli || true + echo "built DiffusionGemma binaries" + else + echo "warning: DiffusionGemma binaries failed to build; bundle will omit them" + fi + exit 0 + + - name: Package bundle (tar.gz) + run: | + set -eux + ASSET="app-${{ inputs.tag }}-linux-${{ matrix.arch }}-cpu.tar.gz" + cp src/LICENSE src/build/bin/ + mkdir -p dist + (cd src/build/bin && tar -czf "${GITHUB_WORKSPACE}/dist/${ASSET}" .) + ls -la dist + + - name: Upload bundle artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: app-${{ inputs.tag }}-linux-${{ matrix.arch }}-cpu + path: dist/app-${{ inputs.tag }}-linux-${{ matrix.arch }}-cpu.tar.gz + if-no-files-found: error + + - name: Evict stale ccache files + # !cancelled(), unlike the save below: on a timeout the job gets a + # single ~5 minute teardown window (measured ~4m50s after process + # kill), shared by every remaining step and not replenished. Evicting + # spends that window on housekeeping; the save is what actually needs + # it, and a 2 GB cache is not quick to write. + if: ${{ !cancelled() }} + continue-on-error: true + run: ccache --evict-older-than 14d + + - name: Save ccache + # Save even when the build failed: the objects compiled before the + # failure are still worth keeping, and a job that saves nothing leaves + # a hole in the cache lineage that widens the next run's tag gap. + # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. + # + # always(), not !cancelled(): a timeout-minutes expiry puts the job on + # the CANCELLATION path, not the failure path, so !cancelled() would + # skip the save on the single most expensive case -- a leg that + # compiled for hours and then hit the cap. + if: ${{ always() }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.ccache + key: ccache-cpu-linux-${{ matrix.arch }}-${{ inputs.tag }}- + + build-windows: + name: windows/${{ matrix.arch }} + # Single x64 runner for both arches: arm64 is cross-compiled with the clang + # toolchain (vcvars amd64_arm64), exactly like ggml-org's release.yml + # windows-cpu job. CUDA stays on MSVC (mandatory for nvcc on Windows), but + # the CPU build uses clang/LLVM to match upstream's proven CPU recipe. + runs-on: windows-2025-vs2026 + strategy: + fail-fast: false + matrix: + include: + - { arch: x64, vcvars: x64, omp_arch: x86_64, cpu_variants: 'ON' } + - { arch: arm64, vcvars: amd64_arm64, omp_arch: aarch64, cpu_variants: 'OFF' } + defaults: + run: + shell: pwsh + steps: + # The parent's resolve job built the source tree (upstream base + any mix + # PRs, with the build number/commit and Unsloth fingerprint baked + # into cmake/build-info.cmake) and uploaded it as an artifact; extract it + # instead of cloning -- no .git needed, the build number is already baked. + - name: Download source @ ${{ inputs.tag }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: ${{ inputs.source_artifact }} + path: srcpkg + - name: Extract source + shell: bash + run: | + set -eux + mkdir -p src + tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 + + - name: ccache + uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 + with: + key: cpu-windows-${{ matrix.arch }}-${{ inputs.tag }} + restore-keys: | + cpu-windows-${{ matrix.arch }} + append-timestamp: false + variant: ccache + max-size: 2G + save: false + + - name: Install Ninja + run: choco install ninja --no-progress + + # Build recipe copied from ggml-org/llama.cpp release.yml (windows-cpu): + # clang via the per-arch LLVM toolchain file, "Ninja Multi-Config", OpenMP, + # BoringSSL, and GGML_CPU_ALL_VARIANTS only on x64 (it is an x86 microarch + # fan-out). vcvarsall sets the env (incl. the amd64_arm64 cross toolchain), + # so this runs in cmd. CMAKE_ARGS mirrors upstream's env.CMAKE_ARGS and + # builds the full tool + server set (no --target), matching the macOS bins. + - name: Build + working-directory: src + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.vcvars }} + cmake -S . -B build -G "Ninja Multi-Config" ^ + -D CMAKE_TOOLCHAIN_FILE=cmake/${{ matrix.arch }}-windows-llvm.cmake ^ + -DLLAMA_BUILD_BORINGSSL=ON ^ + -DGGML_NATIVE=OFF ^ + -DGGML_BACKEND_DL=ON ^ + -DGGML_CPU_ALL_VARIANTS=${{ matrix.cpu_variants }} ^ + -DGGML_OPENMP=ON ^ + -DGGML_RPC=ON ^ + -DLLAMA_BUILD_TESTS=OFF ^ + -DLLAMA_BUILD_EXAMPLES=OFF ^ + -DLLAMA_BUILD_TOOLS=ON ^ + -DLLAMA_BUILD_SERVER=ON ^ + -DCMAKE_C_COMPILER_LAUNCHER=ccache ^ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + if errorlevel 1 exit /b 1 + cmake --build build --config Release + if errorlevel 1 exit /b 1 + + # DiffusionGemma binaries (#24423 mix builds only): best-effort, never fail + # the job (trailing `exit /b 0`). vcvarsall is re-called -- step env does + # not persist -- and the reconfigure reuses the cached clang toolchain. + - name: Build DiffusionGemma binaries (best-effort; mix builds only) + working-directory: src + shell: cmd + run: | + if not exist examples\diffusion-gemma-server ( + echo no DiffusionGemma sources in this tree; skipping + exit /b 0 + ) + call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.vcvars }} + cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON + cmake --build build --config Release --target llama-diffusion-gemma-visual-server llama-diffusion-cli + exit /b 0 + + # Ninja Multi-Config emits to build/bin/Release. Ship the OpenMP runtime + # (GGML_OPENMP=ON) from the VS LLVM redist -- exactly as upstream does -- + # globbing the MSVC version dir so an image bump does not break the path. + # Sort newest-first: older toolsets' libomp lacks entry points current + # clang imports (__kmpc_dispatch_deinit -> STATUS_ENTRYPOINT_NOT_FOUND). + - name: Package bundle (zip) + run: | + $rel = "src/build/bin/Release" + Copy-Item src/LICENSE $rel/ + $redist = "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Redist\MSVC" + $omp = Get-ChildItem "$redist\*\debug_nonredist\${{ matrix.arch }}\Microsoft.VC*.OpenMP.LLVM\libomp140.${{ matrix.omp_arch }}.dll" -ErrorAction SilentlyContinue | + Sort-Object { [version]$_.Directory.Parent.Parent.Parent.Name } -Descending | + Select-Object -First 1 + if (-not $omp) { Write-Error "libomp140.${{ matrix.omp_arch }}.dll not found in the VS LLVM redist"; exit 1 } + Write-Host "shipping OpenMP runtime: $($omp.FullName)" + Copy-Item $omp.FullName $rel/ + New-Item -ItemType Directory -Force -Path dist | Out-Null + $asset = "app-${{ inputs.tag }}-windows-${{ matrix.arch }}-cpu.zip" + Push-Location $rel + 7z a -tzip "$env:GITHUB_WORKSPACE/dist/$asset" . + Pop-Location + Get-ChildItem dist + + # Smoke the packaged x64 bundle: launching an exe loads ggml-base and the + # picked libomp, so --version fails on a bad pick. timeout-minutes bounds + # a loader hard-error that can block instead of exiting. arm64 is + # cross-compiled and cannot run here. + - name: Smoke test bundle (x64 only) + if: matrix.arch == 'x64' + timeout-minutes: 5 + run: | + $rel = "src/build/bin/Release" + & "$rel\llama-server.exe" --version + if ($LASTEXITCODE -ne 0) { Write-Error "llama-server --version failed: $LASTEXITCODE"; exit 1 } + + - name: Upload bundle artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: app-${{ inputs.tag }}-windows-${{ matrix.arch }}-cpu + path: dist/app-${{ inputs.tag }}-windows-${{ matrix.arch }}-cpu.zip + if-no-files-found: error + + - name: Evict stale ccache files + # !cancelled(), unlike the save below: on a timeout the job gets a + # single ~5 minute teardown window (measured ~4m50s after process + # kill), shared by every remaining step and not replenished. Evicting + # spends that window on housekeeping; the save is what actually needs + # it, and a 2 GB cache is not quick to write. + if: ${{ !cancelled() }} + continue-on-error: true + run: ccache --evict-older-than 14d + + - name: Save ccache + # Save even when the build failed: the objects compiled before the + # failure are still worth keeping, and a job that saves nothing leaves + # a hole in the cache lineage that widens the next run's tag gap. + # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. + # + # always(), not !cancelled(): a timeout-minutes expiry puts the job on + # the CANCELLATION path, not the failure path, so !cancelled() would + # skip the save on the single most expensive case -- a leg that + # compiled for hours and then hit the cap. + if: ${{ always() }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}\.ccache + key: ccache-cpu-windows-${{ matrix.arch }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-cuda-windows.yml b/.github/workflows/unsloth-prebuilt-cuda-windows.yml new file mode 100644 index 000000000000..90e28e1749b5 --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-cuda-windows.yml @@ -0,0 +1,310 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: "Unsloth prebuilt: CUDA Windows" + +# Reusable child of unsloth-prebuilt.yml. Builds the per-profile CUDA Windows +# bundles the parent's `resolve` job filtered into `matrix`. Each entry uploads +# a single app-*.zip artifact for the parent's assemble step to pick up. +# +# Mirrors unsloth-prebuilt-cuda.yml (Linux), adapted for Windows: MSVC + Ninja +# toolchain, BoringSSL instead of system OpenSSL, no RPATH (Windows resolves +# DLLs from the executable's directory), and a .zip archive. CUDA compiles +# entirely in software here -- no GPU is present on the runner. + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + commit: + description: 'Upstream commit SHA for that tag, resolved by parent' + required: true + type: string + repo: + description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' + required: false + default: 'ggml-org/llama.cpp' + type: string + source_artifact: + description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' + required: false + default: '' + type: string + matrix: + description: 'Matrix JSON {include:[...]} produced by the parent resolve job' + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: x64/${{ matrix.profile }} + runs-on: ${{ matrix.runner }} + # Hang guard only. Without it the job inherits GitHub's 360-minute default, + # which is above both assemble's `timeout-minutes: 350` and, more to the + # point, the "Wait for the build matrix" step's own 330-minute deadline in + # unsloth-prebuilt.yml -- so a wedged leg burns a runner for an hour after + # the publish it was feeding has already given up. + # + # 345, not something tighter: x64/cuda12-portable legitimately reaches the + # low 300s on a cold ccache. Run 27347317849 took 305.1 minutes and + # SUCCEEDED (286.9 of it genuine compile). Across 62 historical runs of this + # leg the max is 305.1 and nothing falls between 215 and 305, so a tighter + # cap buys no hang detection and would have destroyed that publish -- one + # killed leg fails the bundle-coverage gate and the whole night ships + # nothing. + # + # Note this cannot by itself rescue a slow run: the waiter's 330-minute + # deadline is wall-clock from assemble start and INCLUDES the child's queue + # wait, while timeout-minutes starts at job start and excludes it. Raising + # the waiter deadline is the separate change that would actually save runs. + timeout-minutes: 345 + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix) }} + defaults: + run: + shell: pwsh + steps: + # Report both fixed volumes at job start. The work is split across two: + # the CUDA toolkit, the tool cache and Program Files land on C:, while + # GITHUB_WORKSPACE -- the source tree, the CMake build tree and the ccache + # -- is on D:. A leg that dies mid-build can only be read against the + # volume it was writing to, so both are printed here and again after the + # toolkit install. + # + # No cleanup step, deliberately. One was written and then measured on a + # live windows-2022 runner: D: starts at 147.0 GB free of 150.0 and C: at + # 84.3 of 255.4, and the CUDA 12.8 toolkit is 4.79 GB, so C: never drops + # below about 106 GB. Reclaiming ~33 GB of preinstalled SDKs on every leg + # would have freed the volume that was not under pressure, on every + # release, forever. The 87-minute "runner lost communication" failure was + # not disk exhaustion, and unsloth-prebuilt-retry.yml already covers that + # class of infrastructure loss. + - name: Report disk space + run: | + "workspace: $env:GITHUB_WORKSPACE" + Get-CimInstance Win32_LogicalDisk -Filter 'DriveType = 3' | Sort-Object DeviceID | ForEach-Object { + "{0} {1:N1} GB free of {2:N1} GB" -f $_.DeviceID, ($_.FreeSpace / 1GB), ($_.Size / 1GB) + } + + - name: Checkout build tooling (this repo) + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + path: tooling + + # The parent's resolve job built the source tree (upstream base + any mix + # PRs, with the build number/commit and Unsloth fingerprint baked + # into cmake/build-info.cmake) and uploaded it as an artifact; extract it + # instead of cloning -- no .git needed, the build number is already baked. + - name: Download source @ ${{ inputs.tag }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: ${{ inputs.source_artifact }} + path: srcpkg + - name: Extract source + shell: bash + run: | + set -eux + mkdir -p src + tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 + + - name: ccache + uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 + with: + key: cuda-${{ matrix.cuda }}-windows-${{ matrix.profile }}-${{ inputs.tag }} + restore-keys: | + cuda-${{ matrix.cuda }}-windows-${{ matrix.profile }} + append-timestamp: false + variant: ccache + max-size: 2G + save: false + + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.11" + + - name: Install Ninja + run: choco install ninja --no-progress + + - name: Setup MSVC + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 + + # Jimver's action has no mapping for CUDA 13.3 yet, so for that version we + # fall back to llama.cpp's own install method: curl the individual NVIDIA + # redist component archives and assemble the toolkit by hand. Block copied + # verbatim from ggml-org/llama.cpp .github/actions/windows-setup-cuda + # (the cuda_version == '13.3' case). Any other version uses Jimver. + - name: Install CUDA toolkit 13.3 (NVIDIA redist) + if: matrix.cuda == '13.3' + run: | + mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" + choco install unzip -y + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.3.29-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.5.1.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.3.29-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.3.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.3.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.3.3.1-archive.zip" + unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_crt-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_cudart-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvcc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvrtc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libcublas-windows-x86_64-13.5.1.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libnvvm-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvtx-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_profiler_api-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\visual_studio_integration-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cccl-windows-x86_64-13.3.3.3.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + + - name: Install CUDA toolkit ${{ matrix.cuda }} (Jimver) + if: matrix.cuda != '13.3' + uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 + id: cuda-toolkit + with: + cuda: ${{ matrix.cuda }} + method: 'network' + + - name: Set up CUDA environment (Jimver) + if: matrix.cuda != '13.3' + run: | + echo "CUDA_PATH=$env:CUDA_PATH" >> $env:GITHUB_ENV + echo "CUDA_HOME=$env:CUDA_PATH" >> $env:GITHUB_ENV + + - name: Verify CUDA + run: nvcc --version + + # Headroom going into the build, with the toolkit on C: and the restored + # ccache in the workspace on D:. First numbers to read if a leg dies + # mid-build; the build itself writes to the workspace volume. + - name: Report disk space before build + run: | + "workspace: $env:GITHUB_WORKSPACE" + Get-CimInstance Win32_LogicalDisk -Filter 'DriveType = 3' | Sort-Object DeviceID | ForEach-Object { + "{0} {1:N1} GB free of {2:N1} GB" -f $_.DeviceID, ($_.FreeSpace / 1GB), ($_.Size / 1GB) + } + + - name: Configure + working-directory: src + run: | + # CMAKE_CUDA_ARCHITECTURES is the explicit per-profile arch list (the + # whole point of the matrix). ccache launchers cache nvcc + cl.exe; + # no RPATH knobs -- Windows loads sibling DLLs from the binary's dir. + $archs = "${{ matrix.archs }}".Replace(' ', ';') + cmake -S . -B build -G Ninja ` + -DCMAKE_BUILD_TYPE=Release ` + -DGGML_NATIVE=OFF ` + -DGGML_BACKEND_DL=ON ` + -DGGML_CPU_ALL_VARIANTS=ON ` + -DGGML_RPC=ON ` + -DGGML_CUDA=ON ` + -DGGML_CUDA_CUB_3DOT2=ON ` + -DLLAMA_BUILD_TESTS=OFF ` + -DLLAMA_BUILD_EXAMPLES=OFF ` + -DLLAMA_BUILD_TOOLS=ON ` + -DLLAMA_BUILD_SERVER=ON ` + -DLLAMA_BUILD_BORINGSSL=ON ` + -DCMAKE_CUDA_ARCHITECTURES="$archs" ` + -DCMAKE_C_COMPILER_LAUNCHER=ccache ` + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ` + -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache + + - name: Build + working-directory: src + run: | + # -j 3: this multi-arch nvcc build peaks at ~3 GB host RSS on the + # 4 vCPU / 16 GB GitHub-hosted runners (measured over a 1s sampler), + # so 3 parallel TUs leave ~13 GB free. -j 4 adds no speed (the runner + # is ~2 physical cores + HT), so 3 is the sweet spot. + cmake --build build --config Release -j 3 + + # DiffusionGemma binaries (example targets present only in #24423 mix + # builds): best-effort, never fail the job. See the Linux CUDA child for + # the rationale. package_bundle.py ships the .exe only if it was produced. + - name: Build DiffusionGemma binaries (best-effort; mix builds only) + working-directory: src + run: | + if (-not (Test-Path "examples/diffusion-gemma-server")) { + Write-Host "no DiffusionGemma sources in this tree; skipping" + exit 0 + } + cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON + if ($LASTEXITCODE -ne 0) { + Write-Host "reconfigure for examples failed; skipping DiffusionGemma binaries" + exit 0 + } + cmake --build build --config Release -j 3 ` + --target llama-diffusion-gemma-visual-server llama-diffusion-cli + if ($LASTEXITCODE -ne 0) { + Write-Host "warning: DiffusionGemma binaries failed to build; bundle will omit them" + } else { + Write-Host "built DiffusionGemma binaries" + } + exit 0 + + - name: Package bundle + env: + PLATFORM: windows + ARCH: x64 + BIN_DIR: ${{ github.workspace }}/src/build/bin + SRC_DIR: ${{ github.workspace }}/src + OUT_DIR: ${{ github.workspace }}/dist + TAG: ${{ inputs.tag }} + SOURCE_COMMIT: ${{ inputs.commit }} + SOURCE_REPO: ${{ inputs.repo }} + SOURCE_REF_KIND: ${{ inputs.repo == 'ggml-org/llama.cpp' && 'tag' || 'mix' }} + PROFILE: ${{ matrix.profile }} + LINE: ${{ matrix.line }} + KLASS: ${{ matrix.klass }} + RANK: ${{ matrix.rank }} + TOOLKIT_LINE: ${{ matrix.toolkit_line }} + DOCKER_IMAGE: github-hosted ${{ matrix.runner }}, CUDA ${{ matrix.cuda }} ${{ matrix.cuda == '13.3' && '(NVIDIA redist)' || '(Jimver)' }} + ARCHS: ${{ matrix.archs }} + SMS: ${{ matrix.sms }} + run: python tooling/scripts/unsloth/package_bundle.py + + - name: Upload bundle artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: app-${{ inputs.tag }}-windows-x64-${{ matrix.profile }} + path: dist/app-${{ inputs.tag }}-windows-x64-${{ matrix.profile }}.zip + if-no-files-found: error + + - name: Evict stale ccache files + # !cancelled(), unlike the save below: on a timeout the job gets a + # single ~5 minute teardown window (measured ~4m50s after process + # kill), shared by every remaining step and not replenished. Evicting + # spends that window on housekeeping; the save is what actually needs + # it, and a 2 GB cache is not quick to write. + if: ${{ !cancelled() }} + continue-on-error: true + run: ccache --evict-older-than 14d + + - name: Save ccache + # Save even when the build failed: the objects compiled before the + # failure are still worth keeping, and a job that saves nothing leaves + # a hole in the cache lineage that widens the next run's tag gap. + # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. + # + # always(), not !cancelled(): a timeout-minutes expiry puts the job on + # the CANCELLATION path, not the failure path, so !cancelled() would + # skip the save on the single most expensive case -- a leg that + # compiled for hours and then hit the cap. + if: ${{ always() }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}\.ccache + key: ccache-cuda-${{ matrix.cuda }}-windows-${{ matrix.profile }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-cuda.yml b/.github/workflows/unsloth-prebuilt-cuda.yml new file mode 100644 index 000000000000..b64ebe41f4a1 --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-cuda.yml @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: "Unsloth prebuilt: CUDA" + +# Reusable child of unsloth-prebuilt.yml. Builds the per-profile CUDA bundles +# the parent's `resolve` job filtered into `matrix`. Each entry uploads a +# single app-*.tar.gz artifact for the parent's assemble step to pick up. + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + commit: + description: 'Upstream commit SHA for that tag, resolved by parent' + required: true + type: string + repo: + description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' + required: false + default: 'ggml-org/llama.cpp' + type: string + source_artifact: + description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' + required: false + default: '' + type: string + matrix: + description: 'Matrix JSON {include:[...]} produced by the parent resolve job' + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: ${{ matrix.arch }}/${{ matrix.profile }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix) }} + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: false + swap-storage: true + + - name: Checkout build tooling (this repo) + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + path: tooling + + # The parent's resolve job built the source tree (upstream base + any mix + # PRs, with the build number/commit and Unsloth fingerprint baked + # into cmake/build-info.cmake) and uploaded it as an artifact; extract it + # instead of cloning -- no .git needed, the build number is already baked. + - name: Download source @ ${{ inputs.tag }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: ${{ inputs.source_artifact }} + path: srcpkg + - name: Extract source + shell: bash + run: | + set -eux + mkdir -p src + tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 + + - name: Install build dependencies + run: | + set -eux + sudo apt-get update + sudo apt-get install -y build-essential libssl-dev + + - name: Install ARM64 host compiler (gcc-14) + if: matrix.arch == 'arm64' + run: | + set -eux + sudo apt-get install -y gcc-14 g++-14 + { + echo "CC=gcc-14" + echo "CXX=g++-14" + echo "CUDAHOSTCXX=g++-14" + } >> "$GITHUB_ENV" + + - name: ccache + uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 + with: + key: cuda-${{ matrix.cuda }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ inputs.tag }} + restore-keys: | + cuda-${{ matrix.cuda }}-${{ matrix.arch }}-${{ matrix.profile }} + append-timestamp: false + variant: ccache + max-size: 2G + save: false + + # Jimver has no mapping for CUDA 13.3 yet, so for that version we pull the + # toolkit components straight from NVIDIA's redist CDN (same source Jimver + # and ggml-org use) and assemble a prefix by hand. Component versions are + # from NVIDIA's redistrib_13.3.0.json. Runs on the same ubuntu runner, so + # the glibc floor is unchanged. Any other version uses Jimver. + - name: Install CUDA toolkit 13.3 (NVIDIA redist) + if: matrix.cuda == '13.3' + run: | + set -eux + case "${{ matrix.arch }}" in + x64) PLAT=linux-x86_64 ;; + arm64) PLAT=linux-sbsa ;; + *) echo "unsupported arch ${{ matrix.arch }}" >&2; exit 1 ;; + esac + PREFIX="$HOME/cuda-13.3" + mkdir -p "$PREFIX" + BASE="https://developer.download.nvidia.com/compute/cuda/redist" + for cv in \ + cuda_crt:13.3.33 cuda_cudart:13.3.29 cuda_nvcc:13.3.33 \ + cuda_nvrtc:13.3.33 libcublas:13.5.1.27 libnvvm:13.3.33 \ + cuda_nvtx:13.3.29 cuda_profiler_api:13.3.27 cccl:13.3.3.3.1; do + comp="${cv%:*}"; ver="${cv#*:}" + name="${comp}-${PLAT}-${ver}-archive" + curl -fsSL -o comp.tar.xz "$BASE/${comp}/${PLAT}/${name}.tar.xz" + tar -xf comp.tar.xz + cp -a "${name}/." "$PREFIX"/ + rm -rf comp.tar.xz "${name}" + done + # Redist ships libs under lib/; some CMake CUDA discovery expects lib64. + ln -sfn lib "$PREFIX/lib64" + { + echo "CUDA_PATH=$PREFIX" + echo "CUDA_HOME=$PREFIX" + echo "LD_LIBRARY_PATH=$PREFIX/lib:${LD_LIBRARY_PATH:-}" + } >> "$GITHUB_ENV" + echo "$PREFIX/bin" >> "$GITHUB_PATH" + + - name: Install CUDA toolkit ${{ matrix.cuda }} (Jimver) + if: matrix.cuda != '13.3' + uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 + id: cuda-toolkit + with: + cuda: ${{ matrix.cuda }} + method: 'network' + + - name: Set up CUDA environment (Jimver) + if: matrix.cuda != '13.3' + run: | + echo "CUDA_PATH=${{ steps.cuda-toolkit.outputs.CUDA_PATH }}" >> "$GITHUB_ENV" + echo "CUDA_HOME=${{ steps.cuda-toolkit.outputs.CUDA_PATH }}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${{ steps.cuda-toolkit.outputs.CUDA_PATH }}/lib64:${LD_LIBRARY_PATH:-}" >> "$GITHUB_ENV" + + - name: Verify CUDA + run: nvcc --version + + - name: Configure + working-directory: src + run: | + set -eux + # Three non-obvious flags: + # - CMAKE_INSTALL_RPATH=$ORIGIN + RPATH knobs: the tar.gz layout + # ships sibling .so files in the same directory as the binaries. + # - CMAKE_CUDA_ARCHITECTURES: explicit per-profile arch list, the + # whole point of the matrix. + # - CMAKE_*_COMPILER_LAUNCHER=ccache: Jimver installs nvcc outside + # /usr/local/bin, so PATH-symlinked ccache misses CUDA TUs -- + # setting the launcher explicitly caches nvcc too. + # No LLAMA_FATAL_WARNINGS on this leg, unlike cpu/vulkan/macos. Here it also + # sets nvcc -Werror all-warnings, and the host warnings arrive through an + # -Xcompiler list llama.cpp composes itself, which CMAKE_CXX_FLAGS does not + # reach -- verified on 08-27, where the suppression added that morning was + # absent from the actual nvcc command line. So the warning set here is + # all-or-nothing, and 'all' means every libstdc++ false positive fails a + # release: cuda13-newer and cuda13-portable both died on a GCC 11 + # -Wstringop-overflow inside std::copy, from ggml_cuda_try_fuse. + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_NATIVE=OFF \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_RPC=ON \ + -DGGML_CUDA=ON \ + -DGGML_CUDA_CUB_3DOT2=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_SERVER=ON \ + -DCMAKE_CUDA_ARCHITECTURES="$(echo '${{ matrix.archs }}' | tr ' ' ';')" \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache + + - name: Build + working-directory: src + run: | + set -eux + # Backend modules (CPU variants, CUDA, RPC) build as ggml dependencies, + # so every tool pulls them in. + # -j 3: this multi-arch nvcc build peaks at ~3 GB host RSS on the + # 4 vCPU / 16 GB GitHub-hosted runners (x64 and arm64 alike, measured + # over a 1s sampler), so 3 parallel TUs leave ~13 GB free. -j 4 adds + # no speed (the runner is ~2 physical cores + HT), so 3 is the sweet spot. + cmake --build build --config Release -j 3 + strip build/bin/llama-* || true + + # The DiffusionGemma visual server + cli are example targets that exist + # only when the source tree carries ggml-org/llama.cpp#24423 (a mix build). + # Build them best-effort and NEVER fail the job: the full tool bundle + # above must publish even if these do not compile on a given toolchain. + # package_bundle.py ships them only if they were produced. + - name: Build DiffusionGemma binaries (best-effort; mix builds only) + working-directory: src + run: | + set -u + if [ ! -d examples/diffusion-gemma-server ]; then + echo "no DiffusionGemma sources in this tree; skipping" + exit 0 + fi + cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON \ + || { echo "reconfigure for examples failed; skipping DiffusionGemma binaries"; exit 0; } + if cmake --build build --config Release -j 3 \ + --target llama-diffusion-gemma-visual-server llama-diffusion-cli; then + strip build/bin/llama-diffusion-gemma-visual-server build/bin/llama-diffusion-cli || true + echo "built DiffusionGemma binaries" + else + echo "warning: DiffusionGemma binaries failed to build; bundle will omit them" + fi + exit 0 + + - name: Package bundle + env: + PLATFORM: linux + ARCH: ${{ matrix.arch }} + BIN_DIR: ${{ github.workspace }}/src/build/bin + SRC_DIR: ${{ github.workspace }}/src + OUT_DIR: ${{ github.workspace }}/dist + TAG: ${{ inputs.tag }} + SOURCE_COMMIT: ${{ inputs.commit }} + SOURCE_REPO: ${{ inputs.repo }} + SOURCE_REF_KIND: ${{ inputs.repo == 'ggml-org/llama.cpp' && 'tag' || 'mix' }} + PROFILE: ${{ matrix.profile }} + LINE: ${{ matrix.line }} + KLASS: ${{ matrix.klass }} + RANK: ${{ matrix.rank }} + TOOLKIT_LINE: ${{ matrix.toolkit_line }} + DOCKER_IMAGE: github-hosted ${{ matrix.runner }}, CUDA ${{ matrix.cuda }} ${{ matrix.cuda == '13.3' && '(NVIDIA redist)' || '(Jimver)' }} + ARCHS: ${{ matrix.archs }} + SMS: ${{ matrix.sms }} + run: python3 tooling/scripts/unsloth/package_bundle.py + + - name: Upload bundle artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: app-${{ inputs.tag }}-linux-${{ matrix.arch }}-${{ matrix.profile }} + path: dist/app-${{ inputs.tag }}-linux-${{ matrix.arch }}-${{ matrix.profile }}.tar.gz + if-no-files-found: error + + - name: Evict stale ccache files + # !cancelled(), unlike the save below: on a timeout the job gets a + # single ~5 minute teardown window (measured ~4m50s after process + # kill), shared by every remaining step and not replenished. Evicting + # spends that window on housekeeping; the save is what actually needs + # it, and a 2 GB cache is not quick to write. + if: ${{ !cancelled() }} + continue-on-error: true + run: ccache --evict-older-than 14d + + - name: Save ccache + # Save even when the build failed: the objects compiled before the + # failure are still worth keeping, and a job that saves nothing leaves + # a hole in the cache lineage that widens the next run's tag gap. + # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. + # + # always(), not !cancelled(): a timeout-minutes expiry puts the job on + # the CANCELLATION path, not the failure path, so !cancelled() would + # skip the save on the single most expensive case -- a leg that + # compiled for hours and then hit the cap. + if: ${{ always() }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.ccache + key: ccache-cuda-${{ matrix.cuda }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-deadman.yml b/.github/workflows/unsloth-prebuilt-deadman.yml new file mode 100644 index 000000000000..9aa74bd69904 --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-deadman.yml @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: Unsloth prebuilt dead-man check + +# Watches the output, not the process: a run can go green while publishing +# nothing, which no failure-triggered alert can see. Upstream cuts releases +# several times a day, so two days of silence here is a fault, not a quiet +# upstream. + +on: + schedule: + - cron: '23 9 * * *' + workflow_dispatch: + +permissions: + contents: read + issues: write + +env: + STALE_DAYS: '2' + +jobs: + deadman: + name: Check publish freshness + runs-on: ubuntu-24.04 + steps: + - name: Checkout (for the composite action) + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: { persist-credentials: false } + + - name: Check newest published release + id: c + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + LATEST="$(gh api "repos/${GITHUB_REPOSITORY}/releases?per_page=30" \ + --jq '[.[] | select(.draft==false and .prerelease==false)] + | max_by(.published_at|fromdateiso8601) + | "\(.tag_name)\t\(.published_at)"' 2>/dev/null || true)" + + if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then + # Do not manufacture an alert from an API hiccup. + echo "::warning::could not read the release list; skipping this check" + echo "status=skip" >> "$GITHUB_OUTPUT" + exit 0 + fi + + TAG="${LATEST%%$'\t'*}" + PUB="${LATEST##*$'\t'}" + AGE_D=$(( ( $(date -u +%s) - $(date -u -d "$PUB" +%s) ) / 86400 )) + echo "newest release ${TAG} published ${PUB} (${AGE_D}d ago); threshold ${STALE_DAYS}d" + + if [ "$AGE_D" -le "${STALE_DAYS}" ]; then + echo "status=success" >> "$GITHUB_OUTPUT" + echo "details=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "status=failure" >> "$GITHUB_OUTPUT" + { + echo 'details<> "$GITHUB_OUTPUT" + + - name: Alert + if: ${{ steps.c.outputs.status != 'skip' }} + uses: ./.github/actions/prebuilt-alert + with: + status: ${{ steps.c.outputs.status }} + key: llama-prebuilt-stale + title: 'No llama.cpp prebuilt release published recently' + details: ${{ steps.c.outputs.details }} + token: ${{ github.token }} diff --git a/.github/workflows/unsloth-prebuilt-macos.yml b/.github/workflows/unsloth-prebuilt-macos.yml new file mode 100644 index 000000000000..2bfe3e832d80 --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-macos.yml @@ -0,0 +1,211 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: "Unsloth prebuilt: macOS" + +# Reusable child of unsloth-prebuilt.yml. Builds the macOS slices (arm64 Metal + +# x64 CPU) the parent's `resolve` job placed in `matrix`. Each entry uploads a +# single app-*.tar.gz artifact for the parent's assemble step to pick up. +# +# The only change over ggml-org's own macos build is an explicit +# -DCMAKE_OSX_DEPLOYMENT_TARGET per slice, so the binaries declare their load +# floor instead of inheriting the runner OS. Upstream stopped pinning this on +# arm64 (their macos-26 runner stamps minos=26, which fails to dyld-load on +# macOS < 26); owning the build is how we keep a loadable floor. + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + repo: + description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' + required: false + default: 'ggml-org/llama.cpp' + type: string + source_artifact: + description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' + required: false + default: '' + type: string + matrix: + description: 'Matrix JSON {include:[...]} produced by the parent resolve job' + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: ${{ matrix.build }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix) }} + steps: + - name: Checkout build tooling (this repo) + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + path: tooling + persist-credentials: false + + # The parent's resolve job built the source tree (upstream base + any mix + # PRs, with the build number/commit and Unsloth fingerprint baked + # into cmake/build-info.cmake) and uploaded it as an artifact; extract it + # instead of cloning -- no .git needed, the build number is already baked. + - name: Download source @ ${{ inputs.tag }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: ${{ inputs.source_artifact }} + path: srcpkg + - name: Extract source + shell: bash + run: | + set -eux + mkdir -p src + tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 + + # The macOS legs were the last build jobs without a compiler cache. Metal's + # backend is Objective-C, and the arm64 leg is the only one that builds it + # (the x64 leg passes -DGGML_METAL=OFF), so the OBJC launchers matter here + # in a way they do not elsewhere. Setting a launcher for a language this + # build never enables is inert, so both are set unconditionally. + - name: ccache + uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 + with: + key: macos-${{ matrix.build }}-${{ matrix.deploy_target }}-${{ inputs.tag }} + restore-keys: | + macos-${{ matrix.build }}-${{ matrix.deploy_target }} + append-timestamp: false + variant: ccache + max-size: 2G + save: false + + - name: Build (deployment target ${{ matrix.deploy_target }}) + working-directory: src + run: | + set -euo pipefail + cmake -B build \ + ${{ matrix.defines }} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_OBJC_COMPILER_LAUNCHER=ccache \ + -DCMAKE_OBJCXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ matrix.deploy_target }} \ + -DCMAKE_INSTALL_RPATH='@loader_path' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_RPC_RDMA=OFF + cmake --build build --config Release -j "$(sysctl -n hw.logicalcpu)" + + # ggml-rpc AUTO-DETECTS RDMA: on Apple it find_library(rdma)s and, when the + # runner has one, defaults GGML_RPC_RDMA to ON and links /usr/lib/librdma.dylib + # into libggml-rpc. That library ships with the runner image, not with macOS, + # so the bundle loaded on the builder and died everywhere else: + # + # dyld: Library not loaded: /usr/lib/librdma.dylib + # Referenced from: .../libggml-rpc.0.dylib + # + # (b10639-mix-f6f92fe, the first release carrying upstream b114b4739.) A + # redistributable must not have its configuration decided by whatever happened + # to be installed where it was built, so the value is pinned rather than + # detected. Pinned OFF specifically because RDMA-over-Thunderbolt is not + # something the prebuilt's consumers can use. + - name: Assert the RDMA transport stayed off + working-directory: src + run: | + set -euo pipefail + grep -q '^GGML_RPC_RDMA:BOOL=OFF$' build/CMakeCache.txt || { + echo "::error::GGML_RPC_RDMA is not OFF in the CMake cache; a macOS"\ + "prebuilt that links librdma cannot load on a consumer Mac" + grep -i 'rdma' build/CMakeCache.txt || true + exit 1 + } + # The cache says what was asked for; otool says what was linked. The + # launch gate below cannot see this, because it runs on the one host + # where the library does exist. + # No xargs -r: that is a GNU extension and these are BSD userland runners. + # An empty find just makes otool complain to a discarded stderr and grep + # match nothing, which is the answer we want anyway. + if find build/bin -type f \( -name '*.dylib' -o -perm -u+x \) -print0 \ + | xargs -0 otool -L 2>/dev/null | grep -i 'librdma'; then + echo "::error::a shipped Mach-O still links librdma" + exit 1 + fi + echo "rdma gate passed: GGML_RPC_RDMA=OFF and nothing links librdma" + + - name: Load gate (minos <= ${{ matrix.deploy_target }}, arch, launch) + run: bash tooling/scripts/unsloth/assert_macho_minos.sh src/build/bin "${{ matrix.expect_arch }}" "${{ matrix.deploy_target }}" + + # DiffusionGemma binaries (example targets present only in #24423 mix + # builds): best-effort, never fail the job. Built after the load gate so + # they do not affect the required-binary minos check; the same cached + # deployment target applies. The bundle tars all of build/bin, so anything + # produced here is shipped automatically. + - name: Build DiffusionGemma binaries (best-effort; mix builds only) + working-directory: src + run: | + set -u + if [ ! -d examples/diffusion-gemma-server ]; then + echo "no DiffusionGemma sources in this tree; skipping" + exit 0 + fi + cmake -B build -DLLAMA_BUILD_EXAMPLES=ON \ + || { echo "reconfigure for examples failed; skipping DiffusionGemma binaries"; exit 0; } + if cmake --build build --config Release -j "$(sysctl -n hw.logicalcpu)" \ + --target llama-diffusion-gemma-visual-server llama-diffusion-cli; then + echo "built DiffusionGemma binaries" + else + echo "warning: DiffusionGemma binaries failed to build; bundle will omit them" + fi + exit 0 + + - name: Package bundle + working-directory: src + run: | + set -euo pipefail + TAG="${{ inputs.tag }}" + cp LICENSE build/bin/ + mkdir -p "$GITHUB_WORKSPACE/dist" + # BSD tar (-s) rewrites the leading ./ to llama-/ so the archive + # unpacks into a named dir, matching upstream's own macos tarball layout. + tar -czf "$GITHUB_WORKSPACE/dist/llama-${TAG}-bin-macos-${{ matrix.build }}.tar.gz" \ + -s ",^\.,llama-${TAG}," -C build/bin . + + - name: Upload bundle artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: app-${{ inputs.tag }}-macos-${{ matrix.build }} + path: dist/llama-${{ inputs.tag }}-bin-macos-${{ matrix.build }}.tar.gz + if-no-files-found: error + + - name: Evict stale ccache files + # !cancelled(), unlike the save below: on a timeout the job gets a + # single ~5 minute teardown window (measured ~4m50s after process + # kill), shared by every remaining step and not replenished. Evicting + # spends that window on housekeeping; the save is what actually needs + # it, and a 2 GB cache is not quick to write. + if: ${{ !cancelled() }} + continue-on-error: true + run: ccache --evict-older-than 14d + + - name: Save ccache + # Save even when the build failed: the objects compiled before the + # failure are still worth keeping, and a job that saves nothing leaves + # a hole in the cache lineage that widens the next run's tag gap. + # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. + # + # always(), not !cancelled(): a timeout-minutes expiry puts the job on + # the CANCELLATION path, not the failure path, so !cancelled() would + # skip the save on the single most expensive case -- a leg that + # compiled for hours and then hit the cap. + if: ${{ always() }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.ccache + key: ccache-macos-${{ matrix.build }}-${{ matrix.deploy_target }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-retry.yml b/.github/workflows/unsloth-prebuilt-retry.yml new file mode 100644 index 000000000000..6f90e788baaa --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-retry.yml @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: Unsloth prebuilt retry + +# Re-runs a nightly that stopped for a reason outside the build itself. +# +# Two causes so far, both of which leave a finished build set unpublished: +# +# Cancelled with nothing failing. On 08-05 all 40 builds passed and the +# publish job was cancelled 15s in, with no failing job, no superseding run +# and nothing in the workflow that cancels. Every artifact was still there; +# one rerun published the release. +# +# A runner that dies mid-build. On 08-07 39 of 40 builds passed and +# `CUDA Windows / x64/cuda12-portable` failed after 78 minutes of nvcc with +# no compile error in its log, which simply stopped 46 minutes before the +# job ended. GitHub's own annotation was "The hosted runner lost +# communication with the server". assemble is `needs:` every build, so it +# skipped and no release was cut. +# +# Without this the pipeline sits on a finished build set until a human +# notices, which is the silent no-publish the alerting exists to catch, +# reached one step later. + +on: + workflow_run: + workflows: ['Unsloth prebuilt (full release)'] + types: [completed] + +permissions: + contents: read + actions: write + # The runner-loss test reads each failed job's annotation, which lives on + # the check run rather than in the job log. + checks: read + +jobs: + retry: + name: Rerun an externally broken run once + # run_attempt caps this at one retry: the rerun fires this workflow again + # as attempt 2, which no longer matches. A hand-cancelled workflow_dispatch + # is someone stopping their own build, so leave it stopped. + if: >- + (github.event.workflow_run.conclusion == 'cancelled' + || github.event.workflow_run.conclusion == 'failure') + && github.event.workflow_run.run_attempt == 1 + && github.event.workflow_run.event != 'workflow_dispatch' + runs-on: ubuntu-24.04 + steps: + - name: Rerun the run when nothing about the build actually failed + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: | + set -uo pipefail + + JOBS="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs" \ + --jq '.jobs[] | [.id, .conclusion, .name] | @tsv')" + + # A rerun of a run where nothing succeeded buys nothing: there is no + # salvageable work, and whatever stopped it will stop it again. + if ! cut -f2 <<<"$JOBS" | grep -qx 'success'; then + echo "not retrying ${RUN_ID}: nothing succeeded, so there is no work to salvage" \ + | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + FAILED="$(awk -F'\t' '$2 == "failure"' <<<"$JOBS")" + + # assemble waits on the build matrix from inside the job, so a leg + # that dies fails assemble too. Alongside another failure that is a + # consequence, not a cause; read it only when it failed alone. + # The name tracks assemble's `name:` in unsloth-prebuilt.yml. + INDEPENDENT="$(awk -F'\t' 'NF && $3 != "Assemble metadata + publish"' <<<"$FAILED")" + if [ -n "$INDEPENDENT" ]; then + FAILED="$INDEPENDENT" + fi + + # A cancellation that follows a real failure is a build problem to + # read, not to repeat. + if [ "$CONCLUSION" = cancelled ] && [ -n "$FAILED" ]; then + echo "not retrying ${RUN_ID}: it has a failed job, so the cancel is a consequence" \ + | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + # For an outright failure, retry only when every failed job died + # because its runner went away. A compile error repeats identically + # and must be read, not rerun; a lost runner is infrastructure and a + # rerun is the whole fix. Anything we cannot positively identify as + # runner loss counts as a real failure, so an unreadable annotation + # fails closed and the run stays put. + if [ "$CONCLUSION" = failure ]; then + if [ -z "$FAILED" ]; then + echo "not retrying ${RUN_ID}: it concluded failure with no failed job, so there is nothing to rerun" \ + | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + while IFS=$'\t' read -r JOB_ID _ JOB_NAME; do + [ -n "${JOB_ID:-}" ] || continue + NOTES="$(gh api "repos/${GITHUB_REPOSITORY}/check-runs/${JOB_ID}/annotations" \ + --jq '.[].message' 2>/dev/null)" + if ! grep -qF 'lost communication with the server' <<<"$NOTES"; then + echo "not retrying ${RUN_ID}: \`${JOB_NAME}\` failed for a reason other than runner loss, so read it rather than repeat it" \ + | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo "runner loss: ${JOB_NAME}" | tee -a "$GITHUB_STEP_SUMMARY" + done <<<"$FAILED" + fi + + if gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/rerun-failed-jobs"; then + echo "rerunning ${CONCLUSION} jobs of ${RUN_URL}" | tee -a "$GITHUB_STEP_SUMMARY" + else + echo "::warning::could not rerun ${RUN_URL}; rerun it by hand" | tee -a "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml new file mode 100644 index 000000000000..b650874f8ecf --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -0,0 +1,878 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: "Unsloth prebuilt: ROCm" + +# Reusable child of unsloth-prebuilt.yml. Per-gfx-target ROCm bundles on +# Windows + Ubuntu. +# +# Huge thanks to the lemonade-sdk team -- this is adapted from their workflow: +# https://github.com/lemonade-sdk/llamacpp-rocm/blob/main/.github/workflows/build-llamacpp-rocm.yml +# +# Build mechanics taken from that file: TheRock multi-arch nightly download + +# version auto-detect, gfx target name mapping, HIP/clang cmake invocation, the +# full hardcoded ROCm runtime lib copy list, patchelf RPATH. +# +# Local adaptations: +# - tag is passed in (the parent resolves it against ggml-org upstream) +# instead of lemonade's auto-incrementing b1000+ scheme. +# - artifacts pre-packaged into app---x64-rocm-.{tar.gz|zip} +# so the parent assemble step picks them up via the same merge-multiple +# download pattern it uses for CUDA bundles. +# - test jobs (stx-halo, stx) dropped -- those need self-hosted AMD runners. +# - build-summary + post-build cleanup steps dropped (clutter). + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + repo: + description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' + required: false + default: 'ggml-org/llama.cpp' + type: string + source_artifact: + description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' + required: false + default: '' + type: string + matrix: + description: 'gfx_target matrix JSON ({"gfx_target":[...]}), built by parent' + required: true + type: string + operating_systems: + description: 'OSes to build for (comma-separated: windows,ubuntu)' + required: false + default: 'windows,ubuntu' + type: string + rocm_version: + description: 'TheRock ROCm version (e.g., 10.1.0a20260807), "weekly" or "latest"' + required: false + default: 'weekly' + type: string + rocm_cutoff: + description: 'For "weekly": YYYYMMDD cutoff resolved once by the parent. Blank means each leg computes its own.' + required: false + default: '' + type: string + +permissions: + contents: read + +jobs: + build-windows: + name: windows/${{ matrix.gfx_target }} + runs-on: windows-2022 + if: contains(inputs.operating_systems, 'windows') + strategy: + matrix: ${{ fromJson(inputs.matrix) }} + fail-fast: false + + steps: + - name: Clean up existing directories (safety precaution) + run: | + # Remove existing llama.cpp directory if it exists + if (Test-Path "llama.cpp") { + Write-Host "Removing existing llama.cpp directory..." + Remove-Item -Recurse -Force "llama.cpp" + } + + # Remove existing C:\opt\rocm directory if it exists + if (Test-Path "C:\opt\rocm") { + Write-Host "Removing existing C:\opt\rocm directory..." + Remove-Item -Recurse -Force "C:\opt\rocm" + } + + # Remove any existing ROCm tarball + if (Test-Path "rocm.tar.gz") { + Write-Host "Removing existing rocm.tar.gz..." + Remove-Item -Force "rocm.tar.gz" + } + + Write-Host "Cleanup completed successfully" + + - name: Install Visual Studio Build Tools + run: | + # Retry helper: Invoke-WebRequest's -MaximumRetryCount only retries on + # HTTP status failures (400-599/304), not on the TCP-level connection + # timeouts these download hosts intermittently throw, so wrap each + # download in an explicit catch-all retry with linear backoff. + function Invoke-DownloadWithRetry { + param([string]$Uri, [string]$OutFile, [int]$Retries = 5, [int]$DelaySec = 10) + for ($i = 1; $i -le $Retries; $i++) { + try { + Invoke-WebRequest -Uri $Uri -OutFile $OutFile -ErrorAction Stop + return + } catch { + Write-Host "Download attempt $i/$Retries for $Uri failed: $($_.Exception.Message)" + if ($i -eq $Retries) { throw } + Start-Sleep -Seconds ($DelaySec * $i) + } + } + } + + # Download and install Visual Studio Build Tools + $vsInstallerUrl = "https://aka.ms/vs/17/release/vs_buildtools.exe" + $vsInstallerPath = "$env:TEMP\vs_buildtools.exe" + + Write-Host "Downloading Visual Studio Build Tools..." + Invoke-DownloadWithRetry -Uri $vsInstallerUrl -OutFile $vsInstallerPath + + Write-Host "Installing Visual Studio Build Tools..." + Start-Process -FilePath $vsInstallerPath -ArgumentList "--quiet", "--wait", "--norestart", "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "--add", "Microsoft.VisualStudio.Component.VC.CMake.Project", "--add", "Microsoft.VisualStudio.Component.VC.ATL", "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621" -Wait + + # Clean up installer + Remove-Item $vsInstallerPath -Force + + - name: Install build dependencies + run: | + Write-Host "Installing build dependencies using manual methods..." + + # Retry helper: Invoke-WebRequest's -MaximumRetryCount only retries on + # HTTP status failures (400-599/304), not on the TCP-level connection + # timeouts these download hosts intermittently throw, so wrap each + # download in an explicit catch-all retry with linear backoff. + function Invoke-DownloadWithRetry { + param([string]$Uri, [string]$OutFile, [int]$Retries = 5, [int]$DelaySec = 10) + for ($i = 1; $i -le $Retries; $i++) { + try { + Invoke-WebRequest -Uri $Uri -OutFile $OutFile -ErrorAction Stop + return + } catch { + Write-Host "Download attempt $i/$Retries for $Uri failed: $($_.Exception.Message)" + if ($i -eq $Retries) { throw } + Start-Sleep -Seconds ($DelaySec * $i) + } + } + } + + # Install Ninja + Write-Host "Installing Ninja..." + $ninjaUrl = "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip" + $ninjaPath = "$env:TEMP\ninja-win.zip" + $ninjaDir = "C:\ninja" + New-Item -ItemType Directory -Force -Path $ninjaDir + Invoke-DownloadWithRetry -Uri $ninjaUrl -OutFile $ninjaPath + Expand-Archive -Path $ninjaPath -DestinationPath $ninjaDir -Force + + # Install Strawberry Perl via Chocolatey (already on the runner) instead + # of downloading the MSI from the frequently-unreachable strawberryperl.com + # host that was timing out and failing these builds. It ships preinstalled + # on the windows-2022 image, so this is normally a fast no-op. + Write-Host "Installing Strawberry Perl via Chocolatey..." + choco install strawberryperl -y --no-progress + + # Verify installations + $env:PATH = "C:\ninja;C:\Strawberry\perl\bin;C:\Strawberry\c\bin;$env:PATH" + Write-Host "Verifying installations..." + ninja --version + perl --version + + Write-Host "Manual installation of build dependencies completed" + + - name: Download ROCm nightly tarball + run: | + # Retry helper: Invoke-WebRequest's -MaximumRetryCount only retries on + # HTTP status failures (400-599/304), not on the TCP-level connection + # timeouts these download hosts intermittently throw, so wrap each + # download in an explicit catch-all retry with linear backoff. + function Invoke-DownloadWithRetry { + param([string]$Uri, [string]$OutFile, [int]$Retries = 5, [int]$DelaySec = 10) + for ($i = 1; $i -le $Retries; $i++) { + try { + Invoke-WebRequest -Uri $Uri -OutFile $OutFile -ErrorAction Stop + return + } catch { + Write-Host "Download attempt $i/$Retries for $Uri failed: $($_.Exception.Message)" + if ($i -eq $Retries) { throw } + Start-Sleep -Seconds ($DelaySec * $i) + } + } + } + + # Determine ROCm version to use + $rocmVersion = "${{ inputs.rocm_version }}" + $currentTarget = "${{ matrix.gfx_target }}" + + # Map the build target to the matching TheRock archive family + $archiveTarget = $currentTarget + if ($currentTarget -eq "gfx103X" -or $currentTarget -eq "gfx110X" -or $currentTarget -eq "gfx120X") { + $archiveTarget = "$currentTarget-all" + Write-Host "Using target with -all suffix: $archiveTarget" + } + + # TheRock publishes nightlies to the multi-arch tarball index. The + # static HTML page embeds a JSON `files` array with names and mtimes. + $baseUrl = "https://rocm.nightlies.amd.com/tarball-multi-arch" + if ($rocmVersion -eq "latest" -or $rocmVersion -eq "weekly") { + # weekly: see the Linux job. + $cutoff = "99999999" + if ($rocmVersion -eq "weekly") { + $cutoff = "${{ inputs.rocm_cutoff }}" + if (-not $cutoff) { + # Windows uses its own zone ids; the IANA name is the fallback. + try { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("Pacific Standard Time") } + catch { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("America/Los_Angeles") } + $sf = [System.TimeZoneInfo]::ConvertTimeFromUtc([DateTime]::UtcNow, $tz) + $cutoff = $sf.Date.AddDays(-([int]$sf.DayOfWeek + 1)).ToString("yyyyMMdd") + } + Write-Host "Weekly pin: newest build dated on or before $cutoff (week of the last SF Sunday)" + } else { + Write-Host "Auto-detecting latest ROCm version for target: $currentTarget" + } + $indexHtml = (Invoke-WebRequest "$baseUrl/" -UseBasicParsing).Content + $filesMatch = [regex]::Match($indexHtml, 'const files = (\[.*?\]);', [System.Text.RegularExpressions.RegexOptions]::Singleline) + if (-not $filesMatch.Success) { + Write-Error "Failed to parse file index from $baseUrl/" + exit 1 + } + + # Pick the newest build date and exclude sibling test archives. + $allFiles = $filesMatch.Groups[1].Value | ConvertFrom-Json + $prefix = "therock-dist-windows-$archiveTarget-" + $versionPattern = "^$([regex]::Escape($prefix))\d+\.\d+\.\d+(a|rc)\d+\.tar\.gz$" + $latest = $allFiles | + Where-Object { $_.name -match $versionPattern } | + Where-Object { [regex]::Match($_.name, '(\d{8})\.tar\.gz$').Groups[1].Value -le $cutoff } | + Sort-Object { [regex]::Match($_.name, '(\d{8})\.tar\.gz$').Groups[1].Value } | + Select-Object -Last 1 + if (-not $latest) { + Write-Error "No tarball found for prefix '$prefix' at or before $cutoff at $baseUrl/" + exit 1 + } + $latestFile = $latest.name + Write-Host "Found latest file: $latestFile" + + # Extract version from the filename for environment variable + if ($latestFile -match "therock-dist-windows-$archiveTarget-(\d+\.\d+\.\d+(?:a|rc)\d+)\.tar\.gz") { + $rocmVersion = $matches[1] + Write-Host "Detected latest ROCm version: $rocmVersion" + } else { + Write-Error "Failed to extract ROCm version from latest file: $latestFile" + Write-Error "Expected pattern: therock-dist-windows-$archiveTarget-.tar.gz" + exit 1 + } + + $rocmUrl = "$baseUrl/$latestFile" + } else { + $rocmUrl = "$baseUrl/therock-dist-windows-$archiveTarget-$rocmVersion.tar.gz" + } + + # Store the version for use in other steps + echo "DETECTED_ROCM_VERSION=$rocmVersion" >> $env:GITHUB_ENV + + Write-Host "Downloading ROCm from: $rocmUrl" + Invoke-DownloadWithRetry -Uri $rocmUrl -OutFile "rocm.tar.gz" + + - name: Extract ROCm to C:\opt\rocm + run: | + # Create directory if it doesn't exist + New-Item -ItemType Directory -Force -Path "C:\opt\rocm" + + # Extract the tarball + tar -xzf rocm.tar.gz -C C:\opt\rocm --strip-components=1 + + # Keyed per gfx target and OS: the seven targets compile different device + # code from the same host sources, so a shared cache would mostly miss. + # save: false -- the explicit actions/cache/save step at the end of the job + # runs after packaging, so a failed package step does not persist a cache + # for a bundle that never shipped (same pattern as the CPU/CUDA children). + # + # The ROCm version is in the key, and in the restore prefix, because ccache hashes the compiler into every entry: a cache from another nightly can never hit. + - name: ccache + uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 + with: + key: rocm-windows-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }} + restore-keys: | + rocm-windows-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }} + append-timestamp: false + variant: ccache + max-size: 2G + save: false + + # The parent's resolve job built the source tree (upstream base + any mix + # PRs, with the build number/commit and Unsloth fingerprint baked + # into cmake/build-info.cmake) and uploaded it as an artifact; extract it + # instead of cloning -- no .git needed, the build number is already baked. + - name: Download source @ ${{ inputs.tag }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: ${{ inputs.source_artifact }} + path: srcpkg + - name: Extract source + shell: bash + run: | + set -eux + mkdir -p llama.cpp + tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C llama.cpp --strip-components=1 + + - name: Build Llama.cpp + ROCm + shell: cmd + run: | + + REM Map GPU targets + set "current_target=${{ matrix.gfx_target }}" + echo Input target: %current_target% + + if "%current_target%"=="gfx110X" ( + set "mapped_target=gfx1100;gfx1101;gfx1102;gfx1103" + ) else if "%current_target%"=="gfx103X" ( + set "mapped_target=gfx1030;gfx1031;gfx1032;gfx1034" + ) else if "%current_target%"=="gfx1151" ( + set "mapped_target=gfx1151" + ) else if "%current_target%"=="gfx1150" ( + set "mapped_target=gfx1150" + ) else if "%current_target%"=="gfx120X" ( + set "mapped_target=gfx1200;gfx1201" + ) else ( + set "mapped_target=%current_target%" + ) + echo Mapped target: %mapped_target% + + REM Set up environment variables and PATH + set HIP_PATH=C:\opt\rocm + set HIP_PLATFORM=amd + set PATH=%HIP_PATH%\lib\llvm\bin;%HIP_PATH%\bin;%PATH% + + REM Set up x64 Native Tools Command Prompt environment. + REM Pin to the VS 2022 line (-version "[17.0,18.0)") so vswhere -latest does not + REM pick up the runner image's VS 2026 (MSVC 14.51), whose STL adds constexpr to + REM math builtins like isgreater; that constexpr (implicitly __host__ __device__) + REM collides with clang's HIP __device__ forward declares and breaks the ggml-cuda + REM .cu compile (ggml-org/llama.cpp#22570). VS 2022 (MSVC 14.44) is unaffected. + call "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -latest -version "[17.0,18.0)" -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath > vs_path.txt + set /p VS_PATH= 68% hit rate, gap 13 -> 6%. + # + # always(), not !cancelled(): a timeout-minutes expiry puts the job on + # the CANCELLATION path, not the failure path, so !cancelled() would + # skip the save on the single most expensive case -- a leg that + # compiled for hours and then hit the cap. + if: ${{ always() }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}\.ccache + key: ccache-rocm-windows-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }}- + + build-ubuntu: + name: linux/${{ matrix.gfx_target }} + runs-on: ubuntu-22.04 + if: contains(inputs.operating_systems, 'ubuntu') + strategy: + matrix: ${{ fromJson(inputs.matrix) }} + fail-fast: false + + steps: + - name: Free disk space + # Remove unused runner files to free up disk space + run: curl -fsSL https://raw.githubusercontent.com/kou/arrow/e49d8ae15583ceff03237571569099a6ad62be32/ci/scripts/util_free_space.sh | bash + + - name: Clean up existing directories (safety precaution) + run: | + # Remove existing llama.cpp directory if it exists + if [ -d "llama.cpp" ]; then + echo "Removing existing llama.cpp directory..." + rm -rf llama.cpp + fi + + # Remove existing /opt/rocm directory if it exists + if [ -d "/opt/rocm" ]; then + echo "Removing existing /opt/rocm directory..." + sudo rm -rf /opt/rocm + fi + + # Remove any existing ROCm tarball + if [ -f "rocm.tar.gz" ]; then + echo "Removing existing rocm.tar.gz..." + rm -f rocm.tar.gz + fi + + echo "Cleanup completed successfully" + + - name: Install build dependencies + run: | + echo "Installing build dependencies..." + sudo apt update + sudo apt install -y cmake ninja-build unzip curl + + # Verify installations + echo "Verifying installations..." + cmake --version + ninja --version + echo "Build dependencies installation completed" + + - name: Download and extract ROCm directly to /opt/rocm + run: | + # Determine ROCm version to use + rocm_version="${{ inputs.rocm_version }}" + current_target="${{ matrix.gfx_target }}" + + # Map the build target to the matching TheRock archive family + archive_target="$current_target" + if [[ "$current_target" = "gfx103X" || "$current_target" = "gfx110X" || "$current_target" = "gfx120X" ]]; then + archive_target="${current_target}-all" + echo "Using target with -all suffix: $archive_target" + fi + + # TheRock publishes nightlies to the multi-arch tarball index. The + # static HTML page embeds a JSON `files` array with names and mtimes. + base_url="https://rocm.nightlies.amd.com/tarball-multi-arch" + if [ "$rocm_version" = "latest" ] || [ "$rocm_version" = "weekly" ]; then + # weekly: take the newest alpha up to the Saturday before the last SF Sunday, so a whole week uses one toolchain and the ccache hits. + # The cutoff day must be settled before the first run reads it: TheRock usually publishes the evening before, but has landed as late as 18:33 PT on the named day, which splits the week. + # The parent sends one cutoff for the run, so every leg agrees; the fallback is for a standalone call. + cutoff=99999999 + if [ "$rocm_version" = "weekly" ]; then + cutoff="${{ inputs.rocm_cutoff }}" + [ -n "$cutoff" ] || cutoff="$(TZ=America/Los_Angeles date -d "-$(( $(TZ=America/Los_Angeles date +%w) + 1 )) days" +%Y%m%d)" + echo "Weekly pin: newest build dated on or before $cutoff (week of the last SF Sunday)" + else + echo "Auto-detecting latest ROCm version for target: $current_target" + fi + prefix="therock-dist-linux-${archive_target}-" + files_json=$(curl -s "$base_url/" | tr '\n' ' ' | grep -oP 'const files = \K\[.*?\](?=\s*;)') + if [ -z "$files_json" ]; then + echo "Failed to parse file index from $base_url/" + exit 1 + fi + + # Pick the newest build date and exclude sibling test archives. + latest_file=$(echo "$files_json" | jq -r --arg p "$prefix" --arg c "$cutoff" \ + '[.[] | select(.name | test("^" + $p + "[0-9]+\\.[0-9]+\\.[0-9]+(a|rc)[0-9]+\\.tar\\.gz$")) | select((.name | capture("(?[0-9]{8})\\.tar\\.gz$").d) <= $c)] | sort_by(.name | capture("(?[0-9]{8})\\.tar\\.gz$").d) | last | .name // empty') + if [ -z "$latest_file" ]; then + echo "No tarball found for prefix '$prefix' at or before $cutoff at $base_url/" + exit 1 + fi + echo "Found latest file: $latest_file" + + # Extract version from the filename for environment variable + if [[ "$latest_file" =~ therock-dist-linux-${archive_target}-([0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+)\.tar\.gz ]]; then + rocm_version="${BASH_REMATCH[1]}" + echo "Detected latest ROCm version: $rocm_version" + else + echo "Failed to extract ROCm version from latest file: $latest_file" + echo "Expected pattern: therock-dist-linux-${archive_target}-.tar.gz" + exit 1 + fi + + rocm_url="$base_url/$latest_file" + else + rocm_url="$base_url/therock-dist-linux-${archive_target}-${rocm_version}.tar.gz" + fi + + # Store the version for use in other steps + echo "DETECTED_ROCM_VERSION=$rocm_version" >> $GITHUB_ENV + + echo "Streaming ROCm from: $rocm_url directly to extraction" + + # Create directory if it doesn't exist + sudo mkdir -p /opt/rocm + + # Stream download directly into tar extraction (no intermediate file) + curl -sL "$rocm_url" | sudo tar --use-compress-program=gzip -xf - -C /opt/rocm --strip-components=1 + + - name: Set ROCm environment variables + run: | + echo "Setting ROCm environment variables..." + + # Set environment variables for this step and subsequent steps + echo "HIP_PATH=/opt/rocm" >> $GITHUB_ENV + echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV + echo "HIP_PLATFORM=amd" >> $GITHUB_ENV + echo "HIP_CLANG_PATH=/opt/rocm/llvm/bin" >> $GITHUB_ENV + echo "HIP_INCLUDE_PATH=/opt/rocm/include" >> $GITHUB_ENV + echo "HIP_LIB_PATH=/opt/rocm/lib" >> $GITHUB_ENV + echo "HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode" >> $GITHUB_ENV + + # Update PATH + echo "/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH" >> $GITHUB_PATH + + # Set library paths + echo "LD_LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV + echo "LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:${LIBRARY_PATH:-}" >> $GITHUB_ENV + echo "CPATH=/opt/rocm/include:${CPATH:-}" >> $GITHUB_ENV + echo "PKG_CONFIG_PATH=/opt/rocm/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> $GITHUB_ENV + + echo "ROCm environment variables set successfully" + + # See the Windows job for why the key is per gfx target and ROCm version, and why save: false. + - name: ccache + uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 + with: + key: rocm-linux-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }} + restore-keys: | + rocm-linux-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }} + append-timestamp: false + variant: ccache + max-size: 2G + save: false + + # The action sets this on Windows and macOS but leaves Linux on mtime, and ROCm is re-extracted every run, so mtime is not a reliable compiler identity. + - name: Hash the compiler by content, not mtime + run: ccache --set-config=compiler_check=content + + # The parent's resolve job built the source tree (upstream base + any mix + # PRs, with the build number/commit and Unsloth fingerprint baked + # into cmake/build-info.cmake) and uploaded it as an artifact; extract it + # instead of cloning -- no .git needed, the build number is already baked. + - name: Download source @ ${{ inputs.tag }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: ${{ inputs.source_artifact }} + path: srcpkg + - name: Extract source + shell: bash + run: | + set -eux + mkdir -p llama.cpp + tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C llama.cpp --strip-components=1 + + - name: Build Llama.cpp + ROCm + run: | + # Map GPU targets + current_target="${{ matrix.gfx_target }}" + echo "Input target: $current_target" + + if [ "$current_target" = "gfx110X" ]; then + mapped_target="gfx1100;gfx1101;gfx1102;gfx1103" + elif [ "$current_target" = "gfx103X" ]; then + mapped_target="gfx1030;gfx1031;gfx1032;gfx1034" + elif [ "$current_target" = "gfx1151" ]; then + mapped_target="gfx1151" + elif [ "$current_target" = "gfx1150" ]; then + mapped_target="gfx1150" + elif [ "$current_target" = "gfx120X" ]; then + mapped_target="gfx1200;gfx1201" + else + mapped_target="$current_target" + fi + echo "Mapped target: $mapped_target" + + # Create build directory + cd llama.cpp + mkdir build + cd build + + # Configure the project + # CMAKE_HIP_COMPILER_LAUNCHER is needed here but not on Windows: the + # CXX compiler is ROCm's clang++ (not hipcc), so ggml-hip leaves + # CXX_IS_HIPCC false and calls enable_language(HIP), putting the device + # sources on the HIP language rather than CXX. + cmake .. -G Ninja \ + -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang \ + -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ + -DCMAKE_CXX_FLAGS="-I/opt/rocm/include" \ + -DCMAKE_CROSSCOMPILING=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DGPU_TARGETS="$mapped_target" \ + -DBUILD_SHARED_LIBS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DGGML_HIP=ON \ + -DGGML_OPENMP=OFF \ + -DGGML_CUDA_FORCE_CUBLAS=OFF \ + -DGGML_RPC=ON \ + -DGGML_HIP_ROCWMMA_FATTN=OFF \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_STATIC=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_HIP_COMPILER_LAUNCHER=ccache \ + -DCMAKE_SYSTEM_NAME=Linux + + # Build the project + cmake --build . -j $(nproc) + + - name: Copy ROCm core libs to build directory + run: | + build_bin_path="llama.cpp/build/bin" + rocm_bin_path="/opt/rocm/bin" + + # Copy the rocblas/library folder and all its contents + rocblas_lib_path="/opt/rocm/lib/rocblas/library" + if [ -d "$rocblas_lib_path" ]; then + echo "Copying rocblas/library folder and all contents..." + dest_rocblas_path="$build_bin_path/rocblas/library" + mkdir -p "$(dirname "$dest_rocblas_path")" + cp -r "$rocblas_lib_path" "$(dirname "$dest_rocblas_path")/" + echo "Copied: rocblas/library folder with all contents" + + # List the contents of the copied rocblas/library folder + echo "Contents of rocblas/library:" + find "$dest_rocblas_path" -type f -exec ls -la {} \; | head -20 + else + echo "Warning: rocblas/library folder not found at: $rocblas_lib_path" + fi + + # Copy the hipblaslt/library folder and all its contents + hipblaslt_lib_path="/opt/rocm/lib/hipblaslt/library" + if [ -d "$hipblaslt_lib_path" ]; then + echo "Copying hipblaslt/library folder and all contents..." + dest_hipblaslt_path="$build_bin_path/hipblaslt/library" + mkdir -p "$(dirname "$dest_hipblaslt_path")" + cp -r "$hipblaslt_lib_path" "$(dirname "$dest_hipblaslt_path")/" + echo "Copied: hipblaslt/library folder with all contents" + + # List the contents of the copied hipblaslt/library folder + echo "Contents of hipblaslt/library:" + find "$dest_hipblaslt_path" -type f -exec ls -la {} \; | head -20 + else + echo "Warning: hipblaslt/library folder not found at: $hipblaslt_lib_path" + fi + + # Copy required ROCm libraries to build directory + # If artifacts from ROCm or Llama.cpp change, you may need to update this list + # To get a new list of all libraries, run: + # gather_required_libs.py --rocm-dir /opt/rocm --dest-dir llama.cpp/build/bin + echo "Copying required ROCm libraries to build directory..." + cp -v /opt/rocm/lib/libhipblas.so* "$build_bin_path/" 2>/dev/null || echo "libhipblas.so* not found" + cp -v /opt/rocm/lib/librocblas.so* "$build_bin_path/" 2>/dev/null || echo "librocblas.so* not found" + cp -v /opt/rocm/lib/libamdhip64.so* "$build_bin_path/" 2>/dev/null || echo "libamdhip64.so* not found" + cp -v /opt/rocm/lib/librocsolver.so* "$build_bin_path/" 2>/dev/null || echo "librocsolver.so* not found" + cp -v /opt/rocm/lib/libroctx64.so* "$build_bin_path/" 2>/dev/null || echo "libroctx64.so* not found" + cp -v /opt/rocm/lib/libhipblaslt.so* "$build_bin_path/" 2>/dev/null || echo "libhipblaslt.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_liblzma.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_liblzma.so* not found" + cp -v /opt/rocm/lib/librocprofiler-register.so* "$build_bin_path/" 2>/dev/null || echo "librocprofiler-register.so* not found" + cp -v /opt/rocm/lib/libamd_comgr.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr.so* not found" + cp -v /opt/rocm/lib/libamd_comgr_loader.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr_loader.so* not found" + cp -v /opt/rocm/lib/libhsa-runtime64.so* "$build_bin_path/" 2>/dev/null || echo "libhsa-runtime64.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_numa.so* not found" + cp -v /opt/rocm/lib/librocroller.so* "$build_bin_path/" 2>/dev/null || echo "librocroller.so* not found" + cp -v /opt/rocm/lib/liborigami.so* "$build_bin_path/" 2>/dev/null || echo "liborigami.so* not found" + cp -v /opt/rocm/lib/librocm_kpack.so* "$build_bin_path/" 2>/dev/null || echo "librocm_kpack.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_z.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_zstd.so* not found" + cp -v /opt/rocm/lib/llvm/lib/libLLVM.so* "$build_bin_path/" 2>/dev/null || echo "libLLVM.so* not found" + cp -v /opt/rocm/lib/llvm/lib/libclang-cpp.so* "$build_bin_path/" 2>/dev/null || echo "libclang-cpp.so* not found" + + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_elf.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm_amdgpu.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm_amdgpu.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_bz2.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_bz2.so* not found" + + # libatomic.so.1 is a transitive dependency of librocm_sysdeps_numa + # (libhsa-runtime64 -> numa -> libatomic) but it is a system GCC runtime + # library, not shipped under /opt/rocm. Bundle it so the runtime stays + # self-contained on hosts that lack it. lemonade-sdk/lemonade#1349 hit + # exactly this ("libatomic.so.1: cannot open shared object file"). + sudo apt-get install -y libatomic1 >/dev/null 2>&1 || true + libatomic_path="$(ldconfig -p | awk -F'=> ' '/libatomic\.so\.1/{print $2; exit}')" + cp -v "$libatomic_path" "$build_bin_path/" 2>/dev/null || echo "libatomic.so.1 not found" + + echo "Finished copying required ROCm libraries" + + - name: Set RPATH for portable distribution + run: | + sudo apt-get install -y patchelf + cd llama.cpp/build/bin + # Set RPATH to $ORIGIN so all libraries (including the comgr stub loader) find deps locally + for file in *.so* llama-*; do + [ -f "$file" ] && [ ! -L "$file" ] && patchelf --set-rpath '$ORIGIN' "$file" 2>/dev/null || true + done + + - name: List build artifacts (including ROCm files) + run: | + cd llama.cpp/build/bin + echo "Final build artifacts (including ROCm library files):" + ls -la + + - name: Package bundle (tar.gz) + run: | + set -eux + ASSET="app-${{ inputs.tag }}-linux-x64-rocm-${{ matrix.gfx_target }}.tar.gz" + mkdir -p dist + (cd llama.cpp/build/bin && tar -czf "${GITHUB_WORKSPACE}/dist/${ASSET}" .) + ls -la dist + + - name: Upload bundle artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: app-${{ inputs.tag }}-linux-x64-rocm-${{ matrix.gfx_target }} + path: dist/app-${{ inputs.tag }}-linux-x64-rocm-${{ matrix.gfx_target }}.tar.gz + if-no-files-found: error + + # See the Windows job: logs whether the HIP-language TUs actually cache. + - name: ccache stats + continue-on-error: true + run: ccache --show-stats -v + + - name: Evict stale ccache files + # !cancelled(), unlike the save below: on a timeout the job gets a + # single ~5 minute teardown window (measured ~4m50s after process + # kill), shared by every remaining step and not replenished. Evicting + # spends that window on housekeeping; the save is what actually needs + # it, and a 2 GB cache is not quick to write. + if: ${{ !cancelled() }} + continue-on-error: true + run: ccache --evict-older-than 14d + + - name: Save ccache + # Save even when the build failed: the objects compiled before the + # failure are still worth keeping, and a job that saves nothing leaves + # a hole in the cache lineage that widens the next run's tag gap. + # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. + # + # always(), not !cancelled(): a timeout-minutes expiry puts the job on + # the CANCELLATION path, not the failure path, so !cancelled() would + # skip the save on the single most expensive case -- a leg that + # compiled for hours and then hit the cap. + if: ${{ always() }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.ccache + key: ccache-rocm-linux-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-vulkan.yml b/.github/workflows/unsloth-prebuilt-vulkan.yml new file mode 100644 index 000000000000..2d1d1631b443 --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-vulkan.yml @@ -0,0 +1,409 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: "Unsloth prebuilt: Vulkan" + +# Reusable child of unsloth-prebuilt.yml. Builds the Vulkan bundles for +# Linux x64/arm64 + Windows x64. Each matrix entry uploads a single +# app-*.{tar.gz|zip} artifact for the parent's assemble step to pick up. +# +# Mirrors oobabooga/llama-cpp-binaries' build-wheels-vulkan.yml build recipe +# (the CPU recipe plus GGML_VULKAN=ON and a Vulkan SDK install), adapted to +# this repo's conventions: app----vulkan archives packaged +# straight from build/bin like the ROCm/macOS children (no embedded +# UNSLOTH_PREBUILT_INFO.json -- assemble_metadata.py derives the manifest entry +# from the filename), MSVC + BoringSSL on Windows, $ORIGIN RPATH on Linux. +# +# The Vulkan loader (libvulkan.so.1 / vulkan-1.dll) is a system/driver library +# resolved by the OS at runtime, so -- like llama-cpp-binaries -- it is not +# bundled; only ggml's own libggml-vulkan backend module ships in the archive. +# Both Linux legs use ubuntu-22.04 to keep a glibc 2.35 / GLIBCXX <= 3.4.30 +# floor. The arm64 leg supplies newer header-only SDK pieces at build time. + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + repo: + description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' + required: false + default: 'ggml-org/llama.cpp' + type: string + source_artifact: + description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' + required: false + default: '' + type: string + +permissions: + contents: read + +env: + # LunarG SDK version used by ggml-org's Windows release build. + VULKAN_VERSION: 1.4.357.0 + +jobs: + build-linux: + name: linux/${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { arch: x64, runner: ubuntu-22.04 } + - { arch: arm64, runner: ubuntu-22.04-arm } + steps: + # The parent's resolve job built the source tree (upstream base + any mix + # PRs, with the build number/commit and Unsloth fingerprint baked + # into cmake/build-info.cmake) and uploaded it as an artifact; extract it + # instead of cloning -- no .git needed, the build number is already baked. + - name: Download source @ ${{ inputs.tag }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: ${{ inputs.source_artifact }} + path: srcpkg + - name: Extract source + shell: bash + run: | + set -eux + mkdir -p src + tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 + + - name: Install Vulkan SDK + build dependencies + run: | + set -eux + if [ "${{ matrix.arch }}" = x64 ]; then + wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add - + sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list + sudo apt-get update -y + sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libssl-dev ninja-build + else + sudo apt-get update -y + sudo apt-get install -y build-essential libvulkan-dev spirv-headers libssl-dev ninja-build + fi + + # Match the CPU arm64 leg so the bundle retains Jammy's loader and + # libstdc++ floors. Jammy's gcc cannot target armv9.2-a+sme. + - name: Toolchain (clang 19 on arm64) + if: matrix.arch == 'arm64' + run: | + set -eux + wget -q https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 19 + sudo apt-get install -y libomp-19-dev + { + echo "CC=clang-19" + echo "CXX=clang++-19" + } >> "$GITHUB_ENV" + + - name: ccache + uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 + with: + key: vulkan-linux-${{ matrix.arch }}-${{ inputs.tag }} + restore-keys: | + vulkan-linux-${{ matrix.arch }} + append-timestamp: false + variant: ccache + max-size: 2G + save: false + + # Ubuntu 22.04 has an arm64 Vulkan loader, but its headers are too old for + # the current backend and it has no glslc package. Install matching pinned + # headers and build the shader compiler shipped with the LunarG SDK. + - name: Install Vulkan headers (arm64) + if: matrix.arch == 'arm64' + run: | + set -eux + package="$RUNNER_TEMP/vulkan-headers.deb" + staging="$RUNNER_TEMP/vulkan-headers" + curl -fsSL \ + "https://packages.lunarg.com/vulkan/pool/main/v/vulkan-headers/vulkan-headers_1.4.313.0~rc1-1lunarg22.04-1_all.deb" \ + -o "$package" + echo "587b2d8e79416b394170ab61557c98765570cd153730f819a917866d78f45e1a $package" | sha256sum -c - + dpkg-deb -x "$package" "$staging" + sudo cp -a "$staging/usr/." /usr/local/ + + - name: Build glslc (arm64) + if: matrix.arch == 'arm64' + run: | + set -eux + archive="$RUNNER_TEMP/shaderc.tar.gz" + source="$RUNNER_TEMP/shaderc" + curl -fsSL \ + "https://packages.lunarg.com/vulkan/pool/main/s/shaderc/shaderc_2025.2~rc1-1lunarg22.04.orig.tar.gz" \ + -o "$archive" + echo "59c0c478f2f40a076e610587d099e39ed059cb7319fe464f8ba1bd07c6bf02c5 $archive" | sha256sum -c - + mkdir -p "$source" + tar -xzf "$archive" -C "$source" + cmake -S "$source" -B "$source/build" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DSHADERC_SKIP_TESTS=ON \ + -DSHADERC_SKIP_EXAMPLES=ON \ + -DSHADERC_SKIP_COPYRIGHT_CHECK=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + cmake --build "$source/build" --target glslc_exe -j "$(nproc)" + sudo install -m 0755 "$source/build/glslc/glslc" /usr/local/bin/glslc + glslc --version + + - name: Configure + working-directory: src + run: | + set -eux + # Build recipe mirrors llama-cpp-binaries' Vulkan wheel: the CPU recipe + # (backend-DL + all CPU variants + RPC) plus GGML_VULKAN. RPATH=$ORIGIN + # so the bundle's sibling .so files resolve from the binary's own dir. + # LLAMA_FATAL_WARNINGS below is -Werror. The arm64 image compiles with + # clang-19 against GCC 12's libstdc++, where std::stable_sort still + # reaches the deprecated get_temporary_buffer; GCC buries that in a + # system header, clang reports it at our instantiation. That failed + # this leg on 08-27 over a deprecation in code we do not own, so that + # one diagnostic is off. Every other warning stays fatal. + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_NATIVE=OFF \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_RPC=ON \ + -DGGML_VULKAN=ON \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_SERVER=ON \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + + - name: Build + working-directory: src + run: | + set -eux + # Backend modules (CPU variants, Vulkan, RPC) build as ggml + # dependencies, so every tool pulls them in. + cmake --build build --config Release -j "$(nproc)" + strip build/bin/llama-* || true + + - name: Bundle OpenMP runtime (arm64) + if: matrix.arch == 'arm64' + run: cp /usr/lib/llvm-19/lib/libomp.so.5 src/build/bin/ + + # DiffusionGemma binaries (example targets present only in #24423 mix + # builds): best-effort, never fail the job. See the CUDA child for the + # rationale. The bundle tars all of build/bin, so anything produced here + # is shipped automatically. + - name: Build DiffusionGemma binaries (best-effort; mix builds only) + working-directory: src + run: | + set -u + if [ ! -d examples/diffusion-gemma-server ]; then + echo "no DiffusionGemma sources in this tree; skipping" + exit 0 + fi + cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON \ + || { echo "reconfigure for examples failed; skipping DiffusionGemma binaries"; exit 0; } + if cmake --build build --config Release -j "$(nproc)" \ + --target llama-diffusion-gemma-visual-server llama-diffusion-cli; then + strip build/bin/llama-diffusion-gemma-visual-server build/bin/llama-diffusion-cli || true + echo "built DiffusionGemma binaries" + else + echo "warning: DiffusionGemma binaries failed to build; bundle will omit them" + fi + exit 0 + + - name: Package bundle (tar.gz) + run: | + set -eux + ASSET="app-${{ inputs.tag }}-linux-${{ matrix.arch }}-vulkan.tar.gz" + cp src/LICENSE src/build/bin/ + mkdir -p dist + (cd src/build/bin && tar -czf "${GITHUB_WORKSPACE}/dist/${ASSET}" .) + ls -la dist + + - name: Upload bundle artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: app-${{ inputs.tag }}-linux-${{ matrix.arch }}-vulkan + path: dist/app-${{ inputs.tag }}-linux-${{ matrix.arch }}-vulkan.tar.gz + if-no-files-found: error + + - name: Evict stale ccache files + # !cancelled(), unlike the save below: on a timeout the job gets a + # single ~5 minute teardown window (measured ~4m50s after process + # kill), shared by every remaining step and not replenished. Evicting + # spends that window on housekeeping; the save is what actually needs + # it, and a 2 GB cache is not quick to write. + if: ${{ !cancelled() }} + continue-on-error: true + run: ccache --evict-older-than 14d + + - name: Save ccache + # Save even when the build failed: the objects compiled before the + # failure are still worth keeping, and a job that saves nothing leaves + # a hole in the cache lineage that widens the next run's tag gap. + # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. + # + # always(), not !cancelled(): a timeout-minutes expiry puts the job on + # the CANCELLATION path, not the failure path, so !cancelled() would + # skip the save on the single most expensive case -- a leg that + # compiled for hours and then hit the cap. + if: ${{ always() }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.ccache + key: ccache-vulkan-linux-${{ matrix.arch }}-${{ inputs.tag }}- + + build-windows: + name: windows/x64 + runs-on: windows-2022 + defaults: + run: + shell: pwsh + steps: + # The parent's resolve job built the source tree (upstream base + any mix + # PRs, with the build number/commit and Unsloth fingerprint baked + # into cmake/build-info.cmake) and uploaded it as an artifact; extract it + # instead of cloning -- no .git needed, the build number is already baked. + - name: Download source @ ${{ inputs.tag }} + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: ${{ inputs.source_artifact }} + path: srcpkg + - name: Extract source + shell: bash + run: | + set -eux + mkdir -p src + tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 + + - name: ccache + uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 + with: + key: vulkan-windows-x64-${{ inputs.tag }} + restore-keys: | + vulkan-windows-x64 + append-timestamp: false + variant: ccache + max-size: 2G + save: false + + - name: Install Ninja + run: choco install ninja --no-progress + + - name: Setup MSVC + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 + + - name: Install Vulkan SDK + run: | + curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe" + & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install + Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}" + Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin" + + - name: Configure + working-directory: src + run: | + # Same recipe as Linux plus BoringSSL (statically linked, no system + # OpenSSL on the runner). No RPATH -- Windows loads sibling DLLs from + # the binary's directory. CMAKE_PREFIX_PATH points at the Vulkan SDK + # so CMake finds its SPIRV-Headers config (set via env, not -D, so the + # backslashes survive); mirrors llama-cpp-binaries' Vulkan wheel. + # GGML_OPENMP=OFF keeps this MSVC bundle self-contained: a default-ON + # MSVC build would link vcomp140.dll (not shipped). Upstream sidesteps + # this by building its Windows Vulkan artifact with GGML_CPU=OFF (no + # OpenMP at all); we ship a full bundle, so we disable OpenMP instead + # (the CPU backend is a GPU-offload fallback and uses ggml's threadpool). + $env:CMAKE_PREFIX_PATH = $env:VULKAN_SDK + cmake -S . -B build -G Ninja ` + -DCMAKE_BUILD_TYPE=Release ` + -DGGML_NATIVE=OFF ` + -DGGML_BACKEND_DL=ON ` + -DGGML_CPU_ALL_VARIANTS=ON ` + -DGGML_OPENMP=OFF ` + -DGGML_RPC=ON ` + -DGGML_VULKAN=ON ` + -DLLAMA_BUILD_TESTS=OFF ` + -DLLAMA_BUILD_EXAMPLES=OFF ` + -DLLAMA_BUILD_TOOLS=ON ` + -DLLAMA_BUILD_SERVER=ON ` + -DLLAMA_BUILD_BORINGSSL=ON ` + -DCMAKE_C_COMPILER_LAUNCHER=ccache ` + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + + - name: Build + working-directory: src + run: | + cmake --build build --config Release -j 3 + + # DiffusionGemma binaries (#24423 mix builds only): best-effort, never + # fail the job. See the CUDA child for the rationale. + - name: Build DiffusionGemma binaries (best-effort; mix builds only) + working-directory: src + run: | + if (-not (Test-Path "examples/diffusion-gemma-server")) { + Write-Host "no DiffusionGemma sources in this tree; skipping" + exit 0 + } + cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON + if ($LASTEXITCODE -ne 0) { + Write-Host "reconfigure for examples failed; skipping DiffusionGemma binaries" + exit 0 + } + cmake --build build --config Release -j 3 ` + --target llama-diffusion-gemma-visual-server llama-diffusion-cli + if ($LASTEXITCODE -ne 0) { + Write-Host "warning: DiffusionGemma binaries failed to build; bundle will omit them" + } else { + Write-Host "built DiffusionGemma binaries" + } + exit 0 + + - name: Package bundle (zip) + shell: bash + run: | + set -eux + ASSET="app-${{ inputs.tag }}-windows-x64-vulkan.zip" + cp src/LICENSE src/build/bin/ + mkdir -p dist + (cd src/build/bin && 7z a -tzip "${GITHUB_WORKSPACE}/dist/${ASSET}" .) + ls -la dist + + - name: Upload bundle artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: app-${{ inputs.tag }}-windows-x64-vulkan + path: dist/app-${{ inputs.tag }}-windows-x64-vulkan.zip + if-no-files-found: error + + - name: Evict stale ccache files + # !cancelled(), unlike the save below: on a timeout the job gets a + # single ~5 minute teardown window (measured ~4m50s after process + # kill), shared by every remaining step and not replenished. Evicting + # spends that window on housekeeping; the save is what actually needs + # it, and a 2 GB cache is not quick to write. + if: ${{ !cancelled() }} + continue-on-error: true + run: ccache --evict-older-than 14d + + - name: Save ccache + # Save even when the build failed: the objects compiled before the + # failure are still worth keeping, and a job that saves nothing leaves + # a hole in the cache lineage that widens the next run's tag gap. + # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. + # + # always(), not !cancelled(): a timeout-minutes expiry puts the job on + # the CANCELLATION path, not the failure path, so !cancelled() would + # skip the save on the single most expensive case -- a leg that + # compiled for hours and then hit the cap. + if: ${{ always() }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}\.ccache + key: ccache-vulkan-windows-x64-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt.yml b/.github/workflows/unsloth-prebuilt.yml new file mode 100644 index 000000000000..834ff5449093 --- /dev/null +++ b/.github/workflows/unsloth-prebuilt.yml @@ -0,0 +1,1317 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: Unsloth prebuilt (full release) + +# Atomic daily build for unslothai/llama.cpp. One cron, one workflow run, one +# release per upstream b#### tag. Splits the heavy build work into six reusable +# children: +# unsloth-prebuilt-cuda.yml -- Linux CUDA bundles (x64 + arm64, matrix profiles) +# unsloth-prebuilt-cuda-windows.yml -- CUDA Windows bundles (x64, matrix profiles) +# unsloth-prebuilt-rocm.yml -- ROCm bundles (Windows + Ubuntu, per gfx target) +# unsloth-prebuilt-macos.yml -- macOS bundles (arm64 Metal + x64 CPU) +# unsloth-prebuilt-cpu.yml -- CPU-only bundles (Linux + Windows, x64 + arm64) +# unsloth-prebuilt-vulkan.yml -- Vulkan bundles (Linux x64/arm64 + Windows x64) +# +# Atomicity: the `assemble` job depends on all children. GitHub's default +# `needs` semantics require all needs to succeed -- if any matrix entry in +# any child fails, `assemble` skips and no release is published. The +# installer needs the full bundle set at the same tag or it'll dispatch to +# something that isn't there. +# +# Mix builds: scripts/unsloth/pr-set.json can list ggml-org/llama.cpp or +# unslothai/llama.cpp PRs (each pinned to an exact commit) to merge into the +# build. `resolve` merges the open ones onto the base tag and uploads the +# merged tree as a workflow artifact that the children extract instead of +# cloning; the release is tagged b####-mix-. Empty list = vanilla +# upstream build. + +on: + schedule: + - cron: '13 20 * * *' # ~1PM San Francisco PDT / ~12PM PST (UTC; not DST adjusted) + workflow_dispatch: + inputs: + tag: + description: 'ggml-org tag (b#### or "latest")' + default: 'latest' + required: true + type: string + min_age_hours: + description: 'For "latest": only build a release public for at least this many hours (blank = default 6)' + default: '' + required: false + type: string + only_profile: + description: 'CUDA profile to build' + default: 'all' + required: false + type: choice + options: [all, cuda12-legacy, cuda12-older, cuda12-newer, cuda12-portable, cuda13-older, cuda13-newer, cuda13-portable] + operating_systems: + description: 'OSes for ROCm builds' + default: 'windows,ubuntu' + required: false + type: string + gfx_target: + description: 'GPU targets for ROCm builds' + default: 'gfx1151,gfx1150,gfx120X,gfx110X,gfx103X,gfx90a,gfx908' + required: false + type: string + rocm_version: + description: 'ROCm version, "weekly" (newest alpha as of the last SF Sunday) or "latest"' + default: 'weekly' + required: false + type: string + publish: + description: 'Publish to GitHub Releases' + default: false + required: false + type: boolean + keep_artifacts: + description: 'Keep this run''s artifacts even if it publishes nothing (artifact-only test runs)' + default: false + required: false + type: boolean + +permissions: + contents: write + +env: + # Supply-chain aging: when resolving "latest", only build an upstream release + # that has been public for at least this many hours, so a malicious or broken + # release has time to be caught and yanked before we compile and ship it. An + # explicit b#### tag (manual run) skips it. Per-run override via min_age_hours. + UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS: "6" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.tag || 'scheduled' }} + # A nightly still in flight when the next one starts is stuck, not busy: a + # healthy run finishes in a couple of hours. Leaving this false lets one + # wedged run block every following nightly with nothing to show for it. + # Only the schedule supersedes; a manual dispatch never kills a live nightly. + cancel-in-progress: ${{ github.event_name == 'schedule' }} + +jobs: + resolve: + name: Resolve tag + runs-on: ubuntu-24.04 + # Read-only: resolve merges third-party PR content but pushes nothing; + # only assemble needs the workflow-level contents:write (publish). + permissions: + contents: read + outputs: + tag: ${{ steps.r.outputs.tag }} + repo: ${{ steps.r.outputs.repo }} + base: ${{ steps.r.outputs.base }} + prs: ${{ steps.r.outputs.prs }} + source_artifact: ${{ steps.r.outputs.source_artifact }} + commit: ${{ steps.r.outputs.commit }} + ggml_tree: ${{ steps.r.outputs.ggml_tree }} + ggml_version: ${{ steps.r.outputs.ggml_version }} + exists: ${{ steps.r.outputs.exists }} + cuda_matrix: ${{ steps.r.outputs.cuda_matrix }} + win_cuda_matrix: ${{ steps.r.outputs.win_cuda_matrix }} + rocm_matrix: ${{ steps.r.outputs.rocm_matrix }} + rocm_cutoff: ${{ steps.r.outputs.rocm_cutoff }} + macos_matrix: ${{ steps.r.outputs.macos_matrix }} + env: + GH_TOKEN: ${{ github.token }} + steps: + # Shallow by default (only pr-set.json is needed); a mix build unshallows + # in-place to merge the PRs. + - name: Checkout (pr-set.json + mix merge workspace) + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - id: r + run: | + set -euo pipefail + REQ='${{ github.event.inputs.tag || 'latest' }}' + ONLY='${{ github.event.inputs.only_profile || 'all' }}' + GFX_DEFAULT='gfx1151,gfx1150,gfx120X,gfx110X,gfx103X,gfx90a,gfx908' + GFX='${{ github.event.inputs.gfx_target }}'; GFX="${GFX:-$GFX_DEFAULT}" + OS_LIST='${{ github.event.inputs.operating_systems || 'windows,ubuntu' }}' + AGE_H='${{ github.event.inputs.min_age_hours }}' + + # publish does delete-then-create on the whole release, and the installer + # treats the published manifest as the authoritative bundle set: a partial + # build drops bundles and silently downgrades uncovered hosts to slow + # source builds. Refuse to publish unless the full default set is built; + # publish=false test runs may use subsets. + if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ inputs.publish }}" = "true" ]; then + [ "$ONLY" = "all" ] || { echo "refusing to publish only_profile=$ONLY: a partial CUDA set clobbers manifest coverage. Use only_profile=all to publish, or publish=false for an artifact-only test." >&2; exit 1; } + [ "$GFX" = "$GFX_DEFAULT" ] || { echo "refusing to publish gfx_target='$GFX': publish requires the full default set ($GFX_DEFAULT); use publish=false for a subset test." >&2; exit 1; } + case "$OS_LIST" in + *windows*ubuntu*|*ubuntu*windows*) : ;; + *) echo "refusing to publish operating_systems='$OS_LIST': both windows and ubuntu ROCm bundles are required. Use publish=false for a subset test." >&2; exit 1 ;; + esac + fi + [ -n "$AGE_H" ] || AGE_H="${UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS:-6}" + if [ "$REQ" = "latest" ]; then + # Newest published b#### build release that has been public for + # >= AGE_H hours -- the supply-chain aging window. GitHub does not + # guarantee the list order, so pick the max by published_at + # explicitly rather than trusting `first`. + # Select on the tag shape, not on prerelease: since 08-21 upstream + # marks the b#### builds prerelease and keeps that flag clear only + # for the semver v#.#.# releases, which are not what we build. + CUTOFF="$(date -u -d "-${AGE_H} hours" +%s)" + BASE="$(gh api 'repos/ggml-org/llama.cpp/releases?per_page=100' \ + | jq -r --argjson cutoff "$CUTOFF" '[.[] | select(.draft==false) | select(.tag_name|test("^b[0-9]+$")) | select((.published_at|fromdateiso8601) <= $cutoff)] | max_by(.published_at|fromdateiso8601) | .tag_name')" + if [ -z "$BASE" ] || [ "$BASE" = "null" ]; then + echo "refusing: no ggml-org b#### release older than ${AGE_H}h in the last 100 releases" >&2 + exit 1 + fi + echo "selected $BASE (aged >= ${AGE_H}h)" + else + BASE="$REQ" # explicit b#### override skips the aging filter + fi + printf '%s' "$BASE" | grep -qE '^b[0-9]+$' || { echo "refusing non-release tag '$BASE'" >&2; exit 1; } + + # Resolve the PR mix set (scripts/unsloth/pr-set.json): PR commit + # urls from ggml-org/llama.cpp or unslothai/llama.cpp (no other + # repos). Pins are mandatory -- only an exact, reviewed commit + # is ever built, so a PR author pushing more commits cannot change + # what the nightly ships. Non-open PRs are still merged in -- upstream + # tags lag merges, so dropping a pin on merge leaves the arch in + # neither the base nor the mix; see the state gate below and the + # pr-set.json _doc. unsloth-pr-set-lint.yml runs the same checks on every + # push that edits the file, but that is only a tripwire -- a red + # lint does not stop the schedule, so the gate must live here. + # An entry is a bare url string (required) or {"url", "required": false}. + # Bare strings stay valid, so the file needs no migration. + jq -e '.prs | type == "array" and all(.[]; + type == "string" + or (type == "object" and (.url | type == "string") + and ((if .required == null then true else .required end) | type == "boolean")))' \ + scripts/unsloth/pr-set.json >/dev/null \ + || { echo "scripts/unsloth/pr-set.json: .prs must be an array of PR url strings, or {url, required} objects" >&2; exit 1; } + PRS='[]' + URL_RE='^https://github\.com/(ggml-org|unslothai)/llama\.cpp/pull/([0-9]+)/commits/([0-9a-f]{40})/?$' + while read -r url REQUIRED; do + [[ "$url" =~ $URL_RE ]] || { echo "refusing malformed PR url '$url' (expected https://github.com/{ggml-org,unslothai}/llama.cpp/pull//commits/<40-hex-sha>)" >&2; exit 1; } + SRC="${BASH_REMATCH[1]}/llama.cpp"; NUM="${BASH_REMATCH[2]}"; SHA="${BASH_REMATCH[3]}" + # Abort naming the entry on a gh failure (typo'd numbers 404). + # Title can contain spaces, so it can't ride a space-delimited + # read -- pull each field out on its own. + PR_JSON="$(gh api "repos/${SRC}/pulls/${NUM}")" \ + || { echo "refusing ${SRC}#${NUM}: could not fetch PR metadata (nonexistent PR number in '$url', or a transient API failure); fix the pin in scripts/unsloth/pr-set.json or retry" >&2; exit 1; } + STATE="$(jq -r '.state' <<<"$PR_JSON")" + HEAD="$(jq -r '.head.sha' <<<"$PR_JSON")" + N_COMMITS="$(jq -r '.commits' <<<"$PR_JSON")" + TITLE="$(jq -r '.title' <<<"$PR_JSON")" + MERGED_AT="$(jq -r '.merged_at // ""' <<<"$PR_JSON")" + if [ "$STATE" != "open" ]; then + # Merging upstream does NOT put an arch in the build: upstream tags + # lag their merges, and BASE is then aged a further + # UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS on top of that. 26841 merged + # at 11:07 with the newest tag cut at 07:53, so dropping the pin on + # merge left the arch in neither the base nor the mix, and repinning + # was impossible because the old gate refused a non-open pin. So a + # non-open pin keeps being merged: once BASE contains the commit the + # merge is an empty no-op, and the pin can be deleted at leisure. + # "required": false keeps the old rot-away behaviour for an entry + # that should disappear the moment it stops being open. + if [ "$REQUIRED" = "false" ]; then + echo "::warning::skipping optional pin ${SRC}#${NUM} (${STATE}): $url" + continue + fi + if [ -n "$MERGED_AT" ]; then + echo "::warning::${SRC}#${NUM} merged upstream at ${MERGED_AT}; still mixing its pinned commit until a base tag contains it" + else + # Closed unmerged means upstream declined it. Nothing stops it + # shipping now, so the warning is the only signal -- drop the pin + # once you have decided you do not want that code. + echo "::warning::${SRC}#${NUM} is closed without being merged (upstream declined it); still mixing its pinned commit because the entry is required" + fi + fi + # A pin pasted from the wrong PR would build arbitrary code while + # the manifest blames PR #; require the pinned commit to be + # a commit of that PR. The commits listing is capped at 250 by + # the API; past that, skip rather than false-fail. + if [ "$N_COMMITS" -gt 250 ]; then + echo "note: ${SRC}#${NUM} has ${N_COMMITS} commits (over the API listing cap); skipping pin membership check" + elif ! gh api "repos/${SRC}/pulls/${NUM}/commits" --paginate --jq '.[].sha' | grep -qx "$SHA"; then + echo "refusing ${SRC}#${NUM}: pinned commit ${SHA} is not a commit of that PR (wrong paste, or force-pushed away); fix the pin in scripts/unsloth/pr-set.json" >&2 + exit 1 + fi + [ "$SHA" = "$HEAD" ] || echo "note: ${SRC}#${NUM} is pinned to ${SHA} but its head has moved to ${HEAD}" + echo "including ${SRC}#${NUM} @ ${SHA}" + PRS="$(jq -c --arg r "$SRC" --arg n "$NUM" --arg s "$SHA" --arg u "$url" --arg t "$TITLE" '. + [{repo: $r, number: ($n|tonumber), sha: $s, url: $u, title: $t}]' <<<"$PRS")" + done < <(jq -r '.prs[] | if type == "string" then {url: ., required: true} else . end + | "\(.url)\t\(if .required == null then true else .required end)"' scripts/unsloth/pr-set.json | tr '\t' ' ') + + # Decide the tag and source repo first; the source tree is built + # further down, only if this release isn't already published. + if [ "$(jq length <<<"$PRS")" = 0 ]; then + # Plain build: pristine upstream base, no merge. REPO stays the real + # upstream repo even though we ship our own stamped tarball. + TAG="$BASE" + REPO='ggml-org/llama.cpp' + else + # The synthetic tag embeds a hash of the pinned repo#number:sha + # triples in listed order (merge order can matter for conflicts), so + # a pin update or a reorder yields a new tag and build, while the + # "exists" check below still skips rebuilding an already-published + # set. The repo is part of the key so PR #n in the two repos can't + # hash to the same set. The merged tree exists only as this repo's + # release assets, never upstream. + SETHASH="$(jq -r 'map("\(.repo)#\(.number):\(.sha)") | join("\n")' <<<"$PRS" | sha256sum | cut -c1-7)" + TAG="${BASE}-mix-${SETHASH}" + REPO="$GITHUB_REPOSITORY" + fi + SRC_ARTIFACT="app-source-${TAG}" + COMMIT="" + GGML_TREE="" + GGML_VERSION="" + + # Only PUBLISHED releases count as "exists"; a leftover draft (from + # a failed prior publish) should not block today's rebuild. + EXISTS=false + if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft 2>/dev/null || true)" = "false" ]; then + EXISTS=true + fi + + # Build the source tree every child compiles, ONCE, here -- but only + # when something will actually be built. On a scheduled no-op (the aged + # "latest" was already published) skip the whole checkout/merge/tar + + # upload; a manual dispatch always rebuilds. Check out the upstream base, + # merge any pinned PRs (mix builds), then bake the build number/commit + # and the Unsloth fingerprint into cmake/build-info.cmake. The + # result is uploaded as the app-source-* artifact every child extracts -- + # one identical tree everywhere, no child clones, fingerprint patch in one + # place. Nothing is pushed (GITHUB_TOKEN may never push commits touching + # .github/workflows, which upstream history routinely does). + if [ "$EXISTS" != "true" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + git remote add upstream https://github.com/ggml-org/llama.cpp.git + # The upstream checkout below takes scripts/unsloth/ away. Copy the + # whole dir out, not file by file: see the note above the step. + cp -r scripts/unsloth "${RUNNER_TEMP}/us" + ADDITIVE_MERGE="${RUNNER_TEMP}/us/additive_merge.py" + if [ "$(jq length <<<"$PRS")" != 0 ]; then + # Merges need a merge-base, so unshallow first. + if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then + git fetch -q --unshallow --no-tags origin + fi + git fetch -q --no-tags upstream "refs/tags/${BASE}:refs/tags/${BASE}" + git checkout -q --detach "refs/tags/${BASE}" + # Each pin is fetched from the repo its PR lives in (repo/number/ + # sha are machine fields, so the space-delimited read is safe; + # titles stay out of this loop). + while read -r SRC NUM SHA; do + # The PR's own repo first, then our refs/pins mirror, which is + # the only copy left once an author force-pushes the reviewed + # commit out of the PR. That is what took the nightly down on + # 07-31, and it is unrecoverable without a ref of our own. + git fetch -q --no-tags "https://github.com/${SRC}.git" "$SHA" 2>/dev/null \ + || git fetch -q --no-tags origin "refs/pins/${SHA}" 2>/dev/null \ + || { echo "could not fetch commit ${SHA} for ${SRC}#${NUM}; it is gone from ${SRC} and was never mirrored to refs/pins -- update or remove the pin in scripts/unsloth/pr-set.json" >&2; exit 1; } + if ! git rev-parse --verify -q "${SHA}^{commit}" >/dev/null; then + echo "fetched something for ${SRC}#${NUM} but ${SHA} is still missing" >&2; exit 1 + fi + # diff3 so additive_merge.py can see the merge base and refuse + # anything that is not a pure add/add. + if ! git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + -c merge.conflictStyle=diff3 \ + merge --no-ff --no-edit -m "Merge ${SRC}#${NUM} @ ${SHA}" "$SHA"; then + # The recurring conflict is two PRs adding a line to the same + # architecture table, where the answer is always "keep both". + # additive_merge.py resolves only that, and refuses when + # either side edited existing text, so a real disagreement + # still hard-fails here rather than being papered over. + if python3 "$ADDITIVE_MERGE" \ + && [ -z "$(git diff --name-only --diff-filter=U)" ]; then + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit -q --no-edit + echo "::warning::${SRC}#${NUM} needed an additive merge; every conflict was a pure add/add and both sides were kept" + else + git merge --abort 2>/dev/null + echo "${SRC}#${NUM} (${SHA}) does not merge cleanly onto ${BASE} + the PRs listed before it; reorder or drop it in scripts/unsloth/pr-set.json" >&2; exit 1 + fi + fi + done < <(jq -r '.[] | "\(.repo) \(.number) \(.sha)"' <<<"$PRS") + echo "MERGED_PINS=1" >> "$GITHUB_ENV" + else + # Plain build: only the base tree is needed. Shallow is fine -- the + # build number is baked below, so no git history is needed at build time. + git fetch -q --depth 1 --no-tags upstream "refs/tags/${BASE}:refs/tags/${BASE}" + git checkout -q --detach "refs/tags/${BASE}" + fi + COMMIT="$(git rev-parse HEAD)" + # ABI key for anything compiled against our ggml (whisper.cpp slim + # bundles). The tree id changes only when ggml/ contents change, so + # a release that touches nothing under ggml/ does not force a + # rebuild downstream. The -mix- tag suffix is a hash of the PR set, + # not a ggml identity: it stays constant while the base tag moves. + GGML_TREE="$(git rev-parse HEAD:ggml)" + GGML_VERSION="$(sed -nE 's/^set\(GGML_VERSION_(MAJOR|MINOR|PATCH) ([0-9]+)\)$/\2/p' ggml/CMakeLists.txt | paste -sd. -)" + # Warn, do not fail: these only tighten downstream pairing, and + # whisper falls back to the old comparison when they are absent. + # Killing a 39-job release over a metadata field would be worse + # than publishing without it. + printf '%s' "$GGML_TREE" | grep -qE '^[0-9a-f]{40}$' \ + || { echo "::warning::could not resolve the ggml tree id"; GGML_TREE=""; } + printf '%s' "$GGML_VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' \ + || { echo "::warning::could not parse GGML_VERSION from ggml/CMakeLists.txt"; GGML_VERSION=""; } + echo "ggml tree ${GGML_TREE:-unknown} (version ${GGML_VERSION:-unknown})" + + # The tarball ships without .git, where cmake/build-info.cmake falls + # back to its defaults; bake real values into those defaults so + # llama-server --version doesn't report b0 (unknown). The build number + # stays the BASE's b#### number (the upstream tag's commit count), so + # anything comparing versions against upstream releases keeps working; + # BUILD_COMMIT identifies the (possibly merged) head. + COUNT="${BASE#b}" + SHORT="$(git rev-parse --short HEAD)" + sed -i "s/^set(BUILD_NUMBER 0)$/set(BUILD_NUMBER ${COUNT})/" cmake/build-info.cmake + sed -i "s/^set(BUILD_COMMIT \"unknown\")$/set(BUILD_COMMIT \"${SHORT}\")/" cmake/build-info.cmake + grep -q "set(BUILD_NUMBER ${COUNT})" cmake/build-info.cmake \ + && grep -q "set(BUILD_COMMIT \"${SHORT}\")" cmake/build-info.cmake \ + || { echo "cmake/build-info.cmake no longer has the expected fallback lines; cannot bake the build number into the source artifact" >&2; exit 1; } + + # Unsloth fingerprint: fold "Compiled by the Unsloth team" + # into BUILD_TARGET, which cmake bakes into LLAMA_BUILD_TARGET, so it + # prints on llama-server --version ("built with ... for ...") and shows + # up in `strings` on every binary that links common. Append so it wraps + # whatever BUILD_TARGET the build computes. Keep this string byte-identical + # to MARK in the assemble job's verify gate, which re-checks it landed. + grep -q 'set(BUILD_TARGET' cmake/build-info.cmake \ + || { echo "cmake/build-info.cmake has no BUILD_TARGET line to stamp" >&2; exit 1; } + printf '\n# Unsloth fingerprint: shows in --version and strings.\nset(BUILD_TARGET "${BUILD_TARGET} (Compiled by the Unsloth team)")\n' >> cmake/build-info.cmake + grep -q 'Compiled by the Unsloth team' cmake/build-info.cmake \ + || { echo "failed to stamp the Unsloth fingerprint into cmake/build-info.cmake" >&2; exit 1; } + + tar -czf "${RUNNER_TEMP}/llama.cpp-source-${TAG}.tar.gz" --exclude-vcs --transform "s,^\.,llama.cpp-${TAG}," . + echo "prepared ${TAG} (${COMMIT}, build ${COUNT})" + else + echo "release ${TAG} already published; skipping source prep (nothing to build)" + fi + + # ubuntu-22.04 is x64 (glibc 2.35). arm64 only has ubuntu-24.04-arm + # available on GitHub-hosted runners (glibc 2.39). arm64 is cuda13-only + # (no cuda12 SBSA) and ships the single "portable" coverage class. + # cuda12 installs via Jimver; cuda13 is pinned to 13.3 (matching + # upstream) which Jimver lacks, so those install via NVIDIA redist in + # the build children -- on the same runners, so the glibc floor holds. + ALL='[ + {"profile":"cuda12-legacy", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"legacy", "rank":5, "cuda":"12.8.0","toolkit_line":"12.8","archs":"50-virtual 61-virtual","sms":"50 52 60 61"}, + {"profile":"cuda12-older", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"older", "rank":10,"cuda":"12.8.0","toolkit_line":"12.8","archs":"70 75 80 86 89"}, + {"profile":"cuda12-newer", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"newer", "rank":20,"cuda":"12.8.0","toolkit_line":"12.8","archs":"86 89 90 100 120"}, + {"profile":"cuda12-portable","arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"portable","rank":30,"cuda":"12.8.0","toolkit_line":"12.8","archs":"70 75 80 86 89 90 100 120"}, + {"profile":"cuda13-older", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda13","klass":"older", "rank":40,"cuda":"13.3","toolkit_line":"13.3","archs":"75 80 86 89"}, + {"profile":"cuda13-newer", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda13","klass":"newer", "rank":50,"cuda":"13.3","toolkit_line":"13.3","archs":"86 89 90 100 120"}, + {"profile":"cuda13-portable","arch":"x64", "runner":"ubuntu-22.04", "line":"cuda13","klass":"portable","rank":60,"cuda":"13.3","toolkit_line":"13.3","archs":"75 80 86 89 90 100 120"}, + {"profile":"cuda13-portable","arch":"arm64","runner":"ubuntu-24.04-arm","line":"cuda13","klass":"portable","rank":60,"cuda":"13.3","toolkit_line":"13.3","archs":"90 100 120 121"} + ]' + if [ "$ONLY" = "all" ]; then + CUDA_INCLUDE="$(echo "$ALL" | jq -c .)" + else + CUDA_INCLUDE="$(echo "$ALL" | jq -c --arg p "$ONLY" '[.[] | select(.profile==$p)]')" + fi + # CUDA Windows reuses the x64 profiles (same arch lists / CUDA + # versions), just on a Windows runner. arm64 has no CUDA Windows target. + WIN_CUDA_INCLUDE="$(echo "$CUDA_INCLUDE" | jq -c '[.[] | select(.arch=="x64") | .runner="windows-2022"]')" + [ -n "$GFX" ] || { echo "refusing empty gfx_target (would publish a CUDA-only release labeled CUDA + ROCm)" >&2; exit 1; } + ROCM_MATRIX="$(jq -cn --arg g "$GFX" '{gfx_target: ($g | split(",") | map(gsub("^\\s+|\\s+$"; "")))}')" + + # One weekly cutoff for the whole run, before fan-out. See the ROCm child for why it is the Saturday before. + # Each leg reads its own clock otherwise, so a run crossing the SF Sat-to-Sun boundary would mix two toolchains in one release. + ROCM_CUTOFF="$(TZ=America/Los_Angeles date -d "-$(( $(TZ=America/Los_Angeles date +%w) + 1 )) days" +%Y%m%d)" + + # macOS slices are static: two fixed runners with per-slice deployment + # targets. arm64 builds on macos-26 (newest Metal SDK; avoids the + # M5/A19 "error compiling source" the macos-14 SDK emits) while both + # slices pin 13.3 to match upstream's Ventura compatibility floor. + MACOS_INCLUDE='[ + {"build":"arm64","runner":"macos-26", "expect_arch":"arm64", "deploy_target":"13.3","defines":"-DGGML_METAL_EMBED_LIBRARY=ON"}, + {"build":"x64", "runner":"macos-15-intel","expect_arch":"x86_64","deploy_target":"13.3","defines":"-DGGML_METAL=OFF"} + ]' + MACOS_INCLUDE="$(echo "$MACOS_INCLUDE" | jq -c .)" + + { + echo "tag=$TAG" + echo "repo=$REPO" + echo "base=$BASE" + echo "prs=$PRS" + echo "source_artifact=$SRC_ARTIFACT" + echo "commit=$COMMIT" + echo "ggml_tree=$GGML_TREE" + echo "ggml_version=$GGML_VERSION" + echo "exists=$EXISTS" + echo "cuda_matrix={\"include\":$CUDA_INCLUDE}" + echo "win_cuda_matrix={\"include\":$WIN_CUDA_INCLUDE}" + echo "rocm_matrix=$ROCM_MATRIX" + echo "rocm_cutoff=$ROCM_CUTOFF" + echo "macos_matrix={\"include\":$MACOS_INCLUDE}" + } >> "$GITHUB_OUTPUT" + echo "Resolved $REQ -> $TAG ($COMMIT); prs=$PRS; source_artifact=${SRC_ARTIFACT:-none}; release exists=$EXISTS; only=$ONLY; gfx=$GFX" + + # A bad pin resolution can still build fine, so it must be caught before the source artifact ships. See merge_checks.py. + # Its own step, not more script in `resolve`: GitHub caps one workflow string at 21000 chars and that step is near it. See check_workflow_scalars.py. + # That is also why `resolve` copies all of scripts/unsloth/ to ${RUNNER_TEMP}/us in one line rather than one cp per script: every check added + # here would otherwise cost another line inside the capped block, and going over silently disables the whole workflow. + - name: Check the merged tree for silently wrong resolutions + if: ${{ env.MERGED_PINS == '1' }} + run: | + set -euo pipefail + if ! python3 "${RUNNER_TEMP}/us/merge_checks.py" --root . ; then + echo "::error::the pinned PRs merged, but merge_checks.py found a resolution that is silently wrong; see the log for file and line" >&2 + exit 1 + fi + + # merge_checks.py asserts the ABSENCE of two known-bad shapes. This asserts the PRESENCE of what each pin carries, which is a different question and + # the one that goes unanswered when a pin rots into a no-op or a resolution quietly drops an arch registration. Free, so it runs before the compile gate. + - name: Check every pin still contributes what it carries + if: ${{ env.MERGED_PINS == '1' }} + # Through env, never interpolated into the script: `prs` carries PR + # titles, which are third-party text, and `${{ }}` pastes them into the + # shell source before bash ever sees it. + env: + PRS: ${{ steps.r.outputs.prs }} + BASE: ${{ steps.r.outputs.base }} + run: | + set -euo pipefail + if ! python3 "${RUNNER_TEMP}/us/pin_contract.py" --root . --base "$BASE" \ + --prs-json "$PRS" --report "${RUNNER_TEMP}/pin_contract.json" ; then + echo "::error::the pinned PRs merged, but the merged tree is missing code a pin carries; see the log for the pin and file" >&2 + exit 1 + fi + + # The gap this closes, observed 09-03: ggml-org#27754 merged with zero conflicts and did not compile, because upstream had added a parameter to + # build_attn_mha and the pin's new build_attn_sparse still called the old signature. Nothing before this point can see that, and without it the release + # dies in the CUDA leg after the 38-job fan-out. CPU only: a cold `llama` build took 59s at -j4 with no ccache, against 20-60 minutes for a CUDA build. + # mtmd is in the gate because `llama` alone is not enough: observed 09-04, ggml-org#25731 built `llama` clean while tools/mtmd did not compile at all, + # upstream having made mtmd_image_preprocessor::preprocess const while the pin's Inkling subclass stayed non-const, so it overrode nothing and the + # vision and audio towers were abstract. Every vision pin lands in mtmd, so a gate that skips it cannot see the whole class. + - name: Compile gate (CPU, llama and mtmd targets) + if: ${{ env.MERGED_PINS == '1' }} + run: | + set -euo pipefail + cmake -B "${RUNNER_TEMP}/gate" -DCMAKE_BUILD_TYPE=Release \ + -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_CURL=OFF > /dev/null + if ! cmake --build "${RUNNER_TEMP}/gate" --target llama mtmd -j "$(nproc)" ; then + echo "::error::the pinned PRs merged cleanly and the merged tree does not compile; fix or drop the pin rather than letting the build matrix find this" >&2 + exit 1 + fi + + # The stamped source tree (every build): every build child extracts this + # instead of cloning, and assemble ships it as the release's source-tarball + # asset, so a source build reproduces the same fingerprinted binary. Only + # produced when we build, so guard on the same condition as the build jobs + # (resolve skips source prep on a scheduled no-op, leaving no tarball). + - name: Upload source artifact + if: ${{ steps.r.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ steps.r.outputs.source_artifact }} + path: ${{ runner.temp }}/llama.cpp-source-${{ steps.r.outputs.tag }}.tar.gz + if-no-files-found: error + retention-days: 7 + + build-cuda: + name: CUDA + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-cuda.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + repo: ${{ needs.resolve.outputs.repo }} + source_artifact: ${{ needs.resolve.outputs.source_artifact }} + commit: ${{ needs.resolve.outputs.commit }} + matrix: ${{ needs.resolve.outputs.cuda_matrix }} + + build-windows-cuda: + name: CUDA Windows + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-cuda-windows.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + repo: ${{ needs.resolve.outputs.repo }} + source_artifact: ${{ needs.resolve.outputs.source_artifact }} + commit: ${{ needs.resolve.outputs.commit }} + matrix: ${{ needs.resolve.outputs.win_cuda_matrix }} + + build-rocm: + name: ROCm + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-rocm.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + repo: ${{ needs.resolve.outputs.repo }} + source_artifact: ${{ needs.resolve.outputs.source_artifact }} + matrix: ${{ needs.resolve.outputs.rocm_matrix }} + operating_systems: ${{ github.event.inputs.operating_systems || 'windows,ubuntu' }} + rocm_version: ${{ github.event.inputs.rocm_version || 'weekly' }} + rocm_cutoff: ${{ needs.resolve.outputs.rocm_cutoff }} + + build-macos: + name: macOS + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-macos.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + repo: ${{ needs.resolve.outputs.repo }} + source_artifact: ${{ needs.resolve.outputs.source_artifact }} + matrix: ${{ needs.resolve.outputs.macos_matrix }} + + build-cpu: + name: CPU + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-cpu.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + repo: ${{ needs.resolve.outputs.repo }} + source_artifact: ${{ needs.resolve.outputs.source_artifact }} + + build-vulkan: + name: Vulkan + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-vulkan.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + repo: ${{ needs.resolve.outputs.repo }} + source_artifact: ${{ needs.resolve.outputs.source_artifact }} + + assemble: + name: Assemble metadata + publish + # Only `resolve`, so this job starts alongside the build matrix instead of + # after it. It used to `needs:` every build child, which meant it began + # queueing for a runner only once the last leg went green: on the reference + # run that queue wait was 109 minutes to then run a 10-second job. Starting + # early overlaps the wait with the build. The cost is one runner slot held + # for the length of the run. + needs: [resolve] + # No `if: always()`. Atomicity is unchanged in effect, but it is now + # enforced by the "Wait for the build matrix" step below rather than by + # `needs:`: that step blocks until every sibling build job is finished and + # exits non-zero unless all of them succeeded, so a failed leg still + # publishes nothing. The installer needs the full bundle set or it'll fail. + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-24.04 + # Consumed by the reclaim job below: 'true' only when THIS run's publish + # step actually ran and succeeded. + outputs: + published: ${{ steps.publish.outcome == 'success' }} + # The waiter holds this job open for the whole build. GitHub's 6h job cap + # would kill it with no useful message; stop short of that deliberately, + # leaving room for the download/verify/publish steps that follow. + timeout-minutes: 350 + # A job-level block replaces the workflow-level one, so contents:write has + # to be repeated here; actions:read is what lets the waiter read the run's + # job list (the `alert` job already needs it for the same reason). + permissions: + contents: write + actions: read + env: + GH_TOKEN: ${{ github.token }} + steps: + # Reimplements the `needs:` success check that the trimmed-down `needs:` + # above gave away. It has to be at least as strict as `needs:` was -- + # publishing a partial release is far worse than publishing late. + - name: Wait for the build matrix + env: + # Same expression that gates the verify/publish steps below. + PUBLISH_INTENT: ${{ github.event_name == 'schedule' || inputs.publish }} + run: | + set -euo pipefail + + # Build jobs live in called workflows, so they appear in this run's + # job list as " / ", e.g. + # "CUDA / x64/cuda12-legacy". One prefix per build child; every one of + # them must be represented or we are not looking at a complete run. + PREFIXES=('CUDA / ' 'CUDA Windows / ' 'ROCm / ' 'macOS / ' 'CPU / ' 'Vulkan / ') + SELF='Assemble metadata + publish' + ALERT='Report pipeline health' + + POLL=60 + DEADLINE=$(( $(date +%s) + 330 * 60 )) + # Called-workflow job records do not all exist the moment the run + # starts, so a missing prefix is only fatal once this has passed. + STARTUP_DEADLINE=$(( $(date +%s) + 45 * 60 )) + API_FAILS=0 + + while :; do + if ! JOBS="$(gh api --paginate \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \ + --jq '.jobs[] | [.name, .status, (.conclusion // "none")] | @tsv' 2>/dev/null)"; then + # A blip in the jobs API must not throw away a finished build. + API_FAILS=$(( API_FAILS + 1 )) + if [ "$API_FAILS" -ge 10 ]; then + echo "ERROR: the jobs API failed 10 times running; cannot confirm the build matrix finished" >&2 + exit 1 + fi + echo "jobs API call failed (${API_FAILS}/10); retrying in ${POLL}s" + sleep "$POLL" + continue + fi + API_FAILS=0 + + # Everything in the run except this job and the alert job that + # reports on it. `resolve` stays in the set; it is already a + # `needs:`, so it is a free consistency check. + SIBS="$(printf '%s\n' "$JOBS" | awk -F'\t' -v self="$SELF" -v alert="$ALERT" 'NF && $1 != self && $1 != alert')" + + MISSING="" + for p in "${PREFIXES[@]}"; do + printf '%s\n' "$SIBS" | awk -F'\t' -v p="$p" 'index($1, p) == 1 { found = 1 } END { exit !found }' \ + || MISSING="${MISSING} '${p}'" + done + if [ -n "$MISSING" ]; then + if [ "$(date +%s)" -gt "$STARTUP_DEADLINE" ]; then + echo "ERROR: no job records for build child(ren):${MISSING}; refusing to publish without confirming they ran" >&2 + exit 1 + fi + echo "waiting for job records to appear for:${MISSING}" + sleep "$POLL" + continue + fi + + # Fail on the first finished leg that did not succeed rather than + # sitting on a runner for another hour to reach the same answer. + # `skipped` is the one conclusion that needs a judgement call: a + # publish run is pinned by `resolve` to the full default matrix, so + # a skipped leg there means something is wrong and must block, but a + # publish=false subset dispatch (say operating_systems=ubuntu) skips + # legs on purpose and `needs:` tolerated that before. + BAD="$(printf '%s\n' "$SIBS" | awk -F'\t' -v strict="$PUBLISH_INTENT" ' + !NF || $2 != "completed" { next } + $3 == "success" { next } + $3 == "skipped" && strict != "true" { next } + { printf " %s (%s)\n", $1, $3 } + ')" + if [ -n "$BAD" ]; then + echo "ERROR: refusing to publish, these build jobs did not succeed:" >&2 + printf '%s\n' "$BAD" >&2 + exit 1 + fi + + TOTAL="$(printf '%s\n' "$SIBS" | grep -c . || true)" + PENDING="$(printf '%s\n' "$SIBS" | awk -F'\t' 'NF && $2 != "completed"' | grep -c . || true)" + if [ "$PENDING" -eq 0 ]; then + echo "all ${TOTAL} build jobs finished and succeeded" + break + fi + if [ "$(date +%s)" -gt "$DEADLINE" ]; then + echo "ERROR: timed out waiting for the build matrix; ${PENDING} of ${TOTAL} jobs still running" >&2 + exit 1 + fi + echo "${PENDING} of ${TOTAL} build jobs still running; polling again in ${POLL}s" + sleep "$POLL" + done + + - name: Checkout build tooling (this repo) + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + path: tooling + + # The first download attempt occasionally hits a transient ECONNRESET + # on GitHub's ListArtifacts API. Retrying once is cheaper than + # re-running the whole hour-long build matrix. + - name: Download built bundles + id: download + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + continue-on-error: true + with: + path: dist + pattern: app-* + merge-multiple: true + + - name: Download built bundles (retry) + if: steps.download.outcome == 'failure' + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + path: dist + pattern: app-* + merge-multiple: true + + # Fingerprint gate: every bundle must carry the fingerprint the build children + # fold into BUILD_TARGET (-> LLAMA_BUILD_TARGET, printed by `--version`). That + # string is compiled into common (build-info.cpp); on Linux/macOS it lands in + # llama-server's own .rodata, but on a Windows shared build it can live in the + # sibling llama-common DLL instead -- so scan every file in the bundle, not + # just the exe. Nothing but a compiled binary carries this string (no metadata + # file does), so a hit means the brand shipped. Scan bytes rather than running + # `--version`: cross-compiled (ROCm) and Windows binaries can't run on this + # Linux runner. MARK must stay byte-identical to the resolve job's stamp string. + # Refuse to publish if any bundle is unbranded. + - name: Verify Unsloth fingerprint in every bundle + run: | + set -euo pipefail + MARK='Compiled by the Unsloth team' + shopt -s nullglob + tmp="$(mktemp -d)" + fail=0 + checked=0 + for arc in dist/app-*.tar.gz dist/app-*.zip dist/llama-*-bin-macos-*.tar.gz; do + d="$tmp/extract" + rm -rf "$d"; mkdir -p "$d" + case "$arc" in + *.zip) unzip -qo "$arc" -d "$d" ;; + *.tar.gz) tar -xzf "$arc" -C "$d" ;; + esac + if grep -arq "$MARK" "$d"; then + checked=$((checked + 1)) + else + echo "ERROR: $(basename "$arc"): no file carries the Unsloth fingerprint" >&2; fail=1 + fi + done + rm -rf "$tmp" + [ "$checked" -gt 0 ] || { echo "ERROR: no bundles found to verify (expected app-*/llama-*-bin-macos-* in dist/)" >&2; exit 1; } + [ "$fail" = 0 ] || { echo "ERROR: refusing to publish unbranded binaries" >&2; exit 1; } + echo "fingerprint verified in $checked bundles" + + # Ship the stamped source tree as release assets so the installer's + # source-build fallback reproduces the same fingerprinted binary. The resolve + # job's app-source-* artifact lands the tag-named tarball in dist/ (via the + # app-* download above); copy it to the commit name too. Both are the exact + # local bytes assemble_metadata hashes into the sha256 index. + - name: Fetch source archives + run: | + set -eux + TAG='${{ needs.resolve.outputs.tag }}' + SHA='${{ needs.resolve.outputs.commit }}' + cp "dist/llama.cpp-source-${TAG}.tar.gz" "dist/llama.cpp-source-commit-${SHA}.tar.gz" + + - name: Generate manifest + sha256 index + # prs carries PR titles (arbitrary text), so pass it through env + # rather than inlining it: a title with a quote would otherwise break out + # of the shell command. Same handling in the publish step below. + env: + PRS_JSON: ${{ needs.resolve.outputs.prs }} + run: | + set -eux + python3 tooling/scripts/unsloth/assemble_metadata.py \ + --tag '${{ needs.resolve.outputs.tag }}' \ + --source-repo '${{ needs.resolve.outputs.repo }}' \ + --base-tag '${{ needs.resolve.outputs.base }}' \ + --pr-set "$PRS_JSON" \ + --ggml-tree '${{ needs.resolve.outputs.ggml_tree }}' \ + --ggml-version '${{ needs.resolve.outputs.ggml_version }}' \ + --commit '${{ needs.resolve.outputs.commit }}' \ + --dist dist --out dist \ + --publish-repo "$GITHUB_REPOSITORY" + ls -la dist + + - name: Verify full bundle coverage before publish + if: ${{ (github.event_name == 'schedule' || inputs.publish) && needs.resolve.outputs.exists != 'true' }} + run: | + set -eu + TAG='${{ needs.resolve.outputs.tag }}' + fail=0 + # A partial CUDA set (cuda13 without cuda12) silently strands the + # cuda12-runtime majority; require matching x64 coverage on both lines. + # cuda12-legacy has no cuda13 sibling, so `-nE ... p` drops it from + # this cross-line parity check; its presence is required separately below. + for os in linux windows; do + ext=$([ "$os" = windows ] && echo zip || echo tar.gz) + c12=$(ls dist/app-*-"$os"-x64-cuda12-*."$ext" 2>/dev/null | sed -nE 's/.*cuda12-(older|newer|portable)\..*/\1/p' | sort -u | paste -sd, -) + c13=$(ls dist/app-*-"$os"-x64-cuda13-*."$ext" 2>/dev/null | sed -nE 's/.*cuda13-(older|newer|portable)\..*/\1/p' | sort -u | paste -sd, -) + echo "$os x64 coverage: cuda12=[$c12] cuda13=[$c13]" + [ -n "$c12" ] && [ "$c12" = "$c13" ] || { echo "ERROR: $os x64 cuda12/cuda13 coverage mismatch" >&2; fail=1; } + done + # Children passing is not the same as files landing in dist/ (a + # download-artifact anomaly leaves green jobs and missing bundles), + # so assert presence of every line not covered by the parity check + # above (the non-CUDA-x64 lines, plus cuda12-legacy). The resolve-time + # input guard already pins publish runs to the full default matrix, so + # these names are exact. + for f in \ + "app-${TAG}-linux-x64-cuda12-legacy.tar.gz" \ + "app-${TAG}-windows-x64-cuda12-legacy.zip" \ + "llama-${TAG}-bin-macos-arm64.tar.gz" \ + "llama-${TAG}-bin-macos-x64.tar.gz" \ + "app-${TAG}-linux-arm64-cuda13-portable.tar.gz" \ + "app-${TAG}-linux-x64-cpu.tar.gz" \ + "app-${TAG}-windows-x64-cpu.zip" \ + "app-${TAG}-linux-arm64-cpu.tar.gz" \ + "app-${TAG}-windows-arm64-cpu.zip" \ + "app-${TAG}-linux-x64-vulkan.tar.gz" \ + "app-${TAG}-linux-arm64-vulkan.tar.gz" \ + "app-${TAG}-windows-x64-vulkan.zip"; do + [ -s "dist/$f" ] || { echo "ERROR: missing $f in dist/" >&2; fail=1; } + done + # Default ROCm set (mirrors the gfx_target input default): both OSes + # per family, or AMD hosts of that family silently lose their bundle. + for gfx in gfx1151 gfx1150 gfx120X gfx110X gfx103X gfx90a gfx908; do + [ -s "dist/app-${TAG}-linux-x64-rocm-${gfx}.tar.gz" ] || { echo "ERROR: missing linux rocm ${gfx} bundle in dist/" >&2; fail=1; } + [ -s "dist/app-${TAG}-windows-x64-rocm-${gfx}.zip" ] || { echo "ERROR: missing windows rocm ${gfx} bundle in dist/" >&2; fail=1; } + done + [ "$fail" = 0 ] || { echo "ERROR: refusing to publish a partial release" >&2; exit 1; } + + - name: Publish GitHub release + id: publish + # Last resort cap. The uploader has its own 90m phase deadline; if that + # ever fails to trip, this still fails the step with hours of job budget + # left instead of letting the 350m job cap kill it with no message, and + # it lets the rescue artifact step below run. + timeout-minutes: 120 + if: ${{ (github.event_name == 'schedule' || inputs.publish) && needs.resolve.outputs.exists != 'true' }} + # prs carries PR titles (arbitrary text), so pass it through env + # rather than inlining it: a title with a quote would otherwise break out + # of the shell command. + env: + PRS_JSON: ${{ needs.resolve.outputs.prs }} + run: | + set -eux + TAG='${{ needs.resolve.outputs.tag }}' + REPO="$GITHUB_REPOSITORY" + PRS="$PRS_JSON" + BASE='${{ needs.resolve.outputs.base }}' + # Link the base to the upstream release tag (always resolves). Each PR + # line is " (#<n>, commit <sha>)": GitHub does not expand a bare + # reference into the PR title inline (only on hover), so we bake the + # title in ourselves. #<n> links to the PR in its home repo (full URL, + # so it resolves no matter which repo hosts the release, and GitHub + # still attaches its hovercard); non-upstream PRs spell the repo in + # the link text so an unslothai pin can't read as an upstream one. + # <sha> links to that pin's commit-in-PR URL. We deliberately do not + # link the merged commit: in mix mode it is a throwaway merge made on + # the runner and never pushed anywhere, so any repo@sha URL for it 404s. + NOTES="Automated Unsloth llama.cpp CUDA + ROCm + Vulkan + macOS + CPU prebuild for upstream [${BASE}](https://github.com/ggml-org/llama.cpp/releases/tag/${BASE})" + if [ "$(jq length <<<"$PRS")" = 0 ]; then + NOTES="${NOTES}." + else + PR_LIST="$(jq -r 'map("- \(.title) ([\(if .repo == "ggml-org/llama.cpp" then "" else .repo end)#\(.number)](https://github.com/\(.repo)/pull/\(.number)), commit [\(.sha[0:7])](\(.url)))") | join("\n")' <<<"$PRS")" + NOTES="$(printf '%s, merged with:\n\n%s' "$NOTES" "$PR_LIST")" + fi + + # Atomic publish: upload as draft (hidden from the anon GitHub API + # the installer uses), then flip draft=false only once every asset + # landed. Leftover drafts from failed runs are detected as "not + # exists" by resolve and rebuilt on the next run. + if [ "$(gh release view "$TAG" --repo "$REPO" --json isDraft --jq .isDraft 2>/dev/null || true)" = "true" ]; then + gh release delete "$TAG" --repo "$REPO" --yes + fi + # Create the draft empty, then upload through the uploader instead of + # passing dist/* here. `gh release create` uses a fixed 5-worker pool + # with no per-connection timeout, so a few wedged PUTs block the whole + # set: in run 31335302864 six large bundles stalled at ~0.03 MB/s and + # held the pool for 3h45m, while the other 25 assets took 37s total. + for attempt in 1 2 3; do + if gh release create "$TAG" --repo "$REPO" --draft \ + --title "llama.cpp prebuilt $TAG" \ + --notes "$NOTES"; then + break + fi + if [ "$attempt" = 3 ]; then + echo "ERROR: could not create draft release $TAG" >&2 + exit 1 + fi + sleep $(( attempt * 10 )) + done + + bash tooling/scripts/unsloth/upload_release_assets.sh \ + --tag "$TAG" --repo "$REPO" --dist dist + + # Reached only after the uploader verified every asset is present, + # byte-identical and in state "uploaded". + gh release edit "$TAG" --repo "$REPO" --draft=false + + # Debug fallback for a failed publish. This used to upload on EVERY run, + # ahead of the publish gate, which cost ~688 MiB of Actions artifact + # storage per run for a bundle nothing reads: no download-artifact in this + # repo references it, an org-wide code search for `actions/artifacts` + # returns nothing, and Studio's installer resolves release ASSETS + # (install_llama_prebuilt.py -> release_asset_download_url). Uploading + # ahead of the gate also published unverified builds -- on a public repo + # any signed-in user can download an artifact -- for runs that + # deliberately never released. + # + # Kept but moved after publish and gated on failure(), so it still rescues + # a failed-publish run without re-running the 40-job matrix, and costs + # nothing when the run is green. + - name: Upload full release set (artifacts) + # always() && !success(), not failure(): a cancelled assemble -- its own + # timeout, or a superseding nightly -- is not on the failure path, so + # failure() would skip the rescue bundle on exactly the runs a human + # most wants it for. Nothing else in this job competes for the teardown + # window, so attempting the upload there is free; if dist/ was never + # populated, if-no-files-found: warn keeps that quiet. + if: ${{ always() && !success() }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: unsloth-prebuilt-${{ needs.resolve.outputs.tag }} + path: dist/* + # warn, not error: an early failure can leave dist/ absent, and a + # missing debug bundle must not turn a diagnosable failure into a + # confusing second one. + if-no-files-found: warn + # rerun-failed-jobs reuses the run id, and artifacts persist across + # attempts, so a second failed attempt would 409 on the duplicate name. + overwrite: true + retention-days: 7 + + # whisper.cpp slim bundles are compiled against this release's ggml and + # ship no libggml*, so a new ggml means whisper has to republish. Without + # this it only finds out on its own cron, and Studio reports "no + # compatible prebuilt" until then. + - name: Notify whisper.cpp + if: ${{ (github.event_name == 'schedule' || inputs.publish) && needs.resolve.outputs.exists != 'true' }} + # Never fail a published release because the notification did not land. + continue-on-error: true + env: + DISPATCH_TOKEN: ${{ secrets.WHISPER_DISPATCH_TOKEN }} + TAG: ${{ needs.resolve.outputs.tag }} + GGML_TREE: ${{ needs.resolve.outputs.ggml_tree }} + run: | + # No -x here: it would trace the token into the log. + set -eu + if [ -z "${DISPATCH_TOKEN:-}" ]; then + echo "::warning::WHISPER_DISPATCH_TOKEN is not set; whisper.cpp will only pick this up on its own schedule" + exit 0 + fi + PAYLOAD="$(jq -cn --arg t "$TAG" --arg g "$GGML_TREE" --arg r "$GITHUB_RUN_ID" \ + '{event_type: "llama-published", client_payload: {llama_tag: $t, ggml_tree: $g, run_id: $r}}')" + CODE="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${DISPATCH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/unslothai/whisper.cpp/dispatches \ + -d "$PAYLOAD")" + # 204 is "event accepted", not "whisper republished". The dead-man + # check is what catches a dropped or ignored event. + if [ "$CODE" = "204" ]; then + echo "notified whisper.cpp of ${TAG}" + else + echo "::warning::whisper.cpp dispatch returned HTTP ${CODE}; it will fall back to its own schedule" + fi + + # Reclaim this run's Actions artifact storage once the release is canonical. + # + # Every prebuilt is stored TWICE: as an Actions artifact (billed against the + # Actions storage quota) and as a release asset (not billed against it). + # Measured on run 31218133438: 36 of its 40 artifacts, 7.67 GiB, are byte-for + # -byte already on the release. On 2026-08-07 that duplication reached ~492 + # GiB org-wide, 449 GiB of it here, and GitHub stopped scheduling Actions runs + # across unslothai/* while personal-account repos ran the same workflows + # normally. Deleting artifacts restored scheduling within ~45 seconds. + # + # Its OWN job rather than a step of assemble, for two reasons: + # + # * Cancellation. As a trailing step, a cancel landing inside its ~13 second + # window left assemble `cancelled` with a partial app-* set, and + # unsloth-prebuilt-retry.yml reruns a cancelled run that has no failed job + # -- rerunning assemble itself, which then restarts at "Download built + # bundles", dies at the coverage gate, and reports "Nothing was published" + # for a release that HAD published. As a separate job, assemble has already + # succeeded, so a rerun re-runs only this job, and re-running it is a no-op: + # deleted artifacts are simply absent from the second listing. + # * Least privilege. `actions: write` also grants Actions CACHE deletion, so + # holding it across assemble's unzip/tar/python steps would put every + # ccache this pipeline depends on within reach of that job. Here it is + # confined to a job whose only action is deleting artifacts. + # + # Skipped, not failed, when there is nothing to reclaim: `needs.assemble + # .outputs.published` is 'true' only when THIS run's publish step ran and + # succeeded, so a dispatch that skips publishing (publish:false, or the tag + # already released) never reaches the delete -- its artifacts would otherwise + # be name-matched against a DIFFERENT build's release assets, since artifact + # names are keyed only on the tag. + # + # Residual, accepted: `published` proves THIS run's publish step succeeded, not + # that the release object is this run's. A scheduled run and a same-tag + # dispatch sit in different concurrency groups, so a dispatch can delete the + # scheduled run's draft and create its own; the scheduled run's + # `gh release edit --draft=false` then succeeds against the dispatch's release. + # The payloads are equivalent -- the resolve guard pins every publish run to + # the full default matrix, and the tag encodes base plus pr-set hash -- so the + # assets match whichever run wrote them. + reclaim: + name: Reclaim artifact storage + # Every build leg, not just assemble. + # assemble needs only resolve, so one failing leg fails it immediately while slower legs are still building, and with always() this job then deleted the app-source-* artifact out from under them. + # On 08-27 one arm64 CPU failure became ten: nine ROCm legs died on "Artifact not found" seconds later, which reads as a ROCm fault and is not. + needs: [resolve, build-cuda, build-windows-cuda, build-rocm, build-macos, build-cpu, build-vulkan, assemble] + # always(), so a run that publishes NOTHING still cleans up after itself. + # Gating this on `published` leaked every non-publishing run's bundles: a + # workflow_dispatch defaults to publish:false, and a cancelled run never + # reaches publish either. On 08-10 that was 5.9 GiB from one cancelled run + # plus 7.1 GiB from one publish:false run, both deleted by hand. The two + # cases are handled by different steps below -- a published run deletes only + # what is provably on the release, an unpublished one has nothing to match + # against and deletes its own bundles outright. + if: ${{ always() }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: write # delete this run's artifacts + contents: read # read the release asset list + steps: + - name: Delete artifacts already published as release assets + if: ${{ needs.assemble.outputs.published == 'true' }} + # Never fail a published release over cleanup. + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.resolve.outputs.tag }} + run: | + set -euo pipefail + repo="$GITHUB_REPOSITORY" + + if [ -z "${TAG:-}" ]; then + echo "no tag resolved; leaving artifacts untouched" + exit 0 + fi + + # Gate 1: the release must exist and be published, not a draft. + draft="$(gh release view "$TAG" --repo "$repo" --json isDraft -q .isDraft 2>/dev/null || echo missing)" + if [ "$draft" != "false" ]; then + echo "release $TAG is '$draft', not a published release; leaving artifacts untouched" + exit 0 + fi + assets="$RUNNER_TEMP/reclaim-assets.txt" + arts="$RUNNER_TEMP/reclaim-arts.tsv" + gh release view "$TAG" --repo "$repo" --json assets -q '.assets[].name' | sort > "$assets" + echo "release $TAG has $(wc -l < "$assets") assets" + + # Gate 2: only THIS run's artifacts are even considered, so the step + # cannot reach another run's -- including a concurrent build's. + gh api "repos/$repo/actions/runs/$GITHUB_RUN_ID/artifacts" --paginate \ + -q '.artifacts[] | select(.expired==false) | "\(.id)\t\(.size_in_bytes)\t\(.name)"' > "$arts" || true + echo "this run has $(grep -c . "$arts" || true) live artifacts" + + freed=0; deleted=0; kept=0; failed=0 + while IFS="$(printf '\t')" read -r id size name; do + [ -z "${id:-}" ] && continue + # Gate 3: delete only what is provably already on the release. + # Build children upload `app-<tag>-<platform>`; assemble publishes + # it as `.tar.gz` (linux/macos) or `.zip` (windows). Anything that + # does not match is KEPT -- that is what protects a partial publish. + # By design this also keeps the macOS bundles (published under + # `llama-<tag>-bin-macos-*`) and the source tarball: 56 MiB of the + # 7.42 GiB, a cheap price for never deleting something early. + # -F: fixed string. Without it every `.` in the name is a regex + # wildcard, and gfx_target / only_profile are free-text + # workflow_dispatch inputs that flow into artifact names. + if grep -qxF -- "${name}.tar.gz" "$assets" || grep -qxF -- "${name}.zip" "$assets"; then + # < /dev/null so the command can never consume the loop's stdin + # and silently truncate the sweep to one artifact. + if err="$(gh api -X DELETE "repos/$repo/actions/artifacts/$id" --silent < /dev/null 2>&1)"; then + freed=$(( freed + size )); deleted=$(( deleted + 1 )) + else + # Keep the reason: a 403 from a permissions regression and a + # transient blip need different responses, and this step is + # continue-on-error so nothing else surfaces it. + printf ' could not delete %s: %s\n' "$name" "$err" + failed=$(( failed + 1 )) + fi + else + printf ' KEEP %s (no matching release asset)\n' "$name" + kept=$(( kept + 1 )) + fi + done < "$arts" + + echo "deleted $deleted artifacts, freed $(( freed / 1048576 )) MiB, kept $kept, failed $failed" + if [ "$failed" -gt 0 ]; then + echo "::warning::$failed artifact(s) could not be deleted; storage will be reclaimed by retention instead" + fi + { + echo "### Artifact storage reclaimed" + echo "" + echo "| metric | value |" + echo "| --- | --- |" + echo "| release | \`$TAG\` |" + echo "| artifacts deleted | $deleted |" + echo "| storage freed | $(( freed / 1048576 )) MiB |" + echo "| kept (no release asset) | $kept |" + echo "| delete failures | $failed |" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Delete artifacts of a run that published nothing + # The other step's name-match against release assets is meaningless + # here: either no release was written, or the tag belongs to a DIFFERENT + # run's release. So the rule is simply that nothing will ever consume + # these -- a publish:false dispatch is a test, and a cancelled or failed + # run is not resumable past the missing legs -- and they are deleted. + # + # Accepted: unsloth-prebuilt-retry.yml can rerun a cancelled run whose + # artifacts this step deleted. That rerun fails loudly at assemble's + # coverage gate rather than publishing a partial set, which is the + # failure mode this pipeline already prefers. Pass keep_artifacts:true + # on a dispatch whose bundles you intend to download by hand. + if: ${{ needs.assemble.outputs.published != 'true' && inputs.keep_artifacts != true }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + repo="$GITHUB_REPOSITORY" + arts="$RUNNER_TEMP/reclaim-unpublished.tsv" + + # Same containment as the published path: only THIS run's artifacts + # are listed, so the step cannot reach a concurrent build's. + gh api "repos/$repo/actions/runs/$GITHUB_RUN_ID/artifacts" --paginate \ + -q '.artifacts[] | select(.expired==false) | "\(.id)\t\(.size_in_bytes)\t\(.name)"' > "$arts" || true + n="$(grep -c . "$arts" || true)" + echo "run published nothing; deleting its $n live artifact(s)" + + freed=0; deleted=0; failed=0 + while IFS="$(printf '\t')" read -r id size name; do + [ -z "${id:-}" ] && continue + if err="$(gh api -X DELETE "repos/$repo/actions/artifacts/$id" --silent < /dev/null 2>&1)"; then + freed=$(( freed + size )); deleted=$(( deleted + 1 )) + else + printf ' could not delete %s: %s\n' "$name" "$err" + failed=$(( failed + 1 )) + fi + done < "$arts" + + echo "deleted $deleted artifacts, freed $(( freed / 1048576 )) MiB, failed $failed" + if [ "$failed" -gt 0 ]; then + echo "::warning::$failed artifact(s) could not be deleted; storage will be reclaimed by retention instead" + fi + { + echo "### Artifact storage reclaimed (unpublished run)" + echo "" + echo "| metric | value |" + echo "| --- | --- |" + echo "| artifacts deleted | $deleted |" + echo "| storage freed | $(( freed / 1048576 )) MiB |" + echo "| delete failures | $failed |" + } >> "$GITHUB_STEP_SUMMARY" + + # ── Keep the ccache budget inside the repo limit ── + # + # ccache entries are immutable, so every run writes a NEW cache per + # (cuda, os, profile) and finds the previous one by restore-keys prefix. + # restore-keys returns only the MOST RECENT match, so older generations can + # never be selected again -- they are pure landfill. Measured 2026-08-08: + # 122 caches / 31.72 GiB, of which only 40 / 9.13 GiB were reachable. + # + # That matters for build time, not tidiness. The repo cache limit is 50 GB + # and there are ~40 prefixes; once the total hits the limit GitHub evicts by + # its own LRU, which can take a LIVE cache. A partial cache is far worse + # than none: measured locally over 513 real translation units, capping a + # cache to 30% of what the build needs drops the hit rate to 14.2% and + # flips hits from direct to preprocessed -- the exact 3-direct / + # 52-preprocessed signature seen in CI when the cap was 500 MB, which cost + # a 204-minute build instead of 54. + # + # Keeping 2 rather than 1: the second generation is the fallback when a job + # dies before saving, which is precisely the hole that forces a cold build. + - name: Prune superseded ccache generations + # Unchanged trigger: this job now runs on every outcome for artifact + # cleanup, but cache pruning stays on published runs only. A cancelled + # run's children may not have written their caches yet, and pruning + # "superseded" generations against a half-written set could drop a live + # one -- a partial cache is worse than a stale one. + if: ${{ needs.assemble.outputs.published == 'true' }} + # Never fail a published release over cache housekeeping. + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + KEEP: 2 + run: | + set -euo pipefail + repo="$GITHUB_REPOSITORY" + all="$RUNNER_TEMP/caches.tsv" + gh api --paginate "repos/$repo/actions/caches?per_page=100" \ + -q '.actions_caches[] | "\(.id)\t\(.created_at)\t\(.size_in_bytes)\t\(.key)"' > "$all" || true + total=$(awk -F'\t' '{s+=$3} END {printf "%d", s+0}' "$all") + echo "$(grep -c . "$all" || true) caches, $(( total / 1073741824 )) GiB" + + # Group by the restore-keys prefix: the key minus its -<tag>- suffix. + # Newest first, so anything past $KEEP is unreachable by restore-keys. + # The ROCm version comes off too, else every weekly toolchain becomes its own group and keeps 2 caches that can never hit again. + freed=0; deleted=0 + while IFS=$'\t' read -r id created size key; do + [ -z "${id:-}" ] && continue + pre="$(printf '%s' "$key" | sed -E 's/-b[0-9]+(-mix-[0-9a-f]+)?-?$//; s/-[0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+$//')" + printf '%s\t%s\t%s\t%s\n' "$pre" "$created" "$id" "$size" + done < "$all" | sort -t"$(printf '\t')" -k1,1 -k2,2r > "$RUNNER_TEMP/grouped.tsv" + + prev=""; n=0 + while IFS=$'\t' read -r pre created id size; do + [ -z "${pre:-}" ] && continue + if [ "$pre" != "$prev" ]; then prev="$pre"; n=1; else n=$(( n + 1 )); fi + [ "$n" -le "$KEEP" ] && continue + if gh api -X DELETE "repos/$repo/actions/caches/$id" --silent < /dev/null 2>/dev/null; then + freed=$(( freed + size )); deleted=$(( deleted + 1 )) + fi + done < "$RUNNER_TEMP/grouped.tsv" + + after=$(( total - freed )) + echo "pruned $deleted superseded caches, freed $(( freed / 1073741824 )) GiB" + { + echo "### ccache budget" + echo "" + echo "| metric | value |" + echo "| --- | --- |" + echo "| kept per prefix | $KEEP |" + echo "| caches pruned | $deleted |" + echo "| freed | $(( freed / 1073741824 )) GiB |" + echo "| cache total after | $(( after / 1073741824 )) GiB of 50 GB |" + } >> "$GITHUB_STEP_SUMMARY" + if [ "$after" -gt 42949672960 ]; then + echo "::warning::ccache total is $(( after / 1073741824 )) GiB of the 50 GB repo limit; at the limit GitHub evicts by LRU and a partially evicted cache collapses the hit rate. Lower KEEP to 1, or raise the limit." + fi + + # Without this a failed scheduled run is silent: GitHub emails only whoever + # last touched the cron file, which is how three nightlies failed unnoticed. + # Runs on publish-intent runs only, so subset test dispatches stay quiet. + alert: + name: Report pipeline health + needs: [resolve, build-cuda, build-windows-cuda, build-rocm, build-macos, build-cpu, build-vulkan, assemble] + if: ${{ always() && (github.event_name == 'schedule' || inputs.publish) }} + runs-on: ubuntu-24.04 + permissions: + contents: read + issues: write + actions: read + steps: + - name: Checkout (for the composite action) + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: { persist-credentials: false } + + - name: Summarise run + id: s + env: + GH_TOKEN: ${{ github.token }} + NEEDS_JSON: ${{ toJSON(needs) }} + run: | + set -uo pipefail + # 'skipped' is healthy: a scheduled no-op skips every build job. + # 'cancelled' is not. On 08-05 every build passed and the publish job + # was cancelled 15s in by something outside the run, so nothing was + # released and this reported success: the silent no-publish we exist + # to catch. Anything that is not success or skipped counts. + FAILED="$(jq -r 'to_entries | map(select(.value.result != "success" and .value.result != "skipped") | "\(.key) (\(.value.result))") | join(", ")' <<<"$NEEDS_JSON")" + if [ -z "$FAILED" ]; then + echo "status=success" >> "$GITHUB_OUTPUT" + echo "details=" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "status=failure" >> "$GITHUB_OUTPUT" + + # Quote the actual refusal so the issue says why, not just that a job failed. + REASON="$(gh run view "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" --log-failed 2>/dev/null \ + | grep -aE 'refusing |does not merge cleanly onto |could not fetch commit |ERROR: ' \ + | sed -E 's/^[0-9-]+T[0-9:.]+Z //' | cut -c1-300 | head -5 || true)" + + { + echo 'details<<ALERT_EOF' + echo "**Failed jobs:** ${FAILED}" + if [ -n "$REASON" ]; then + echo + echo 'Reported reason:' + echo + echo '```' + echo "$REASON" + echo '```' + fi + echo + echo "Nothing was published; the previous release remains \`Latest\`." + echo 'ALERT_EOF' + } >> "$GITHUB_OUTPUT" + + - name: Alert + uses: ./.github/actions/prebuilt-alert + with: + status: ${{ steps.s.outputs.status }} + key: llama-prebuilt-nightly + title: 'Nightly llama.cpp prebuilt is failing' + details: ${{ steps.s.outputs.details }} + token: ${{ github.token }} diff --git a/.github/workflows/unsloth-repin-bot.yml b/.github/workflows/unsloth-repin-bot.yml new file mode 100644 index 000000000000..a074992511a0 --- /dev/null +++ b/.github/workflows/unsloth-repin-bot.yml @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: Unsloth repin bot + +# Preflight says the pins stopped merging. This does the mechanical half of the +# fix: merge the base tag into each pin branch we own, resolve the add/add +# collisions that cause almost all of these, and open a PR moving the pins. +# +# It opens a PR and stops. It never merges it, never touches a branch belonging +# to somebody else, and never repins past a commit a human reviewed -- the pin +# file exists to guarantee that only reviewed code ships, and a bot that can +# widen it on its own has removed the guarantee. + +on: + workflow_run: + workflows: ['Unsloth pin preflight'] + types: [completed] + workflow_dispatch: + +permissions: + contents: read + issues: write + +concurrency: + group: unsloth-repin-bot + cancel-in-progress: false + +env: + # Branch the bot parks its proposal on. Reused every run so a week of + # breakage is one PR to review, not seven. + REPIN_BRANCH: unsloth/auto-repin + +jobs: + repin: + name: Repin against the current base tag + # A preflight that passed has nothing to fix. + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'failure' }} + runs-on: ubuntu-24.04 + steps: + # Without this, checkout leaves a github.com extraheader carrying + # GITHUB_TOKEN, which would win over the REPIN_TOKEN in our push URLs. + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: { persist-credentials: false } + + - name: Resolve base tag + id: base + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + AGE_H="${UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS:-6}" + CUTOFF="$(date -u -d "-${AGE_H} hours" +%s)" + # Same base tag the nightly resolves: newest aged b#### build. + # Upstream marks those prerelease since 08-21, so match the tag shape. + BASE="$(gh api 'repos/ggml-org/llama.cpp/releases?per_page=100' \ + | jq -r --argjson cutoff "$CUTOFF" '[.[] | select(.draft==false) | select(.tag_name|test("^b[0-9]+$")) | select((.published_at|fromdateiso8601) <= $cutoff)] | max_by(.published_at|fromdateiso8601) | .tag_name')" + if [ -z "$BASE" ] || [ "$BASE" = "null" ]; then + echo "::warning::no aged upstream release found; nothing to repin onto" + echo "base=" >> "$GITHUB_OUTPUT"; exit 0 + fi + echo "base $BASE" + echo "base=$BASE" >> "$GITHUB_OUTPUT" + + - name: Merge and repin + id: repin + if: ${{ steps.base.outputs.base != '' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + python3 scripts/unsloth/repin.py \ + --pr-set scripts/unsloth/pr-set.json \ + --base "${{ steps.base.outputs.base }}" \ + --work "${RUNNER_TEMP}/repin" \ + --report "${RUNNER_TEMP}/repin.json" \ + --markdown "${RUNNER_TEMP}/repin.md" + CHANGED="$(jq -r '.changed' "${RUNNER_TEMP}/repin.json")" + BLOCKED="$(jq -r '[.results[] | select(.action == "conflict" or .action == "third-party")] | length' "${RUNNER_TEMP}/repin.json")" + echo "changed=$CHANGED" >> "$GITHUB_OUTPUT" + echo "blocked=$BLOCKED" >> "$GITHUB_OUTPUT" + + - name: Push the merged branches and open the PR + id: push + if: ${{ steps.repin.outputs.changed != '' && steps.repin.outputs.changed != '0' }} + env: + # Pushing to danielhanchen/llama.cpp is cross-repo, which GITHUB_TOKEN + # cannot do at all, and these merges carry upstream's own workflow + # changes, which needs workflow write. Without the secret the bot + # still reports; it just cannot act. + REPIN_TOKEN: ${{ secrets.REPIN_TOKEN }} + GH_TOKEN: ${{ github.token }} + BASE: ${{ steps.base.outputs.base }} + run: | + set -uo pipefail + if [ -z "${REPIN_TOKEN:-}" ]; then + echo "::warning::REPIN_TOKEN is not set; reporting the repin instead of pushing it" + echo "mode=report" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Push every merged branch before touching the pin file: a pin whose + # commit is not on a remote is a pin the nightly cannot fetch. + FAILED="" + while read -r path head_repo head_ref new_sha; do + echo "pushing ${new_sha:0:10} to ${head_repo}:${head_ref}" + # No -x anywhere in this step; the URL carries the token. + if ! git -C "$path" push \ + "https://x-access-token:${REPIN_TOKEN}@github.com/${head_repo}.git" \ + "HEAD:refs/heads/${head_ref}" 2>&1 | sed "s/${REPIN_TOKEN}/***/g"; then + FAILED="${FAILED} ${head_repo}:${head_ref}" + fi + done < <(jq -r '.results[] | select(.action == "repin") + | "\(.repo_path) \(.head_repo) \(.head_ref) \(.new_sha)"' "${RUNNER_TEMP}/repin.json") + + if [ -n "$FAILED" ]; then + echo "::error::could not push:${FAILED}" + echo "mode=pushfail" >> "$GITHUB_OUTPUT" + echo "failed=${FAILED}" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git config user.name 'unsloth-repin-bot' + git config user.email 'unsloth-repin-bot@users.noreply.github.com' + git checkout -q -B "${REPIN_BRANCH}" + git add scripts/unsloth/pr-set.json + git commit -qm "Repin PR set onto ${BASE}" + # Force: the branch is a rolling proposal against whatever base tag is + # current, so yesterday's version is not worth preserving. + git push -q --force \ + "https://x-access-token:${REPIN_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + "HEAD:refs/heads/${REPIN_BRANCH}" 2>&1 | sed "s/${REPIN_TOKEN}/***/g" + + { + cat "${RUNNER_TEMP}/repin.md" + echo + echo "Opened automatically after [pin preflight](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/workflows/unsloth-pin-preflight.yml) failed. Review the resolutions above before merging; the bot does not merge its own PRs." + } > "${RUNNER_TEMP}/body.md" + + EXISTING="$(GH_TOKEN="${REPIN_TOKEN}" gh pr list --repo "${GITHUB_REPOSITORY}" \ + --head "${REPIN_BRANCH}" --state open --json number --jq '.[0].number' 2>/dev/null || true)" + if [ -n "$EXISTING" ] && [ "$EXISTING" != "null" ]; then + GH_TOKEN="${REPIN_TOKEN}" gh pr edit "$EXISTING" --repo "${GITHUB_REPOSITORY}" \ + --title "Repin PR set onto ${BASE}" --body-file "${RUNNER_TEMP}/body.md" >/dev/null + echo "updated PR #${EXISTING}" + echo "pr=${EXISTING}" >> "$GITHUB_OUTPUT" + else + URL="$(GH_TOKEN="${REPIN_TOKEN}" gh pr create --repo "${GITHUB_REPOSITORY}" \ + --base master --head "${REPIN_BRANCH}" \ + --title "Repin PR set onto ${BASE}" --body-file "${RUNNER_TEMP}/body.md" 2>&1 | tail -1)" + echo "opened ${URL}" + echo "pr=${URL}" >> "$GITHUB_OUTPUT" + fi + echo "mode=pushed" >> "$GITHUB_OUTPUT" + + - name: Report + id: report + if: ${{ always() && steps.base.outputs.base != '' }} + env: + CHANGED: ${{ steps.repin.outputs.changed }} + BLOCKED: ${{ steps.repin.outputs.blocked }} + OUTCOME: ${{ steps.repin.outcome }} + MODE: ${{ steps.push.outputs.mode }} + PR: ${{ steps.push.outputs.pr }} + FAILED: ${{ steps.push.outputs.failed }} + run: | + set -uo pipefail + # Only shout when a human has something to do. A run that repinned + # everything and opened a PR is already visible as a PR. + STATUS=success + # A crash in the repin step leaves no report at all, which must not + # read as "nothing to do" -- that is the green-run-does-nothing hole. + [ "${OUTCOME:-}" = "success" ] || STATUS=failure + { + echo 'details<<ALERT_EOF' + cat "${RUNNER_TEMP}/repin.md" 2>/dev/null \ + || echo "The repin step did not finish (outcome: ${OUTCOME:-unknown}); no report was produced." + echo + case "${MODE:-}" in + pushed) echo "Proposed in ${PR}." ;; + report) STATUS=failure + echo "\`REPIN_TOKEN\` is not configured, so nothing was pushed. Reproduce locally:" + echo + echo '```' + echo "git checkout <pin sha> && git merge <base tag>" + echo "python3 scripts/unsloth/additive_merge.py --repo ." + echo '```' ;; + pushfail) STATUS=failure + echo "Could not push:${FAILED}. \`REPIN_TOKEN\` likely lacks contents or workflow write on those repositories." ;; + *) [ "${CHANGED:-0}" = "0" ] && echo "Nothing could be repinned automatically." ;; + esac + if [ "${BLOCKED:-0}" != "0" ]; then + STATUS=failure + echo + echo "${BLOCKED} pin(s) need a human, see the table above." + fi + echo 'ALERT_EOF' + } >> "$GITHUB_OUTPUT" + echo "status=${STATUS}" >> "$GITHUB_OUTPUT" + + - name: Alert + if: ${{ always() && steps.report.outputs.status != '' }} + uses: ./.github/actions/prebuilt-alert + with: + status: ${{ steps.report.outputs.status }} + key: llama-repin-bot + title: 'Pins need a manual repin' + details: ${{ steps.report.outputs.details }} + token: ${{ github.token }} diff --git a/.github/workflows/unsloth-upstream-sync-guard.yml b/.github/workflows/unsloth-upstream-sync-guard.yml new file mode 100644 index 000000000000..20dca54895d3 --- /dev/null +++ b/.github/workflows/unsloth-upstream-sync-guard.yml @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +name: "Unsloth: upstream sync guard" + +# Holds the two properties that make a fork sync cheap, and that silently broke once. +# +# The 08-07 sync (PR #80) was squash-merged, so upstream 82bb48500 never became an ancestor of +# master. The files arrived; the ancestry did not. For three weeks every merge involving a +# master-derived branch three-way merged against a 2026-06-10 base and manufactured conflicts +# in files nobody had touched -- 539 of them, against 21 with the correct base. Nothing was red +# while that was true, which is the whole reason this exists. +# +# Uses the compare API rather than a checkout: the ancestry question is one request, and a +# full-history checkout of this repository is neither fast nor free. + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +jobs: + guard: + name: Upstream sync invariants + runs-on: ubuntu-24.04 + env: + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: { fetch-depth: 1 } + + - name: Check the recorded sync point is still an ancestor of master + run: | + set -euo pipefail + FILE=scripts/unsloth/upstream-sync.json + SHA="$(jq -r .commit "$FILE")" + TAG="$(jq -r .tag "$FILE")" + case "$SHA" in + [0-9a-f]*) [ "${#SHA}" -eq 40 ] || { echo "::error file=$FILE::commit must be a 40-hex sha"; exit 1; } ;; + *) echo "::error file=$FILE::commit must be a 40-hex sha"; exit 1 ;; + esac + + # compare(base...head): 'ahead' or 'identical' means base is an ancestor of head. + # 'diverged' or 'behind' means it is not, which is what a squash-merged sync looks like. + STATUS="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${SHA}...master" --jq .status)" + echo "compare ${SHA:0:10} (${TAG}) ...master -> ${STATUS}" + case "$STATUS" in + ahead|identical) echo "ancestry OK" ;; + *) + echo "::error file=$FILE::upstream ${TAG} (${SHA:0:10}) is NOT an ancestor of master (compare says '${STATUS}')." + echo "::error::A sync PR was almost certainly squash- or rebase-merged. Squashing drops the upstream parent, so the merge base stays stale and every later merge invents hundreds of conflicts. Re-land the sync with a merge commit." + exit 1 ;; + esac + + - name: Check the fork still owns only CI + run: | + set -euo pipefail + FILE=scripts/unsloth/upstream-sync.json + SHA="$(jq -r .commit "$FILE")" + + # The compare API caps its file list. Say so rather than pass on a truncated answer. + RESP="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${SHA}...master")" + TOTAL="$(jq -r '.files | length' <<<"$RESP")" + if [ "$TOTAL" -ge 300 ]; then + echo "::error file=$FILE::compare returned ${TOTAL} files, at or over the API cap, so this check cannot be trusted. The fork delta should be well under 100 paths; if it is genuinely this large the invariant has already broken." + exit 1 + fi + + STRAY="$(jq -r '.files[].filename' <<<"$RESP" | grep -vE '^(\.github/|scripts/unsloth/)' || true)" + if [ -n "$STRAY" ]; then + echo "::error file=$FILE::the fork now diverges from upstream outside .github/ and scripts/unsloth/:" + echo "$STRAY" | sed 's/^/ /' + echo "::error::Syncs are provably additive only while this fork owns no llama.cpp source. Land source changes upstream, or pin them through scripts/unsloth/pr-set.json, rather than carrying them on master." + exit 1 + fi + echo "fork delta is ${TOTAL} path(s), all under .github/ or scripts/unsloth/" diff --git a/common/arg.cpp b/common/arg.cpp index 74241f931285..5d58afb3005f 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1315,6 +1315,11 @@ bool common_params_parse(int argc, char ** argv, common_params & params, llama_e exit(0); } params.lr.init(); + + if (!common_exact_concurrency_init(ctx_arg.params)) { + ctx_arg.params = params_org; + return false; + } } catch (const std::invalid_argument & ex) { fprintf(stderr, "%s\n", ex.what()); ctx_arg.params = params_org; @@ -1728,6 +1733,25 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.cache_ram_mib = value; } ).set_env("LLAMA_ARG_CACHE_RAM").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI})); + add_opt(common_arg( + {"--preempt-ram"}, "N", + string_format("with a unified KV cache, park a slot in host RAM instead of failing every slot when the cache fills; " + "N is the maximum host RAM for parked sequences in MiB (default: %d, -1 - no limit, 0 - disable)", params.preempt_ram_mib), + [](common_params & params, int value) { + params.preempt_ram_mib = value; + } + ).set_env("LLAMA_ARG_PREEMPT_RAM").set_examples({LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"--preempt-async"}, + {"--no-preempt-async"}, + "copy a parked sequence out of and back into the KV cache on a stream of its own: the copy out " + "overlaps with the slots that keep decoding, while a copy back in, and a kv-full retry behind a " + "copy out that has not landed, wait for it (default: enabled, needs a backend that can copy " + "asynchronously, otherwise the copies are synchronous as before)", + [](common_params & params, bool value) { + params.preempt_async = value; + } + ).set_env("LLAMA_ARG_PREEMPT_ASYNC").set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"-kvu", "--kv-unified"}, {"-no-kvu", "--no-kv-unified"}, diff --git a/common/common.cpp b/common/common.cpp index d162a38800e0..944028da7180 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1,4 +1,5 @@ #include "ggml.h" +#include "ggml-backend.h" #include "gguf.h" #include "build-info.h" @@ -1289,6 +1290,12 @@ struct common_init_result::impl { common_init_result::common_init_result(common_params & params, bool model_only) : pimpl(new impl{}) { + // [TAG_EXACT_CONCURRENCY] before any context exists, so one is never created under a figure the explicit bound does not cover + if (!model_only && !common_exact_concurrency_init(params)) { + COM_ERR("%s", "LLAMA_EXACT_CONCURRENCY: refusing to load the model, see the error above\n"); + return; + } + auto mparams = common_model_params_to_llama(params); auto cparams = common_context_params_to_llama(params); @@ -1337,6 +1344,11 @@ common_init_result::common_init_result(common_params & params, bool model_only) return; } + if (!common_exact_concurrency_model(params, model)) { + COM_ERR("%s", "LLAMA_EXACT_CONCURRENCY: refusing to create a context, see the error above\n"); + return; + } + const llama_vocab * vocab = llama_model_get_vocab(model); // load and optionally apply lora adapters @@ -1403,6 +1415,12 @@ common_init_result::common_init_result(common_params & params, bool model_only) pimpl->context.reset(lctx); + if (!common_exact_concurrency_context(params, lctx)) { + COM_ERR("%s", "LLAMA_EXACT_CONCURRENCY: refusing to serve this context, see the error above\n"); + pimpl->context.reset(); + return; + } + set_process_priority(params.cpuparams.priority); pimpl->threadpools.init(lctx, params); @@ -1433,6 +1451,148 @@ std::vector<llama_adapter_lora_ptr> & common_init_result::lora() { return pimpl->lora; } +// [TAG_EXACT_CONCURRENCY] +bool common_exact_concurrency() { + static const bool enabled = []() { + const char * val = getenv("LLAMA_EXACT_CONCURRENCY"); + return val && atoi(val) != 0; + }(); + + return enabled; +} + +int common_exact_decode_width(const common_params & params) { + const int64_t n_slots = std::max(1, params.n_parallel); + + const int64_t n_draft = std::max(0, (int) common_speculative_n_max(¶ms.speculative)); + + // the product is handed to a backend as an int; one that overflows is reported, not wrapped + const int64_t n_cols = n_slots*(1 + n_draft); + + return n_cols > INT32_MAX ? -1 : (int) n_cols; +} + +bool common_exact_batch_geometry(int n_batch, int n_ubatch, int n_decode_width, int * n_batch_min) { + // an unset ubatch is the whole batch, and a ubatch never exceeds it + const int n_ub = std::min(n_batch, n_ubatch <= 0 ? n_batch : n_ubatch); + + const int n_min = n_ub + std::max(0, n_decode_width); + + if (n_batch_min) { + *n_batch_min = n_min; + } + + return n_batch >= n_min; +} + +// [TAG_EXACT_CONCURRENCY] the refusals that need the loaded model, run before a context exists +bool common_exact_concurrency_model(const common_params & params, const llama_model * model) { + if (!common_exact_concurrency() || params.mmproj.path.empty()) { + return true; + } + + // the paged pool places a cell from the sequence and the position alone, and M-RoPE gives every token of one image the same temporal position, so the second of them lands on the first one's cell and the batch is refused at the first image + const llama_rope_type rope_type = llama_model_rope_type(model); + + if (rope_type == LLAMA_ROPE_TYPE_MROPE || rope_type == LLAMA_ROPE_TYPE_IMROPE) { + COM_ERR("%s", "LLAMA_EXACT_CONCURRENCY does not support M-RoPE together with a projector: the tokens of one image share a temporal position and the paged pool would give them one cell\n"); + return false; + } + + return true; +} + +bool common_exact_concurrency_init(const common_params & params) { + if (!common_exact_concurrency()) { + return true; + } + + // DFlash drafting turns causal attention off on its draft context, which the paged attention needs; say so instead of asserting in the graph. DSpark is the same. + for (const auto type : params.speculative.types) { + if (type == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) { + COM_ERR("%s", "LLAMA_EXACT_CONCURRENCY does not support --spec-type draft-dflash or draft-dspark: both disable causal attention on the draft, which the paged attention needs\n"); + return false; + } + } + + const int n_cols = common_exact_decode_width(params); + + if (n_cols < 0) { + COM_ERR("LLAMA_EXACT_CONCURRENCY: a decode step of %d slots with %d draft tokens each is too wide to report\n", + std::max(1, params.n_parallel), std::max(0, (int) common_speculative_n_max(¶ms.speculative))); + return false; + } + + const char * bound = getenv("GGML_CUDA_BATCH_INVARIANT_MAX_COLS"); + if (bound) { + const int max_cols = atoi(bound); + if (max_cols > 0 && max_cols < n_cols) { + COM_ERR("GGML_CUDA_BATCH_INVARIANT_MAX_COLS is %d but LLAMA_EXACT_CONCURRENCY needs at " + "least %d to cover a decode step of %d slots, above which a matmul is left " + "batched and its rows depend on the other rows in the ubatch. Raise it to %d, " + "set it to 0 for no bound, or unset it to let it default to %d.\n", + max_cols, n_cols, std::max(1, params.n_parallel), n_cols, n_cols); + return false; + } + } + + // a prompt is added to a batch in whole ubatches, so a batch that cannot hold one beside a decode step of every slot would leave a prefill shorter ubatches than it gets alone, and the mode would report itself as on while a shared step changed the prompt's arithmetic + // a causal context clamps the batch to the context size, so that is the batch a prefill really gets; an unset -c is only known once the context exists, which common_exact_concurrency_context() checks + const int n_batch_eff = params.n_ctx > 0 ? std::min(params.n_ctx, params.n_batch) : params.n_batch; + + int n_batch_min = 0; + + if (!common_exact_batch_geometry(n_batch_eff, params.n_ubatch, n_cols, &n_batch_min)) { + COM_ERR("LLAMA_EXACT_CONCURRENCY needs a batch of at least %d tokens for a %d-token ubatch " + "and a decode step of %d slots (%d columns), but the batch is %d: a prefill beside " + "a running slot would be split into shorter ubatches than the same prompt gets alone. " + "Raise -b to %d (and -c to at least that), or lower -ub.\n", + n_batch_min, std::min(n_batch_eff, params.n_ubatch <= 0 ? n_batch_eff : params.n_ubatch), + std::max(1, params.n_parallel), n_cols, n_batch_eff, n_batch_min); + return false; + } + + // the batch splitter isolates prompts by width, so tell it how wide one sequence's decode step is; this also covers a caller that decodes before creating a context + if (!llama_set_exact_decode_tokens((uint32_t) (n_cols / std::max(1, params.n_parallel))) || + !llama_set_exact_decode_width((uint32_t) n_cols)) { + COM_ERR("%s", "LLAMA_EXACT_CONCURRENCY: the decode width could not be reported, see the error above\n"); + return false; + } + + return true; +} + +bool common_exact_concurrency_context(const common_params & params, const llama_context * ctx) { + if (!common_exact_concurrency()) { + return true; + } + + const int n_cols = common_exact_decode_width(params); + + if (n_cols < 0) { + return false; // already reported by common_exact_concurrency_init() + } + + // the context clamps the batch to the context size and the ubatch to the batch, and an unset -c takes its size from the model or from the fit to device memory, so this is the geometry a prefill really gets + const int n_batch = (int) llama_n_batch(ctx); + const int n_ubatch = (int) llama_n_ubatch(ctx); + + int n_batch_min = 0; + + if (!common_exact_batch_geometry(n_batch, n_ubatch, n_cols, &n_batch_min)) { + COM_ERR("LLAMA_EXACT_CONCURRENCY needs a batch of at least %d tokens for a %d-token ubatch " + "and a decode step of %d slots (%d columns), but the context was created with a batch " + "of %d: a context of %d tokens clamps it, so a prefill beside a running slot would be " + "split into shorter ubatches than the same prompt gets alone. Raise -c to at least %d " + "(-fitc as well when the context was fitted to device memory), or lower -ub.\n", + n_batch_min, n_ubatch, std::max(1, params.n_parallel), n_cols, n_batch, + (int) llama_n_ctx(ctx), n_batch_min); + return false; + } + + return true; +} + common_init_result_ptr common_init_from_params(common_params & params, bool model_only) { common_init_result_ptr res(new common_init_result(params, model_only)); diff --git a/common/common.h b/common/common.h index 63d0badd0f74..7eb1c059d341 100644 --- a/common/common.h +++ b/common/common.h @@ -630,6 +630,8 @@ struct common_params { int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unset int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. + int32_t preempt_ram_mib = 8192; // host RAM for parked (preempted) sequences: -1 = no limit, 0 = disable preemption + bool preempt_async = true; // park and restore on a stream of their own, off the decode loop std::string hostname = "127.0.0.1"; std::string public_path = ""; // NOLINT @@ -947,6 +949,23 @@ using common_init_result_ptr = std::unique_ptr<common_init_result>; common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false); +// [TAG_EXACT_CONCURRENCY] true when LLAMA_EXACT_CONCURRENCY is set for this process +bool common_exact_concurrency(); + +int common_exact_decode_width(const common_params & params); + +// [TAG_EXACT_CONCURRENCY] whether a batch of this shape holds a whole prompt ubatch beside a decode step of every slot, which a prefill needs to be split into the ubatches it would get alone; n_batch_min reports the batch size that would +bool common_exact_batch_geometry(int n_batch, int n_ubatch, int n_decode_width, int * n_batch_min = nullptr); + +// report that width to the CUDA backend, refusing a smaller explicit GGML_CUDA_BATCH_INVARIANT_MAX_COLS; false if the configuration must not run +bool common_exact_concurrency_init(const common_params & params); + +// the same for what only the loaded model tells: false if the model must not be served in exact mode +bool common_exact_concurrency_model(const common_params & params, const struct llama_model * model); + +// the same for the geometry the created context settled on, which the context size may have clamped below what -b and -ub asked for +bool common_exact_concurrency_context(const common_params & params, const struct llama_context * ctx); + struct llama_model_params common_model_params_to_llama ( common_params & params); struct llama_context_params common_context_params_to_llama(const common_params & params); diff --git a/common/speculative.cpp b/common/speculative.cpp index 2db381d58086..82fc175281e7 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -810,7 +810,9 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { result.push_back(id); - if (params.n_max <= (int) result.size()) { + // the per-call bound comes from the caller's remaining context, so it stops the loop as well as the configured maximum + if ((params.n_max <= (int) result.size()) || + (dp.n_max > 0 && dp.n_max <= (int) result.size())) { drafting[seq_id] = false; n_drafting--; continue; @@ -1193,7 +1195,8 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { const int32_t n = (int32_t) dp.n_past; - const int32_t n_draft = params.n_max; + // the caller's remaining context bounds the block as well as the configured maximum: the whole block is decoded before any truncation + const int32_t n_draft = dp.n_max > 0 ? std::min(params.n_max, dp.n_max) : params.n_max; const int32_t n_block_tokens = n_draft + (is_dspark && sample_from_anchor ? 0 : 1); i_block_beg[seq_id] = batch.n_tokens; @@ -1691,7 +1694,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { result.push_back(id); - if (params.n_max <= (int) result.size()) { + // the per-call bound comes from the caller's remaining context, so it stops the loop as well as the configured maximum + if ((params.n_max <= (int) result.size()) || + (dp.n_max > 0 && dp.n_max <= (int) result.size())) { drafting[seq_id] = false; n_drafting--; continue; diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index cc3f8cd36e35..84a2f8458ea2 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -62,6 +62,8 @@ extern "C" { GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor); GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); + // whether the buffer copies a strided set of rows in one call (see ggml_backend_tensor_set_2d); without it the generic path issues one transfer per row + GGML_API bool ggml_backend_buffer_supports_2d (ggml_backend_buffer_t buffer); GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer); GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); @@ -125,6 +127,8 @@ extern "C" { GGML_API void ggml_backend_event_free(ggml_backend_event_t event); GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend); GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event); + // non-blocking: true once everything recorded before the event has completed. Backends without a query implementation fall back to a blocking synchronize. + GGML_API bool ggml_backend_event_query(ggml_backend_event_t event); GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event); // @@ -190,6 +194,8 @@ extern "C" { GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size); GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + // whether ggml_backend_event_query() on this device really is non-blocking, rather than falling back to a blocking synchronize + GGML_API bool ggml_backend_dev_supports_event_query(ggml_backend_dev_t device); GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft); GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op); diff --git a/ggml/include/ggml-cuda.h b/ggml/include/ggml-cuda.h index 1cd81eeaebcd..897da6ca5f82 100644 --- a/ggml/include/ggml-cuda.h +++ b/ggml/include/ggml-cuda.h @@ -38,6 +38,9 @@ GGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char GGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total); GGML_BACKEND_API bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size); + +// [TAG_EXACT_CONCURRENCY] report the widest ubatch a decode step of this process can build, so the column policy covers it; call before the first graph is computed +GGML_BACKEND_API void ggml_backend_cuda_set_exact_decode_width(int n_cols); GGML_BACKEND_API void ggml_backend_cuda_unregister_host_buffer(void * buffer); GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cuda_reg(void); diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index ef05905cf9ab..241f5bfb1dd7 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -8,7 +8,7 @@ extern "C" { #endif - #define GGML_BACKEND_API_VERSION 2 + #define GGML_BACKEND_API_VERSION 3 // // Backend buffer type @@ -215,6 +215,9 @@ extern "C" { ggml_backend_event_t (*event_new) (ggml_backend_dev_t dev); void (*event_free) (ggml_backend_dev_t dev, ggml_backend_event_t event); void (*event_synchronize) (ggml_backend_dev_t dev, ggml_backend_event_t event); + + // (optional) non-blocking completion test for an event. Kept last: a missing entry is NULL and ggml_backend_event_query() then blocks instead. + bool (*event_query) (ggml_backend_dev_t dev, ggml_backend_event_t event); }; struct ggml_backend_device { diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 3ec40fb1af7f..d531ae4b5fa2 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -193,6 +193,7 @@ static const ggml_backend_device_i ggml_backend_meta_device_iface = { /* .event_new = */ nullptr, /* .event_free = */ nullptr, /* .event_synchronize = */ nullptr, + /* .event_query = */ NULL, }; static bool ggml_backend_dev_is_meta(ggml_backend_dev_t dev) { diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 40e50c5c9dbd..1d156e8f0fe6 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -184,6 +184,10 @@ bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) { return ggml_backend_buft_is_host(ggml_backend_buffer_get_type(buffer)); } +bool ggml_backend_buffer_supports_2d(ggml_backend_buffer_t buffer) { + return buffer->iface.set_tensor_2d != NULL && buffer->iface.get_tensor_2d != NULL; +} + void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) { GGML_ASSERT(buffer); buffer->usage = usage; @@ -560,6 +564,18 @@ void ggml_backend_event_synchronize(ggml_backend_event_t event) { event->device->iface.event_synchronize(event->device, event); } +bool ggml_backend_event_query(ggml_backend_event_t event) { + GGML_ASSERT(event); + + if (event->device->iface.event_query == NULL) { + // no way to ask: the honest answer is to wait for it and then say yes + ggml_backend_event_synchronize(event); + return true; + } + + return event->device->iface.event_query(event->device, event); +} + void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event) { GGML_ASSERT(backend); GGML_ASSERT(backend->iface.event_wait != NULL); @@ -636,6 +652,11 @@ bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_t return device->iface.supports_op(device, op); } +bool ggml_backend_dev_supports_event_query(ggml_backend_dev_t device) { + GGML_ASSERT(device); + return device->iface.event_query != NULL; +} + bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) { GGML_ASSERT(device); return device->iface.supports_buft(device, buft); diff --git a/ggml/src/ggml-blas/ggml-blas.cpp b/ggml/src/ggml-blas/ggml-blas.cpp index e4b5bd254747..7271b6b632b6 100644 --- a/ggml/src/ggml-blas/ggml-blas.cpp +++ b/ggml/src/ggml-blas/ggml-blas.cpp @@ -469,6 +469,7 @@ static const struct ggml_backend_device_i ggml_backend_blas_device_i = { /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL, + /* .event_query = */ NULL, }; // backend reg interface diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index c2745014a192..20c0e59df711 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -2660,6 +2660,10 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten return true; case GGML_OP_FLASH_ATTN_EXT: { + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads + if (op->src[5]) { + return false; + } #ifdef ASCEND_310P // FA not support on 310p device return false; @@ -2952,6 +2956,7 @@ static const ggml_backend_device_i ggml_backend_cann_device_interface = { /* .event_new = */ ggml_backend_cann_device_event_new, /* .event_free = */ ggml_backend_cann_device_event_free, /* .event_synchronize = */ ggml_backend_cann_device_event_synchronize, + /* .event_query = */ NULL, }; // backend reg diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 8cece71f186f..7ea548bcbce8 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -474,6 +474,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st return ggml_is_contiguous(op->src[0]); case GGML_OP_SSM_SCAN: return ggml_get_op_params_i32(op, 0) == 1 || op->src[3]->ne[0] == 1; + // [TAG_EXACT_CONCURRENCY] note: FLASH_ATTN_EXT with src[5], the page table, is deliberately still accepted: the CPU ignores it, but it is the reference test-backend-ops uses default: return true; } @@ -500,6 +501,7 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = { /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL, + /* .event_query = */ NULL, }; // CPU backend - backend (reg) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 7d14ce9067ee..b3006642ad48 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -49,6 +49,11 @@ #define GGML_CUDA_CC_PASCAL 600 #define GGML_CUDA_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products +// [TAG_BATCH_INVARIANT] 0 = off, 1 = split every batched matmul, 2 = split only where it changes bits +int ggml_cuda_batch_invariant(); +// widest batch the split applies to, 0 = no bound; bounding it gives up prompt-phase invariance only +int ggml_cuda_batch_invariant_max_cols(); + #define GGML_CUDA_CC_VOLTA 700 #define GGML_CUDA_CC_TURING 750 #define GGML_CUDA_CC_AMPERE 800 diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index 7442bc22af20..6d035c4be38a 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -1105,7 +1105,9 @@ void launch_fattn( // Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped. // Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or // multiple sequences of possibly different lengths. - if (!use_sparse && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) { + // [TAG_BATCH_INVARIANT] without this scan the KV loop runs to K->ne[1], which grows with the other sequences; the mask bounds it by the sequence's own extent + const bool batch_invariant_KV_max = ggml_cuda_batch_invariant() != 0; + if (!use_sparse && !dst->src[5] && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || batch_invariant_KV_max)) { const int64_t s31 = mask->nb[1] / sizeof(half2); const int64_t s33 = mask->nb[3] / sizeof(half2); @@ -1163,6 +1165,13 @@ void launch_fattn( if (ntiles_dst % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles. dst_tmp_meta.alloc((size_t(blocks_num.x) * ncols * (2 + DV/2))); } + } else if (dst->src[5] || ggml_cuda_batch_invariant()) { + // [TAG_BATCH_INVARIANT] the KV split between blocks, and so the order the partials combine in, follows K->ne[1]: pin it to one block per tile + parallel_blocks = 1; + + blocks_num.x = ntiles_x; + blocks_num.y = parallel_blocks; + blocks_num.z = ntiles_z_gqa*K->ne[2]*Q->ne[3]; } else { // parallel_blocks must not be larger than what the tensor size allows: parallel_blocks = std::min(parallel_blocks, ntiles_KV); @@ -1229,7 +1238,7 @@ void launch_fattn( V_data, mask ? ((const char *) mask->data) : nullptr, sinks ? ((const char *) sinks->data) : nullptr, - KV_max.ptr, + dst->src[5] ? (const int *) dst->src[5]->data : KV_max.ptr, !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuh index 57a285565913..fc35c0f6358d 100644 --- a/ggml/src/ggml-cuda/fattn-vec.cuh +++ b/ggml/src/ggml-cuda/fattn-vec.cuh @@ -16,7 +16,7 @@ static constexpr __device__ int ggml_cuda_fattn_vec_get_nthreads_device() { #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wpass-failed" #endif // __clang__ -template<int D, int ncols, ggml_type type_K, ggml_type type_V, bool use_logit_softcap> // D == head size +template<int D, int ncols, ggml_type type_K, ggml_type type_V, bool use_logit_softcap, bool paged = false> // D == head size __launch_bounds__(ggml_cuda_fattn_vec_get_nthreads_device(), 1) static __global__ void flash_attn_ext_vec( const char * Q_ptr, @@ -247,13 +247,24 @@ static __global__ void flash_attn_ext_vec( #endif // V_DOT2_F32_F16_AVAILABLE } - const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11; + // in the paged specialization KV_max carries [count, physical page IDs...] per query; the loop and each warp's recurrence follow logical positions, never physical addresses + static_assert(!paged || ncols == 1, "paged attention has one query per block"); + const int * pages = paged ? KV_max + (sequence*int(ne01.z) + ic0)*(1 + ne11/FATTN_KQ_STRIDE) : nullptr; + const int k_VKQ_max = paged ? pages[0]*FATTN_KQ_STRIDE : (KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11); + const char * K_base = K; + const char * V_base = V; + const half * mask_base = maskh; K += blockIdx.y*nthreads * nb11; V += blockIdx.y*nthreads * nb21; maskh += blockIdx.y*nthreads; for (int k_VKQ_0 = blockIdx.y*nthreads; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nthreads, - // Increment pointers after each loop: K += gridDim.y*nthreads*nb11, V += gridDim.y*nthreads*nb21, maskh += gridDim.y*nthreads) { + if constexpr (paged) { + const int physical = pages[1 + k_VKQ_0/FATTN_KQ_STRIDE]*FATTN_KQ_STRIDE + k_VKQ_0%FATTN_KQ_STRIDE; + K = K_base + int64_t(physical)*nb11; + V = V_base + int64_t(physical)*nb21; + maskh = mask_base + physical; + } // Calculate KQ tile and keep track of new maximum KQ values: float KQ_reg[ncols]; // KQ in registers. diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index ae217fbd9df1..118aed95115a 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -590,6 +590,11 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const // 192 satisfies % 64 == 0 but has no vec instance (DKQ != DV); force it onto the MMA path. const bool can_use_vector_kernel = Q->ne[0] <= 256 && Q->ne[0] % 64 == 0 && Q->ne[0] != 192 && K->ne[1] % FATTN_KQ_STRIDE == 0; + // [TAG_BATCH_INVARIANT] every choice below switches on Q->ne[1] or K->ne[1], both of which grow with the other sequences, so pin the kernel a batch of one would use + if (ggml_cuda_batch_invariant() && can_use_vector_kernel && Q->ne[1] == 1) { + return BEST_FATTN_KERNEL_VEC; + } + // If Turing tensor cores are available, use them: if (turing_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { if (can_use_vector_kernel) { @@ -702,6 +707,52 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ggml_cuda_set_device(ctx.device); + + if (dst->src[5]) { + GGML_ASSERT(dst->src[0]->ne[0] == 256 && dst->src[2]->ne[0] == 256); + GGML_ASSERT(dst->src[1]->type == GGML_TYPE_F16 && dst->src[2]->type == GGML_TYPE_F16); + GGML_ASSERT(dst->src[3] && dst->src[0]->ne[3] == 1); + GGML_ASSERT(dst->src[5]->type == GGML_TYPE_I32 && ggml_is_contiguous(dst->src[5])); + GGML_ASSERT(dst->src[5]->ne[0] == 1 + dst->src[1]->ne[1]/FATTN_KQ_STRIDE); + GGML_ASSERT(dst->src[5]->ne[1] == dst->src[0]->ne[1]); + float softcap; + memcpy(&softcap, (const float *) dst->op_params + 2, sizeof(softcap)); + GGML_ASSERT(softcap == 0.0f); + fattn_kernel_t kernel = flash_attn_ext_vec<256, 1, GGML_TYPE_F16, GGML_TYPE_F16, false, true>; + launch_fattn<256, 1, 1>(ctx, dst, kernel, 4, 0, 128, false, false, false, /*use_sparse =*/ false); + return; + } + + // [TAG_BATCH_INVARIANT] attend one query row at a time, as a batch of one would + const int fattn_max_cols = ggml_cuda_batch_invariant_max_cols(); + if (ggml_cuda_batch_invariant() && dst->src[0]->ne[1] > 1 && dst->src[0]->ne[3] == 1 && + (fattn_max_cols <= 0 || dst->src[0]->ne[1] <= fattn_max_cols)) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * mask = dst->src[3]; + + for (int64_t i = 0; i < Q->ne[1]; ++i) { + ggml_tensor Q_row = *Q; + Q_row.ne[1] = 1; + Q_row.data = (char *) Q->data + i*Q->nb[1]; + + ggml_tensor mask_row; + ggml_tensor dst_row = *dst; + // ne[2] runs to the end of dst so the F16 K/V scratch behind dst stays in place + dst_row.ne[2] = dst->ne[2] - i; + dst_row.data = (char *) dst->data + i*dst->nb[2]; + dst_row.src[0] = &Q_row; + if (mask) { + mask_row = *mask; + mask_row.ne[1] = 1; + mask_row.data = (char *) mask->data + i*mask->nb[1]; + dst_row.src[3] = &mask_row; + } + + ggml_cuda_flash_attn_ext(ctx, &dst_row); + } + return; + } + switch (ggml_cuda_get_best_fattn_kernel(ggml_cuda_get_device(), dst)) { case BEST_FATTN_KERNEL_NONE: GGML_ABORT("fatal error"); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 38bd4c9a07e6..85d29851cb1a 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -1757,6 +1757,11 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up, } static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) { + // [TAG_BATCH_INVARIANT] mul_mat+GLU is fused for a single destination column only, so leaving it on would give a solo request a different code path from a batched one + if (ggml_cuda_batch_invariant()) { + return false; + } + ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; const ggml_tensor * dst = tensor; @@ -1784,6 +1789,10 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) { } static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) { + if (ggml_cuda_batch_invariant()) { + return false; + } + ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; const ggml_tensor * dst = tensor; @@ -1812,60 +1821,281 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) { return use_mul_mat_vec_q; } -static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - GGML_TENSOR_BINARY_OP_LOCALS +// [TAG_BATCH_INVARIANT] the token count picks the matmul and how its K loop is split, so the same request produces different bits. GGML_CUDA_BATCH_INVARIANT: +// 1 - compute every destination column on its own, exactly as a batch of one would +// 2 - split off only the columns whose batch-of-one configuration differs from the batched one +static bool ggml_cuda_exact_concurrency() { + static const bool exact = []() { + const char * value = getenv("LLAMA_EXACT_CONCURRENCY"); + return value && atoi(value) != 0; + }(); + return exact; +} - const int32_t hint = ggml_get_op_params_i32(dst, 1); - if (hint == GGML_HINT_SRC0_IS_HADAMARD && ggml_cuda_op_fwht(ctx, src1, dst)) { +int ggml_cuda_batch_invariant() { + static const int mode = []() { + if (ggml_cuda_exact_concurrency()) { return 2; } + const char * val = getenv("GGML_CUDA_BATCH_INVARIANT"); + return val ? atoi(val) : 0; + }(); + return mode; +} + +// [TAG_EXACT_CONCURRENCY] the widest decode ubatch the caller says it can build, 0 if it never said +static std::atomic<int> g_exact_decode_width{0}; + +void ggml_backend_cuda_set_exact_decode_width(int n_cols) { + // monotonic: the widest figure ever reported stays, whatever order the reports arrive in + int cur = g_exact_decode_width.load(std::memory_order_relaxed); + + while (n_cols > cur && !g_exact_decode_width.compare_exchange_weak(cur, n_cols, std::memory_order_relaxed)) { + } +} + +int ggml_cuda_batch_invariant_max_cols() { + // [TAG_EXACT_CONCURRENCY] prompt ubatches hold one sequence, so a prefill already matches its solo run; an explicit bound always wins + static const int explicit_cols = []() { + const char * val = getenv("GGML_CUDA_BATCH_INVARIANT_MAX_COLS"); + return val ? atoi(val) : -1; + }(); + + if (explicit_cols >= 0) { + return explicit_cols; + } + + if (!ggml_cuda_exact_concurrency()) { + return 0; + } + + const int width = g_exact_decode_width.load(std::memory_order_relaxed); + + return width > 0 ? width : 16; +} + +// [TAG_EXACT_CONCURRENCY] a batch wider than the bound is left batched, so say so once. Only when nothing reported a decode width: with one, wider batches are single-sequence prefills. +static void ggml_cuda_warn_above_exact_bound(const char * op, int64_t ncols, int max_cols) { + if (!ggml_cuda_exact_concurrency()) { + return; + } + + if (g_exact_decode_width.load(std::memory_order_relaxed) > 0) { return; } + static std::atomic_flag warned = ATOMIC_FLAG_INIT; + if (warned.test_and_set(std::memory_order_relaxed)) { + return; + } + + GGML_LOG_WARN("%s: LLAMA_EXACT_CONCURRENCY is set, but this %s is %d columns wide while " + "GGML_CUDA_BATCH_INVARIANT_MAX_COLS is %d, so it is left batched and its result depends " + "on the other columns in the ubatch. Raise the bound, set it to 0 for no bound, or call " + "ggml_backend_cuda_set_exact_decode_width() with the widest decode this process builds. " + "Reported once.\n", __func__, op, (int) ncols, max_cols); +} + +enum ggml_cuda_mm_path { + GGML_CUDA_MM_CUBLAS_UNSUPPORTED, + GGML_CUDA_MM_MMVF, + GGML_CUDA_MM_MMVF_TRANSPOSED, + GGML_CUDA_MM_MMF, + GGML_CUDA_MM_MMVQ, + GGML_CUDA_MM_MMQ, + GGML_CUDA_MM_CUBLAS, +}; + +static ggml_cuda_mm_path ggml_cuda_mul_mat_path( + int cc, int warp_size, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst, int64_t ne11) { // If src0 is a temporary compute buffer it may have some padding that needs to be cleared for mul_mat_vec_q or mul_mat_q. // But if src0 is also a view of another tensor then this cannot be done safely because it may overwrite valid tensor data. // Therefore, in such cases use cuBLAS. const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && src0->view_src; if (bad_padding_clear || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { - ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); - return; + return GGML_CUDA_MM_CUBLAS_UNSUPPORTED; } - - const int cc = ggml_cuda_info().devices[ctx.device].cc; - const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - if (ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, ne11)) { // The custom F16 vector kernel can be used over batched cuBLAS GEMM. // But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention) - ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); - return; + return GGML_CUDA_MM_MMVF; } // A transposed vector can still use MMVQ (i.e. ne01 == 1) - if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1 + if (src0->ne[1] == 1 && ne11 > MMVF_MAX_BATCH_SIZE && dst->ne[2] == 1 && dst->ne[3] == 1 && src0->type == GGML_TYPE_F32 && ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) && ggml_cuda_should_use_mmvf(src1->type, cc, src1->ne, src1->nb, /*ne11 =*/ 1)) { - ggml_tensor dst_vec = *dst; - dst_vec.ne[0] = ne11; - dst_vec.ne[1] = 1; - dst_vec.nb[1] = dst_vec.nb[0]*ne11; - dst_vec.nb[2] = dst_vec.nb[1]; - dst_vec.nb[3] = dst_vec.nb[1]; - ggml_cuda_mul_mat_vec_f(ctx, src1, src0, nullptr, &dst_vec); - return; + return GGML_CUDA_MM_MMVF_TRANSPOSED; } if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11, /*mul_mat_id =*/ false)) { - ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst); - return; + return GGML_CUDA_MM_MMF; } if (ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { - ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); - return; + return GGML_CUDA_MM_MMVQ; } if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { - ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); + return GGML_CUDA_MM_MMQ; + } + return GGML_CUDA_MM_CUBLAS; +} + +static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); + +// [TAG_BATCH_INVARIANT] the widest slice of columns that can be recomputed in one launch while every column still sums as a batch of one; always below ncols_dst, so the recursion ends +static int64_t ggml_cuda_mul_mat_invariant_width( + int cc, int warp_size, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst, + ggml_cuda_mm_path path_one, int64_t ncols_dst) { + if (path_one != GGML_CUDA_MM_MMVF && path_one != GGML_CUDA_MM_MMVQ) { + return 1; + } + const int64_t widest = path_one == GGML_CUDA_MM_MMVF ? MMVF_MAX_BATCH_SIZE : MMVQ_MAX_BATCH_SIZE; + for (int64_t w = std::min<int64_t>(ncols_dst - 1, widest); w > 1; --w) { + if (ggml_cuda_mul_mat_path(cc, warp_size, src0, src1, dst, w) != path_one) { + continue; + } + if (path_one == GGML_CUDA_MM_MMVQ && !ggml_cuda_mmvq_matches_single_column(src0->type, cc, w)) { + continue; + } + return w; + } + return 1; +} + +static bool ggml_cuda_mul_mat_split_columns( + ggml_backend_cuda_context & ctx, int cc, int warp_size, + const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + // recurrent output projections broadcast one weight matrix over sequence planes, so normalize each plane before applying the column policy + if (ggml_cuda_exact_concurrency() && src0->ne[2] == 1 && src0->ne[3] == 1 && + (dst->ne[2] > 1 || dst->ne[3] > 1) && + src1->ne[2] == dst->ne[2] && src1->ne[3] == dst->ne[3]) { + for (int64_t i3 = 0; i3 < dst->ne[3]; ++i3) { + for (int64_t i2 = 0; i2 < dst->ne[2]; ++i2) { + ggml_tensor src_plane = *src1; + ggml_tensor dst_plane = *dst; + src_plane.ne[2] = src_plane.ne[3] = 1; + dst_plane.ne[2] = dst_plane.ne[3] = 1; + src_plane.data = (char *) src1->data + i2*src1->nb[2] + i3*src1->nb[3]; + dst_plane.data = (char *) dst->data + i2*dst->nb[2] + i3*dst->nb[3]; + ggml_cuda_mul_mat(ctx, src0, &src_plane, &dst_plane); + } + } + return true; + } + + const int64_t ncols_dst = dst->ne[1]; + if (ncols_dst <= 1 || src1->ne[1] != ncols_dst) { + return false; + } + if (src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + const int max_cols = ggml_cuda_batch_invariant_max_cols(); + if (max_cols > 0 && ncols_dst > max_cols) { + ggml_cuda_warn_above_exact_bound("MUL_MAT", ncols_dst, max_cols); + return false; + } + + // mode 1 recomputes one column at a time; mode 2, which exact concurrency runs under, uses the widest slices that keep the batch-of-one arithmetic + int64_t width = 1; + if (ggml_cuda_batch_invariant() >= 2) { + const ggml_cuda_mm_path path_one = ggml_cuda_mul_mat_path(cc, warp_size, src0, src1, dst, 1); + const ggml_cuda_mm_path path_batched = ggml_cuda_mul_mat_path(cc, warp_size, src0, src1, dst, ncols_dst); + if (path_one == path_batched) { + // same implementation, but it still has to sum in the same order + if (path_batched == GGML_CUDA_MM_MMVF) { + return false; // the block size follows K alone + } + if (path_batched == GGML_CUDA_MM_MMVQ && + ggml_cuda_mmvq_matches_single_column(src0->type, cc, ncols_dst)) { + return false; + } + } + width = ggml_cuda_mul_mat_invariant_width(cc, warp_size, src0, src1, dst, path_one, ncols_dst); + if (width >= ncols_dst) { + width = 1; + } + } + + for (int64_t i = 0; i < ncols_dst; i += width) { + const int64_t n = std::min(width, ncols_dst - i); + + ggml_tensor src1_col = *src1; + ggml_tensor dst_col = *dst; + + src1_col.ne[1] = n; + src1_col.nb[2] = n*src1_col.nb[1]; + src1_col.nb[3] = n*src1_col.nb[1]; + src1_col.data = (char *) src1->data + i*src1->nb[1]; + + dst_col.ne[1] = n; + dst_col.nb[2] = n*dst_col.nb[1]; + dst_col.nb[3] = n*dst_col.nb[1]; + dst_col.data = (char *) dst->data + i*dst->nb[1]; + + ggml_cuda_mul_mat(ctx, src0, &src1_col, &dst_col); + } + return true; +} + +static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS + + const int32_t hint = ggml_get_op_params_i32(dst, 1); + if (hint == GGML_HINT_SRC0_IS_HADAMARD && ggml_cuda_op_fwht(ctx, src1, dst)) { return; } - ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); + + const int cc = ggml_cuda_info().devices[ctx.device].cc; + const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; + + if (ggml_cuda_batch_invariant() && ggml_cuda_mul_mat_split_columns(ctx, cc, warp_size, src0, src1, dst)) { + return; + } + + switch (ggml_cuda_mul_mat_path(cc, warp_size, src0, src1, dst, ne11)) { + case GGML_CUDA_MM_CUBLAS_UNSUPPORTED: + case GGML_CUDA_MM_CUBLAS: + ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst); + return; + case GGML_CUDA_MM_MMVF: + ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); + return; + case GGML_CUDA_MM_MMVF_TRANSPOSED: { + ggml_tensor dst_vec = *dst; + dst_vec.ne[0] = ne11; + dst_vec.ne[1] = 1; + dst_vec.nb[1] = dst_vec.nb[0]*ne11; + dst_vec.nb[2] = dst_vec.nb[1]; + dst_vec.nb[3] = dst_vec.nb[1]; + ggml_cuda_mul_mat_vec_f(ctx, src1, src0, nullptr, &dst_vec); + return; + } + case GGML_CUDA_MM_MMF: + ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst); + return; + case GGML_CUDA_MM_MMVQ: + ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); + return; + case GGML_CUDA_MM_MMQ: + ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); + return; + } + GGML_ABORT("fatal error"); +} + +static bool ggml_cuda_mul_mat_id_splits_tokens(const ggml_tensor * dst) { + if (!ggml_cuda_batch_invariant()) { + return false; + } + const int64_t ntokens = dst->ne[2]; + if (ntokens <= 1) { + return false; + } + const int max_cols = ggml_cuda_batch_invariant_max_cols(); + if (max_cols > 0 && ntokens > max_cols) { + ggml_cuda_warn_above_exact_bound("MUL_MAT_ID", ntokens, max_cols); + return false; + } + return true; } // returns true when ggml_cuda_mul_mat_id takes the fallback path that requires stream synchronization @@ -1878,9 +2108,12 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c return true; } - if (dst->ne[2] <= MMVQ_MAX_BATCH_SIZE) { + // [TAG_BATCH_INVARIANT] a split node runs as ntokens single-token calls, so the path that decides whether the stream is synchronized is the single-token one + const int64_t ntokens = ggml_cuda_mul_mat_id_splits_tokens(dst) ? 1 : dst->ne[2]; + + if (ntokens <= MMVQ_MAX_BATCH_SIZE) { if (ggml_is_quantized(src0->type)) { - if (dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc)) { + if (ntokens <= get_mmvq_mmid_max_batch(src0->type, cc)) { return false; } } else if (GGML_CUDA_CC_IS_AMD(cc)) { @@ -1888,17 +2121,51 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c } } - if (ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[2], /*n_experts=*/src0->ne[2])) { + if (ggml_cuda_should_use_mmq(src0->type, cc, ntokens, /*n_experts=*/src0->ne[2])) { return false; } - if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) { + if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, ntokens, /*mul_mat_id=*/true)) { return false; } return true; } +static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +// [TAG_BATCH_INVARIANT] recompute dst one token at a time: every implementation below groups the ubatch's tokens by the expert they routed to, so shapes depend on the other tokens +static void ggml_cuda_mul_mat_id_split_tokens(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * ids = dst->src[2]; + + const int64_t ntokens = dst->ne[2]; + + for (int64_t i = 0; i < ntokens; ++i) { + ggml_tensor src1_token = *src1; + ggml_tensor ids_token = *ids; + ggml_tensor dst_token = *dst; + + src1_token.ne[2] = 1; + src1_token.nb[3] = src1_token.nb[2]; + src1_token.data = (char *) src1->data + i*src1->nb[2]; + + ids_token.ne[1] = 1; + ids_token.nb[2] = ids_token.nb[1]; + ids_token.nb[3] = ids_token.nb[1]; + ids_token.data = (char *) ids->data + i*ids->nb[1]; + + dst_token.ne[2] = 1; + dst_token.nb[3] = dst_token.nb[2]; + dst_token.data = (char *) dst->data + i*dst->nb[2]; + + dst_token.src[1] = &src1_token; + dst_token.src[2] = &ids_token; + + ggml_cuda_mul_mat_id(ctx, &dst_token); + } +} + static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; @@ -1911,6 +2178,18 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor * const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + // [TAG_BATCH_INVARIANT] + if (ggml_cuda_mul_mat_id_splits_tokens(dst)) { + GGML_ASSERT(ne3 == 1 && src1->ne[3] == 1 && ids->ne[2] == 1 && ids->ne[3] == 1); + // a quantized expert matrix takes the single-token MMVQ path at every token count and can put the tokens on its sample axis in one launch; anything else goes token by token + if (ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst); + return; + } + ggml_cuda_mul_mat_id_split_tokens(ctx, dst); + return; + } + // [TAG_MUL_MAT_ID_CUDA_GRAPHS] if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { static_assert(MMVQ_MAX_BATCH_SIZE == MMVF_MAX_BATCH_SIZE); @@ -3456,9 +3735,10 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } - //topk-moe - if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || - cgraph->nodes[i]->op == GGML_OP_ARGSORT) { + // [TAG_BATCH_INVARIANT] the routing fusion passes its memory-range check only for a one-token ubatch, so a solo request takes the fused top-k kernel and a batched one the long chain + if (!ggml_cuda_batch_invariant() && + (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || + cgraph->nodes[i]->op == GGML_OP_ARGSORT)) { ggml_cuda_topk_moe_args args; const bool can_fuse = ggml_cuda_topk_moe_fusion(cgraph, i, args); std::vector<ggml_op> ops; @@ -5570,6 +5850,21 @@ static void ggml_backend_cuda_device_event_synchronize(ggml_backend_dev_t dev, g CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context)); } +static bool ggml_backend_cuda_device_event_query(ggml_backend_dev_t dev, ggml_backend_event_t event) { + GGML_UNUSED(dev); + + const cudaError_t err = cudaEventQuery((cudaEvent_t)event->context); + + // not an error, and nothing to clear: cudaEventQuery() returns cudaErrorNotReady without recording it, so collecting one here would consume somebody else's + if (err == cudaErrorNotReady) { + return false; + } + + CUDA_CHECK(err); + + return true; +} + static const ggml_backend_device_i ggml_backend_cuda_device_interface = { /* .get_name = */ ggml_backend_cuda_device_get_name, /* .get_description = */ ggml_backend_cuda_device_get_description, @@ -5586,6 +5881,7 @@ static const ggml_backend_device_i ggml_backend_cuda_device_interface = { /* .event_new = */ ggml_backend_cuda_device_event_new, /* .event_free = */ ggml_backend_cuda_device_event_free, /* .event_synchronize = */ ggml_backend_cuda_device_event_synchronize, + /* .event_query = */ ggml_backend_cuda_device_event_query, }; // backend reg @@ -5687,6 +5983,10 @@ static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, con if (strcmp(name, "ggml_backend_get_features") == 0) { return (void *)ggml_backend_cuda_get_features; } + // [TAG_EXACT_CONCURRENCY] + if (strcmp(name, "ggml_backend_cuda_set_exact_decode_width") == 0) { + return (void *)ggml_backend_cuda_set_exact_decode_width; + } return nullptr; } diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 6305230b1f92..0435ac197c28 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -580,6 +580,20 @@ static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int return 1; } +// [TAG_BATCH_INVARIANT] +bool ggml_cuda_mmvq_matches_single_column(enum ggml_type type, int cc, int64_t ncols_dst) { + if (ncols_dst < 1 || ncols_dst > MMVQ_MAX_BATCH_SIZE) { + return false; + } + const mmvq_parameter_table_id table_id = get_device_table_id(cc); + if (table_id == MMVQ_PARAMETERS_GB10) { + // There nwarps also depends on the K loop trip count, which the caller does not pass in. + return ncols_dst == 1; + } + // blocks_per_iter, which assigns K blocks to threads, is proportional to nwarps; rows_per_cuda_block only changes which rows a block owns, not the order within a row + return calc_nwarps(type, 1, table_id) == calc_nwarps(type, (int) ncols_dst, table_id); +} + template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false> __launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1) static __global__ void mul_mat_vec_q( @@ -616,9 +630,10 @@ static __global__ void mul_mat_vec_q( uint32_t sample_dst; ggml_cuda_pdl_sync(); - channel_x = ncols_dst == 1 && ids ? ids[channel_dst] : fastdiv(channel_dst, channel_ratio); - channel_y = ncols_dst == 1 && ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst; sample_dst = blockIdx.z; + // [TAG_BATCH_INVARIANT] with ids, a sample is a token: every token goes on the z axis of one single-column launch, so each (token, expert slot) block runs the single-token configuration + channel_x = ncols_dst == 1 && ids ? ids[sample_dst*ids_stride + channel_dst] : fastdiv(channel_dst, channel_ratio); + channel_y = ncols_dst == 1 && ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst; const uint32_t sample_x = fastdiv(sample_dst, sample_ratio); const uint32_t sample_y = sample_dst; @@ -1422,7 +1437,11 @@ void ggml_cuda_mul_mat_vec_q( GGML_ASSERT( nb0 == ts_dst); GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type)); - GGML_ASSERT(!ids || ne12 <= MMVQ_MAX_BATCH_SIZE); + // [TAG_BATCH_INVARIANT] a multi-token MUL_MAT_ID becomes one launch of the single-token configuration with the tokens on the sample axis, so the count is not bounded by the column templates + const bool tokens_as_samples = ids && ne2 > 1 && ggml_cuda_batch_invariant(); + + GGML_ASSERT(!ids || ne12 <= MMVQ_MAX_BATCH_SIZE || tokens_as_samples); + GGML_ASSERT(!tokens_as_samples || !fusion); const float * src1_d = (const float *) src1->data; const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr; @@ -1512,6 +1531,17 @@ void ggml_cuda_mul_mat_vec_q( const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0; + if (tokens_as_samples) { + GGML_ASSERT(ne03 == 1 && ne13 == 1 && ne3 == 1); + // one column, one sample per token: y advances by s12 per token, dst by s2, x not at all + mul_mat_vec_q_switch_type( + src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00, + ne01, 1, s01, stride_col_y, stride_col_dst, + ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, + 1, ne2, s03, s12, s2, ids_stride, stream); + return; + } + mul_mat_vec_q_switch_type( src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst, diff --git a/ggml/src/ggml-cuda/mmvq.cuh b/ggml/src/ggml-cuda/mmvq.cuh index 5605bf7a4e60..688c944c1f90 100644 --- a/ggml/src/ggml-cuda/mmvq.cuh +++ b/ggml/src/ggml-cuda/mmvq.cuh @@ -4,6 +4,9 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11); +// [TAG_BATCH_INVARIANT] true when an MMVQ launch of ncols_dst columns sums each destination element in the same order as a single-column launch, i.e. when nwarps is unchanged +bool ggml_cuda_mmvq_matches_single_column(enum ggml_type type, int cc, int64_t ncols_dst); + // Returns the maximum batch size for which MMVQ should be used for MUL_MAT_ID, // based on the quantization type and GPU architecture (compute capability). int get_mmvq_mmid_max_batch(ggml_type type, int cc); diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index 2fc0fe9fdbb7..1a09deffd56a 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -58,10 +58,12 @@ #define cudaDeviceSynchronize hipDeviceSynchronize #define cudaError_t hipError_t #define cudaErrorMemoryAllocation hipErrorOutOfMemory +#define cudaErrorNotReady hipErrorNotReady #define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled #define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled #define cudaEventCreateWithFlags hipEventCreateWithFlags #define cudaEventDisableTiming hipEventDisableTiming +#define cudaEventQuery hipEventQuery #define cudaEventRecord hipEventRecord #define cudaEventSynchronize hipEventSynchronize #define cudaEvent_t hipEvent_t diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 6d725c7ec196..ebecf679950e 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -46,10 +46,12 @@ #define cudaDeviceSynchronize musaDeviceSynchronize #define cudaError_t musaError_t #define cudaErrorMemoryAllocation musaErrorMemoryAllocation +#define cudaErrorNotReady musaErrorNotReady #define cudaErrorPeerAccessAlreadyEnabled musaErrorPeerAccessAlreadyEnabled #define cudaErrorPeerAccessNotEnabled musaErrorPeerAccessNotEnabled #define cudaEventCreateWithFlags musaEventCreateWithFlags #define cudaEventDisableTiming musaEventDisableTiming +#define cudaEventQuery musaEventQuery #define cudaEventRecord musaEventRecord #define cudaEventSynchronize musaEventSynchronize #define cudaEvent_t musaEvent_t diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp index 61c31d6f2912..b6305eb2a240 100644 --- a/ggml/src/ggml-et/ggml-et.cpp +++ b/ggml/src/ggml-et/ggml-et.cpp @@ -1267,6 +1267,11 @@ static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggm (op->src[1]->ne[1] % op->src[4]->ne[1] == 0); break; case GGML_OP_FLASH_ATTN_EXT: + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads + if (op->src[5]) { + supported = false; + break; + } if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && op->src[2] && (op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr && @@ -1685,6 +1690,7 @@ static const struct ggml_backend_device_i ggml_backend_et_device_i = { /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL, + /* .event_query = */ NULL, }; /* diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index a39df2a878c5..48325cad4f2b 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -5930,7 +5930,8 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons break; case GGML_OP_FLASH_ATTN_EXT: - supp = ggml_hexagon_supported_flash_attn_ext(sess, op); + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads + supp = op->src[5] == nullptr && ggml_hexagon_supported_flash_attn_ext(sess, op); break; case GGML_OP_SET_ROWS: @@ -6036,6 +6037,7 @@ static const struct ggml_backend_device_i ggml_backend_hexagon_device_i = { /* .event_new = */ ggml_backend_hexagon_device_event_new, /* .event_free = */ ggml_backend_hexagon_device_event_free, /* .event_synchronize = */ ggml_backend_hexagon_device_event_synchronize, + /* .event_query = */ NULL, }; //** backend registry diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index afd6f521011e..590df1bd24bd 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1703,6 +1703,10 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_ROLL: return ggml_is_contiguous(op->src[0]); case GGML_OP_FLASH_ATTN_EXT: + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads; walking the pool in physical order here would be silently wrong + if (op->src[5] != NULL) { + return false; + } // for new head sizes, add checks here if (op->src[0]->ne[0] != 32 && op->src[0]->ne[0] != 40 && diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 3bd6abd06fdc..69af3678fa06 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -821,6 +821,7 @@ static ggml_backend_device_i ggml_backend_metal_device_i = { /* .event_new = */ ggml_backend_metal_device_event_new, /* .event_free = */ ggml_backend_metal_device_event_free, /* .event_synchronize = */ ggml_backend_metal_device_event_synchronize, + /* .event_query = */ NULL, }; // backend registry diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 3002835e8aea..0ed3e8fc5c77 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -8745,6 +8745,10 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_MEAN: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_FLASH_ATTN_EXT: { + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads + if (op->src[5]) { + return false; + } // The E17 compilers segfault while building FA kernels, skip E17 for now if (adreno_e17_compiler_quirks(backend_ctx)) { return false; @@ -12410,6 +12414,7 @@ struct ggml_backend_device_i ggml_backend_opencl_device_i = { /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL, + /* .event_query = */ NULL, }; } diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index a7956227830b..e640e666138d 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -1171,6 +1171,10 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { break; } case GGML_OP_FLASH_ATTN_EXT: { + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads + if (op->src[5]) { + return true; + } float scale = 1.0f; float max_bias = 0.0f; float logit_softcap = 0.0f; @@ -1495,6 +1499,7 @@ static const struct ggml_backend_device_i ggml_backend_openvino_device_interface /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL, + /* .event_query = */ NULL, }; struct ggml_backend_openvino_reg_context { diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index cc7d7206933f..fa5f1cfb6ede 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -2185,7 +2185,10 @@ static ggml_backend_buffer_type_t ggml_backend_rpc_device_get_buffer_type(ggml_b static bool ggml_backend_rpc_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { GGML_UNUSED(dev); - GGML_UNUSED(op); + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads; the remote end is not asked, so it is not claimed here + if (op->op == GGML_OP_FLASH_ATTN_EXT && op->src[5]) { + return false; + } //TODO: call the remote backend and cache the results return true; } @@ -2233,6 +2236,7 @@ static const struct ggml_backend_device_i ggml_backend_rpc_device_i = { /* .event_new = */ ggml_backend_rpc_device_event_new, /* .event_free = */ ggml_backend_rpc_device_event_free, /* .event_synchronize = */ ggml_backend_rpc_device_event_synchronize, + /* .event_query = */ NULL, }; // backend reg interface diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 4091f73a4674..05b66a3a9e58 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6649,7 +6649,8 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_SOLVE_TRI: return op->src[0]->ne[0] <= SYCL_SOLVE_TRI_MAX_N && op->src[1]->ne[0] <= SYCL_SOLVE_TRI_MAX_K; case GGML_OP_FLASH_ATTN_EXT: - return ggml_sycl_flash_attn_ext_supported(device, op); + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads + return op->src[5] == nullptr && ggml_sycl_flash_attn_ext_supported(device, op); default: return false; } @@ -6755,6 +6756,7 @@ static const ggml_backend_device_i ggml_backend_sycl_device_interface = { /* .event_new = */ ggml_backend_sycl_device_event_new, /* .event_free = */ ggml_backend_sycl_device_event_free, /* .event_synchronize = */ ggml_backend_sycl_device_event_synchronize, + /* .event_query = */ NULL, }; // backend reg diff --git a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp b/ggml/src/ggml-virtgpu/ggml-backend-device.cpp index 987ce9dd110c..13a70c594df6 100644 --- a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +++ b/ggml/src/ggml-virtgpu/ggml-backend-device.cpp @@ -157,4 +157,5 @@ const ggml_backend_device_i ggml_backend_remoting_device_interface = { /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL, + /* .event_query = */ NULL, }; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 8f37f65b8ab9..75d8014ec56f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -19350,6 +19350,10 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm } case GGML_OP_FLASH_ATTN_EXT: { + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads + if (op->src[5]) { + return false; + } bool coopmat2 = device->coopmat2; uint32_t HSK = op->src[1]->ne[0]; uint32_t HSV = op->src[2]->ne[0]; @@ -20027,6 +20031,7 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = { /* .event_new = */ ggml_backend_vk_device_event_new, /* .event_free = */ ggml_backend_vk_device_event_free, /* .event_synchronize = */ ggml_backend_vk_device_event_synchronize, + /* .event_query = */ NULL, }; static const char * ggml_backend_vk_reg_get_name(ggml_backend_reg_t reg) { diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index f06a9c872db9..38f2cfca7bc5 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -4425,6 +4425,12 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const break; case GGML_OP_FLASH_ATTN_EXT: { + // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads + if (op->src[5]) { + supports_op = false; + break; + } + // conservative support checks for whether the more resource-intensive shader paths // can be used, to avoid cases where flash_attn is assigned to the CPU later on supports_op = src0->type == GGML_TYPE_F32 && @@ -4679,6 +4685,7 @@ static struct ggml_backend_device_i ggml_backend_webgpu_device_i = { /* .event_new = */ ggml_backend_webgpu_device_event_new, /* .event_free = */ ggml_backend_webgpu_device_event_free, /* .event_synchronize = */ ggml_backend_webgpu_device_event_synchronize, + /* .event_query = */ NULL, }; /* End GGML Backend Device Interface */ diff --git a/ggml/src/ggml-zdnn/ggml-zdnn.cpp b/ggml/src/ggml-zdnn/ggml-zdnn.cpp index 4007ac9dfc7d..bbd74fb9d5aa 100644 --- a/ggml/src/ggml-zdnn/ggml-zdnn.cpp +++ b/ggml/src/ggml-zdnn/ggml-zdnn.cpp @@ -547,6 +547,7 @@ static ggml_backend_device_i ggml_backend_zdnn_device_i = { /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL, + /* .event_query = */ NULL, }; // diff --git a/ggml/src/ggml-zendnn/ggml-zendnn.cpp b/ggml/src/ggml-zendnn/ggml-zendnn.cpp index ec7ce233145a..89c6c36a0f19 100644 --- a/ggml/src/ggml-zendnn/ggml-zendnn.cpp +++ b/ggml/src/ggml-zendnn/ggml-zendnn.cpp @@ -781,6 +781,7 @@ static const struct ggml_backend_device_i ggml_backend_zendnn_device_i = { /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL, + /* .event_query = */ NULL, }; // backend reg interface diff --git a/include/llama.h b/include/llama.h index ef7a012c43a1..dfb19eae8da4 100644 --- a/include/llama.h +++ b/include/llama.h @@ -804,6 +804,27 @@ extern "C" { // Check if the memory supports shifting LLAMA_API bool llama_memory_can_shift(llama_memory_t mem); + // [TAG_EXACT_CONCURRENCY] cells the memory allocates in one indivisible unit: 1 ordinarily, + // larger where a mode places cells in blocks, and then n contiguous tokens occupy + // round_up(n, granularity) cells. A caller deciding whether the pool has room must round the + // same way. round_up is the contiguous case only: a block is held for as long as any cell in + // it is live, so a sequence left with holes by a partial llama_memory_seq_rm still holds every + // block that has one, which can be far more than round_up of what it has left. Removing + // positions 1 to 510 of a 512-token sequence leaves two live cells holding two whole blocks. + LLAMA_API uint32_t llama_memory_alloc_granularity(llama_memory_t mem); + + // [TAG_PREEMPT] run the in-place update a seq_add() recorded, which llama_decode() would otherwise run at the start of the next batch + // takes the context because the update is a graph; returns true when one was run + LLAMA_API bool llama_memory_update(struct llama_context * ctx); + + // [TAG_EXACT_CONCURRENCY] the most tokens one sequence contributes to a decode step: 1, or 1 plus the draft length. Never lowered; false when a column bound cannot cover it. + LLAMA_API bool llama_set_exact_decode_tokens(uint32_t n_tokens); + LLAMA_API uint32_t llama_exact_decode_tokens(void); + + // [TAG_EXACT_CONCURRENCY] the widest decode ubatch this process can build, in columns; never lowered, and false when GGML_CUDA_BATCH_INVARIANT_MAX_COLS is below it + LLAMA_API bool llama_set_exact_decode_width(uint32_t n_cols); + LLAMA_API uint32_t llama_exact_decode_width(void); + // // State / sessions // @@ -936,6 +957,46 @@ extern "C" { llama_seq_id dest_seq_id, llama_state_seq_flags flags); + // [TAG_STATE_ASYNC] asynchronous per-sequence state transfer, polled with llama_state_seq_copy_done(). + // Until it completes the caller must not touch the buffer, free the cells read, or decode what is written. + struct llama_state_seq_copy; + + // NULL when the backends cannot copy asynchronously, or cannot say whether a copy has finished without waiting for it; the caller then uses the synchronous calls + LLAMA_API struct llama_state_seq_copy * llama_state_seq_copy_init(struct llama_context * ctx); + LLAMA_API void llama_state_seq_copy_free(struct llama_state_seq_copy * cpy); + + // size the transfer's host buffer, keeping no contents; NULL on failure. Grow-only: page-locking is far too slow to redo per transfer, so only llama_state_seq_copy_buf_free() frees it. + LLAMA_API uint8_t * llama_state_seq_copy_buf_resize (struct llama_state_seq_copy * cpy, size_t size); + LLAMA_API uint8_t * llama_state_seq_copy_buf (struct llama_state_seq_copy * cpy); + LLAMA_API size_t llama_state_seq_copy_buf_size (struct llama_state_seq_copy * cpy); + LLAMA_API size_t llama_state_seq_copy_buf_capacity(struct llama_state_seq_copy * cpy); + LLAMA_API void llama_state_seq_copy_buf_free (struct llama_state_seq_copy * cpy); + + // true when the buffer held right now is page-locked. False while no buffer is held: ask llama_state_seq_copy_buf_can_pin() instead. + LLAMA_API bool llama_state_seq_copy_buf_is_pinned(struct llama_state_seq_copy * cpy); + + LLAMA_API bool llama_state_seq_copy_buf_can_pin(struct llama_state_seq_copy * cpy); + + // issue the copies; the bytes covered, 0 on failure. size must be within llama_state_seq_copy_buf_size(), and LLAMA_STATE_SEQ_FLAGS_ON_DEVICE is refused. + LLAMA_API size_t llama_state_seq_copy_get( + struct llama_state_seq_copy * cpy, + size_t size, + llama_seq_id seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_copy_set( + struct llama_state_seq_copy * cpy, + size_t size, + llama_seq_id dest_seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_copy_n_copies(struct llama_state_seq_copy * cpy); + + LLAMA_API int64_t llama_state_seq_copy_sync_us(struct llama_state_seq_copy * cpy); + + LLAMA_API bool llama_state_seq_copy_done(struct llama_state_seq_copy * cpy); + LLAMA_API void llama_state_seq_copy_wait(struct llama_state_seq_copy * cpy); + // // Decoding // diff --git a/scripts/unsloth/additive_merge.py b/scripts/unsloth/additive_merge.py new file mode 100644 index 000000000000..5364dc1b7c10 --- /dev/null +++ b/scripts/unsloth/additive_merge.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Resolve merge conflicts that are provably pure add/add, and only those. + +The conflict that keeps breaking the nightly is always the same shape: upstream +registers a new architecture in a fallthrough group and one of our pinned PRs +registers another one at the same spot. Neither side changed a line the other +side touched -- both only added, at a place where the merge base had nothing. +The union of the two additions is the resolution, and it is mechanical. + +Anything else is left conflicted and reported. In particular a conflict where +the merge base is non-empty means at least one side *edited* shared text, and +picking a side or unioning them is a guess. This script never guesses. + +The two additions are compared on their CONTENT, not on the braces around it. +A case arm is `case X:`, a body, and `} break;`, and two arms for different +architectures share that last part whatever they do. Treating the scaffolding +as evidence that the same change was made twice refuses exactly the conflict +this script exists for; see STRUCTURAL below. + +Reads a conflicted work tree, writes resolutions in place, exits 0 if every +conflict in every file was resolved and 1 otherwise. `--report` emits JSON +describing what it did for the caller to quote in a PR body. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +OURS = "<<<<<<< " +BASE = "||||||| " +SEP = "=======" +THEIRS = ">>>>>>> " + + +class Unresolvable(Exception): + """A conflict this script is not allowed to decide.""" + + +def parse_conflicts(lines: list[str]) -> list[tuple[int, int, list[str], list[str], list[str]]]: + """Split diff3-style content into (start, end, ours, base, theirs) regions. + + Raises Unresolvable if the markers do not nest as diff3 promises, which + means the file is not in the state we think it is. + """ + regions = [] + i = 0 + n = len(lines) + while i < n: + if not lines[i].startswith(OURS): + i += 1 + continue + start = i + ours: list[str] = [] + base: list[str] = [] + theirs: list[str] = [] + cur = ours + seen_base = False + i += 1 + while True: + if i >= n: + raise Unresolvable(f"unterminated conflict starting at line {start + 1}") + ln = lines[i] + if ln.startswith(OURS): + raise Unresolvable(f"nested conflict marker at line {i + 1}") + if ln.startswith(BASE): + cur = base + seen_base = True + elif ln.rstrip("\n") == SEP: + cur = theirs + elif ln.startswith(THEIRS): + i += 1 + break + else: + cur.append(ln) + i += 1 + if not seen_base: + # Without the base section we cannot tell add/add from edit/edit. + raise Unresolvable( + f"conflict at line {start + 1} has no base section; " + "re-checkout with --conflict=diff3" + ) + regions.append((start, i, ours, base, theirs)) + return regions + + +def nonblank(lines: list[str]) -> list[str]: + return [ln.strip() for ln in lines if ln.strip()] + + +# A line that closes or opens a block and nothing else. Two INDEPENDENT case +# arms in the same switch share these by construction -- `{`, `} break;`, `}` +# are what a case arm is made of, not what makes it that case arm -- so finding +# them on both sides says nothing about whether the two sides added the same +# construct. Matching them as "shared" is what refused the real add/add of +# PROJECTOR_TYPE_KIMIK3 next to PROJECTOR_TYPE_DEEPSEEK4V in tools/mtmd/clip.cpp +# with "one change made twice: {, } break;", when the two arms had no line of +# actual content in common. +# +# Deliberately narrow: braces, brackets, parens, semicolons and commas, around +# at most one bare block-terminating keyword. `break;` matches, `return true;` +# does not, and anything naming a type, a constant or a function does not. +STRUCTURAL = re.compile(r"^[\s{}()\[\];,]*(?:break|continue|return|pass)?[\s{}()\[\];,]*$") + + +def identifying(lines: list[str]) -> set[str]: + """The lines that say WHICH construct this is, ignoring block scaffolding.""" + return {ln for ln in nonblank(lines) if not STRUCTURAL.match(ln)} + + +# `case FOO:`, `case FOO :`, `default:`. A fallthrough label may carry no body +# at all, which is the shape the nightly hits most often. +CASE_LABEL = re.compile(r"^(?:case\s+[^:]+|default\s*):") + + +def case_arms(lines: list[str]) -> set[str] | None: + """The case labels this side adds, or None if it is not a run of case arms. + + None, not an empty set: "adds no case arm" and "adds case arms, none of + which the other side adds" have to be told apart, and only the second one + licenses the union below. + """ + ident = [ln for ln in nonblank(lines) if not STRUCTURAL.match(ln)] + if not ident or not CASE_LABEL.match(ident[0]): + return None + return {ln for ln in ident if CASE_LABEL.match(ln)} + + +def resolve_region(ours: list[str], base: list[str], theirs: list[str]) -> list[str]: + """Return the union, or raise if this region is not a pure add/add.""" + if nonblank(base): + raise Unresolvable( + "merge base is not empty, so at least one side edited existing text" + ) + if not nonblank(ours) or not nonblank(theirs): + # One side added and the other added nothing: git would not have + # conflicted, so seeing this means the region is not what we expect. + raise Unresolvable("one side of the conflict is empty") + if ours == theirs: + # Both sides added byte-identical text; one copy is the resolution. + return list(ours) + ours_arms, theirs_arms = case_arms(ours), case_arms(theirs) + if ours_arms and theirs_arms and ours_arms.isdisjoint(theirs_arms): + # Both sides added case arms, and not one label is on both sides. Two + # arms of the same switch labelled differently are two constructs, so + # any line they happen to share is body text, not a duplicate: the real + # tools/mtmd/clip.cpp collision has a KIMIK3 arm and a DEEPSEEK4V arm + # that both set `hparams.rope_theta = 10000.0f;`, and refusing on that + # coincidence is what the shared-line check is for, backwards. + # + # The same change made twice would keep its label, so it lands in the + # check below instead. This is the one place where a shared line is + # allowed, and it is allowed because the labels prove the arms are + # distinct -- a duplicated label would not even compile. + return list(theirs) + list(ours) + shared = identifying(ours) & identifying(theirs) + if shared: + # Overlapping content is the signature of one construct added twice, + # not two independent additions. Unioning it would duplicate code. + # Scaffolding lines are excluded above, so what is left is content both + # sides genuinely wrote, which is the thing that makes this a duplicate. + raise Unresolvable( + "both sides add the same line(s), so this is one change made twice: " + + ", ".join(sorted(shared)[:3]) + ) + if not identifying(ours) or not identifying(theirs): + # Everything one side added is scaffolding, so there is no content to + # tell the two additions apart and the exclusion above has nothing left + # to work with. Refuse rather than union braces onto braces. + raise Unresolvable( + "one side adds only block scaffolding, so the two additions cannot " + "be told apart" + ) + # Upstream first, then ours: the same order a human repin produces. + return list(theirs) + list(ours) + + +def decide_file(path: Path) -> tuple[str, list[dict]]: + """Return the resolved content and a per-hunk record, without writing.""" + lines = path.read_text(encoding="utf-8", errors="surrogateescape").splitlines(keepends=True) + regions = parse_conflicts(lines) + if not regions: + raise Unresolvable("no conflict markers found") + + out: list[str] = [] + prev = 0 + hunks = [] + for start, end, ours, base, theirs in regions: + resolution = resolve_region(ours, base, theirs) + out.extend(lines[prev:start]) + out.extend(resolution) + prev = end + hunks.append( + { + "ours": "".join(ours), + "theirs": "".join(theirs), + "resolution": "".join(resolution), + } + ) + out.extend(lines[prev:]) + return "".join(out), hunks + + +def conflicted_files(repo: Path) -> list[str]: + r = subprocess.run( + ["git", "diff", "--name-only", "--diff-filter=U"], + cwd=repo, capture_output=True, text=True, check=True, + ) + return [f for f in r.stdout.splitlines() if f] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--repo", default=".", help="conflicted work tree") + ap.add_argument("--report", help="write a JSON report here") + ap.add_argument("--dry-run", action="store_true", help="decide, but do not write") + args = ap.parse_args() + + repo = Path(args.repo).resolve() + files = conflicted_files(repo) + report: dict = {"resolved": [], "refused": [], "ok": False} + + if not files: + report["refused"].append({"file": "-", "reason": "no conflicted files"}) + + # Decide every file before writing any of them. A refusal on the second + # file must not leave the first one already rewritten on disk: the caller + # would then be looking at a tree that is neither the conflict nor the + # resolution. + pending: list[tuple[Path, str]] = [] + for f in files: + try: + content, hunks = decide_file(repo / f) + pending.append((repo / f, content)) + report["resolved"].append({"file": f, "hunks": hunks}) + except Unresolvable as e: + report["refused"].append({"file": f, "reason": str(e)}) + except OSError as e: + report["refused"].append({"file": f, "reason": f"cannot read: {e}"}) + + report["ok"] = bool(files) and not report["refused"] + + if report["ok"] and not args.dry_run: + for p, content in pending: + p.write_text(content, encoding="utf-8", errors="surrogateescape") + subprocess.run(["git", "add", "--"] + files, cwd=repo, check=True) + + for r in report["resolved"]: + print(f"resolved {r['file']}") + for r in report["refused"]: + print(f"refused {r['file']}: {r['reason']}", file=sys.stderr) + + if args.report: + Path(args.report).write_text(json.dumps(report, indent=2)) + return 0 if report["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/assemble_metadata.py b/scripts/unsloth/assemble_metadata.py new file mode 100644 index 000000000000..87fa3cebe1e4 --- /dev/null +++ b/scripts/unsloth/assemble_metadata.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Assemble the release-level sidecars for an Unsloth llama.cpp prebuilt release. + +Produces, matching the schema consumed by unslothai/unsloth's installer: + - llama-prebuilt-manifest.json : describes every locally-built bundle in this + release (CUDA x64/arm64 profiles + ROCm Linux/Windows per gfx target + + macOS arm64/x64 + CPU Linux/Windows x64+arm64 + Vulkan Linux x64/arm64 and + Windows x64), with the dispatch metadata the installer needs to pick the + right one. + - llama-prebuilt-sha256.json : a cross-OS integrity index covering both the + locally-built bundles AND the upstream ggml-org assets the installer still + pulls (arm64 CPU + the Windows CUDA cudart/runtime) + the source tarballs. + +Run after the build matrix has dropped the app-*.{tar.gz,zip} bundles into --dist. +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import hashlib +import json +import os +import re +import sys +import tarfile +import time +import urllib.error +import urllib.request +import zipfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +UPSTREAM_REPO = "ggml-org/llama.cpp" + +BUNDLE_RE = re.compile( + r"^app-(?P<tag>[^/]+)-(?P<platform>linux|windows)-(?P<arch>x64|arm64)-(?P<profile>cuda1[23]-(?:older|newer|portable|legacy))\.(?P<ext>tar\.gz|zip)$" +) + +ROCM_BUNDLE_RE = re.compile( + r"^app-(?P<tag>[^/]+)-(?P<platform>linux|windows)-x64-rocm-(?P<gfx>gfx[0-9a-zA-Z]+)\.(?P<ext>tar\.gz|zip)$" +) + +# CPU-only and Vulkan bundles, built locally by unsloth-prebuilt-cpu.yml / +# unsloth-prebuilt-vulkan.yml. Like ROCm/macOS they are raw build/bin archives +# with no embedded UNSLOTH_PREBUILT_INFO.json, so everything in the manifest +# entry is derived from the filename. CPU covers Linux/Windows x64 + arm64; +# Vulkan covers Linux x64 + arm64 and Windows x64. +CPU_BUNDLE_RE = re.compile( + r"^app-(?P<tag>[^/]+)-(?P<platform>linux|windows)-(?P<arch>x64|arm64)-cpu\.(?P<ext>tar\.gz|zip)$" +) + +VULKAN_BUNDLE_RE = re.compile( + r"^app-(?P<tag>[^/]+)-(?P<target>linux-(?:x64|arm64)|windows-x64)-vulkan\.(?P<ext>tar\.gz|zip)$" +) + +# macOS slices are built by unsloth-prebuilt-macos.yml and land in dist/ under +# upstream's own naming (the installer expects that name). They carry no +# embedded UNSLOTH_PREBUILT_INFO.json, so -- like ROCm -- everything is derived +# from the filename. +MACOS_BUNDLE_RE = re.compile( + r"^llama-(?P<tag>[^/]+)-bin-macos-(?P<arch>arm64|x64)\.tar\.gz$" +) + +# Per-(platform, arch) dispatch keys for the published manifest + sha256 index. +# Linux x64 keeps the historical "linux-cuda" so older unsloth installers stay +# compatible; the others get distinct kinds so installers cleanly ignore a +# bundle they can't run instead of trying to launch the wrong binary. +KIND_BY_CUDA = { + ("linux", "x64"): {"manifest": "linux-cuda", "sha": "linux-cuda-app"}, + ("linux", "arm64"): {"manifest": "linux-arm64-cuda", "sha": "linux-arm64-cuda-app"}, + ("windows", "x64"): {"manifest": "windows-cuda", "sha": "windows-cuda-app"}, +} + +KIND_BY_ROCM_PLATFORM = { + "linux": {"manifest": "linux-rocm", "sha": "linux-rocm-app"}, + "windows": {"manifest": "windows-rocm", "sha": "windows-rocm-app"}, +} + +# CPU + Vulkan slices. These supersede the upstream ggml-org CPU/Vulkan +# passthroughs (we now build them ourselves). The manifest kinds match what the +# installer selects per (platform, arch): x64 keeps the historical +# linux-cpu/windows-cpu, arm64 uses linux-arm64/windows-arm64 (the same kinds +# the installer's upstream-fallback path used). The "-app" sha kinds mark them +# as locally-built bundles. +KIND_BY_CPU = { + ("linux", "x64"): {"manifest": "linux-cpu", "sha": "linux-cpu-app"}, + ("linux", "arm64"): {"manifest": "linux-arm64", "sha": "linux-arm64-app"}, + ("windows", "x64"): {"manifest": "windows-cpu", "sha": "windows-cpu-app"}, + ("windows", "arm64"): {"manifest": "windows-arm64", "sha": "windows-arm64-app"}, +} + +KIND_BY_VULKAN_TARGET = { + "linux-x64": {"manifest": "linux-vulkan", "sha": "linux-vulkan-app"}, + "linux-arm64": {"manifest": "linux-vulkan", "sha": "linux-arm64-vulkan-app"}, + "windows-x64": {"manifest": "windows-vulkan", "sha": "windows-vulkan-app"}, +} + +# macOS slices: install_kind / sha-index kind / manifest bundle_profile per arch. +# We build these ourselves now (upstream's arm64 release stamps minos=26 and +# won't dyld-load on macOS < 26), so they are recorded as locally-built bundles +# rather than upstream passthroughs. +MACOS_SLICE = { + "arm64": {"manifest": "macos-arm64", "sha": "macos-arm64-app", "profile": "macos-metal-arm64"}, + "x64": {"manifest": "macos-x64", "sha": "macos-x64-app", "profile": "macos-cpu-x64"}, +} + +# Mapping from the umbrella gfx target name (as it appears in the asset +# filename) to the concrete gfx architectures it compiles for. Mirrors the +# `mapped_target` switch in unsloth-prebuilt-rocm.yml; kept duplicated so the +# manifest can stay self-describing without parsing the workflow. +ROCM_TARGET_MAP = { + "gfx1151": ["gfx1151"], + "gfx1150": ["gfx1150"], + "gfx120X": ["gfx1200", "gfx1201"], + "gfx110X": ["gfx1100", "gfx1101", "gfx1102", "gfx1103"], + "gfx103X": ["gfx1030", "gfx1031", "gfx1032", "gfx1034"], + "gfx90a": ["gfx90a"], + "gfx908": ["gfx908"], +} + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def read_bundle_info(bundle: Path) -> dict: + """Read the UNSLOTH_PREBUILT_INFO.json embedded in a built bundle. + + Linux/macOS bundles are .tar.gz; Windows bundles are .zip -- dispatch on the + extension so the Windows CUDA bundles can be read too. + """ + target = "UNSLOTH_PREBUILT_INFO.json" + if bundle.name.endswith(".zip"): + with zipfile.ZipFile(bundle) as zf: + for n in zf.namelist(): + if n.endswith(target): + return json.loads(zf.read(n)) + else: + with tarfile.open(bundle, "r:gz") as tar: + for m in tar.getmembers(): + if m.isfile() and m.name.endswith(target): + return json.loads(tar.extractfile(m).read()) + sys.exit(f"ERROR: {bundle.name} has no {target}") + + +def _request(url: str, token: str | None) -> urllib.request.Request: + req = urllib.request.Request(url, headers={"User-Agent": "unsloth-prebuilt-assembler"}) + if token and "api.github.com" in url: + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Accept", "application/vnd.github+json") + return req + + +def _with_retry(fn, *, attempts: int = 4, base: float = 2.0): + for i in range(attempts): + try: + return fn() + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + code = getattr(e, "code", None) + # give up on the last try or on a non-transient 4xx (429 is transient) + if i == attempts - 1 or (code is not None and 400 <= code < 500 and code != 429): + raise + time.sleep(base * (2 ** i)) + + +def http_json(url: str, token: str | None) -> object: + def go(): + with urllib.request.urlopen(_request(url, token), timeout=120) as resp: + return json.loads(resp.read()) + return _with_retry(go) + + +def sha256_url(url: str, token: str | None) -> str: + def go(): + h = hashlib.sha256() + with urllib.request.urlopen(_request(url, token), timeout=300) as resp: + for chunk in iter(lambda: resp.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + return _with_retry(go) + + +def upstream_assets(tag: str, token: str | None) -> dict[str, dict]: + """name -> {url, digest} for the upstream release at `tag`.""" + data = http_json(f"https://api.github.com/repos/{UPSTREAM_REPO}/releases/tags/{tag}", token) + out: dict[str, dict] = {} + for asset in data.get("assets", []): # type: ignore[union-attr] + out[asset["name"]] = { + "url": asset["browser_download_url"], + "digest": asset.get("digest"), # "sha256:<hex>" since 2024, else None + } + return out + + +def asset_digest_or_hash(asset: dict, token: str | None) -> str: + """Prefer GitHub's published asset digest; stream-hash as fallback.""" + raw = (asset.get("digest") or "").strip().lower() + if raw.startswith("sha256:"): + h = raw.split(":", 1)[1] + if len(h) == 64 and all(c in "0123456789abcdef" for c in h): + return h + return sha256_url(asset["url"], token) + + +def build_artifacts( + cuda_bundles: list[tuple[str, str, str, dict]], + rocm_bundles: list[tuple[str, str, str]], + macos_bundles: list[tuple[str, str]], + cpu_bundles: list[tuple[str, str, str]], + vulkan_bundles: list[tuple[str, str]], +) -> list[dict]: + """cuda_bundles: list of (asset_name, platform, arch, embedded UNSLOTH_PREBUILT_INFO). + rocm_bundles: list of (asset_name, platform, gfx_target). + macos_bundles: list of (asset_name, arch). + cpu_bundles: list of (asset_name, platform, arch). + vulkan_bundles: list of (asset_name, target). + + CUDA fields come from each bundle's own embedded metadata, so the manifest + can never disagree with what was actually compiled. ROCm, macOS, CPU and + Vulkan bundles are raw archives (no embedded info), so their manifest + entries are derived from the filename + the ROCM_TARGET_MAP / MACOS_SLICE + tables. + """ + artifacts = [] + for asset_name, platform, arch, info in cuda_bundles: + artifacts.append({ + "asset_name": asset_name, + "install_kind": KIND_BY_CUDA[(platform, arch)]["manifest"], + "bundle_profile": info["bundle_profile"], + "runtime_line": info["runtime_line"], + "coverage_class": info["coverage_class"], + "supported_sms": info["supported_sms"], + "min_sm": info["min_sm"], + "max_sm": info["max_sm"], + "rank": info["bundle_rank"], + "toolkit_version": info["toolkit_line"], + }) + for asset_name, platform, gfx in rocm_bundles: + artifacts.append({ + "asset_name": asset_name, + "install_kind": KIND_BY_ROCM_PLATFORM[platform]["manifest"], + "gfx_target": gfx, + "mapped_targets": ROCM_TARGET_MAP.get(gfx, [gfx]), + }) + for asset_name, arch in macos_bundles: + # No runtime_line/coverage_class for macOS (no CUDA/ROCm runtime to + # match); emitted as explicit null so the key set stays stable, and a + # fixed rank since there is a single slice per arch. + artifacts.append({ + "asset_name": asset_name, + "install_kind": MACOS_SLICE[arch]["manifest"], + "bundle_profile": MACOS_SLICE[arch]["profile"], + "runtime_line": None, + "coverage_class": None, + "rank": 50, + }) + # CPU + Vulkan: no CUDA/ROCm runtime to match, so runtime_line/coverage_class + # are explicit null (stable key set). A single slice per (backend, platform, + # arch), so a fixed rank; CPU ranks last (1000) as the universal fallback, + # matching the installer's own direct-scan rank for a CPU bundle. + for asset_name, platform, arch in cpu_bundles: + artifacts.append({ + "asset_name": asset_name, + "install_kind": KIND_BY_CPU[(platform, arch)]["manifest"], + "bundle_profile": f"{platform}-cpu-{arch}", + "runtime_line": None, + "coverage_class": None, + "rank": 1000, + }) + for asset_name, target in vulkan_bundles: + platform, arch = target.rsplit("-", 1) + artifacts.append({ + "asset_name": asset_name, + "install_kind": KIND_BY_VULKAN_TARGET[target]["manifest"], + "bundle_profile": f"{platform}-vulkan-{arch}", + "runtime_line": None, + "coverage_class": None, + "rank": 60, + }) + return artifacts + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--tag", required=True) + ap.add_argument("--ref", default=None, + help="git ref the source was built from; defaults to refs/tags/<tag>") + ap.add_argument("--source-repo", default=UPSTREAM_REPO, + help="repo holding the source ref: upstream, or the publish repo for merged mix tags") + ap.add_argument("--base-tag", default=None, + help="upstream release tag the build is based on; defaults to --tag (differs for mix builds)") + ap.add_argument("--pr-set", default="[]", + help='JSON array of merged PRs: [{"repo":..,"number":..,"sha":..,"url":..,"title":..},..]') + ap.add_argument("--commit", required=True) + ap.add_argument("--ggml-tree", default=None, + help="git tree id of ggml/ in the built source; ABI key for paired builds") + ap.add_argument("--ggml-version", default=None) + ap.add_argument("--dist", required=True, type=Path, help="dir holding the built app-*.tar.gz bundles") + ap.add_argument("--out", required=True, type=Path, help="dir to write the two JSON sidecars into") + ap.add_argument("--publish-repo", required=True, help="repo the bundles+manifest are published to") + ap.add_argument("--token", default=None, help="GitHub token (else $GH_TOKEN/$GITHUB_TOKEN)") + args = ap.parse_args() + + token = args.token or os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + tag, commit, short = args.tag, args.commit, args.commit[:7] + ref = args.ref or f"refs/tags/{tag}" + source_repo = args.source_repo + base_tag = args.base_tag or tag + pr_set = json.loads(args.pr_set) + # Upstream release assets exist only for a vanilla build of an upstream tag + # (a mix build's merged tree exists in no repo, only in its release assets). + is_upstream_release = source_repo == UPSTREAM_REPO and ref == f"refs/tags/{tag}" and not pr_set + ref_kind = "tag" if is_upstream_release else "mix" if pr_set else "ref" + source_ref = tag if ref == f"refs/tags/{tag}" else ref + args.out.mkdir(parents=True, exist_ok=True) + + def base_entry(kind: str, repo: str, digest: str) -> dict: + return { + "kind": kind, + "repo": repo, + "sha256": digest, + "source_commit": commit, + "source_commit_short": short, + "upstream_tag": base_tag, + } + + sha_artifacts: dict[str, dict] = {} + + # 1a) locally-built CUDA bundles (Linux x64/arm64 .tar.gz + Windows x64 + # .zip): hash in parallel. All carry embedded UNSLOTH_PREBUILT_INFO.json. + found: list[tuple[str, str, str, dict]] = [] + cuda_paths = sorted(args.dist.glob("app-*-linux-*.tar.gz")) + sorted(args.dist.glob("app-*-windows-*.zip")) + for p in cuda_paths: + m = BUNDLE_RE.match(p.name) + if not m: + continue + found.append((p.name, m.group("platform"), m.group("arch"), read_bundle_info(p))) + if not found: + print(f"ERROR: no app-* CUDA bundles in {args.dist}", file=sys.stderr) + return 1 + with ThreadPoolExecutor(max_workers=4) as pool: + local_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), found)) + for (name, platform, arch, _info), digest in zip(found, local_digests): + sha_artifacts[name] = base_entry(KIND_BY_CUDA[(platform, arch)]["sha"], args.publish_repo, digest) + + # 1b) locally-built ROCm bundles (linux .tar.gz + windows .zip): hash in + # parallel. No embedded metadata; we derive everything from the filename. + rocm_found: list[tuple[str, str, str]] = [] + for p in sorted(list(args.dist.glob("app-*-rocm-*.tar.gz")) + list(args.dist.glob("app-*-rocm-*.zip"))): + m = ROCM_BUNDLE_RE.match(p.name) + if not m: + continue + rocm_found.append((p.name, m.group("platform"), m.group("gfx"))) + if rocm_found: + with ThreadPoolExecutor(max_workers=4) as pool: + rocm_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), rocm_found)) + for (name, platform, _gfx), digest in zip(rocm_found, rocm_digests): + sha_artifacts[name] = base_entry(KIND_BY_ROCM_PLATFORM[platform]["sha"], args.publish_repo, digest) + else: + # Warning, not error: ROCm can legitimately be empty when a dispatch run + # narrows operating_systems to skip both Windows and Ubuntu. The daily + # schedule always builds the full set, so this fires only on manual runs. + print("WARNING: no app-*-rocm-*.{tar.gz,zip} bundles found", file=sys.stderr) + + # 1c) locally-built macOS slices (arm64 Metal + x64 CPU): hash in parallel. + # No embedded metadata; we derive everything from the filename. We build + # these ourselves now, so they are NOT recorded as upstream passthroughs in + # section 2. + macos_found: list[tuple[str, str]] = [] + for p in sorted(args.dist.glob("llama-*-bin-macos-*.tar.gz")): + m = MACOS_BUNDLE_RE.match(p.name) + if not m: + continue + macos_found.append((p.name, m.group("arch"))) + if macos_found: + with ThreadPoolExecutor(max_workers=4) as pool: + macos_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), macos_found)) + for (name, arch), digest in zip(macos_found, macos_digests): + sha_artifacts[name] = base_entry(MACOS_SLICE[arch]["sha"], args.publish_repo, digest) + else: + # Like ROCm: warn rather than error, so a partial dispatch run still + # assembles. The daily schedule always builds both slices. + print("WARNING: no llama-*-bin-macos-*.tar.gz bundles found", file=sys.stderr) + + # 1d) locally-built CPU + Vulkan bundles (Linux .tar.gz + Windows .zip). + # No embedded metadata; everything is derived from the filename. These + # replace the upstream ggml-org CPU/Vulkan passthroughs that section 2 used + # to record -- the release now ships our own builds for these slices. + def scan_bundles(regex) -> list[tuple[str, "re.Match[str]"]]: + out: list[tuple[str, "re.Match[str]"]] = [] + for p in sorted(list(args.dist.glob("app-*.tar.gz")) + list(args.dist.glob("app-*.zip"))): + m = regex.match(p.name) + if m: + out.append((p.name, m)) + return out + + cpu_found = [(name, m.group("platform"), m.group("arch")) for name, m in scan_bundles(CPU_BUNDLE_RE)] + if cpu_found: + with ThreadPoolExecutor(max_workers=4) as pool: + cpu_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), cpu_found)) + for (name, platform, arch), digest in zip(cpu_found, cpu_digests): + sha_artifacts[name] = base_entry(KIND_BY_CPU[(platform, arch)]["sha"], args.publish_repo, digest) + else: + print("WARNING: no app-*-cpu.{tar.gz,zip} bundles found", file=sys.stderr) + + vulkan_found = [(name, m.group("target")) for name, m in scan_bundles(VULKAN_BUNDLE_RE)] + if vulkan_found: + with ThreadPoolExecutor(max_workers=4) as pool: + vulkan_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), vulkan_found)) + for (name, target), digest in zip(vulkan_found, vulkan_digests): + sha_artifacts[name] = base_entry( + KIND_BY_VULKAN_TARGET[target]["sha"], args.publish_repo, digest + ) + else: + print("WARNING: no app-*-vulkan.{tar.gz,zip} bundles found", file=sys.stderr) + + # 2) upstream per-OS bundles: read GitHub's published asset.digest from the + # API response; fall back to a streaming hash if a digest is missing. + # macOS and the locally-built CPU/Vulkan slices are absent here on + # purpose -- we build those ourselves (1c/1d). + # A mix build has no upstream release for its tag, so the whole section + # is skipped; its uncovered hosts fall back to a source build of the + # merged tree instead of a vanilla upstream binary missing the PRs. + if not is_upstream_release: + print(f"WARNING: {source_repo}@{ref} is not an upstream release tag; " + "skipping upstream asset index entries", file=sys.stderr) + else: + assets = upstream_assets(tag, token) + wanted: list[tuple[str, str]] = [] # (name, kind) + for name in sorted(assets): + if re.fullmatch(r"cudart-llama-bin-win-cuda-\d+\.\d+-x64\.zip", name): + wanted.append((name, "windows-cuda-upstream")) + # The win-cuda BINARY zips must be recorded under their own names too: + # the installer resolves an attempt's hash by exact asset name first + # and only then falls back to the cudart alias, so without these + # entries every Windows CUDA binary gets paired with the cudart digest + # and fails download verification. + elif re.fullmatch( + rf"llama-{re.escape(tag)}-bin-win-cuda-\d+\.\d+-x64\.zip", name + ): + wanted.append((name, "windows-cuda-upstream")) + # x64 CPU and all current Vulkan targets are no longer passthroughs -- + # we build them ourselves (section 1d above). arm64 CPU is now built too + # (1d emits the locally-built linux-arm64/windows-arm64 bundles), but the + # installer still selects the upstream arm64 asset until it is switched to + # those bundles; keep these passthrough checksums until that installer + # flip lands, then drop them. + for name, kind in ( + (f"llama-{tag}-bin-ubuntu-arm64.tar.gz", "linux-arm64-upstream"), + (f"llama-{tag}-bin-win-cpu-arm64.zip", "windows-arm64-upstream"), + ): + if name not in assets: + print(f"WARNING: upstream asset {name} not found at {tag}; skipping", file=sys.stderr) + continue + wanted.append((name, kind)) + for name, kind in wanted: + sha_artifacts[name] = base_entry(kind, UPSTREAM_REPO, asset_digest_or_hash(assets[name], token)) + + # 3) source tarballs: prefer a local copy in dist -- the workflow downloads + # them from codeload so the published asset and its recorded checksum are + # the exact same bytes. Fall back to stream-hashing codeload if absent + # (e.g. a standalone/local run that didn't pre-fetch them). codeload + # doesn't expose pre-computed digests, so we always hash the content. + source_jobs = [ + (f"llama.cpp-source-{tag}.tar.gz", "upstream-source", + f"https://codeload.github.com/{source_repo}/tar.gz/{ref}"), + (f"llama.cpp-source-commit-{commit}.tar.gz", "exact-source", + f"https://codeload.github.com/{source_repo}/tar.gz/{commit}"), + ] + + def source_digest(name: str, url: str) -> str: + local = args.dist / name + return sha256_file(local) if local.is_file() else sha256_url(url, token) + + with ThreadPoolExecutor(max_workers=2) as pool: + source_digests = list(pool.map(lambda j: source_digest(j[0], j[2]), source_jobs)) + for (name, kind, _url), digest in zip(source_jobs, source_digests): + sha_artifacts[name] = base_entry(kind, source_repo, digest) + + # 4) manifest, then hash it into the index. Both sidecars share the same + # source-description header; merged_prs records the exact pinned PR SHAs + # a mix build compiled (empty for vanilla builds). + common = { + "schema_version": 1, + "component": "llama.cpp", + "source_repo": source_repo, + "source_repo_url": f"https://github.com/{source_repo}", + "source_ref_kind": ref_kind, + "requested_source_ref": source_ref, + "resolved_source_ref": source_ref, + "source_commit": commit, + "source_commit_short": short, + "upstream_repo": UPSTREAM_REPO, + "upstream_tag": base_tag, + "merged_prs": pr_set, + # ABI key for anything compiled against this release's ggml, e.g. + # whisper.cpp slim bundles. Changes only when ggml/ contents change. + # The -mix- tag suffix hashes the PR set, not ggml, so it stays + # constant while the base tag (and ggml with it) moves. + "ggml_tree": args.ggml_tree, + "ggml_version": args.ggml_version, + } + manifest = { + **common, + "generated_at_utc": _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "artifacts": build_artifacts(found, rocm_found, macos_found, cpu_found, vulkan_found), + } + manifest_path = args.out / "llama-prebuilt-manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2)) + sha_artifacts["llama-prebuilt-manifest.json"] = base_entry( + "published-manifest", args.publish_repo, sha256_file(manifest_path) + ) + + sha256_doc = { + **common, + "release_tag": tag, + "artifacts": sha_artifacts, + } + (args.out / "llama-prebuilt-sha256.json").write_text(json.dumps(sha256_doc, indent=2)) + + print(f"wrote manifest ({len(manifest['artifacts'])} artifacts) and sha256 index " + f"({len(sha_artifacts)} entries) to {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/unsloth/assert_macho_minos.sh b/scripts/unsloth/assert_macho_minos.sh new file mode 100755 index 000000000000..01a1536c9a0d --- /dev/null +++ b/scripts/unsloth/assert_macho_minos.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# Pre-publish gate for the Unsloth macOS llama.cpp prebuilt. Fails the build +# unless every shipped Mach-O declares a minimum macOS <= the pinned deployment +# target (so it dyld-loads on that floor or newer), carries the expected arch slice, and +# actually launches. This is what keeps a runner/SDK bump from silently shipping +# a minos=26 binary that fails on older Macs. +# +# Usage: assert_macho_minos.sh <bin_dir> <expect_arch> [max_minos] +# expect_arch: arm64 | x86_64 max_minos: default 14.0 +set -uo pipefail + +BIN_DIR="${1:?bin dir required}" +EXPECT_ARCH="${2:?expected arch required}" +MAX_MINOS="${3:-14.0}" + +fail() { echo "::error::$*"; exit 1; } +# Compare dotted major.minor as major*100+minor (14.0 -> 1400). +ver_key() { local v="${1%%-*}"; awk -F. '{printf "%d", $1*100 + ($2==""?0:$2)}' <<<"$v"; } +MAX_KEY="$(ver_key "$MAX_MINOS")" + +command -v vtool >/dev/null 2>&1 || fail "vtool not found (Xcode command line tools required)" + +# macOS ships bash 3.2, which has no `mapfile`; read into the array portably. +MACHOS=() +while IFS= read -r _macho; do MACHOS+=("$_macho"); done < <(find "$BIN_DIR" -type f \( -name '*.dylib' -o -name 'llama-server' -o -name 'llama-quantize' -o -name 'llama-cli' \) 2>/dev/null) +[ "${#MACHOS[@]}" -gt 0 ] || fail "no Mach-O binaries found under $BIN_DIR" + +for macho in "${MACHOS[@]}"; do + minos="$(vtool -show-build "$macho" 2>/dev/null | awk '/minos/{print $2; exit}')" + [ -n "$minos" ] || fail "$(basename "$macho") has no LC_BUILD_VERSION/minos" + if [ "$(ver_key "$minos")" -gt "$MAX_KEY" ]; then + fail "$(basename "$macho") minos=$minos exceeds deployment target $MAX_MINOS" + fi + if ! lipo -archs "$macho" 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECT_ARCH"; then + fail "$(basename "$macho") is missing the $EXPECT_ARCH slice (got: $(lipo -archs "$macho" 2>/dev/null))" + fi +done +echo "static check passed: ${#MACHOS[@]} Mach-O files, all minos<=$MAX_MINOS, arch=$EXPECT_ARCH" + +# Runtime launch forces dyld to resolve every linked dylib (incl. Metal). +for tool in llama-cli llama-quantize; do + bin="$(find "$BIN_DIR" -type f -name "$tool" 2>/dev/null | head -1)" + [ -n "$bin" ] || fail "$tool not found under $BIN_DIR" +done +CLI="$(find "$BIN_DIR" -type f -name llama-cli | head -1)" +QUANT="$(find "$BIN_DIR" -type f -name llama-quantize | head -1)" +"$CLI" --version >/dev/null 2>&1 || fail "llama-cli failed to launch (dyld load / symbol error)" +# llama-quantize's usage() ends in exit(1), so --help is non-zero by design. +# A dyld/symbol failure dies before main and prints nothing, so verify the +# binary actually reached main (printed usage) rather than trusting the code. +q_out="$("$QUANT" --help 2>&1 || true)" +printf '%s\n' "$q_out" | grep -q "usage:" || fail "llama-quantize failed to launch (dyld load / symbol error)" +echo "runtime launch passed: llama-cli --version and llama-quantize --help both ran" diff --git a/scripts/unsloth/carry_vintage.py b/scripts/unsloth/carry_vintage.py new file mode 100755 index 000000000000..0f469db9cca9 --- /dev/null +++ b/scripts/unsloth/carry_vintage.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Report which carry files are an unmodified older copy of the upstream PR. + +A carry branch replays an upstream PR onto an aged base tag. +When that PR moves, the question before every refresh is the same one: have we actually changed this file, or are we just holding a stale copy of theirs? +Answering it by hand means diffing every file the PR touches and reading each hunk, which is what made the 08-27 GLM-5-Next refresh expensive, and two of those hand answers were wrong. + +The mechanical answer: if our version of a file is byte-identical to the version at SOME commit of the upstream PR, then we never edited it, and their newer copy supersedes ours with nothing lost. +That is a fact about blob hashes, not a judgement. +Files we really did change match no upstream commit and are reported as diverged, which is correct: on 08-27 gguf-py/gguf/tensor_mapping.py did not match, because it genuinely carried qwen4exp additions as well. + +This only reports. +It does not resolve, stage or write anything, because "upstream superseded ours" is not the same as "we want upstream's", and holding a deliberately older vintage is a legitimate decision this script cannot see. + +Use it to decide whether a refresh should merge or simply rebuild: + + python3 scripts/unsloth/carry_vintage.py \\ + --carry <carry sha> --pr-ref refs/pull/27754/head --base refs/tags/b10639 + +When every file is SUPERSEDED, rebuilding the carry from the PR head avoids the merge, and its conflicts, entirely. +A file the PR head still has and the carry does not is reported as OMITTED and blocks that advice, because a rebuild would restore it; a file the PR DELETED is merely ABSENT and blocks nothing. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys + + +def git(*args: str) -> str: + r = subprocess.run(["git", *args], capture_output=True, text=True) + if r.returncode: + raise RuntimeError(" ".join(args) + ": " + r.stderr.strip()) + return r.stdout.strip() + + +def blob(rev: str, path: str) -> str | None: + """The tree entry as "mode oid", or None if the rev has no such path. + + Mode, not just the oid: a carry that only chmods a file it took verbatim has the same content as upstream, so an oid comparison calls it superseded and a rebuild silently drops the mode change. + """ + r = subprocess.run(["git", "ls-tree", "--full-tree", "-z", rev, "--", path], + capture_output=True, text=True) + if r.returncode != 0 or not r.stdout.strip(): + return None + mode, _type, oid = r.stdout.split("\0")[0].split("\t", 1)[0].split() + return f"{mode} {oid}" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--carry", required=True, help="carry branch commit") + ap.add_argument("--pr-ref", required=True, help="upstream PR head ref or sha") + ap.add_argument("--base", required=True, help="base tag the PR forked from") + ap.add_argument("--max-commits", type=int, default=60, + help="how far back through the PR to look for a match") + ap.add_argument("--report", metavar="PATH", help="write a JSON summary here") + a = ap.parse_args() + + head = git("rev-parse", a.pr_ref) + fork = git("merge-base", head, a.base) + # --no-renames, because rename detection hides exactly the path that matters here. + # `git diff --name-only` prints only the NEW name of a rename, so a PR moving `old` to `new` never puts `old` in this list. + # A carry that deliberately keeps `old` is then never looked at: `new` comes back SUPERSEDED, nothing diverges, and the summary says a rebuild is equivalent when a rebuild deletes the file the carry is holding. + # Without detection the rename is a delete plus an add, so `old` is classified - DIVERGED, since our copy is the fork's and matches no upstream vintage. + files = [f for f in git("diff", "--name-only", "--no-renames", + fork, head).split("\n") if f] + # fork..head, not head: `--max-count` caps the output, it does not bound the walk, so a bare `head` runs straight past the fork point into the base branch. + # A file our carry deliberately holds at the BASE version then matches a pre-fork commit and is called SUPERSEDED, and the summary says rebuilding from the PR head is equivalent - it is not, it re-adds what the carry dropped. + # Only commits of the PR itself are vintages. + history = [c for c in git("rev-list", f"--max-count={a.max_commits}", + f"{fork}..{head}").split("\n") if c] + + # Paths the CARRY changed that the PR never touched at all. + # Everything above reasons only about files in the PR's diff, so a carry-only edit is invisible to it: every PR path can be SUPERSEDED, nothing diverges, and the summary says a rebuild from the PR head is equivalent while the rebuild drops that edit. + # Same failure as OMITTED, arrived at from the other side. + # Diffed from --base rather than the fork point because a carry replays the PR onto the base tag, so that is what its own delta is against; if a carry is ever based on something else, the extra paths only ever withhold the rebuild advice, which is the safe direction to be wrong in. + touched = set(files) + carry_only = [f for f in git("diff", "--name-only", "--no-renames", + a.base, a.carry).split("\n") + if f and f not in touched] + + superseded, diverged, absent, omitted = [], [], [], [] + for path in files: + ours = blob(a.carry, path) + if ours is None: + # Two very different reasons a path is missing from the carry. + # If the PR DELETED it, the carry agrees and a rebuild reproduces that. + # If it still exists at the PR head, the carry dropped it on purpose, and a rebuild would re-add it - the same mistake as calling a file held at the base version superseded. + (absent if blob(head, path) is None else omitted).append(path) + continue + if ours == blob(head, path): + superseded.append({"path": path, "vintage": head, "current": True}) + continue + # Bounding the walk to fork..head is not enough on its own. + # A PR with several commits usually does not touch every file in its first one, so the commits BEFORE the one that first changed this path still carry the fork's blob - inside the range. + # A carry holding the file at the base version matches one of those and is called SUPERSEDED again, and the summary again says a rebuild is equivalent when it would overwrite exactly what the carry is holding. + # Our copy being the fork's copy is not evidence the PR ever produced it, so it is never a vintage; such a file falls through to DIVERGED, which is where a file needing a human decision belongs. + at_fork = blob(fork, path) + hit = None if ours == at_fork else \ + next((c for c in history if blob(c, path) == ours), None) + if hit: + superseded.append({"path": path, "vintage": hit, "current": False}) + else: + diverged.append(path) + + print(f"carry {a.carry[:10]} vs {a.pr_ref} ({head[:10]}), {len(files)} file(s) touched") + print() + for e in superseded: + note = "already at PR head" if e["current"] else f"our copy is upstream {e['vintage'][:10]}" + print(f" SUPERSEDED {e['path']}\n {note}") + for p in diverged: + print(f" DIVERGED {p}\n matches no upstream vintage; we changed it, or we are " + "holding the base version on purpose. Keep it") + for p in omitted: + print(f" OMITTED {p}\n exists at the PR head, not in the carry; " + "a rebuild would re-add it") + for p in absent: + print(f" ABSENT {p}\n deleted by the PR, not in the carry") + for p in carry_only: + print(f" CARRY ONLY {p}\n changed by the carry, untouched by the PR; " + "a rebuild would drop it") + print() + if diverged: + print(f"{len(diverged)} file(s) genuinely diverge. A refresh has to merge, " + "and those files are the only ones needing judgement.") + if omitted: + print(f"{len(omitted)} file(s) the PR head still has are missing from the " + "carry. Rebuilding would restore them, so it is NOT equivalent to " + "merging; keep or re-drop each one deliberately.") + if carry_only: + print(f"{len(carry_only)} file(s) the carry changed are outside the " + "PR entirely. Rebuilding from the PR head would drop them, so it " + "is NOT equivalent to merging; carry each one across deliberately.") + # Matching an older commit of the PR proves only that we never edited the file, not that we want the newest one: the carry may be holding that vintage on purpose, which this cannot see. + # Rebuilding moves it to head, so the unqualified "nothing is lost" below has to stand down and say so. + older = [e for e in superseded if not e["current"]] + if older: + print(f"{len(older)} file(s) sit at an OLDER vintage than the PR head. " + "We never edited them, but a rebuild still moves them to head; " + "confirm no carry is holding one of them deliberately.") + if not diverged and not omitted and not carry_only and not older: + print("Nothing diverges. Rebuilding the carry from the PR head is " + "equivalent to merging it, without the conflicts.") + + if a.report: + with open(a.report, "w") as fh: + json.dump({"head": head, "superseded": superseded, + "diverged": diverged, "omitted": omitted, + "absent": absent, "carry_only": carry_only}, fh, indent=2) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/check_workflow_scalars.py b/scripts/unsloth/check_workflow_scalars.py new file mode 100644 index 000000000000..24f6a2e1b158 --- /dev/null +++ b/scripts/unsloth/check_workflow_scalars.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Fail when a workflow string is close to GitHub's 21000 character cap. + +GitHub's template compiler refuses any single string in a workflow file longer than 21000 characters. +The whole file then fails to compile, and the failure is close to invisible: + + - the run has zero jobs, no annotations and an empty check suite, so there is nothing to click on; + - `gh run view` says only "This run likely failed because of a workflow file issue"; + - the run is reported against whatever event triggered it, even an event the workflow does not subscribe to, because compilation never got as far as reading `on:`; + - and nothing local catches it. yaml parses the file, actionlint passes it, and so does GitHub's own published parser (@actions/workflow-parser). The limit is enforced only server-side. + +On 08-27 a 14-line explanatory comment added inside the `resolve` job's script took it from 20503 to 21620 characters and silently disabled the entire release workflow for four pushes. +The only way to see the real error was to fire a workflow_dispatch at the ref, which returns it as a 422: + + (Line: 125, Col: 14): Exceeded max expression length 21000 + +Comments inside a `run:` block scalar are part of the string and count against the limit. +Comments in the YAML around it do not, so prose belongs above a step rather than inside it. +Past that, the fix is to split the script into more steps: a step boundary costs nothing and resets the budget. + +Measuring the parsed scalar is not exactly what GitHub measures, so the warn threshold is deliberately well below the cap rather than a character-perfect model of it. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import yaml + +# What GitHub enforces, and the point at which a script is big enough that the next edit to it can cross the line without anyone thinking about size. +CAP = 21000 +WARN = 20000 + + +def scalars(node, path: str = ""): + """Every string in the document, with a path that names where it lives.""" + if isinstance(node, str): + yield path, node + elif isinstance(node, dict): + for k, v in node.items(): + yield from scalars(v, f"{path}.{k}") + elif isinstance(node, list): + for i, v in enumerate(node): + yield from scalars(v, f"{path}[{i}]") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--root", default=".", help="repository root") + ap.add_argument("--cap", type=int, default=CAP, help="hard limit, fails") + ap.add_argument("--warn", type=int, default=WARN, help="soft limit, warns") + a = ap.parse_args() + + files = sorted(Path(a.root, ".github/workflows").glob("*.y*ml")) + if not files: + print(f"no workflows under {a.root}/.github/workflows", file=sys.stderr) + return 1 + + over, near = [], [] + for f in files: + try: + doc = yaml.safe_load(f.read_text()) + except yaml.YAMLError as e: + print(f"::error file={f}::not valid YAML: {e}", file=sys.stderr) + return 1 + for path, s in scalars(doc): + if len(s) > a.cap: + over.append((len(s), f, path)) + elif len(s) > a.warn: + near.append((len(s), f, path)) + + for n, f, path in sorted(near, reverse=True): + print(f"::warning file={f}::{path} is {n} characters, within " + f"{a.cap - n} of GitHub's {a.cap} character limit. Move any " + "prose out of the block scalar into YAML comments above the " + "step, or split the script into another step.") + for n, f, path in sorted(over, reverse=True): + print(f"::error file={f}::{path} is {n} characters, over GitHub's " + f"{a.cap} character limit. GitHub will refuse to compile this " + "file and every run of it will fail with no jobs and no " + "annotation. Split the script into another step; a step " + "boundary resets the budget.", file=sys.stderr) + + biggest = max((len(s) for f in files for _, s in scalars(yaml.safe_load(f.read_text()))), + default=0) + print(f"{len(files)} workflow(s), largest string {biggest} of {a.cap}" + f"{f', {len(near)} within {a.cap - a.warn} of the limit' if near else ''}") + return 1 if over else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/feature-checks.json b/scripts/unsloth/feature-checks.json new file mode 100644 index 000000000000..93fe8d3c18f5 --- /dev/null +++ b/scripts/unsloth/feature-checks.json @@ -0,0 +1,104 @@ +{ + "_doc": [ + "The test that proves each shipped feature works. Read by feature_matrix.py.", + "", + "Keyed by FEATURE, with the pin that currently carries it, and NOT the other", + "way round. When upstream absorbs a feature the pin is deleted, and deleting", + "the check with it would put the blind spot back somewhere else: the feature", + "is still in the release, it just arrives through the base tag now. So an", + "entry outlives its `owner`, and `owner` becomes null rather than the entry", + "being removed.", + "", + "This is the half that cannot be derived. pin_contract.py reads a pin's own", + "diff and proves the merge kept it, which needs no upkeep but can only ever", + "prove the MERGE lost nothing -- a regression inside the pin regenerates a", + "smaller contract that passes. What a feature has to DO is a human sentence.", + "", + "Every pin in pr-set.json must appear in `features` or in `unchecked`. The", + "lint enforces that, so adding a pin forces a decision instead of a silence.", + "`unchecked` is a recorded reason, not a hole.", + "", + "kinds:", + " arch test-llama-archs -a <arch> builds a synthetic model of", + " the architecture, decodes 128", + " tokens on every device and", + " compares against CPU", + " backend-op test-backend-ops test -o <OP> runs the op against the CPU", + " reference implementation", + " mtmd test-mtmd-impl projector registry, no model", + "", + "A probe that exits 0 having run nothing is a failure, not a pass: both", + "harnesses do exactly that for an excluded arch or a misspelled op name.", + "feature_matrix.py rejects skip markers and requires a non-zero case count.", + "", + "No runner in the prebuild pipeline has a GPU, so the nightly runs this on", + "CPU and every backend-op check is DEFERRED there: named and counted, never", + "reported as passing. The kernels are exactly where a merge goes wrong", + "silently, so before accepting a carry PR that touches one, build it on a", + "GPU box and run:", + "", + " python3 scripts/unsloth/feature_matrix.py --build-dir build --gpu", + "", + "and paste the output into the PR. That is the only place those checks run." + ], + "schema": 1, + "features": { + "inkling": { + "owner": "ggml-org#25731", + "checks": [ + { "kind": "arch", "arch": "inkling" }, + { "kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED" }, + { "kind": "mtmd", "projector": "inkling" } + ] + }, + "glm5next": { + "owner": "ggml-org#27754", + "checks": [ + { "kind": "arch", "arch": "glm5next" }, + { "kind": "backend-op", "op": "LIGHTNING_INDEXER" } + ] + }, + "diffusion-gemma": { + "owner": "ggml-org#24423", + "checks": [ + { "kind": "arch", "arch": "diffusion-gemma" } + ] + }, + "kimi-k3": { + "owner": "unslothai#70", + "checks": [ + { "kind": "arch", "arch": "kimi-k3" }, + { "kind": "mtmd", "projector": "kimik3" } + ] + }, + "iq1-narrow-grids": { + "owner": "unslothai#61", + "checks": [ + { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xs" }, + { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xxs" }, + { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xxxs" } + ] + }, + "qwen4exp-mtp": { + "owner": "unslothai#144", + "checks": [ + { "kind": "arch", "arch": "qwen4exp" }, + { "kind": "backend-op", "op": "TOPK_QSA" } + ] + }, + "projector-registry": { + "owner": "unslothai#176", + "checks": [ + { "kind": "mtmd", "projector": "*" } + ] + } + }, + "unchecked": { + "unslothai#95": "sampling penalties indexed by token id; behaviour is covered by test-sampling, and there is no feature surface of its own to probe", + "unslothai#137": "batched readahead for lazily read gather tables; a throughput change with no observable output difference", + "unslothai#149": "GGML_CUDA_ENABLE_UNIFIED_MEMORY=0 env parsing; needs a CUDA or HIP host, and no runner in the pipeline has one", + "unslothai#152": "per-run mmap of a context's tensors; a memory-layout change with no observable output difference", + "unslothai#157": "cudaMemcpyDefault in the ggml_cuda_cpy 2D fast path; needs a CUDA host", + "unslothai#158": "ROCm_Host compute buffer type on HIP integrated GPUs; needs a ROCm host" + } +} diff --git a/scripts/unsloth/feature_matrix.py b/scripts/unsloth/feature_matrix.py new file mode 100644 index 000000000000..00b392ca1ec5 --- /dev/null +++ b/scripts/unsloth/feature_matrix.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Run the test that proves each shipped feature works, against a built tree. + +pin_contract.py proves the merge did not lose a pin's code. That is a different +question from whether the feature works, and neither one implies the other: the +Inkling banded-attention kernel merged against upstream's sparse attention is +thirteen hunks of CUDA template parameter threading, where a mistake gives +wrong attention output and every static check passes. + +Keyed by FEATURE, not by pin. When upstream absorbs a feature and the pin is +deleted, removing the check with it would put the blind spot back in a +different place -- the feature is still in the release, it just arrives through +the base tag now. So the manifest binds a feature to its current pin and +survives that pin going away. + +A PASS HAS TO BE POSITIVE EVIDENCE. Both harnesses exit 0 having done nothing: + + test-llama-archs -a diffusion-gemma # excluded -> prints SKIP, exits 0 + test-backend-ops test -o TYPO # matches nothing, exits 0 + +so every probe rejects skip markers and requires a non-zero count of cases it +actually ran. Without that this file is decoration. + +CPU only under CUDA_VISIBLE_DEVICES="" is what CI can do, since no runner in +the prebuild pipeline has a GPU. Run it with the variable unset on a GPU box to +get the comparison that matters for kernels. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +# Output that means "this did not run" from a process that exited 0. +SKIP_RE = re.compile(r"\bSKIP\b|not supported|unsupported|no tests|0 tests", re.I) + + +class Unproven(Exception): + """The probe exited 0 without demonstrating anything.""" + + +class NeedsGPU(Exception): + """Nothing is wrong; this check cannot be answered on this machine. + + test-backend-ops compares a backend against the CPU reference, so with no + accelerator present it has nothing to compare and prints "Skipping CPU + backend". Reporting that as a pass would be a lie and reporting it as a + failure would block every nightly, since no runner in the prebuild pipeline + has a GPU. It is counted and named instead. + """ + + +def bins(build_dir: Path) -> Path: + for c in (build_dir / "bin", build_dir): + if (c / "test-backend-ops").exists() or (c / "test-llama-archs").exists(): + return c + raise SystemExit(f"no test binaries under {build_dir}") + + +def run(cmd: list[str], cwd: Path, gpu: bool) -> tuple[int, str]: + env = None + if not gpu: + import os + env = dict(os.environ, CUDA_VISIBLE_DEVICES="", HIP_VISIBLE_DEVICES="") + r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + return r.returncode, (r.stdout or "") + (r.stderr or "") + + +def probe_arch(check: dict, b: Path, gpu: bool) -> str: + """A synthetic model of this architecture decodes, and matches CPU.""" + arch = check["arch"] + rc, out = run([str(b / "test-llama-archs"), "-a", arch, "-s", "1234"], b, gpu) + if rc != 0: + raise Unproven(f"test-llama-archs -a {arch} exited {rc}") + # The arch's own rows, not the header and not another arch's. + rows = [ln for ln in out.splitlines() if ln.strip().startswith("|") and f"|{arch:>16}|" in ln + or (ln.strip().startswith("|") and ln.split("|")[1].strip() == arch)] + if not rows: + raise Unproven(f"test-llama-archs printed no row for {arch}; it is not in the harness") + ok = [r for r in rows if "OK" in r] + if not ok: + raise Unproven(f"every {arch} row was skipped, so nothing was decoded: {rows[0].strip()}") + return f"{len(ok)}/{len(rows)} device rows decoded and matched CPU" + + +def probe_backend_op(check: dict, b: Path, gpu: bool) -> str: + """The op exists in the backend and matches the CPU reference.""" + if not gpu: + raise NeedsGPU("test-backend-ops compares against CPU, so with no " + "accelerator it skips every backend and proves nothing") + cmd = [str(b / "test-backend-ops"), "test", "-o", check["op"]] + if check.get("params"): + cmd += ["-p", check["params"]] + rc, out = run(cmd, b, gpu) + if rc != 0: + raise Unproven(f"{' '.join(cmd[1:])} exited {rc}") + m = re.search(r"(\d+)/(\d+) tests passed", out) + if not m: + raise Unproven(f"{check['op']} produced no test count; the filter matched nothing") + passed, total = int(m.group(1)), int(m.group(2)) + if total == 0: + raise Unproven(f"{check['op']} matched 0 cases; the op name is stale") + if passed != total: + raise Unproven(f"{check['op']}: {passed}/{total} passed") + return f"{passed}/{total} cases matched the CPU reference" + + +def probe_mtmd(check: dict, b: Path, gpu: bool) -> str: + """The projector registry is intact, including this projector's entry.""" + rc, out = run([str(b / "test-mtmd-impl"), "test_projector_registry"], b, gpu) + if rc != 0: + raise Unproven(f"test-mtmd-impl exited {rc}") + m = re.search(r"assertions\s*:\s*(\d+)", out) + if not m or int(m.group(1)) == 0: + raise Unproven("test_projector_registry ran no assertions; the filter matched nothing") + # The registry test walks the whole enum, so it proves the table is sound. + # That the specific projector is IN the enum is pin_contract.py's job. + return f"projector registry intact over {m.group(1)} assertions" + + +PROBES = {"arch": probe_arch, "backend-op": probe_backend_op, "mtmd": probe_mtmd} + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--build-dir", required=True) + ap.add_argument("--feature-checks", required=True) + ap.add_argument("--only", help="one feature id") + ap.add_argument("--gpu", action="store_true", + help="let the probes see the GPU; CI has none, so the default " + "hides it and the comparison is CPU-only") + ap.add_argument("--report") + args = ap.parse_args() + + b = bins(Path(args.build_dir).resolve()) + doc = json.loads(Path(args.feature_checks).read_text()) + report: dict = {"gpu": args.gpu, "features": [], "ok": False, "deferred": 0} + failed = 0 + deferred = 0 + + for name, feat in sorted(doc["features"].items()): + if args.only and name != args.only: + continue + entry = {"feature": name, "owner": feat.get("owner"), + "results": [], "problems": [], "deferred": []} + for check in feat["checks"]: + kind = check["kind"] + label = f"{kind}:{check.get('arch') or check.get('op') or check.get('projector')}" + try: + if kind not in PROBES: + raise Unproven(f"unknown check kind {kind!r}") + entry["results"].append({"check": label, "evidence": PROBES[kind](check, b, args.gpu)}) + except NeedsGPU as e: + entry["deferred"].append(f"{label}: {e}") + deferred += 1 + except Unproven as e: + entry["problems"].append(f"{label}: {e}") + except OSError as e: + entry["problems"].append(f"{label}: cannot run: {e}") + report["features"].append(entry) + if entry["problems"]: + failed += 1 + print(f"FAIL {name}", file=sys.stderr) + for p in entry["problems"]: + print(f" {p}", file=sys.stderr) + elif entry["results"]: + print(f"ok {name}: " + "; ".join(r["evidence"] for r in entry["results"]) + + (f" [{len(entry['deferred'])} needs a GPU]" if entry["deferred"] else "")) + else: + # Nothing was shown either way. Not a failure here, but it must not + # read as one of the ok lines. + print(f"-- {name}: nothing provable without a GPU " + f"({len(entry['deferred'])} check(s) deferred)") + + for pin, why in sorted(doc.get("unchecked", {}).items()): + print(f"note {pin} has no runtime check: {why}") + + report["ok"] = failed == 0 + report["deferred"] = deferred + if args.report: + Path(args.report).write_text(json.dumps(report, indent=2)) + if failed: + print(f"\n{failed} feature(s) could not be shown to work", file=sys.stderr) + return 1 + # Say what was NOT proven in the same breath as what was. A run that only + # ever prints a success line teaches the reader that green means covered. + tail = f", {deferred} check(s) need a GPU and were not run" if deferred else "" + print(f"\nall {len(report['features'])} features demonstrated" + + (" on GPU" if args.gpu else " on CPU") + tail) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/merge_checks.py b/scripts/unsloth/merge_checks.py new file mode 100755 index 000000000000..a01a6fad574a --- /dev/null +++ b/scripts/unsloth/merge_checks.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Post-merge checks for the two mistakes a clean build does not catch. + +Both of these were made for real on 08-27, resolving GLM-5-Next against the qwen4exp carry, and both survived compilation: + + 1. A resolver unioned two byte-identical additions and produced the same `MODEL_ARCH.GLM5NEXT` key twice in the tensor map. + Python keeps the last definition of a duplicate key, silently, so the file imports, the build passes, and the converter reads the wrong mapping. + + 2. The same union kept both an arch in a shared fallthrough condition AND a dedicated `else if (arch == LLM_ARCH_GLM5NEXT)` arm below it. + The shared condition matches first, so the dedicated arm is dead. + It compiles, and the model runs with the indexer cache that arm was supposed to build. + +Neither is a merge resolver. +They decide nothing and rewrite nothing. +They turn a silent wrong answer into a loud one, which is the property that was missing. + +Both are deliberately narrow, because a check that fires wrongly blocks a release just as effectively as a bad merge: + + - An arch is only consumed by an EARLIER arm that is a pure disjunction of `arch == LLM_ARCH_*` terms. + A conditional arm may not run, so what follows it stays reachable. + The later arm is then dead if it is a disjunction whose alternatives are all consumed, or a plain conjunction requiring a consumed arch, since its other conditions can only narrow it further. + Anything mixing `||` and `&&` is left alone, and only a term that is exactly `arch == X` counts, so `arch != X` is never read as requiring that arch. + + - The unreachable-arm check reads one physical line, so a condition split across lines is skipped rather than analysed. + Checked against the whole of src/: a line-joining variant finds exactly the same zero findings, because every multiline arch condition there is a standalone `if` with no `else if` chain below it. + Widening the regex would add false-positive surface on the release path and buy nothing today, so it stays narrow and this is recorded as a known limitation rather than fixed. + + - Chains are grouped by BRACE DEPTH, not by indentation. + This is an accuracy fix, not a widening: measured over all 181 src/**/*.cpp, depth and indentation report the same zero findings, so nothing new fires and the good tree stays clean, but depth keeps 339 arms in chains against 260 and is right in both directions where they differ. + Indentation drops the enclosing chain at any nested `if`, which silences the check on the exact 08-27 shape, and it glues two unrelated `if`s at one indent into a single chain whenever the second opener is a skipped multiline condition, which reports a reachable arm as dead and blocks a release on good code. + + - The duplicate-key check compares keys by their source text, so it only looks at keys whose value cannot change between evaluations: literals, names, attributes and tuples of those. `{fresh(): 1, fresh(): 2}` reads as one key twice and is really two entries, and a finding here stops the nightly. + + - There is deliberately NO duplicate-C++-definition check. + The obvious version keys on function name and flags legitimate overloads: it reported `llama_model_base::create_tensor`, which is two different signatures. + A real duplicate is an ODR violation the compiler already rejects, so the only gain would be failing sooner, which does not justify a false positive on the release path. + +Exits 0 when clean, 1 when anything is found. +`--report` emits JSON. +""" + +from __future__ import annotations + +import argparse +import ast +import collections +import json +import re +import sys +from pathlib import Path + +ARCH = re.compile(r"arch == (LLM_ARCH_\w+)") +COND = re.compile(r"^\s*(?:\}\s*)?else if \((.*)\)\s*\{\s*$|^\s*if \((.*)\)\s*\{\s*$") +PURE_TERM = re.compile(r"arch == LLM_ARCH_\w+") +# Encoding prefixes a raw string may carry: LR"(...)", u8R"(...)" and so on. +_RAW_PREFIX = ("", "L", "u", "U", "u8") + + +# Node types whose value does not depend on when the expression is evaluated. +# An allowlist, not a denylist, so an expression shape nobody thought about is +# treated as unstable and simply not checked, rather than blocking a release. +_STABLE = (ast.Constant, ast.Name, ast.Attribute, ast.Tuple, ast.Load, + ast.UnaryOp, ast.USub, ast.UAdd, ast.Invert) + + +def _stable_key(node: ast.AST) -> bool: + """True when this key expression names the same object every evaluation. + + `{fresh(): 1, fresh(): 2}` unparses to the same text twice and is still two + entries, so a call anywhere in the key means the two are not comparable by + text. The keys this check exists for, `MODEL_ARCH.GLM5NEXT` and plain + literals, are all stable. + """ + return all(isinstance(n, _STABLE) for n in ast.walk(node)) + + +def duplicate_dict_keys(path: Path) -> list[str]: + """Keys defined twice in one dict literal. Always at best dead code.""" + try: + tree = ast.parse(path.read_text()) + except SyntaxError as e: + return [f"{path}:{e.lineno}: does not parse: {e.msg}"] + out = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Dict): + continue + keys = [ast.unparse(k) for k in node.keys + if k is not None and _stable_key(k)] + for key, n in collections.Counter(keys).items(): + if n > 1: + out.append(f"{path}:{node.lineno}: key {key} defined {n} times " + "in one dict; Python keeps only the last") + return out + + +def _pure_arch_disjunction(cond: str) -> bool: + """True when the condition is only `arch == X` terms joined by `||`.""" + if "&&" in cond: + return False + terms = [t.strip() for t in cond.split("||")] + return bool(terms) and all(PURE_TERM.fullmatch(t) for t in terms) + + +def _raw_delim(text: str, i: int) -> str | None: + """The delimiter of a raw string starting at `i`, or None if one does not. + + `i` indexes the `R`. + The delimiter is what sits between `R"` and `(`, and the literal ends only at `)delim"`, which is the whole reason a raw string cannot be found with a plain regex. + """ + if not text.startswith('R"', i): + return None + j = text.find("(", i + 2) + if j == -1: + return None + delim = text[i + 2:j] + if len(delim) > 16 or any(c in ' ()\\\t\n' for c in delim): + return None + # An `R` glued to an identifier is part of that identifier, not a prefix. + k = i + while k > 0 and (text[k - 1].isalnum() or text[k - 1] == "_"): + k -= 1 + return delim if text[k:i] in _RAW_PREFIX else None + + +def _decommented(text: str) -> list[str]: + """The file with comments and literals blanked, line structure preserved. + + Braces inside a string literal or a comment are not braces. + src/ is full of both (llama-chat.cpp alone embeds dozens of `{` in template strings), so counting them raw would desynchronise the depth for the rest of the file. + + Scanned once, left to right, rather than by substituting one construct at a time. + Order cannot fix a substitution pass: blanking raw strings first lets an `R"(` written inside a comment swallow everything to the next `)"`, and blanking comments first lets a `//` inside a raw string end the line. + Only position decides which construct is real, and a scan is what knows it. + """ + blank = lambda s: re.sub(r"[^\n]", " ", s) # noqa: E731 + out: list[str] = [] + i, n = 0, len(text) + while i < n: + delim = _raw_delim(text, i) + if delim is not None: + close = f'){delim}"' + k = text.find(close, i + 2 + len(delim) + 1) + end = n if k == -1 else k + len(close) + elif text.startswith("//", i): + k = text.find("\n", i) + end = n if k == -1 else k + elif text.startswith("/*", i): + k = text.find("*/", i + 2) + end = n if k == -1 else k + 2 + elif text[i] in "\"'": + q, j = text[i], i + 1 + while j < n and text[j] != q and text[j] != "\n": + j += 2 if text[j] == "\\" else 1 + end = min(j + 1, n) + else: + out.append(text[i]) + i += 1 + continue + out.append(blank(text[i:end])) + i = end + return "".join(out).split("\n") + + +def _depths(line: str, start: int) -> tuple[int, int]: + """(brace depth after this line, lowest depth reached inside it).""" + d = lo = start + for ch in line: + if ch == "{": + d += 1 + elif ch == "}": + d -= 1 + lo = min(lo, d) + return d, lo + + +def if_else_chains(text: str) -> list[list[tuple[int, str]]]: + """Group `if` / `else if` conditions into chains by BRACE DEPTH. + + Indentation is not the structure. + Keying chains on it, and resetting on any change, means a nested `if` inside an arm replaces the enclosing chain, so the outer arms after it are analysed as a fresh chain and an arch the outer chain already matched looks unmatched. + That is a gate that stops gating on a shape that is ordinary C++: the arch dispatch at llama-model.cpp:2434 is exactly one nested `if` away from it. + + The same key also mis-JOINS. + Two unrelated `if`s at the same indentation become one chain whenever the second one's opener is a condition the regex skips, and then a perfectly reachable arm is reported unreachable, which blocks a release on good code. + Depth gets both right: a chain lives at the depth its `if` opened at, and ends when a brace takes the file back past it. + """ + raw = text.split("\n") + clean = _decommented(text) + open_chains: dict[int, list[tuple[int, str]]] = {} + done: list[list[tuple[int, str]]] = [] + + def flush(key: int) -> None: + c = open_chains.pop(key, None) + if c and len(c) > 1: + done.append(c) + + # A chain's closing brace and its `else if` are often on separate lines, which llama.cpp does in src/llama-quant.cpp:461 among others. + # Closing the chain the moment the brace line dedents would end it one line before the arm that continues it, and the duplicate arch arm after it would then be a fresh chain with nothing taken yet, so the gate passes it. + # Ending a chain is therefore deferred one line: the next line either continues it, or it really is over. + # Blank lines do not decide either way. + pending: set[int] = set() + + depth = 0 + for i, (line, cline) in enumerate(zip(raw, clean)): + after, lo = _depths(cline, depth) + stripped = cline.lstrip() + # Matched on the decommented line: COND anchors on the `{` ending the line, so `if (arch == X) { // shared` matched nothing on the raw line and the arm vanished from the chain. + # _decommented blanks in place, so the spans still index the raw line and the condition text below is taken from there, intact. + # A line that blanked away entirely is commented-out code and opens nothing. + m = COND.search(cline) if stripped else None + # `} else if (...) {` and a bare `else if (...) {` after its own `}` line both continue the chain that lives at the depth this line dips to; a plain `if` opens one at the depth it starts from. + cont = bool(m) and (stripped.startswith("}") or stripped.startswith("else")) + key = lo if cont else depth + if stripped: + if cont: + pending.discard(key) # this line continues it after all + for k in sorted(pending, reverse=True): + flush(k) + pending.clear() + # Any chain whose closing brace this line just passed is over, unless the next line turns out to continue it. + for k in [k for k in sorted(open_chains, reverse=True) if k >= lo]: + if not (cont and k == key): + pending.add(k) + if m: + g = 1 if m.group(1) is not None else 2 + cond = line[m.start(g):m.end(g)] + if cont and key in open_chains: + open_chains[key].append((i + 1, cond)) + else: + flush(key) + open_chains[key] = [(i + 1, cond)] + # A line that both dedents and opens a chain at the same key would otherwise leave that key pending and flush the chain it just opened on the next line. + pending.discard(key) + depth = after + for k in sorted(open_chains, reverse=True): + flush(k) + return done + + +def _blocked_by(cond: str, taken: set[str]) -> set[str]: + """Arches that make this arm dead, given what earlier arms already took. + + A pure disjunction is dead only when EVERY alternative is taken. + A plain conjunction is dead as soon as ONE of its `arch == X` conjuncts is, since the other conditions can only narrow it further: an earlier unconditional `arch == X` arm makes a later `arch == X && enabled` unreachable, and that arm is not a pure disjunction so it used to be skipped entirely. + + Anything mixing `||` and `&&` is left alone rather than guessed at, and only a term that is exactly `arch == X` counts, so `!(arch == X)` and `arch != X` cannot be read as requiring that arch. + """ + archs = set(ARCH.findall(cond)) + if not archs: + return set() + if _pure_arch_disjunction(cond): + return archs if archs <= taken else set() + if "||" in cond: + return set() + hit = set() + for term in cond.split("&&"): + m = ARCH.fullmatch(term.strip().strip("()").strip()) + if m and m.group(1) in taken: + hit.add(m.group(1)) + return hit + + +def unreachable_arch_arms(path: Path) -> list[str]: + """`else if` arms whose arch a preceding pure-disjunction arm already took.""" + out = [] + for chain in if_else_chains(path.read_text()): + taken: set[str] = set() + for lineno, cond in chain: + dead = _blocked_by(cond, taken) + if dead: + out.append(f"{path}:{lineno}: unreachable, " + f"{', '.join(sorted(dead))} already matched earlier " + "in this if/else chain") + # Only an unconditional arm consumes an arch. + # A conditional one may not run, so what follows it can still be reached. + if _pure_arch_disjunction(cond): + taken |= set(ARCH.findall(cond)) + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--root", default=".", help="tree to check") + ap.add_argument("--report", metavar="PATH", help="write a JSON summary here") + a = ap.parse_args() + root = Path(a.root) + + findings: list[str] = [] + scanned = {"python": 0, "cpp": 0} + for p in sorted(root.glob("gguf-py/**/*.py")): + scanned["python"] += 1 + findings += duplicate_dict_keys(p) + for p in sorted(root.glob("src/**/*.cpp")): + scanned["cpp"] += 1 + findings += unreachable_arch_arms(p) + + print(f"merge_checks: scanned {scanned['python']} python and {scanned['cpp']} c++ files") + for f in findings: + print(f" {f}") + if a.report: + Path(a.report).write_text(json.dumps( + {"ok": not findings, "scanned": scanned, "findings": findings}, indent=2)) + if findings: + print(f"merge_checks: {len(findings)} problem(s)", file=sys.stderr) + return 1 + print("merge_checks: clean") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/package_bundle.py b/scripts/unsloth/package_bundle.py new file mode 100644 index 000000000000..dca4ccee9d11 --- /dev/null +++ b/scripts/unsloth/package_bundle.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Cross-platform packager for Unsloth llama.cpp prebuilt bundles. + +Curates the shipped executables, their local dynamic-library closure, and the +dynamically-loaded ggml backend modules; writes the in-bundle metadata +(BUILD_INFO.txt / UNSLOTH_PREBUILT_INFO.json); archives the result. + +The curation and archive engine is OS-generic: adding a new OS means +implementing one PlatformStrategy (its dependency-walk tool, lib-name +convention, backend glob, and archive format), not writing a new packaging +script. + +The CUDA runtime (libcudart/libcublas, cudart DLLs) is intentionally NOT +bundled: the installer pairs it with the user's PyTorch runtime, selected by +runtime_line. + +Linux is the CI-validated path. macOS/Windows strategies follow the correct +platform conventions (otool/@loader_path/tar.gz; dir-local DLLs/zip) but have +not yet been exercised on their runners. + +Configuration is read from the environment (see read_config). Runs both inside +the build workflow and standalone for local testing. +""" +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +# Force C locale so tool output (readelf/otool) is not localized. +_C_ENV = {**os.environ, "LC_ALL": "C", "LANG": "C"} + + +def _run(cmd: list[str]) -> str: + # Fail loudly: a missing/erroring readelf|otool would otherwise yield an + # empty closure and silently ship a bundle with missing libraries. + try: + r = subprocess.run(cmd, capture_output=True, text=True, env=_C_ENV) + except FileNotFoundError: + sys.exit(f"ERROR: required tool '{cmd[0]}' not found") + if r.returncode != 0: + sys.exit(f"ERROR: {' '.join(cmd)} failed (rc={r.returncode}): {r.stderr.strip()}") + return r.stdout + + +class PlatformStrategy: + name = "generic" + exe_suffix = "" + archive_ext = ".tar.gz" + rpath = "" + # Required core: these must exist or packaging fails loudly. Every other + # executable the build produced is discovered and shipped too (see curate). + binaries = ["llama-server", "llama-cli", "llama-quantize"] + lib_suffix_re = r"\.so(\.\d+)*$" # POSIX shared-lib suffix to exclude; Windows keys on .exe + + def shipped_binaries(self) -> list[str]: + return [b + self.exe_suffix for b in self.binaries] + + def is_executable(self, path: Path) -> bool: + """True if `path` is a program to ship (not a shared library).""" + if not path.is_file(): + return False + if self.exe_suffix: # Windows: an executable is exactly a .exe + return path.suffix.lower() == self.exe_suffix + return not re.search(self.lib_suffix_re, path.name) and os.access(path, os.X_OK) + + def local_needed(self, path: Path, bin_dir: Path) -> list[str]: + """Names of dynamic libs `path` needs that are *local* (live in bin_dir).""" + raise NotImplementedError + + def backend_patterns(self) -> list[str]: + """Globs for the dlopen'd ggml backend modules (not found via the walk).""" + raise NotImplementedError + + def supports_symlinks(self) -> bool: + return True + + def archive(self, stage: Path, out_path: Path) -> None: + raise NotImplementedError + + +class LinuxStrategy(PlatformStrategy): + name = "linux" + rpath = "$ORIGIN" + + def local_needed(self, path: Path, bin_dir: Path) -> list[str]: + # Locale-independent: key only on the (NEEDED) tag and the [name]. + needed = re.findall(r"\(NEEDED\)[^\[]*\[([^\]]+)\]", _run(["readelf", "-d", str(path)])) + return [n for n in needed if (bin_dir / n).exists() or (bin_dir / n).is_symlink()] + + def backend_patterns(self) -> list[str]: + return ["libggml-cpu-*.so*", "libggml-cuda.so*", "libggml-rpc.so*"] + + def archive(self, stage: Path, out_path: Path) -> None: + with tarfile.open(out_path, "w:gz") as tar: + tar.add(stage, arcname=".") + + +class MacOSStrategy(PlatformStrategy): + name = "macos" + rpath = "@loader_path" + lib_suffix_re = r"\.dylib$" + + def local_needed(self, path: Path, bin_dir: Path) -> list[str]: + out = _run(["otool", "-L", str(path)]) + deps: list[str] = [] + for line in out.splitlines()[1:]: # first line echoes the file path + m = re.match(r"\s+(\S+)\s+\(", line) + if not m: + continue + ref = m.group(1) + base = os.path.basename(ref) + # @rpath/@loader_path/relative refs that exist locally are "ours" + if (ref.startswith("@") or not ref.startswith("/")) and (bin_dir / base).exists(): + deps.append(base) + return deps + + def backend_patterns(self) -> list[str]: + return ["libggml-*.dylib"] + + def archive(self, stage: Path, out_path: Path) -> None: + with tarfile.open(out_path, "w:gz") as tar: + tar.add(stage, arcname=".") + + +class WindowsStrategy(PlatformStrategy): + name = "windows" + exe_suffix = ".exe" + archive_ext = ".zip" + rpath = "" # Windows resolves DLLs from the executable's directory + + # No portable readelf/otool equivalent; the project's own DLLs live beside + # the binaries in build/bin/Release, so bundle those by name convention. + LOCAL_DLL_PREFIXES = ("ggml", "llama", "mtmd") + + def local_needed(self, path: Path, bin_dir: Path) -> list[str]: + return [ + p.name for p in bin_dir.glob("*.dll") + if p.name.lower().startswith(self.LOCAL_DLL_PREFIXES) + ] + + def backend_patterns(self) -> list[str]: + return ["ggml-cpu-*.dll", "ggml-cuda.dll", "ggml-rpc.dll"] + + def supports_symlinks(self) -> bool: + return False + + def archive(self, stage: Path, out_path: Path) -> None: + with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as z: + for p in sorted(stage.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(stage).as_posix()) + + +STRATEGIES = {s.name: s for s in (LinuxStrategy(), MacOSStrategy(), WindowsStrategy())} + + +def _copy_one(strategy: PlatformStrategy, bin_dir: Path, stage: Path, name: str) -> None: + src, dst = bin_dir / name, stage / name + if dst.exists() or dst.is_symlink(): + return + if strategy.supports_symlinks() and src.is_symlink(): + target = os.readlink(src) + os.symlink(target, dst) + _copy_one(strategy, bin_dir, stage, os.path.basename(target)) + elif src.exists(): + shutil.copy2(src, dst, follow_symlinks=True) + + +def curate(strategy: PlatformStrategy, bin_dir: Path, stage: Path) -> None: + roots: list[Path] = [] + required = strategy.shipped_binaries() + for b in required: + if not (bin_dir / b).exists(): + sys.exit(f"ERROR: missing {bin_dir / b}") + shutil.copy2(bin_dir / b, stage / b) + roots.append(stage / b) + + # Ship every other executable the build produced, so a curated GPU bundle + # carries the same tool set as the full-build cpu/macos/rocm tarballs (which + # tar all of build/bin). Each becomes a root too, so any library only it + # needs is pulled into the closure. Runtime libraries that live outside + # bin_dir (e.g. the CUDA runtime) are never pulled in: the walk stays local. + required_set = set(required) + for p in sorted(bin_dir.iterdir()): + if p.name in required_set or not strategy.is_executable(p): + continue + _copy_one(strategy, bin_dir, stage, p.name) + roots.append(stage / p.name) + + # Backend modules are dlopen'd, so they never appear in the NEEDED graph; + # copy them explicitly and treat them as extra roots so their own local + # dependencies get pulled into the closure too. + for pat in strategy.backend_patterns(): + for match in sorted(bin_dir.glob(pat)): + _copy_one(strategy, bin_dir, stage, match.name) + roots.append(stage / match.name) + + # Walk the local NEEDED closure from every root, scanning each lib once. + queue = list(roots) + while queue: + for need in strategy.local_needed(queue.pop(), bin_dir): + if not (stage / need).exists() and not (stage / need).is_symlink(): + _copy_one(strategy, bin_dir, stage, need) + queue.append(stage / need) + + +def detect_nvcc_sms() -> tuple[str, list[str], str]: + if not shutil.which("nvcc"): + return "unavailable", [], "nvcc not found" + r = subprocess.run(["nvcc", "--list-gpu-arch"], capture_output=True, text=True, env=_C_ENV) + if r.returncode != 0: + return "unavailable", [], f"nvcc failed (rc={r.returncode})" + sms = sorted(set(re.findall(r"compute_(\d+)", r.stdout)), key=int) + return "available", sms, f"detected {len(sms)} SM targets" + + +def write_metadata(stage: Path, strategy: PlatformStrategy, cfg: dict, sms: list[str]) -> None: + short = cfg["commit"][:7] + min_sm, max_sm = min(map(int, sms)), max(map(int, sms)) + nvcc_status, nvcc_sms, nvcc_msg = detect_nvcc_sms() + # sm_103 (B300 / GB300 Blackwell Ultra) has no native build, but it JIT-runs + # on the bundled compute_100 PTX, so any bundle that ships sm_100 also covers + # it. Declare it in supported_sms (not the native nvcc build) so every + # platform's manifest agrees -- Windows and arm64 reuse these x64 profiles. + supported_sms = list(sms) + if "100" in supported_sms and "103" not in supported_sms: + supported_sms = sorted([*supported_sms, "103"], key=int) + note = f"CUDA {cfg['line'].removeprefix('cuda')} {cfg['klass']} bundle." + + licenses = [f"Third-party licenses bundled with this llama.cpp prebuilt ({cfg['tag']}).", + f"Source: https://github.com/{cfg['source_repo']} @ {cfg['commit']}", ""] + src = Path(cfg["src_dir"]) + if (src / "LICENSE").is_file(): + licenses += ["=== llama.cpp LICENSE ===", (src / "LICENSE").read_text(), ""] + lic_dir = src / "licenses" + if lic_dir.is_dir(): + for lic in sorted(lic_dir.glob("*")): + if lic.is_file(): + licenses += [f"=== {lic.name} ===", lic.read_text(), ""] + (stage / "THIRD_PARTY_LICENSES.txt").write_text("\n".join(licenses)) + + info = { + "upstream_tag": cfg["tag"], + "source_repo": cfg["source_repo"], + "source_repo_url": f"https://github.com/{cfg['source_repo']}", + "source_ref_kind": cfg["source_ref_kind"], + "requested_source_ref": cfg["tag"], + "resolved_source_ref": cfg["tag"], + "source_commit": cfg["commit"], + "source_commit_short": short, + "platform": f"{strategy.name}-{cfg['arch']}-cuda", + "bundle_profile": cfg["profile"], + "runtime_line": cfg["line"], + "coverage_class": cfg["klass"], + "bundle_rank": int(cfg["rank"]), + "toolkit_line": cfg["toolkit_line"], + "docker_image": cfg["docker_image"], + "supported_sms": supported_sms, + "nvcc_validation_status": nvcc_status, + "nvcc_detected_sms": nvcc_sms, + "nvcc_validation_message": nvcc_msg, + "min_sm": min_sm, + "max_sm": max_sm, + "notes": note, + "build_shared_libs": True, + "ggml_backend_dl": True, + "ggml_cpu_all_variants": True, + "rpath": strategy.rpath, + } + (stage / "UNSLOTH_PREBUILT_INFO.json").write_text(json.dumps(info, indent=2)) + + build_info = [ + f"llama.cpp version: {cfg['tag']}", + f"requested source ref: {cfg['tag']}", + f"resolved source ref: {cfg['tag']}", + f"variant: {cfg['profile']}", + f"runtime line: {cfg['line']}", + f"coverage class: {cfg['klass']}", + f"bundle rank: {cfg['rank']}", + f"docker image: {cfg['docker_image']}", + "backend: CUDA", + f"toolkit version: {cfg['toolkit_line']}", + f"supported sms: {','.join(supported_sms)}", + f"nvcc validation: {nvcc_status}", + f"min sm: {min_sm}", + f"max sm: {max_sm}", + f"os: {strategy.name}", + f"arch: {cfg['arch']}", + "build_shared_libs: ON", + "ggml_backend_dl: ON", + "ggml_cpu_all_variants: ON", + "ggml_cuda_nccl: OFF", + f"rpath: {strategy.rpath}", + "llama_openssl: ON", + "openssl_linkage: dynamic", + "cxx_runtime: dynamic", + f"source commit: {cfg['commit']}", + f"source commit short: {short}", + f"built at (UTC): {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}", + f"notes: {note}", + ] + (stage / "BUILD_INFO.txt").write_text("\n".join(build_info) + "\n") + + +def read_config() -> dict: + def need(k: str) -> str: + v = os.environ.get(k) + if not v: + sys.exit(f"ERROR: missing required env {k}") + return v + + return { + "bin_dir": need("BIN_DIR"), + "src_dir": need("SRC_DIR"), + "out_dir": need("OUT_DIR"), + "tag": need("TAG"), + "commit": need("SOURCE_COMMIT"), + "profile": need("PROFILE"), + "line": need("LINE"), + "klass": need("KLASS"), + "rank": need("RANK"), + "toolkit_line": need("TOOLKIT_LINE"), + "archs": need("ARCHS"), + # Advertised compute capabilities; optional (empty -> derived from ARCHS in main()). + "sms": os.environ.get("SMS", ""), + "platform": os.environ.get("PLATFORM", "linux"), + "arch": os.environ.get("ARCH", "x64"), + "docker_image": os.environ.get("DOCKER_IMAGE", ""), + "source_repo": os.environ.get("SOURCE_REPO", "ggml-org/llama.cpp"), + "source_ref_kind": os.environ.get("SOURCE_REF_KIND", "tag"), + } + + +def main() -> int: + cfg = read_config() + strategy = STRATEGIES.get(cfg["platform"]) + if strategy is None: + sys.exit(f"ERROR: unknown PLATFORM '{cfg['platform']}' (have {sorted(STRATEGIES)})") + + # supported_sms is the concrete coverage, which for a PTX floor (e.g. the + # cuda12-legacy "50-virtual 61-virtual") is wider than the arch int itself. + # So suffixed profiles declare it via SMS; all-real profiles fall back to + # each ARCHS entry's leading SM number. + if cfg["sms"]: + sms = [s for s in re.split(r"[ ;,]+", cfg["sms"]) if s] + else: + sms = [re.match(r"\d+", a).group() for a in re.split(r"[ ;,]+", cfg["archs"]) if a] + bin_dir = Path(cfg["bin_dir"]) + out_dir = Path(cfg["out_dir"]) + out_dir.mkdir(parents=True, exist_ok=True) + + stage = Path(tempfile.mkdtemp()) + try: + curate(strategy, bin_dir, stage) + write_metadata(stage, strategy, cfg, sms) + + asset = f"app-{cfg['tag']}-{strategy.name}-{cfg['arch']}-{cfg['profile']}{strategy.archive_ext}" + out_path = out_dir / asset + strategy.archive(stage, out_path) + + print(f"wrote {out_path}") + for p in sorted(stage.iterdir()): + print(f" {p.name}") + finally: + shutil.rmtree(stage, ignore_errors=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/unsloth/pin_contract.py b/scripts/unsloth/pin_contract.py new file mode 100644 index 000000000000..bf07d58408e3 --- /dev/null +++ b/scripts/unsloth/pin_contract.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Assert the merged tree still contains what each pin carries. + +The nightly proves the pins MERGED. That is not the same as proving they are in +the release, and the difference has cost us three outages: + + * ggml-org#28133 was squash-merged upstream. The pinned commit stopped being + an ancestor of the base tag, so the merge was not a no-op -- it re-applied + code the base already had. It happened to conflict, which is the only + reason anybody noticed. A pin in that state that merges quietly ships + nothing and nothing says so. + * an additive resolution can keep the wrong side, or a later pin can land on + top of an earlier one, and the arch registration the pin exists for is + simply not in the tree any more. It still compiles. + * a pin can rot into contributing nothing at all while its entry stays in + pr-set.json for weeks. + +So: derive from each pin's OWN diff what it puts in the tree, then check the +merged tree still has it. Nothing to maintain -- the expectation comes out of +the commit, so a repin regenerates it. + +Four assertions per pin, cheapest first: + + symbols every LLM_ARCH_/GGML_OP_/PROJECTOR_TYPE_/... name the pin + introduces, in each file it introduces it to. Per FILE, not per + tree: LLM_ARCH_INKLING surviving in llama-arch.h while its arm was + dropped from llama-model.cpp is exactly the failure being looked + for, and a tree-wide grep passes it. + files every file the pin adds still exists. + lines every non-comment code line the pin adds is still in that file. + Catches a resolution that ate a hunk without touching a symbol. + redundancy + a pin whose added lines the BASE TAG already has is work upstream + took. Reported, never fatal -- upstream landing a feature overnight + must not stop that night's release. + +What this CANNOT do, stated plainly so nobody reads more into a pass than is +there: the contract is re-derived from the pin, so it can only ever prove the +MERGE did not lose something. A regression inside the pin itself regenerates a +smaller contract that passes. Proving a feature works is feature_matrix.py's +job, and it needs a build. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +PIN_RE = re.compile( + r"^https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/commits/([0-9a-f]{40})/?$" +) + +# Identifier families that name a FEATURE. Deliberately not "every new symbol": +# a helper function renamed by a later upstream commit is not a lost feature, +# but a missing LLM_ARCH_ entry always is. These are the tables that decide +# whether an architecture, an op, a projector or a quant type exists at all. +SYMBOL_FAMILIES = ( + "LLM_ARCH_", "LLM_TENSOR_", "LLM_KV_", "LLM_TYPE_", + "PROJECTOR_TYPE_", "GGML_OP_", "GGML_TYPE_", "LLAMA_FTYPE_", +) +SYMBOL_RE = re.compile(r"\b(?:" + "|".join(SYMBOL_FAMILIES) + r")[A-Z0-9_]+\b") + +# The subset that names a whole feature rather than one of its tensors. Used +# only to keep --emit readable; the check itself uses all of SYMBOL_FAMILIES. +HEADLINE = ("LLM_ARCH_", "GGML_OP_", "GGML_TYPE_", "PROJECTOR_TYPE_", "LLAMA_FTYPE_") + +# A line worth tracking for survival. Comments and short punctuation drift with +# every reformat and would make the check noise; a substantial code line does +# not move on its own. +TRIVIAL_RE = re.compile(r"^\s*(?://|/\*|\*|\*/|#\s|$)") +MIN_LINE = 12 + +# Comments are stripped before anything is read off a line. A pin that merely +# NAMES an arch in a comment has not registered it, and holding the comment's +# wording as a contract fails the moment upstream rewords it. Observed on +# unslothai#70, whose comment mentions GGML_OP_SSM_SCAN to explain why it does +# NOT use it. +COMMENT_RE = re.compile(r"//.*$|/\*.*?\*/|(?<!\S)#(?!\s*(?:include|define|if|el|endif|pragma)).*$") + +# Binary and generated paths whose "lines" are meaningless. +SKIP_SUFFIXES = (".npy", ".png", ".jpg", ".gguf", ".bin", ".safetensors", ".ico", ".pdf") + + +class Failure(Exception): + """A pin whose contract the merged tree does not satisfy.""" + + +def git(args: list[str], cwd: Path | None = None, check: bool = True) -> str: + r = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) + if check and r.returncode != 0: + raise RuntimeError(f"git {' '.join(args[:3])}...: {r.stderr.strip()[:300]}") + return r.stdout + + +def blob(rev: str, path: str, cwd: Path) -> str | None: + """The tree entry as "mode oid", or None if the rev has no such path. + + Lifted from carry_vintage.py, mode included for the same reason: a change + that only chmods a file it otherwise took verbatim has identical content, + and an oid-only comparison would call that "contributed nothing". + """ + r = subprocess.run(["git", "ls-tree", "--full-tree", "-z", rev, "--", path], + cwd=cwd, capture_output=True, text=True) + if r.returncode != 0 or not r.stdout.strip(): + return None + mode, _type, oid = r.stdout.split("\0")[0].split("\t", 1)[0].split() + return f"{mode} {oid}" + + +def load_effective(prs_json: str) -> list[dict]: + """The pin list the resolve step actually merged. + + Not the same as pr-set.json: resolve drops an optional pin once its PR is + no longer open, and re-reading the file would then check a pin that is not + in the tree and report it missing. The step already has the effective list + as an output, so take it rather than recomputing the filter here and + getting it subtly different. + """ + return [{"url": p.get("url", ""), "src": p["repo"].split("/")[0], + "num": int(p["number"]), "sha": p["sha"], "required": True} + for p in json.loads(prs_json)] + + +def load_pins(pr_set: Path) -> list[dict]: + data = json.loads(pr_set.read_text()) + pins = [] + for entry in data["prs"]: + url = entry if isinstance(entry, str) else entry["url"] + m = PIN_RE.match(url) + if not m: + raise SystemExit(f"malformed pin: {url}") + pins.append({"url": url, "src": m.group(1), "num": int(m.group(2)), + "sha": m.group(3), + "required": True if isinstance(entry, str) + else entry.get("required", True)}) + return pins + + +def derive(pin: dict, base: str, cwd: Path) -> dict: + """What this pin puts in the tree, read off its own diff against the base. + + The fork point is merge-base(pin, base), not the pin's parent: a pin that + has already had the base merged into it (which repin.py and every carry + branch produce) would otherwise look like it contributed all of upstream. + """ + fork = git(["merge-base", pin["sha"], base], cwd).strip() + diff = git(["diff", "--no-renames", fork, pin["sha"]], cwd) + + symbols: dict[str, set[str]] = defaultdict(set) + lines: dict[str, list[str]] = defaultdict(list) + cur = None + for ln in diff.split("\n"): + if ln.startswith("+++ b/"): + cur = ln[6:] + elif ln.startswith("+++ "): + cur = None # /dev/null: a deletion + elif cur and ln.startswith("+") and not ln.startswith("+++"): + text = ln[1:] + stripped = text.strip() + if len(stripped) >= MIN_LINE and not TRIVIAL_RE.match(stripped): + lines[cur].append(stripped) + code = COMMENT_RE.sub("", text).strip() + if code: + symbols[cur].update(SYMBOL_RE.findall(code)) + + # Only symbols the base does not ALREADY have in that file are evidence of + # this pin. Upstream naming an arch in a file the pin also touches is not + # something the pin is owed. + new_symbols: dict[str, list[str]] = {} + for path, names in symbols.items(): + fresh = sorted(n for n in names + if n not in git(["show", f"{base}:{path}"], cwd, check=False)) + if fresh: + new_symbols[path] = fresh + + status = git(["diff", "--name-status", "--no-renames", fork, pin["sha"]], cwd) + added, owned = [], [] + for ln in status.split("\n"): + if not ln.strip(): + continue + code, path = ln.split("\t", 1) + owned.append(path) + if code.startswith("A"): + added.append(path) + + return { + "fork": fork, + "symbols": new_symbols, + "added_files": added, + "owned_paths": owned, + "lines": {p: v for p, v in lines.items() + if not p.endswith(SKIP_SUFFIXES)}, + } + + +def redundancy(contract: dict, base: str, cwd: Path) -> tuple[int, int]: + """How much of what this pin adds the base tag already has. + + This is the pr-set.json retirement rule, mechanised: "delete the entry once + a base tag carries the work". Upstream almost always SQUASHES, so the + pinned commit never becomes an ancestor and no ancestry test will ever say + the work landed; comparing the text is the only thing that can. + + Measured on the real set at b10775, the separation is not close: the pin + that upstream had already absorbed (ggml-org#28133) scored 99%, and the + highest live pin scored 33%. + """ + total = hit = 0 + for path, wanted in contract["lines"].items(): + text = git(["show", f"{base}:{path}"], cwd, check=False) + total += len(wanted) + hit += sum(1 for w in wanted if w in text) + return hit, total + + +def check(pin: dict, contract: dict, root: Path, base: str, cwd: Path, + threshold: float) -> list[str]: + problems = [] + + for path, names in sorted(contract["symbols"].items()): + target = root / path + text = target.read_text(errors="replace") if target.is_file() else "" + for name in names: + if name not in text: + problems.append( + f"{name} is missing from {path}; the pin adds it there and " + "the merged tree does not have it") + + for path in contract["added_files"]: + if not (root / path).exists(): + problems.append(f"{path} is added by the pin and missing from the merged tree") + + for path, wanted in sorted(contract["lines"].items()): + target = root / path + if not target.is_file(): + continue # already reported, or a deletion + text = target.read_text(errors="replace") + lost = [w for w in wanted if w not in text] + if not lost: + continue + kept = len(wanted) - len(lost) + ratio = kept / len(wanted) + if ratio < threshold: + problems.append( + f"{path} kept {kept}/{len(wanted)} of the lines this pin adds " + f"({ratio:.0%}); first missing: {lost[0][:90]}") + + return problems + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--root", default=".", help="the merged tree to check") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--pr-set", help="scripts/unsloth/pr-set.json") + src.add_argument("--prs-json", help="the resolve step's `prs` output: the pins it " + "actually merged, optional ones already dropped") + ap.add_argument("--base", required=True, help="upstream base tag the mix was built on") + ap.add_argument("--git-dir", help="repo the pin commits are reachable from " + "(default: --root)") + ap.add_argument("--threshold", type=float, default=1.0, + help="fraction of a pin's added lines that must survive per file") + ap.add_argument("--redundant-at", type=float, default=0.95, + help="report a pin whose added lines the base tag already has " + "at this fraction or more (never fatal)") + ap.add_argument("--report", help="write a JSON report here") + ap.add_argument("--emit", action="store_true", + help="print the derived contracts and check nothing") + args = ap.parse_args() + + root = Path(args.root).resolve() + cwd = Path(args.git_dir).resolve() if args.git_dir else root + pins = (load_pins(Path(args.pr_set)) if args.pr_set + else load_effective(args.prs_json)) + + report: dict = {"base": args.base, "ok": False, "pins": [], "notices": []} + failed = 0 + notices: list[str] = [] + + for pin in pins: + name = f"{pin['src']}#{pin['num']}" + try: + contract = derive(pin, args.base, cwd) + except RuntimeError as e: + report["pins"].append({"pin": name, "sha": pin["sha"], "problems": [str(e)]}) + print(f"ERROR {name}: {e}", file=sys.stderr) + failed += 1 + continue + + entry = { + "pin": name, + "sha": pin["sha"], + "fork": contract["fork"], + "symbols": contract["symbols"], + "added_files": contract["added_files"], + "line_count": sum(len(v) for v in contract["lines"].values()), + "problems": [], + } + + if args.emit: + report["pins"].append(entry) + # Only the families that NAME a feature are printed. Every symbol + # is still checked; a new file legitimately contributes a hundred + # LLM_TENSOR_ names and listing them buries the one that matters. + sym = sorted({s for v in contract["symbols"].values() for s in v + if s.startswith(HEADLINE)}) + print(f"{name:>18} {entry['line_count']:>5} lines, " + f"{len(contract['added_files'])} new files, symbols: " + f"{', '.join(sym) if sym else '-'}") + continue + + problems = check(pin, contract, root, args.base, cwd, args.threshold) + hit, total = redundancy(contract, args.base, cwd) + entry["problems"] = problems + entry["redundant_lines"] = [hit, total] + + if total and hit / total >= args.redundant_at: + note = (f"the base tag already has {hit}/{total} ({hit / total:.0%}) of the " + "lines this pin adds; upstream has taken this work and the entry " + "should be deleted from pr-set.json") + entry["notices"] = [note] + notices.append(f"{name}: {note}") + + report["pins"].append(entry) + if problems: + failed += 1 + print(f"FAIL {name}", file=sys.stderr) + for p in problems: + print(f" {p}", file=sys.stderr) + else: + print(f"ok {name}: {len(contract['symbols'])} file(s) with new symbols, " + f"{entry['line_count']} line(s) accounted for") + + report["ok"] = failed == 0 or args.emit + report["notices"] = notices + if args.report: + Path(args.report).write_text(json.dumps(report, indent=2)) + if args.emit: + return 0 + + # Notices after the verdict lines, never mixed into them: "upstream took + # this, drop the entry" is housekeeping and must not read as a failure. + for n in notices: + print(f"note {n}") + if failed: + print(f"\n{failed} pin(s) are not intact in the merged tree", file=sys.stderr) + return 1 + print(f"\nall {len(pins)} pins are intact in the merged tree" + + (f", {len(notices)} can be retired" if notices else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/pin_merge.py b/scripts/unsloth/pin_merge.py new file mode 100755 index 000000000000..0ebe184ccd0b --- /dev/null +++ b/scripts/unsloth/pin_merge.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Three-way merge pr-set.json one pin at a time, and refuse anything ambiguous. + +Two repins in flight always collide. +Both edit adjacent lines of the same JSON list, so git merges them as text and reports a conflict over lines that have nothing to do with each other. +On 08-27 that happened twice in one hour, while landing the qwen4exp, Inkling and GLM-5-Next repins: each merge invalidated the next, and each one was resolved by hand into exactly what a per-element merge would have produced. + +Pin ORDER is load-bearing. +resolve merges pins sequentially, so a later pin sees the tree the earlier ones produced, and reordering the list silently changes the composition. +This never reorders: it walks the base list positionally and takes whichever side moved each entry. +That also means a pin added or removed on one side is refused rather than aligned, because guessing where an inserted pin belongs is exactly the kind of guess that would change composition order. +A side that REORDERS the list is refused for the same reason, and for a sharper one: position i would no longer name the same pin on both sides, so merging it field-wise would splice one PR's `required` onto another PR's url. + +Usable as a git merge driver: + + git config merge.pinset.name 'pr-set.json pin-wise merge' + git config merge.pinset.driver 'python3 scripts/unsloth/pin_merge.py %O %A %B' + echo 'scripts/unsloth/pr-set.json merge=pinset' >> .gitattributes + +The driver contract is to write the result over %A (ours) and exit 0, or leave it alone and exit non-zero to fall back to a normal conflict. +That fallback is the whole safety story: a refusal costs a hand resolution, which is the status quo, and never a wrong pin set. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + + +class Ambiguous(Exception): + """A pin set difference this script is not allowed to decide.""" + + +MISSING = object() + +# The pin url shape repin.py already enforces. +# The owner matters: ggml-org#125 and unslothai#125 are different PRs. +PIN_ID = re.compile(r"^https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/commits/") + + +def pins(doc: dict) -> list[str]: + """The pin URLs, in order. Entries are a bare string or {url, required}.""" + return [p if isinstance(p, str) else p["url"] for p in doc["prs"]] + + +def fields(entry: str | dict) -> dict: + """An entry in dict form. A bare url string is just {"url": url}.""" + return {"url": entry} if isinstance(entry, str) else dict(entry) + + +def ident(entry: str | dict) -> str: + """What a pin IS, independent of which commit it currently points at. + + A repin changes only the sha, so (owner, PR number) is the stable identity. + Anything that does not look like a pin url is its own identity, which is the conservative reading: an unrecognised url can only ever cause a refusal. + """ + url = fields(entry).get("url") + m = PIN_ID.match(url) if isinstance(url, str) else None + return f"{m.group(1)}#{m.group(2)}" if m else repr(url) + + +def refuse_reorder(b: list, o: list, t: list) -> None: + """Refuse if either side moved an entry that base holds somewhere else. + + Merging by position assumes position i means the same pin on all three sides. + A reorder breaks that assumption silently: base [A, B] with ours making A optional and theirs reordering to [B, A] merges position 0 as "url moved to B, required moved to false" and produces B(required=false), so the release skips the wrong PR and the driver still exits 0. + + Realigning by identity instead is not safe. + A duplicated entry, or a reorder combined with a repin, leaves more than one alignment consistent with the diff, and choosing one is a guess about composition order, which is load-bearing here. + Refusing costs the hand resolution that was the status quo; guessing costs a wrong build nothing downstream can see. + """ + bid = [ident(e) for e in b] + for side, entries in (("ours", o), ("theirs", t)): + for i, e in enumerate(entries): + k = ident(e) + if k != bid[i] and k in bid: + raise Ambiguous( + f"pin {i} on {side} is {k}, which base holds at position " + f"{bid.index(k)}: the list was reordered, and merging " + "reordered entries by position would take fields from " + "different PRs") + + +def refuse_duplicates(what: str, entries: list) -> None: + """Two entries of one PR are indistinguishable, so nothing can align them.""" + ids = [ident(e) for e in entries] + dupes = sorted({k for k in ids if ids.count(k) > 1}) + if dupes: + raise Ambiguous( + f"{what} names {', '.join(dupes)} more than once; two entries of " + "one PR cannot be told apart, so a swap between them reads as no " + "change and a later merge would splice their fields together") + + +def merge_keys(what: str, bd: dict, od: dict, td: dict) -> dict: + """Three-way merge a mapping key by key, refusing only a real clash. + + A key missing on a side is MISSING rather than absent, so "theirs deleted it, ours left it alone" is a deletion both sides agree on, not a no-op. + Ours' key order is kept, then keys only theirs or only base has. + """ + out: dict = {} + for k in list(od) + [k for k in td if k not in od] + \ + [k for k in bd if k not in od and k not in td]: + bv, ov, tv = bd.get(k, MISSING), od.get(k, MISSING), td.get(k, MISSING) + if ov == tv: + v = ov + elif ov == bv: + v = tv + elif tv == bv: + v = ov + else: + raise Ambiguous(f"{what} field {k!r} changed differently on both " + f"sides:\n ours: {ov}\n theirs: {tv}") + if v is not MISSING: + out[k] = v + return out + + +def merge_entry(i: int, b, o, t): + """Three-way merge one pin ENTRY, not just its url. + + Comparing whole entries matters: an entry carries `required` as well as `url`, and comparing only urls makes a `required` flip on one side look like "no change", so rebuilding the list from ours drops it silently. + """ + if o == t: + return o # same on both sides, including untouched + if o == b: + return t # only theirs touched this entry + if t == b: + return o # only ours touched this entry + # Both sides touched it. + # Merging field-wise is only meaningful while all three sides name the SAME PR. + # A repin moves the sha and keeps the identity, which is the case this is for. + # REPLACING the pin with another PR while the other side edits a field is not: base PR100(required=true), ours PR200, theirs PR100 required=false merges url from ours and required from theirs and yields PR200(required=false), so the release skips a PR nobody made optional and the driver still exits 0. + # + # Refused rather than resolved even when both sides agree on the replacement, because base then describes a different PR and every field comparison below is against settings that were never PR200's. + named = {ident(b), ident(o), ident(t)} + if len(named) != 1: + raise Ambiguous( + f"pin {i} names different PRs across the sides (base {ident(b)}, " + f"ours {ident(o)}, theirs {ident(t)}) and both sides edited it: " + "merging their fields would attach one PR's settings to another") + out = merge_keys(f"pin {i}", fields(b), fields(o), fields(t)) + if "url" not in out: + raise Ambiguous(f"pin {i} lost its url") + # Keep the bare-string form when nothing but the url is present, so the file's shape is not rewritten by merging it. + return out["url"] if list(out) == ["url"] else out + + +def merge_pins(base: dict, ours: dict, theirs: dict) -> dict: + b, o, t = base["prs"], ours["prs"], theirs["prs"] + if not len(b) == len(o) == len(t): + raise Ambiguous( + f"pin count differs (base={len(b)} ours={len(o)} theirs={len(t)}); " + "a pin was added or removed, and placing it is an ordering decision" + ) + # Every side, and the result. + # A duplicate on one input defeats the reorder guard below; a duplicate only in the RESULT is made here, by the two sides adding the same PR at different positions. + for what, side in (("base", b), ("ours", o), ("theirs", t)): + refuse_duplicates(what, side) + refuse_reorder(b, o, t) + merged = [merge_entry(i, bx, ox, tx) + for i, (bx, ox, tx) in enumerate(zip(b, o, t))] + refuse_duplicates("the merged pin set", merged) + # Everything outside .prs is three-way merged the same way. + # Rebuilding the document from ours instead would silently drop a change theirs made to a top-level field - `_doc`, or any schema field added later - and this driver REPLACES git's text merge rather than running after it, so nothing downstream would ever notice the loss. + # A real two-sided clash refuses, which costs a hand resolution and never a wrong document. + skel = [{k: (None if k == "prs" else v) for k, v in d.items()} + for d in (base, ours, theirs)] # .prs is merged positionally + out = merge_keys("document", *skel) + out["prs"] = merged # keeps ours' key position + return out + + +def load(path: str) -> dict: + return json.loads(Path(path).read_text()) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + # argparse %-formats help strings, so a literal percent must be doubled. + ap.add_argument("base", help="%%O, the merge base") + ap.add_argument("ours", help="%%A, our version; the result is written here") + ap.add_argument("theirs", help="%%B, their version") + ap.add_argument("--stdout", action="store_true", + help="print the result instead of writing over `ours`") + ap.add_argument("--report", metavar="PATH", help="write a JSON summary here") + a = ap.parse_args() + + report: dict = {"ok": False, "reason": None} + try: + merged = merge_pins(load(a.base), load(a.ours), load(a.theirs)) + except (Ambiguous, KeyError, json.JSONDecodeError) as e: + report["reason"] = str(e) + if a.report: + Path(a.report).write_text(json.dumps(report, indent=2)) + print(f"pin_merge: refused: {e}", file=sys.stderr) + return 1 + + text = json.dumps(merged, indent=2) + "\n" + if a.stdout: + sys.stdout.write(text) + else: + Path(a.ours).write_text(text) + report["ok"] = True + report["pins"] = pins(merged) + if a.report: + Path(a.report).write_text(json.dumps(report, indent=2)) + # stderr, so --stdout emits nothing but the merged document + print(f"pin_merge: merged {len(pins(merged))} pins", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/pr-set.json b/scripts/unsloth/pr-set.json new file mode 100644 index 000000000000..bdca9a2dd569 --- /dev/null +++ b/scripts/unsloth/pr-set.json @@ -0,0 +1,35 @@ +{ + "_doc": [ + "ggml-org/llama.cpp or unslothai/llama.cpp PRs to merge into the nightly prebuilds. Each entry", + "pins an exact commit -- copy the url of the commit you reviewed from the PR's commits tab:", + " https://github.com/ggml-org/llama.cpp/pull/15926/commits/59a3d0cb8f611aa3110ecea3d0afd16b1b18ee06", + "Only that commit is built, even if the author keeps pushing; update the pin to take newer code.", + "An empty list is a plain upstream build.", + "Closed and merged PRs are STILL merged in. Upstream tags lag their merges and the base is", + "aged a further UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS, so a pin dropped on merge leaves its arch", + "in neither the base nor the mix. Once the base tag contains the commit the merge is an empty", + "no-op ONLY if upstream took the PR as a merge commit. Upstream usually squashes, and a squash", + "is not an ancestor of the pinned commit, so the merge re-applies code the base already has and", + "the whole build stops on an unresolvable conflict. Delete the entry once a base tag carries the", + "work, do not wait for it to rot away. A closed-unmerged pin ships code upstream", + "declined -- the resolve log warns, but nothing else stops it, so prune those deliberately.", + "Use {\"url\": \"...\", \"required\": false} for an entry that should be skipped once it is not open.", + "Merging an unslothai PR into fork master drops it from the nightly (the tree is the upstream", + "tag + pins), so keep its pin listed until the change lands upstream." + ], + "prs": [ + "https://github.com/ggml-org/llama.cpp/pull/24423/commits/c6f8d604b67611b73f7965c0bd39d26e7365a489", + "https://github.com/ggml-org/llama.cpp/pull/25731/commits/36df1bf409c8b257689321a971a66973ee817ee1", + "https://github.com/unslothai/llama.cpp/pull/70/commits/883f2c9ba78f3847148454adf025da29385fff3e", + "https://github.com/unslothai/llama.cpp/pull/61/commits/46cbf0e95786fe8f5b7c0e86d57aaf8f8eceea7f", + "https://github.com/unslothai/llama.cpp/pull/95/commits/3db8cb5b2e9bf291057b9f19960e8601a162da81", + "https://github.com/ggml-org/llama.cpp/pull/27754/commits/629b50552801912b3e2078f9799e4d77213197d7", + "https://github.com/unslothai/llama.cpp/pull/137/commits/4e1865e34ec5f6ca39403215c89129c13731be70", + "https://github.com/unslothai/llama.cpp/pull/158/commits/abfc45b9cb21eae4848cb82196e659f42c9a8341", + "https://github.com/unslothai/llama.cpp/pull/157/commits/6c6da89266ba7839d825c9997782af4f4d26b81b", + "https://github.com/unslothai/llama.cpp/pull/149/commits/b65a2dce12c14a489e19a059cb6ee59112f1b733", + "https://github.com/unslothai/llama.cpp/pull/144/commits/a9e9c3c5fed8a0bb5cc617532d0d16b8f59c13e0", + "https://github.com/unslothai/llama.cpp/pull/152/commits/b2b5ed9ff86427a530b762a45d3fdbd453bcd4e8", + "https://github.com/unslothai/llama.cpp/pull/176/commits/09ce1a4d2939844e211f7b4d30a296f4c1aed9a8" + ] +} diff --git a/scripts/unsloth/repin.py b/scripts/unsloth/repin.py new file mode 100644 index 000000000000..ac9028592ee5 --- /dev/null +++ b/scripts/unsloth/repin.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Merge the current base tag into each pinned PR branch we control, and repin. + +The nightly builds an upstream release tag plus a list of pinned PR commits. +Upstream moves several times a day, so a pin that merged yesterday routinely +stops merging today -- that is what broke four nightlies in a week, and every +fix was the same mechanical merge done by hand. + +This does that merge, and only where it is safe to: + + * branches we own (see OWNED). A third-party PR is reported, never pushed to. + * pins that are still their branch head. If the author has pushed past the + pin, merging into the branch would silently widen the release to include + code nobody reviewed, which is the exact property the pin file exists to + hold. Report it and let a human decide. + * conflicts that additive_merge.py can prove are pure add/add. Anything else + is left alone and reported. + +Writes the new pins back to pr-set.json and prints a markdown report. Pushing +and opening the PR is the caller's job; nothing here talks to a remote except +to fetch. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +# Branches we may push to. Anything else gets a report line and no write. +OWNED = ("unslothai/", "danielhanchen/") + +PIN_RE = re.compile( + r"^https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/commits/([0-9a-f]{40})/?$" +) +HERE = Path(__file__).resolve().parent + + +def run(args, cwd=None, check=True, quiet=True): + r = subprocess.run(args, cwd=cwd, capture_output=True, text=True) + if check and r.returncode != 0: + raise RuntimeError(f"{' '.join(args[:4])}... failed: {r.stderr.strip()[:400]}") + if not quiet and r.stdout: + print(r.stdout.rstrip()) + return r + + +def gh_json(path): + r = run(["gh", "api", path], check=False) + if r.returncode != 0: + return None + try: + return json.loads(r.stdout) + except json.JSONDecodeError: + return None + + +def load_pins(pr_set: Path) -> tuple[dict, list[dict]]: + data = json.loads(pr_set.read_text()) + pins = [] + for entry in data["prs"]: + url = entry if isinstance(entry, str) else entry["url"] + required = True if isinstance(entry, str) else entry.get("required", True) + m = PIN_RE.match(url) + if not m: + raise SystemExit(f"malformed pin: {url}") + pins.append( + { + "url": url, + "required": required, + "src": f"{m.group(1)}/llama.cpp", + "num": int(m.group(2)), + "sha": m.group(3), + } + ) + return data, pins + + +def repin_one(pin: dict, base: str, work: Path) -> dict: + """Try to bring one pin up to `base`. Never raises for an expected refusal.""" + out = dict(pin, action="skip", note="", new_sha="", files=[], hunks=[]) + pr = gh_json(f"repos/{pin['src']}/pulls/{pin['num']}") + if pr is None: + out["note"] = "could not read the PR from the API" + return out + if pr.get("state") != "open": + out["note"] = f"PR is {pr.get('state')}; not repinning a closed PR" + return out + + head_repo = (pr.get("head", {}).get("repo") or {}).get("full_name") + head_ref = pr.get("head", {}).get("ref") + head_sha = pr.get("head", {}).get("sha") + out.update(head_repo=head_repo, head_ref=head_ref) + + if not head_repo: + out["note"] = "head repository was deleted" + return out + if not head_repo.startswith(OWNED): + out["action"] = "third-party" + out["note"] = f"`{head_repo}` is not ours; ask the author to merge master" + return out + if head_sha != pin["sha"]: + out["note"] = ( + f"branch head `{head_sha[:10]}` has moved past the pin `{pin['sha'][:10]}`; " + "repinning would pull in unreviewed commits" + ) + return out + + repo = work / f"r{pin['num']}" + run(["git", "clone", "-q", "--filter=blob:none", "--no-checkout", + f"https://github.com/{pin['src']}.git", str(repo)]) + run(["git", "fetch", "-q", "--no-tags", "origin", pin["sha"]], cwd=repo, check=False) + r = run(["git", "fetch", "-q", "--no-tags", + "https://github.com/ggml-org/llama.cpp.git", + f"refs/tags/{base}:refs/tags/{base}"], cwd=repo, check=False) + if r.returncode != 0: + out["note"] = f"could not fetch base tag {base}" + return out + if run(["git", "rev-parse", "--verify", f"{pin['sha']}^{{commit}}"], + cwd=repo, check=False).returncode != 0: + out["note"] = f"pinned commit {pin['sha'][:10]} is gone (force-pushed away)" + return out + + run(["git", "checkout", "-q", "--detach", pin["sha"]], cwd=repo) + if run(["git", "merge-base", "--is-ancestor", f"refs/tags/{base}", "HEAD"], + cwd=repo, check=False).returncode == 0: + out["note"] = f"already contains {base}" + return out + + # diff3 is what makes the add/add proof possible: without the base section + # an edit/edit conflict is indistinguishable from an add/add one. + git_id = ["-c", "user.name=unsloth-repin-bot", + "-c", "user.email=unsloth-repin-bot@users.noreply.github.com"] + m = run(["git", "-c", "merge.conflictStyle=diff3", *git_id, "merge", "--no-ff", + "--no-edit", "-m", f"Merge {base} into {head_ref}", f"refs/tags/{base}"], + cwd=repo, check=False) + + if m.returncode != 0: + report = work / f"res{pin['num']}.json" + rc = subprocess.run( + [sys.executable, str(HERE / "additive_merge.py"), + "--repo", str(repo), "--report", str(report)], + capture_output=True, text=True, + ).returncode + res = json.loads(report.read_text()) if report.exists() else {} + if rc != 0: + out["action"] = "conflict" + refused = res.get("refused", []) + out["files"] = [x["file"] for x in refused if x["file"] != "-"] + if out["files"]: + out["note"] = "; ".join(f"`{x['file']}`: {x['reason']}" for x in refused) + else: + # git refused the merge without leaving a single conflicted + # file, so the conflict report explains nothing. Its stderr is + # the only thing that does, and discarding it turns a + # diagnosable failure into "no conflicted files". + tail = ((m.stderr or "") + (m.stdout or "")).strip().splitlines() + out["note"] = ("merge failed with no conflicts: " + " / ".join(tail[-3:]) + if tail else "merge failed and git said nothing") + run(["git", "merge", "--abort"], cwd=repo, check=False) + return out + out["hunks"] = [ + {"file": f["file"], **h} for f in res.get("resolved", []) for h in f["hunks"] + ] + out["files"] = [f["file"] for f in res.get("resolved", [])] + run(["git", *git_id, "commit", "-q", "--no-edit"], cwd=repo) + + new_sha = run(["git", "rev-parse", "HEAD"], cwd=repo).stdout.strip() + out["action"] = "repin" + out["new_sha"] = new_sha + out["repo_path"] = str(repo) + out["touches_workflows"] = bool( + run(["git", "diff", "--name-only", f"{pin['sha']}..HEAD", "--", + ".github/workflows"], cwd=repo).stdout.strip() + ) + return out + + +def markdown(base: str, results: list[dict]) -> str: + L = [f"Base tag: `{base}`", ""] + L += ["| pin | branch | outcome |", "|---|---|---|"] + for r in results: + pin = f"[`{r['src']}#{r['num']}`](https://github.com/{r['src']}/pull/{r['num']})" + branch = f"`{r.get('head_repo') or '?'}:{r.get('head_ref') or '?'}`" + if r["action"] == "repin": + what = f"repinned `{r['sha'][:10]}` to `{r['new_sha'][:10]}`" + if r["hunks"]: + what += f" ({len(r['hunks'])} add/add hunk(s) resolved)" + elif r["action"] == "conflict": + what = f"**conflict, not resolvable automatically** -- {r['note']}" + elif r["action"] == "third-party": + what = f"not ours -- {r['note']}" + else: + what = r["note"] or "no change" + L.append(f"| {pin} | {branch} | {what} |") + L.append("") + + for r in results: + if not r.get("hunks"): + continue + L += [f"<details><summary>Resolutions for <code>{r['src']}#{r['num']}</code></summary>", ""] + for h in r["hunks"]: + L += [f"`{h['file']}`", "", "```diff"] + L += [f"-{x}" for x in h["ours"].rstrip("\n").split("\n")] + L += [f"-{x}" for x in h["theirs"].rstrip("\n").split("\n")] + L += [f"+{x}" for x in h["resolution"].rstrip("\n").split("\n")] + L += ["```", ""] + L += ["</details>", ""] + + if any(r.get("touches_workflows") for r in results): + L += ["Some merges carry upstream changes under `.github/workflows/`, so the " + "push needs a token with workflow write permission.", ""] + return "\n".join(L) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--pr-set", required=True) + ap.add_argument("--base", required=True, help="upstream release tag to merge in") + ap.add_argument("--work", required=True, help="scratch directory for clones") + ap.add_argument("--report", help="write a JSON report here") + ap.add_argument("--markdown", help="write the human-readable report here") + args = ap.parse_args() + + pr_set = Path(args.pr_set) + work = Path(args.work) + work.mkdir(parents=True, exist_ok=True) + _, pins = load_pins(pr_set) + + results = [] + for pin in pins: + try: + r = repin_one(pin, args.base, work) + except RuntimeError as e: + r = dict(pin, action="skip", note=f"error: {e}", new_sha="", files=[], hunks=[]) + results.append(r) + print(f"{r['src']}#{r['num']}: {r['action']} {r['note']}".rstrip()) + + # Swap the shas in the raw text rather than re-serialising. Rewriting the + # JSON would reflow the whole file and bury a four-character change in a + # whole-file diff, which is the opposite of what a reviewer needs here. + text = pr_set.read_text() + changed = 0 + for r in results: + if r["action"] != "repin": + continue + if r["sha"] not in text: + print(f"::warning::{r['src']}#{r['num']}: pin not found verbatim; not rewritten") + continue + text = text.replace(r["sha"], r["new_sha"]) + changed += 1 + if changed: + pr_set.write_text(text) + + if args.report: + Path(args.report).write_text(json.dumps( + {"base": args.base, "changed": changed, "results": results}, indent=2)) + if args.markdown: + Path(args.markdown).write_text(markdown(args.base, results)) + + blocked = [r for r in results if r["action"] in ("conflict", "third-party")] + print(f"\n{changed} repinned, {len(blocked)} needing a human") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/sync_deletes.py b/scripts/unsloth/sync_deletes.py new file mode 100755 index 000000000000..3b8c53ca6e5d --- /dev/null +++ b/scripts/unsloth/sync_deletes.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Resolve the modify/delete conflicts a fork sync produces, and only those. + +Measured over every merge commit in this fork: 149 file-level conflicts, of which 68 (45 percent) are the same one. +Upstream edits a workflow this fork deleted on purpose, git cannot know which side wins, and a human keeps the deletion. +Every single time: 68 of 68 historical instances resolved by keeping the deletion, with no exceptions. + +That is not a heuristic, it is the fork's stated invariant. +This fork owns no upstream CI. +scripts/unsloth/upstream-sync.json requires the diff from the sync point to master to touch only .github/ and scripts/unsloth/, and verify_upstream_sync.py already treats these deletions as legitimate. +The deletions are policy, so re-applying them is bookkeeping. + +Scope is deliberately tight, because the cost of being wrong is a workflow silently reappearing and firing on the fork: + + - only paths under .github/workflows/ + - only modify/delete conflicts, never content conflicts + - never a file named unsloth-*.yml, which is ours; if one of those is ever in a modify/delete conflict, something is wrong and a human should look + - the delete must be on our side; upstream deleting a file we modified is the opposite situation and is left alone + +The same policy covers a workflow upstream ADDED since the last sync. +That is not a conflict at all, so git merges it in silently and the fork acquires a workflow that starts firing on it. +Replaying sync e8735f35d3 caught exactly this: resolving only the conflicts left .github/workflows/build-wasm.yml in the tree, where the recorded human resolution had deleted it. +With --added handled too, the replay reproduces that tree exactly. + +Anything else is left conflicted. +Exits 0 if it resolved everything it was asked about, 1 if any conflict remains. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +PREFIX = ".github/workflows/" +OURS = "unsloth-" + + +def git(*args: str, cwd: str = ".") -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) + + +def unmerged(cwd: str) -> dict[str, set[int]]: + """path -> set of stages present (1 base, 2 ours, 3 theirs).""" + out: dict[str, set[int]] = {} + r = git("ls-files", "-u", cwd=cwd) + # A listing that could not run gives empty stdout, which reads as "no + # conflicts" and makes this script report that it resolved everything it + # was asked to. Not a repo, or an unreadable index, has to be an error. + if r.returncode != 0: + raise RuntimeError(f"git ls-files -u in {cwd}: {r.stderr.strip()}") + for line in r.stdout.split("\n"): + if not line.strip(): + continue + meta, path = line.split("\t", 1) + stage = int(meta.split()[2]) + out.setdefault(path.strip(), set()).add(stage) + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--repo", default=".") + ap.add_argument("--merge-base", help="also drop upstream workflows added since this rev") + ap.add_argument("--report", metavar="PATH") + a = ap.parse_args() + + try: + stages = unmerged(a.repo) + except RuntimeError as e: + print(f"sync_deletes: {e}", file=sys.stderr) + if a.report: + Path(a.report).write_text(json.dumps( + {"ok": False, "resolved": [], "removed_added": [], + "left": [str(e)]}, indent=2)) + return 1 + resolved, left, added = [], [], [] + for path, st in sorted(stages.items()): + name = path.rsplit("/", 1)[-1] + # stage 2 missing means our side deleted it; stage 3 present means upstream still has it. + # That is the sync case, and only that. + ours_deleted = 2 not in st and 3 in st + if (path.startswith(PREFIX) and not name.startswith(OURS) and ours_deleted): + r = git("rm", "-q", "--force", "--", path, cwd=a.repo) + if r.returncode == 0: + resolved.append(path) + else: + left.append(f"{path}: git rm failed: {r.stderr.strip()}") + else: + why = ("we own this workflow" if name.startswith(OURS) + else "not an upstream workflow path" if not path.startswith(PREFIX) + else "not a delete on our side") + left.append(f"{path}: {why}") + + # Workflows upstream added since the merge base. + # No conflict, so nothing above sees them, and the fork silently gains CI that fires on its own repo. + if a.merge_base: + # Against the WORKING TREE, not HEAD: mid-merge, HEAD is still our pre-merge commit, so merge_base..HEAD describes our side rather than the merge result and finds nothing. + # --no-renames, or an upstream workflow that was RENAMED reads as R rather than A and this filter drops it. + # The fork would then carry the renamed workflow and it would start firing here, which is the exact thing this block exists to stop. + r = git("diff", "--name-status", "--diff-filter=A", "--no-renames", + a.merge_base, "--", PREFIX, cwd=a.repo) + if r.returncode != 0: + # An unusable --merge-base produces empty stdout, which is indistinguishable from "upstream added nothing" if the exit code is ignored. + # This script reporting success is what tells a sync it can proceed, so a listing that never ran has to be a failure, not a quiet zero: otherwise the sync carries every newly added upstream workflow in and they start firing on the fork. + left.append(f"{a.merge_base}: could not list workflows added since " + f"it: {r.stderr.strip()}") + else: + for line in r.stdout.split("\n"): + if not line.strip(): + continue + path = line.split("\t", 1)[1].strip() + name = path.rsplit("/", 1)[-1] + if name.startswith(OURS): + continue + rm = git("rm", "-q", "--force", "--", path, cwd=a.repo) + if rm.returncode == 0: + added.append(path) + else: + # Same reasoning as the resolve loop above: a removal that did not happen is reported, never dropped. + left.append(f"{path}: git rm failed: {rm.stderr.strip()}") + + for p in resolved: + print(f"resolved {p}: kept the fork's deletion") + for p in added: + print(f"removed {p}: upstream added it; this fork carries no upstream workflows") + for p in left: + print(f"left {p}", file=sys.stderr) + if a.report: + Path(a.report).write_text(json.dumps( + {"ok": not left, "resolved": resolved, + "removed_added": added, "left": left}, indent=2)) + return 1 if left else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/test_additive_merge.py b/scripts/unsloth/test_additive_merge.py new file mode 100644 index 000000000000..0f91e9afd12b --- /dev/null +++ b/scripts/unsloth/test_additive_merge.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Tests for additive_merge.py. Run: python3 scripts/unsloth/test_additive_merge.py + +Every case builds a real git conflict rather than a hand-written one, so the +markers are exactly what git produces. +""" +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "additive_merge.py" +FAILS = [] + + +def check(name, cond, extra=""): + print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def git(repo, *args, **kw): + return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], + cwd=repo, capture_output=True, text=True, **kw) + + +def make_conflict(base_txt, ours_txt, theirs_txt): + """Build a real git conflict, return (repo, conflicted file path).""" + d = Path(tempfile.mkdtemp(prefix="am_")) + git(d, "init", "-q", "-b", "main") + f = d / "f.c" + f.write_text(base_txt) + git(d, "add", "-A"); git(d, "commit", "-qm", "base") + git(d, "checkout", "-qb", "side") + f.write_text(theirs_txt) + git(d, "add", "-A"); git(d, "commit", "-qm", "theirs") + git(d, "checkout", "-q", "main") + f.write_text(ours_txt) + git(d, "add", "-A"); git(d, "commit", "-qm", "ours") + git(d, "-c", "merge.conflictStyle=diff3", "merge", "side") + return d, f + + +def run(repo, *extra): + rep = repo / "r.json" + p = subprocess.run([sys.executable, str(SCRIPT), "--repo", str(repo), "--report", str(rep), *extra], + capture_output=True, text=True) + return p.returncode, json.loads(rep.read_text()) if rep.exists() else {} + + +# --- 1. pure add/add: the real recurring shape ------------------------------ +base = "switch (arch) {\n case A:\n break;\n}\n" +ours = "switch (arch) {\n case A:\n case LLM_ARCH_INKLING:\n break;\n}\n" +theirs = "switch (arch) {\n case A:\n case LLM_ARCH_DEEPSEEK4:\n break;\n}\n" +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +txt = f.read_text() +check("add/add resolves", rc == 0 and rep["ok"], rep) +check("add/add unions both labels", + "LLM_ARCH_DEEPSEEK4" in txt and "LLM_ARCH_INKLING" in txt and "<<<<" not in txt, txt) +check("add/add puts upstream first (matches hand repin)", + txt.index("DEEPSEEK4") < txt.index("INKLING"), txt) +check("add/add stages the file", + git(repo, "diff", "--name-only", "--diff-filter=U").stdout.strip() == "") + +# --- 2. edit/edit on a shared line: must refuse ----------------------------- +base = "if (a == X || a == Y) {\n" +ours = "if (a == X || a == Y || a == KIMI) {\n" +theirs = "if (a == X || a == Y || a == MINIMAX) {\n" +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +check("edit/edit refuses", rc == 1 and not rep["ok"]) +check("edit/edit says why", "base is not empty" in (rep["refused"][0]["reason"] if rep["refused"] else ""), + rep) +check("edit/edit leaves markers in place", "<<<<" in f.read_text()) + +# --- 3. same line added twice: must refuse, not duplicate ------------------- +base = "a\nz\n" +ours = "a\ncase FOO:\n break;\nz\n" +theirs = "a\ncase FOO:\n break;\nz\n" +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +check("identical add/add is not a conflict at all", rc == 1 and "no conflicted files" in json.dumps(rep)) + +base = "a\nz\n" +ours = "a\nstatic void helper() {\n log(\"same\");\n}\nz\n" +theirs = "a\nstatic void helper2() {\n log(\"same\");\n}\nz\n" +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +check("overlapping add/add refuses on shared CONTENT", + rc == 1 and "made twice" in json.dumps(rep), rep) +reason = rep["refused"][0]["reason"] if rep.get("refused") else "" +check("overlapping add/add names the content line, not the braces", + reason.endswith('twice: log("same");'), reason) + +# --- 3b. two independent case arms: braces are shared, content is not ------- +# The real tools/mtmd/clip.cpp shape. Refusing this on `{` and `} break;` is +# what took the 09-02 nightly's last pin down. +base = "switch (t) {\n}\n" +ours = ("switch (t) {\n case PROJECTOR_TYPE_KIMIK3:\n {\n" + " builder = std::make_unique<clip_graph_kimik3>(ctx, img);\n" + " } break;\n}\n") +theirs = ("switch (t) {\n case PROJECTOR_TYPE_DEEPSEEK4V:\n {\n" + " builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);\n" + " } break;\n}\n") +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +txt = f.read_text() +check("independent case arms resolve despite shared braces", rc == 0 and rep["ok"], rep) +check("independent case arms keep both", "KIMIK3" in txt and "DEEPSEEK4V" in txt and "<<<<" not in txt, txt) +check("independent case arms keep both bodies once", + txt.count("} break;") == 2 and txt.count("clip_graph_kimik3") == 1, txt) + +# --- 3b2. two case arms that share a body line, which is a coincidence ------ +# The clip.cpp shape after upstream landed DEEPSEEK4V: both arms set the same +# rope_theta, and refusing on that is the shared-line check backwards. +base = "switch (t) {\n}\n" +ours = ("switch (t) {\n case PROJECTOR_TYPE_KIMIK3:\n {\n" + " hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;\n" + " hparams.rope_theta = 10000.0f;\n } break;\n}\n") +theirs = ("switch (t) {\n case PROJECTOR_TYPE_DEEPSEEK4V:\n {\n" + " hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;\n" + " hparams.rope_theta = 10000.0f;\n } break;\n}\n") +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +txt = f.read_text() +check("case arms with a coincidentally shared body line resolve", rc == 0 and rep["ok"], rep) +check("case arms with a shared body line keep both arms", + txt.count("rope_theta") == 2 and "KIMIK3" in txt and "DEEPSEEK4V" in txt, txt) + +# --- 3b3. the SAME arm added twice keeps its label, so it still refuses ----- +base = "switch (t) {\n}\n" +ours = ("switch (t) {\n case PROJECTOR_TYPE_KIMIK3:\n {\n" + " hparams.rope_theta = 10000.0f;\n } break;\n}\n") +theirs = ("switch (t) {\n case PROJECTOR_TYPE_KIMIK3:\n {\n" + " hparams.rope_theta = 50000.0f;\n } break;\n}\n") +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +check("the same case label on both sides still refuses", + rc == 1 and "made twice" in json.dumps(rep), rep) + +# --- 3b4. only one side is case arms: no label proof, ordinary rules apply -- +base = "a\nz\n" +ours = "a\ncase FOO:\n f(1);\n break;\nz\n" +theirs = "a\nstatic void helper() { f(1); }\nz\n" +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +check("one side not a case arm falls back to the shared-line check", + rc == 0 and rep["ok"], rep) + +base = "a\nz\n" +ours = "a\ncase FOO:\n f(1);\n break;\nz\n" +theirs = "a\nstatic void helper();\n f(1);\nz\n" +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +check("one side not a case arm still refuses on a shared content line", + rc == 1 and "made twice" in json.dumps(rep), rep) + +# --- 3c. one side adds only scaffolding: nothing distinguishes the two ------ +base = "a\nz\n" +ours = "a\n}\nz\n" +theirs = "a\ncase BAR:\n break;\nz\n" +repo, f = make_conflict(base, ours, theirs) +rc, rep = run(repo) +check("scaffolding-only addition refuses", + rc == 1 and "scaffolding" in json.dumps(rep), rep) + +# --- 4. one file good, one file bad: refuse the whole merge ---------------- +d = Path(tempfile.mkdtemp(prefix="am_")) +git(d, "init", "-q", "-b", "main") +(d / "good.c").write_text("x\ny\n") +(d / "bad.c").write_text("if (a || b) {\n") +git(d, "add", "-A"); git(d, "commit", "-qm", "base") +git(d, "checkout", "-qb", "side") +(d / "good.c").write_text("x\ncase UP:\ny\n") +(d / "bad.c").write_text("if (a || b || up) {\n") +git(d, "add", "-A"); git(d, "commit", "-qm", "theirs") +git(d, "checkout", "-q", "main") +(d / "good.c").write_text("x\ncase MINE:\ny\n") +(d / "bad.c").write_text("if (a || b || mine) {\n") +git(d, "add", "-A"); git(d, "commit", "-qm", "ours") +git(d, "-c", "merge.conflictStyle=diff3", "merge", "side") +rc, rep = run(d) +check("mixed: overall refuses", rc == 1 and not rep["ok"]) +check("mixed: both files still unmerged in the index", + {ln.split("\t")[-1] for ln in git(d, "ls-files", "-u").stdout.splitlines()} == {"good.c", "bad.c"}, + git(d, "ls-files", "-u").stdout) +check("mixed: the resolvable file is NOT half-written", + "<<<<" in (d / "good.c").read_text(), (d / "good.c").read_text()) + +# --- 5. dry-run writes nothing -------------------------------------------- +base = "switch (arch) {\n case A:\n break;\n}\n" +ours = "switch (arch) {\n case A:\n case MINE:\n break;\n}\n" +theirs = "switch (arch) {\n case A:\n case UP:\n break;\n}\n" +repo, f = make_conflict(base, ours, theirs) +before = f.read_text() +rc, rep = run(repo, "--dry-run") +check("dry-run reports ok", rc == 0 and rep["ok"]) +check("dry-run does not touch the file", f.read_text() == before) + +print() +print(f"{len(FAILS)} failure(s)" + (": " + ", ".join(FAILS) if FAILS else "")) +sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_carry_vintage.py b/scripts/unsloth/test_carry_vintage.py new file mode 100644 index 000000000000..f420dd48472d --- /dev/null +++ b/scripts/unsloth/test_carry_vintage.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Tests for carry_vintage.py. Run: python3 scripts/unsloth/test_carry_vintage.py + +Builds a throwaway repo shaped like a real carry: an upstream base tag, an upstream PR branch with two commits on top of it, and a carry branch that replayed the PR onto the base while deliberately dropping one file's change. + +The case that matters is that dropped file. +Its content equals the BASE version, which is reachable from the PR head, so a vintage search that walks the PR head's whole ancestry finds a "match" in a commit that is not part of the PR at all, calls the file SUPERSEDED, and concludes that rebuilding from the PR head is equivalent to merging - which would re-add exactly what the carry dropped. +""" +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "carry_vintage.py" +FAILS = [] + + +def check(name, cond, extra=""): + print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def git(d, *args): + r = subprocess.run(["git", *args], cwd=d, capture_output=True, text=True) + if r.returncode: + raise RuntimeError(" ".join(args) + ": " + r.stderr) + return r.stdout.strip() + + +def write(d, name, text): + (Path(d) / name).write_text(text) + + +def fixture(): + d = tempfile.mkdtemp(prefix="cv_") + git(d, "init", "-q", "-b", "main") + git(d, "config", "user.email", "t@t") + git(d, "config", "user.name", "t") + write(d, "dropped.txt", "base version\n") + write(d, "taken.txt", "base version\n") + write(d, "edited.txt", "base version\n") + git(d, "add", "-A") + git(d, "commit", "-qm", "base") + base = git(d, "rev-parse", "HEAD") + + # Upstream PR: two commits, touching all three files. + git(d, "checkout", "-q", "-b", "pr") + write(d, "dropped.txt", "upstream v1\n") + write(d, "taken.txt", "upstream v1\n") + write(d, "edited.txt", "upstream v1\n") + git(d, "commit", "-qam", "pr c1") + mid = git(d, "rev-parse", "HEAD") + write(d, "taken.txt", "upstream v2\n") + write(d, "edited.txt", "upstream v2\n") + git(d, "commit", "-qam", "pr c2") + head = git(d, "rev-parse", "HEAD") + + # The carry: PR replayed onto base, with dropped.txt held at the base version on purpose, edited.txt at an older PR vintage, and taken.txt already at the PR head. + git(d, "checkout", "-q", "-b", "carry", base) + write(d, "dropped.txt", "base version\n") + write(d, "taken.txt", "upstream v2\n") + write(d, "edited.txt", "upstream v1\n") + git(d, "commit", "-qam", "carry") + carry = git(d, "rev-parse", "HEAD") + return d, base, mid, head, carry + + +d, base, mid, head, carry = fixture() +report = str(Path(d) / "report.json") +r = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry, "--pr-ref", head, + "--base", base, "--report", report], + cwd=d, capture_output=True, text=True) +check("runs clean", r.returncode == 0, r.stderr) +out = json.loads(Path(report).read_text()) +sup = {e["path"]: e["vintage"] for e in out["superseded"]} + +check("a file held at the BASE version is not called superseded", + "dropped.txt" in out["diverged"], json.dumps(out, indent=1)) +check("no vintage is a commit outside the PR", + all(v in (mid, head) for v in sup.values()), json.dumps(sup, indent=1)) +check("a file already at the PR head is superseded", + sup.get("taken.txt") == head, json.dumps(sup, indent=1)) +check("a file at an older PR commit is superseded at that vintage", + sup.get("edited.txt") == mid, json.dumps(sup, indent=1)) +check("a real divergence still forces a merge", + "has to merge" in r.stdout, r.stdout[-300:]) + +# With the deliberately dropped file removed from the picture, nothing diverges any more. +# edited.txt is still at an older vintage than the PR head, though, so the advice stays qualified: we never edited that file, but a rebuild moves it to head, and only a person knows whether the carry meant to hold it. +git(d, "checkout", "-q", "carry") +write(d, "dropped.txt", "upstream v1\n") +git(d, "commit", "-qam", "take it after all") +carry2 = git(d, "rev-parse", "HEAD") +r2 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry2, "--pr-ref", head, + "--base", base], cwd=d, capture_output=True, text=True) +check("nothing diverges once the dropped file is taken", + r2.returncode == 0 and "has to merge" not in r2.stdout, r2.stdout[-300:]) +check("an older vintage still qualifies the advice", + "OLDER vintage" in r2.stdout and "Nothing diverges" not in r2.stdout, + r2.stdout[-300:]) + +# A carry that deliberately OMITS a file the PR head still has. +# Nothing diverges, every file the carry does have is superseded, and the naive answer is "rebuild from the PR head" - which restores the omitted file and loses the omission. +# A file the PR DELETED is a different case: the carry not having it is agreement, and a rebuild reproduces it exactly. +def omission_fixture(): + d = tempfile.mkdtemp(prefix="cv_") + git(d, "init", "-q", "-b", "main") + git(d, "config", "user.email", "t@t") + git(d, "config", "user.name", "t") + write(d, "keep.txt", "base version\n") + write(d, "doomed.txt", "base version\n") + git(d, "add", "-A") + git(d, "commit", "-qm", "base") + base = git(d, "rev-parse", "HEAD") + + # The PR edits keep.txt, adds win.cmake, and deletes doomed.txt. + git(d, "checkout", "-q", "-b", "pr") + write(d, "keep.txt", "upstream v1\n") + write(d, "win.cmake", "windows-only build tweak\n") + git(d, "rm", "-q", "doomed.txt") + git(d, "add", "-A") + git(d, "commit", "-qm", "pr c1") + head = git(d, "rev-parse", "HEAD") + + # The carry replays it but never took win.cmake. + git(d, "checkout", "-q", "-b", "carry", base) + write(d, "keep.txt", "upstream v1\n") + git(d, "rm", "-q", "doomed.txt") + git(d, "add", "-A") + git(d, "commit", "-qm", "carry without win.cmake") + return d, base, head, git(d, "rev-parse", "HEAD") + + +d2, base2, head2, carry3 = omission_fixture() +# outside the repo, so the "writes nothing" check below sees a clean tree +report2 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") +r3 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry3, "--pr-ref", head2, + "--base", base2, "--report", report2], + cwd=d2, capture_output=True, text=True) +check("runs clean on an omitting carry", r3.returncode == 0, r3.stderr) +out2 = json.loads(Path(report2).read_text()) +check("a file present at the PR head but not in the carry is OMITTED, not absent", + out2.get("omitted") == ["win.cmake"] and "win.cmake" not in out2.get("absent", []), + json.dumps(out2, indent=1)) +check("a file the PR deleted stays merely ABSENT", + out2.get("absent") == ["doomed.txt"], json.dumps(out2, indent=1)) +check("an omitted file suppresses the rebuild recommendation", + "Nothing diverges" not in r3.stdout, r3.stdout[-400:]) +check("and says why rebuilding is not equivalent", + "NOT equivalent" in r3.stdout, r3.stdout[-400:]) +check("the omitting carry still writes nothing", + git(d2, "status", "--porcelain") == "", git(d2, "status", "--porcelain")) + +# Take the omitted file, and the rebuild advice is correct again. +write(d2, "win.cmake", "windows-only build tweak\n") +git(d2, "add", "-A") +git(d2, "commit", "-qm", "take win.cmake after all") +r4 = subprocess.run([sys.executable, str(SCRIPT), "--carry", git(d2, "rev-parse", "HEAD"), + "--pr-ref", head2, "--base", base2], cwd=d2, capture_output=True, text=True) +check("a PR deletion alone still allows the rebuild", + r4.returncode == 0 and "Nothing diverges" in r4.stdout, r4.stdout[-400:]) + +# A multi-commit PR that does not touch every file in its FIRST commit. +# The commits before the one that first changed a path still carry the fork's blob, and they are inside fork..head, so a carry deliberately holding that file at the base version matches one of them and is called SUPERSEDED - the same wrong "rebuilding is equivalent" answer the fork bound was meant to end, one commit further in. +def late_touch_fixture(): + d = tempfile.mkdtemp(prefix="cv_") + git(d, "init", "-q", "-b", "main") + git(d, "config", "user.email", "t@t") + git(d, "config", "user.name", "t") + write(d, "held.txt", "base version\n") + write(d, "early.txt", "base version\n") + git(d, "add", "-A") + git(d, "commit", "-qm", "base") + base = git(d, "rev-parse", "HEAD") + + # c1 touches early.txt only, so held.txt is still the base blob AT c1. + git(d, "checkout", "-q", "-b", "pr") + write(d, "early.txt", "upstream v1\n") + git(d, "commit", "-qam", "pr c1") + # c2 is the first commit to touch held.txt. + write(d, "held.txt", "upstream v2\n") + git(d, "commit", "-qam", "pr c2") + head = git(d, "rev-parse", "HEAD") + + # The carry took early.txt and holds held.txt at the base version. + git(d, "checkout", "-q", "-b", "carry", base) + write(d, "early.txt", "upstream v1\n") + git(d, "commit", "-qam", "carry") + return d, base, head, git(d, "rev-parse", "HEAD") + + +d3, base3, head3, carry4 = late_touch_fixture() +report3 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") +r5 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry4, "--pr-ref", head3, + "--base", base3, "--report", report3], + cwd=d3, capture_output=True, text=True) +check("runs clean on a late-touch PR", r5.returncode == 0, r5.stderr) +out3 = json.loads(Path(report3).read_text()) +check("a file held at base is not superseded by a PR commit that predates its first change", + "held.txt" in out3["diverged"], json.dumps(out3, indent=1)) +check("no in-range commit before the first change counts as a vintage", + all(e["path"] != "held.txt" for e in out3["superseded"]), json.dumps(out3, indent=1)) +check("and the rebuild advice is withheld", + "Nothing diverges" not in r5.stdout, r5.stdout[-400:]) +check("a file the carry really did take is still superseded", + any(e["path"] == "early.txt" for e in out3["superseded"]), json.dumps(out3, indent=1)) +check("the late-touch run still writes nothing", + git(d3, "status", "--porcelain") == "", git(d3, "status", "--porcelain")) + +# A PR that RENAMES a file while the carry deliberately keeps the old path. +# `git diff --name-only` prints only the new name of a detected rename, so the old path never entered the file list at all: the new path came back SUPERSEDED, nothing diverged, and the summary recommended a rebuild - which deletes the path the carry is holding, unreported. +def rename_fixture(): + d = tempfile.mkdtemp(prefix="cv_") + git(d, "init", "-q", "-b", "main") + git(d, "config", "user.email", "t@t") + git(d, "config", "user.name", "t") + # Long enough that git scores the move as a rename rather than add+delete. + write(d, "old.py", "".join(f"line {i}\n" for i in range(40))) + git(d, "add", "-A") + git(d, "commit", "-qm", "base") + base = git(d, "rev-parse", "HEAD") + + git(d, "checkout", "-q", "-b", "pr") + git(d, "mv", "old.py", "new.py") + git(d, "commit", "-qm", "pr renames it") + head = git(d, "rev-parse", "HEAD") + + # The carry takes the new path AND keeps the old one, on purpose. + git(d, "checkout", "-q", "-b", "carry", base) + write(d, "new.py", "".join(f"line {i}\n" for i in range(40))) + git(d, "add", "-A") + git(d, "commit", "-qm", "carry keeps both") + return d, base, head, git(d, "rev-parse", "HEAD") + + +d4, base4, head4, carry5 = rename_fixture() +report4 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") +r6 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry5, "--pr-ref", head4, + "--base", base4, "--report", report4], + cwd=d4, capture_output=True, text=True) +check("runs clean on a renaming PR", r6.returncode == 0, r6.stderr) +out4 = json.loads(Path(report4).read_text()) +check("the old side of a rename is scanned at all", + "old.py" in out4["diverged"] + out4["absent"] + out4["omitted"] + or any(e["path"] == "old.py" for e in out4["superseded"]), + json.dumps(out4, indent=1)) +check("a retained old path is reported as diverged, not silently dropped", + "old.py" in out4["diverged"], json.dumps(out4, indent=1)) +check("a rename does not license the rebuild advice", + "Nothing diverges" not in r6.stdout, r6.stdout[-400:]) +check("the renaming run still writes nothing", + git(d4, "status", "--porcelain") == "", git(d4, "status", "--porcelain")) + +# A carry that changes a file the PR never touched. +# Everything the PR did touch is superseded, so nothing diverges and nothing is omitted, and the rebuild advice was given anyway - while a rebuild from the PR head drops the carry-only change, which is invisible to a scan of the PR's own file list. +def carry_only_fixture(): + d = tempfile.mkdtemp(prefix="cv_") + git(d, "init", "-q", "-b", "main") + git(d, "config", "user.email", "t@t") + git(d, "config", "user.name", "t") + write(d, "theirs.txt", "base version\n") + write(d, "ours_only.txt", "base version\n") + git(d, "add", "-A") + git(d, "commit", "-qm", "base") + base = git(d, "rev-parse", "HEAD") + + # The PR touches theirs.txt and nothing else. + git(d, "checkout", "-q", "-b", "pr") + write(d, "theirs.txt", "upstream v1\n") + git(d, "commit", "-qam", "pr c1") + head = git(d, "rev-parse", "HEAD") + + # The carry takes the PR's change AND makes one of its own, off the PR. + git(d, "checkout", "-q", "-b", "carry", base) + write(d, "theirs.txt", "upstream v1\n") + write(d, "ours_only.txt", "a fix we carry ourselves\n") + git(d, "commit", "-qam", "carry plus our own fix") + return d, base, head, git(d, "rev-parse", "HEAD") + + +d5, base5, head5, carry6 = carry_only_fixture() +report5 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") +r7 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry6, "--pr-ref", head5, + "--base", base5, "--report", report5], + cwd=d5, capture_output=True, text=True) +check("runs clean on a carry with its own change", r7.returncode == 0, r7.stderr) +out5 = json.loads(Path(report5).read_text()) +check("a file only the carry changed is reported", + out5.get("carry_only") == ["ours_only.txt"], json.dumps(out5, indent=1)) +check("a carry-only change withholds the rebuild advice", + "Nothing diverges" not in r7.stdout, r7.stdout[-400:]) +check("and the PR's own file is still superseded", + any(e["path"] == "theirs.txt" for e in out5["superseded"]), json.dumps(out5, indent=1)) +check("the carry-only run still writes nothing", + git(d5, "status", "--porcelain") == "", git(d5, "status", "--porcelain")) + +# A carry that takes the PR's content but changes the file mode. +# Content is identical, so an oid-only comparison called it superseded and recommended a rebuild, which drops the mode change. +def mode_fixture(): + d = tempfile.mkdtemp(prefix="cv_") + git(d, "init", "-q", "-b", "main") + git(d, "config", "user.email", "t@t") + git(d, "config", "user.name", "t") + write(d, "tool.sh", "#!/bin/sh\necho base\n") + git(d, "add", "-A") + git(d, "commit", "-qm", "base") + base = git(d, "rev-parse", "HEAD") + + git(d, "checkout", "-q", "-b", "pr") + write(d, "tool.sh", "#!/bin/sh\necho upstream v1\n") + git(d, "commit", "-qam", "pr c1") + head = git(d, "rev-parse", "HEAD") + + # The carry takes that content verbatim and makes it executable. + git(d, "checkout", "-q", "-b", "carry", base) + write(d, "tool.sh", "#!/bin/sh\necho upstream v1\n") + git(d, "add", "-A") + git(d, "update-index", "--chmod=+x", "tool.sh") + git(d, "commit", "-qm", "carry makes it executable") + return d, base, head, git(d, "rev-parse", "HEAD") + + +d6, base6, head6, carry7 = mode_fixture() +report6 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") +r8 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry7, "--pr-ref", head6, + "--base", base6, "--report", report6], + cwd=d6, capture_output=True, text=True) +check("runs clean on a mode-only carry change", r8.returncode == 0, r8.stderr) +out6 = json.loads(Path(report6).read_text()) +check("a mode-only difference is not a vintage match", + "tool.sh" in out6["diverged"], json.dumps(out6, indent=1)) +check("a mode-only difference withholds the rebuild advice", + "Nothing diverges" not in r8.stdout, r8.stdout[-400:]) + +# A carry holding a file at an older commit of the PR. +# Nothing diverges, so the advice used to be an unqualified "rebuilding is equivalent" - but a rebuild moves that file to head, and the carry may be holding the older vintage on purpose, which is a decision this script cannot see. +def older_vintage_fixture(): + d = tempfile.mkdtemp(prefix="cv_") + git(d, "init", "-q", "-b", "main") + git(d, "config", "user.email", "t@t") + git(d, "config", "user.name", "t") + write(d, "f.txt", "base\n") + git(d, "add", "-A") + git(d, "commit", "-qm", "base") + base = git(d, "rev-parse", "HEAD") + + git(d, "checkout", "-q", "-b", "pr") + write(d, "f.txt", "upstream v1\n") + git(d, "commit", "-qam", "pr c1") + write(d, "f.txt", "upstream v2\n") + git(d, "commit", "-qam", "pr c2") + head = git(d, "rev-parse", "HEAD") + + # The carry stopped at v1. + git(d, "checkout", "-q", "-b", "carry", base) + write(d, "f.txt", "upstream v1\n") + git(d, "commit", "-qam", "carry holds v1") + return d, base, head, git(d, "rev-parse", "HEAD") + + +d7, base7, head7, carry8 = older_vintage_fixture() +r9 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry8, "--pr-ref", head7, + "--base", base7], cwd=d7, capture_output=True, text=True) +check("runs clean on an older-vintage carry", r9.returncode == 0, r9.stderr) +check("an older vintage is still recognised as superseded", + "SUPERSEDED" in r9.stdout, r9.stdout[-400:]) +check("an older vintage withholds the unqualified rebuild advice", + "Nothing diverges" not in r9.stdout, r9.stdout[-400:]) +check("and says a rebuild would move it to head", + "OLDER vintage" in r9.stdout, r9.stdout[-400:]) + +print() +print(f"{len(FAILS)} failure(s)" if FAILS else "all carry_vintage tests passed") +sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_feature_matrix.py b/scripts/unsloth/test_feature_matrix.py new file mode 100644 index 000000000000..ee050f7f761e --- /dev/null +++ b/scripts/unsloth/test_feature_matrix.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Tests for feature_matrix.py. Run: python3 scripts/unsloth/test_feature_matrix.py + +The thing worth testing here is not that a passing probe passes. It is that a +probe which exits 0 having proved NOTHING is reported as a failure, because both +real harnesses do exactly that: + + test-llama-archs -a <excluded arch> prints SKIP, exits 0 + test-backend-ops test -o <typo> matches nothing, exits 0 + +So the fakes below are the real output shapes, verbatim, and the assertions are +about what the script refuses to call a pass. +""" +import json +import os +import stat +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "feature_matrix.py" +FAILS = [] + + +def check(name, cond, extra=""): + print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def fake(dirp: Path, name: str, stdout: str, rc: int = 0): + p = dirp / name + p.write_text("#!/bin/sh\ncat <<'XEOF'\n" + stdout + "\nXEOF\nexit " + str(rc) + "\n") + p.chmod(p.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +# Real output shapes, copied from actual runs. +ARCHS_OK = """main: using seed 1234 +| Model arch.| Device|Config| NMSE vs. CPU|Roundtrip| +|----------------|-------------------------------|------|---------------|---------| +| inkling| NVIDIA B200| MoE| OK (1.82e-11)| SKIP| +| inkling|Intel(R) Xeon(R) Platinum 8559C| MoE| OK (0.00e+00)| SKIP|""" +ARCHS_ALL_SKIP = """main: using seed 1234 +| Model arch.| Device|Config| NMSE vs. CPU|Roundtrip| +|----------------|-------------------------------|------|---------------|---------| +| inkling| NVIDIA B200| Dense|SKIP | SKIP|""" +ARCHS_ABSENT = """main: using seed 1234 +| Model arch.| Device|Config| NMSE vs. CPU|Roundtrip| +|----------------|-------------------------------|------|---------------|---------|""" +OPS_OK = """ FLASH_ATTN_EXT_BANDED(hsk=64): OK + 13/13 tests passed + Backend CUDA0: OK""" +OPS_NOTHING = """Backend 1/2: CUDA0 +Backend 2/2: CPU + Skipping CPU backend +2/2 backends passed +OK""" +MTMD_OK = """test_projector_registry (185 assertion(s)) [PASS] + +tests : 1 +assertions : 185 +failures : 0""" +MTMD_NOTHING = """tests : 0 +assertions : 0 +failures : 0""" + + +def build(archs=ARCHS_OK, ops=OPS_OK, mtmd=MTMD_OK, checks=None): + d = Path(tempfile.mkdtemp(prefix="fm_")) + (d / "bin").mkdir() + fake(d / "bin", "test-llama-archs", archs) + fake(d / "bin", "test-backend-ops", ops) + fake(d / "bin", "test-mtmd-impl", mtmd) + manifest = d / "feature-checks.json" + manifest.write_text(json.dumps({ + "schema": 1, + "features": {"inkling": {"owner": "unslothai#172", "checks": checks or [ + {"kind": "arch", "arch": "inkling"}, + {"kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED"}, + ]}}, + "unchecked": {"unslothai#95": "no feature surface"}, + })) + return d, manifest + + +def run(d, manifest, *extra): + rep = d / "r.json" + p = subprocess.run([sys.executable, str(SCRIPT), "--build-dir", str(d), + "--feature-checks", str(manifest), "--report", str(rep), *extra], + capture_output=True, text=True) + return p.returncode, (json.loads(rep.read_text()) if rep.exists() else {}), p.stdout + p.stderr + + +# --- 1. everything genuinely ran ------------------------------------------ +d, m = build() +rc, rep, out = run(d, m, "--gpu") +check("a real pass passes", rc == 0 and rep["ok"], out) +check("the evidence is recorded, not just the verdict", + "1.82e-11" not in out and "2/2 device rows" in out, out) + +# --- 2. the arch harness skipped the arch and exited 0 --------------------- +d, m = build(archs=ARCHS_ALL_SKIP) +rc, rep, out = run(d, m, "--gpu") +check("an all-SKIP arch run is a failure", rc == 1, out) +check("and says nothing was decoded", "nothing was decoded" in out, out) + +# --- 3. the arch is not in the harness at all ----------------------------- +d, m = build(archs=ARCHS_ABSENT) +rc, rep, out = run(d, m, "--gpu") +check("an arch with no row at all is a failure", rc == 1, out) +check("and says it is not in the harness", "not in the harness" in out, out) + +# --- 4. the op filter matched nothing ------------------------------------- +d, m = build(ops=OPS_NOTHING) +rc, rep, out = run(d, m, "--gpu") +check("an op filter that matched nothing is a failure", rc == 1, out) +check("and says the filter matched nothing", "matched nothing" in out, out) + +# --- 5. the op ran and failed --------------------------------------------- +d, m = build(ops=" 11/13 tests passed\n Backend CUDA0: FAIL") +rc, rep, out = run(d, m, "--gpu") +check("a failing op is a failure", rc == 1 and "11/13" in out, out) + +# --- 6. no GPU: op probes are deferred, not passed and not failed --------- +d, m = build() +rc, rep, out = run(d, m) +check("without a GPU the op probe is deferred", rc == 0 and rep["deferred"] == 1, out) +check("deferral is stated in the summary", "need a GPU" in out, out) +check("deferral is not counted as evidence", + len(rep["features"][0]["results"]) == 1, rep) + +# --- 7. a feature with nothing but GPU checks reads as unproven, not ok --- +d, m = build(checks=[{"kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED"}]) +rc, rep, out = run(d, m) +check("a wholly deferred feature does not print ok", + rc == 0 and "nothing provable without a GPU" in out and "\nok inkling" not in out, out) + +# --- 8. the mtmd probe ran no assertions ---------------------------------- +d, m = build(mtmd=MTMD_NOTHING, checks=[{"kind": "mtmd", "projector": "kimik3"}]) +rc, rep, out = run(d, m, "--gpu") +check("an mtmd run with zero assertions is a failure", rc == 1, out) + +# --- 9. unchecked pins are reported, not hidden --------------------------- +d, m = build() +rc, rep, out = run(d, m, "--gpu") +check("knowingly unchecked pins are printed", "unslothai#95 has no runtime check" in out, out) + +print() +print(f"{len(FAILS)} failure(s)" + (": " + ", ".join(FAILS) if FAILS else "")) +sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_merge_checks.py b/scripts/unsloth/test_merge_checks.py new file mode 100644 index 000000000000..c0b9be1ddd88 --- /dev/null +++ b/scripts/unsloth/test_merge_checks.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Tests for merge_checks.py. Run: python3 scripts/unsloth/test_merge_checks.py + +The positive cases are reduced from the two real 08-27 mistakes. +The negative cases are the shapes that must NOT fire, because a check that blocks a good merge costs a release just as surely as a bad merge does. +""" +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "merge_checks.py" +FAILS = [] + + +def check(name, cond, extra=""): + print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def run(py=None, cpp=None): + d = Path(tempfile.mkdtemp(prefix="mc_")) + (d / "gguf-py" / "gguf").mkdir(parents=True) + (d / "src" / "models").mkdir(parents=True) + (d / "gguf-py" / "gguf" / "t.py").write_text(py or "x = {}\n") + (d / "src" / "t.cpp").write_text(cpp or "int main() { return 0; }\n") + r = subprocess.run([sys.executable, str(SCRIPT), "--root", str(d)], + capture_output=True, text=True) + return r.returncode, r.stdout + r.stderr + + +DUP_KEY = """ +MAP = { + ARCH.QWEN4EXP: {"a": 1}, + ARCH.GLM5NEXT: {"b": 2}, + ARCH.GLM5NEXT: {"b": 2}, +} +""" +OK_KEYS = """ +MAP = { + ARCH.QWEN4EXP: {"a": 1}, + ARCH.GLM5NEXT: {"b": 2}, +} +""" +DEAD_ARM = """ +void f() { + if (arch == LLM_ARCH_FALCON_H1) { + a(); + } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { + b(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" +LIVE_ARM = """ +void f() { + if (arch == LLM_ARCH_GLM5NEXT && hparams.indexer_head_size > 0) { + a(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + b(); + } +} +""" +DISTINCT_ARMS = """ +void f() { + if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35) { + a(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + b(); + } +} +""" + +rc, out = run(py=DUP_KEY) +check("catches a duplicate dict key", rc == 1 and "GLM5NEXT" in out, out) +rc, out = run(py=OK_KEYS) +check("clean on distinct dict keys", rc == 0, out) + +rc, out = run(cpp=DEAD_ARM) +check("catches an unreachable arch arm", rc == 1 and "unreachable" in out, out) +rc, out = run(cpp=LIVE_ARM) +check("does NOT fire when the earlier arm has &&", rc == 0, out) +rc, out = run(cpp=DISTINCT_ARMS) +check("does NOT fire on distinct arches", rc == 0, out) + +rc, out = run() +check("clean tree exits 0", rc == 0, out) + +# A nested `if` inside an arm must not end the enclosing chain. +# Tracking chains by indentation resets on the nested arm and analyses the outer `else if` as a fresh chain, so the dedicated GLM5NEXT arm below a shared fallthrough - the exact 08-27 mistake - stops being reported. +NESTED_DEAD_ARM = """ +void f() { + if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { + if (hparams.indexer_head_size > 0) { + a(); + } else if (hparams.n_expert > 0) { + b(); + } + } else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" +# Two unrelated chains at the same indentation. +# The second one's opener is a multiline condition, which the regex deliberately skips, so an indentation key appends the reachable GLM5NEXT arm to the FIRST chain and calls it dead. +# Nothing here is unreachable, and firing would block a release on good code. +SEPARATE_CHAINS = """ +void f() { + if (arch == LLM_ARCH_GLM5NEXT) { + a(); + } + unrelated(); + if (hparams.moe_every_n_layers > 0 && + il % hparams.moe_every_n_layers == 1) { + b(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" +# A brace inside a string literal or a comment is not a brace. +# Miscounting one shifts the depth for the rest of the file, which would silence every chain after it. +BRACES_IN_LITERALS = """ +void f() { + const char * tmpl = "{% if x %}{{ y }}{% endif %}"; + // a stray } in a comment { + if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { + a(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" + +rc, out = run(cpp=NESTED_DEAD_ARM) +check("catches a dead arm across a nested if", rc == 1 and "unreachable" in out, out) +rc, out = run(cpp=SEPARATE_CHAINS) +check("does NOT glue two chains at the same indentation", rc == 0, out) +rc, out = run(cpp=BRACES_IN_LITERALS) +check("still analyses a chain after braces in a string or comment", + rc == 1 and "unreachable" in out, out) + +# llama.cpp puts `else if` on its own line as often as not, src/llama-quant.cpp among them. +# Ending the chain on the brace line loses the arm that follows, so the duplicate arch started a fresh chain with nothing taken and passed. +NEXT_LINE_ELSE = """ +void f() { + if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { + a(); + } + else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" +# Same, with a blank line in between: still one chain. +NEXT_LINE_ELSE_BLANK = """ +void f() { + if (arch == LLM_ARCH_GLM5NEXT) { + a(); + } + + else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" +# The other direction, which deferring the close could break: a chain that really has ended, followed by an unrelated chain at the same depth. +# Joining them reports a reachable arm as dead and blocks a release on good code. +CLOSED_THEN_NEW = """ +void f() { + if (arch == LLM_ARCH_GLM5NEXT) { + a(); + } else { + b(); + } + g(); + if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } else if (arch == LLM_ARCH_QWEN3NEXT) { + d(); + } +} +""" + +# COND anchors on the `{` that ends the line, so a trailing comment after the brace stopped the arm matching at all and it left the chain silently. +TRAILING_COMMENT = """ +void f() { + if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { // shared setup + a(); + } else if (arch == LLM_ARCH_GLM5NEXT) { /* dedicated */ + c(); + } +} +""" +# The condition text must survive the comment stripping, since a mangled one would fail to parse as a pure disjunction and quietly stop being checked. +COMMENTED_OUT_ARM = """ +void f() { + if (arch == LLM_ARCH_GLM5NEXT) { + a(); + //} else if (arch == LLM_ARCH_GLM5NEXT) { + } else if (arch == LLM_ARCH_QWEN3NEXT) { + c(); + } +} +""" + +# A raw string ends only at its own delimiter, so it can hold a quote and a brace that the ordinary string regex misreads. +# The stray `}` closed the chain early and the duplicate arm after it passed as clean. +RAW_STRING = """ +void f() { + if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { + const char * s = R"foo("})foo"; + a(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" +# A raw string is blanked before comments, so the `//` inside one is text, not the start of a comment, and the brace after it still counts. +RAW_WITH_SLASHES = """ +void f() { + const char * u = R"(https://example.com/{x})"; + if (arch == LLM_ARCH_GLM5NEXT) { + a(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" + +# The other ordering. +# Blanking raw strings before comments let an `R"(` written inside a comment open a literal that ran to the next `)"`, swallowing the duplicate arm in between. +# Neither order fixes this, which is why the scan is positional: whichever construct starts first wins, and here that is the comment. +RAW_INSIDE_COMMENT = """ +void f() { + if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { + // see R"( for the delimiter rules + a(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + const char * s = R"(text)"; + c(); + } +} +""" +# An R glued to an identifier is part of it, not a raw-string prefix. +# Reading CHAR"( as a literal would blank the rest of the chain. +IDENT_ENDING_IN_R = """ +void f() { + if (arch == LLM_ARCH_GLM5NEXT) { + int n = FOOR; + a(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" + +# An unconditional arm takes the arch outright, so a later arm testing the same arch with an extra condition can never run. +# It is not a pure disjunction, so it was skipped and the dead arm passed. +CONJUNCTION_AFTER_PLAIN = """ +void f() { + if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { + a(); + } else if (arch == LLM_ARCH_GLM5NEXT && hparams.n_expert > 0) { + c(); + } +} +""" +# The reverse must stay clean: a CONDITIONAL arm does not consume the arch, so a later arm testing it is genuinely reachable. +PLAIN_AFTER_CONJUNCTION = """ +void f() { + if (arch == LLM_ARCH_GLM5NEXT && hparams.n_expert > 0) { + a(); + } else if (arch == LLM_ARCH_GLM5NEXT) { + c(); + } +} +""" +# A negated test names the arch but does not require it, so it is reachable. +NEGATED_ARM = """ +void f() { + if (arch == LLM_ARCH_GLM5NEXT) { + a(); + } else if (arch != LLM_ARCH_GLM5NEXT && n > 0) { + c(); + } +} +""" +# Only one alternative of a disjunction was taken, so the arm can still run. +PARTIAL_DISJUNCTION = """ +void f() { + if (arch == LLM_ARCH_GLM5NEXT) { + a(); + } else if (arch == LLM_ARCH_GLM5NEXT || arch == LLM_ARCH_QWEN3NEXT) { + c(); + } +} +""" + +# A key that is a CALL is a different object each evaluation, so repeating it is +# two entries, not one. ast.unparse renders both the same, and a finding here +# blocks the nightly, so an unstable key must not be compared by text at all. +DYNAMIC_KEY = """ +MAP = {fresh(): 1, fresh(): 2} +""" +# A walrus rebinds between elements, so the same name is not the same value. +WALRUS_KEY = """ +MAP = {(n := 1): "a", (n := 2): "b"} +""" +# Stable keys that are not enum attributes still have to be caught. +DUP_LITERAL_KEY = """ +MAP = {"a": 1, "b": 2, "a": 3} +""" + +rc, out = run(py=DYNAMIC_KEY) +check("a repeated call key is not reported as a duplicate", rc == 0, out) +rc, out = run(py=WALRUS_KEY) +check("a walrus key is not reported as a duplicate", rc == 0, out) +rc, out = run(py=DUP_LITERAL_KEY) +check("a duplicate literal key is still caught", + rc == 1 and "defined 2 times" in out, out) + +rc, out = run(cpp=CONJUNCTION_AFTER_PLAIN) +check("catches a conditioned arm after an unconditional match of the same arch", + rc == 1 and "unreachable" in out, out) +rc, out = run(cpp=PLAIN_AFTER_CONJUNCTION) +check("a conditional arm does not consume the arch for what follows", + rc == 0, out) +rc, out = run(cpp=NEGATED_ARM) +check("a negated arch test is not read as requiring that arch", rc == 0, out) +rc, out = run(cpp=PARTIAL_DISJUNCTION) +check("a disjunction with one untaken alternative stays reachable", rc == 0, out) + +rc, out = run(cpp=RAW_INSIDE_COMMENT) +check("an R\"( inside a comment does not open a raw string", + rc == 1 and "unreachable" in out, out) +rc, out = run(cpp=IDENT_ENDING_IN_R) +check("an identifier ending in R is not a raw-string prefix", + rc == 1 and "unreachable" in out, out) + +rc, out = run(cpp=RAW_STRING) +check("a brace inside a raw string does not close the chain", + rc == 1 and "unreachable" in out, out) +rc, out = run(cpp=RAW_WITH_SLASHES) +check("a raw string holding // is not treated as a comment", + rc == 1 and "unreachable" in out, out) + +rc, out = run(cpp=TRAILING_COMMENT) +check("catches a dead arm despite a comment after the brace", + rc == 1 and "unreachable" in out, out) +check("and reports the arm that is actually dead", ":5:" in out, out) +rc, out = run(cpp=COMMENTED_OUT_ARM) +check("a commented-out arm is not treated as a live one", rc == 0, out) + +rc, out = run(cpp=NEXT_LINE_ELSE) +check("catches a dead arm when else if starts on the next line", + rc == 1 and "unreachable" in out, out) +rc, out = run(cpp=NEXT_LINE_ELSE_BLANK) +check("a blank line between } and else does not end the chain", + rc == 1 and "unreachable" in out, out) +rc, out = run(cpp=CLOSED_THEN_NEW) +check("a genuinely closed chain does not absorb the next one", rc == 0, out) + +print() +print(f"{len(FAILS)} failure(s)" if FAILS else "all merge_checks tests passed") +sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_pin_contract.py b/scripts/unsloth/test_pin_contract.py new file mode 100644 index 000000000000..3c7b58d62d54 --- /dev/null +++ b/scripts/unsloth/test_pin_contract.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Tests for pin_contract.py. Run: python3 scripts/unsloth/test_pin_contract.py + +Every case builds a real repository with a real base tag, a real pin branch and +a real merge, then damages the merged tree the way a bad resolution damages it. +A hand-written fixture would only prove the checker reads its own output format. +""" +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "pin_contract.py" +FAILS = [] + + +def check(name, cond, extra=""): + print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def git(repo, *args): + return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], + cwd=repo, capture_output=True, text=True) + + +ARCH_H_BASE = """\ +enum llm_arch { + LLM_ARCH_LLAMA, + LLM_ARCH_UNKNOWN, +}; +""" +MODEL_CPP_BASE = """\ +void build_model(llm_arch arch) { + switch (arch) { + case LLM_ARCH_LLAMA: + build_llama(); + break; + } +} +""" + + +def make_repo(): + """A base tag `b1` plus a pin branch adding one architecture, merged.""" + d = Path(tempfile.mkdtemp(prefix="pc_")) + git(d, "init", "-q", "-b", "main") + (d / "src").mkdir() + (d / "src" / "llama-arch.h").write_text(ARCH_H_BASE) + (d / "src" / "llama-model.cpp").write_text(MODEL_CPP_BASE) + git(d, "add", "-A"); git(d, "commit", "-qm", "base") + git(d, "tag", "b1") + + git(d, "checkout", "-qb", "pin") + (d / "src" / "llama-arch.h").write_text( + ARCH_H_BASE.replace(" LLM_ARCH_UNKNOWN,", + " LLM_ARCH_INKLING,\n LLM_ARCH_UNKNOWN,")) + (d / "src" / "llama-model.cpp").write_text( + MODEL_CPP_BASE.replace(" }\n}", + " case LLM_ARCH_INKLING:\n" + " build_inkling_with_banded_bias();\n" + " break;\n }\n}")) + (d / "src" / "inkling.cpp").write_text( + "void build_inkling_with_banded_bias() { do_the_banded_thing(); }\n") + git(d, "add", "-A"); git(d, "commit", "-qm", "add inkling") + sha = git(d, "rev-parse", "HEAD").stdout.strip() + + git(d, "checkout", "-q", "main") + git(d, "merge", "-q", "--no-ff", "--no-edit", "-m", "merge pin", "pin") + + # Outside the work tree on purpose: a test that commits after this would + # otherwise sweep the pin file into the pin's own diff. + pr_set = Path(tempfile.mkdtemp(prefix="pcset_")) / "pr-set.json" + pr_set.write_text(json.dumps({"prs": [ + f"https://github.com/unslothai/llama.cpp/pull/1/commits/{sha}"]})) + return d, pr_set, sha + + +def run(repo, pr_set, *extra): + rep = repo / "r.json" + p = subprocess.run([sys.executable, str(SCRIPT), "--root", str(repo), + "--pr-set", str(pr_set), "--base", "b1", + "--report", str(rep), *extra], + capture_output=True, text=True) + return p.returncode, (json.loads(rep.read_text()) if rep.exists() else {}), p.stderr + + +# --- 1. an intact merge passes ------------------------------------------- +repo, pr_set, sha = make_repo() +rc, rep, err = run(repo, pr_set) +check("intact merge passes", rc == 0 and rep["ok"], err) +check("intact merge finds the new arch", + "LLM_ARCH_INKLING" in json.dumps(rep["pins"][0]["symbols"]), rep) +check("intact merge reports no notices", rep["notices"] == [], rep) + +# --- 2. the arm is dropped from ONE file: a tree-wide grep would pass ------ +# The real shape: LLM_ARCH_INKLING survives in the enum and the dispatch arm +# that makes it do anything is gone. +repo, pr_set, sha = make_repo() +p = repo / "src" / "llama-model.cpp" +p.write_text(MODEL_CPP_BASE) +rc, rep, err = run(repo, pr_set) +check("a dropped dispatch arm fails", rc == 1 and not rep["ok"], err) +check("the failure names the file, not just the symbol", + any("llama-model.cpp" in x for x in rep["pins"][0]["problems"]), rep) +check("the enum copy of the symbol does not rescue it", + "LLM_ARCH_INKLING" in (repo / "src" / "llama-arch.h").read_text()) + +# --- 3. a whole added file goes missing ---------------------------------- +repo, pr_set, sha = make_repo() +(repo / "src" / "inkling.cpp").unlink() +rc, rep, err = run(repo, pr_set) +check("a missing added file fails", rc == 1, err) +check("the failure names the file", + any("inkling.cpp" in x for x in rep["pins"][0]["problems"]), rep) + +# --- 4. a hunk is eaten without touching a symbol ------------------------- +repo, pr_set, sha = make_repo() +(repo / "src" / "inkling.cpp").write_text( + "void build_inkling_with_banded_bias() { }\n") # body gone, name kept +rc, rep, err = run(repo, pr_set) +check("an eaten body fails on line survival", rc == 1, err) +check("line survival names what went missing", + any("do_the_banded_thing" in x for x in rep["pins"][0]["problems"]), rep) + +# --- 5. redundancy: the base already has everything the pin adds ---------- +# Built the way it happens for real: upstream lands the same work, so the base +# tag has it and the pin is not an ancestor of anything. +d = Path(tempfile.mkdtemp(prefix="pc_")) +git(d, "init", "-q", "-b", "main") +(d / "src").mkdir() +(d / "src" / "f.cpp").write_text("int a() { return 1; }\n") +git(d, "add", "-A"); git(d, "commit", "-qm", "root") +git(d, "checkout", "-qb", "pin") +(d / "src" / "f.cpp").write_text( + "int a() { return 1; }\nint the_new_helper() { return 42; }\n") +git(d, "add", "-A"); git(d, "commit", "-qm", "pin work") +sha5 = git(d, "rev-parse", "HEAD").stdout.strip() +git(d, "checkout", "-q", "main") +(d / "src" / "f.cpp").write_text( # upstream squashed the same work + "int a() { return 1; }\nint the_new_helper() { return 42; }\n") +git(d, "add", "-A"); git(d, "commit", "-qm", "upstream squash of the same change") +git(d, "tag", "b1") +ps5 = Path(tempfile.mkdtemp(prefix="pcset_")) / "pr-set.json" +ps5.write_text(json.dumps({"prs": [ + f"https://github.com/unslothai/llama.cpp/pull/1/commits/{sha5}"]})) +rc, rep, err = run(d, ps5) +check("a pin the base already carries is reported", rep["notices"], rep) +check("redundancy says to delete the entry", + "deleted from pr-set.json" in " ".join(rep["notices"]), rep) +check("redundancy is NOT fatal", rc == 0, err) + +# --- 6. --emit checks nothing --------------------------------------------- +repo, pr_set, sha = make_repo() +(repo / "src" / "inkling.cpp").unlink() +rc, rep, err = run(repo, pr_set, "--emit") +check("--emit does not check", rc == 0 and rep["ok"], err) +check("--emit still derives the contract", + rep["pins"][0]["added_files"] == ["src/inkling.cpp"], rep) + +# --- 7. a comment is not a contract --------------------------------------- +# unslothai#70 has a comment naming GGML_OP_SSM_SCAN to say it does NOT use it. +# Holding comment wording would fail the moment upstream rewords it. +repo, pr_set, sha = make_repo() +git(repo, "checkout", "-q", "pin") +(repo / "src" / "note.cpp").write_text( + "// unlike LLM_ARCH_MISTRAL this one does its own thing\nint g() { return 0; }\n") +git(repo, "add", "-A"); git(repo, "commit", "-qm", "comment") +sha7 = git(repo, "rev-parse", "HEAD").stdout.strip() +git(repo, "checkout", "-q", "main") +git(repo, "merge", "-q", "--no-ff", "--no-edit", "-m", "m2", "pin") +(repo / "src" / "note.cpp").write_text( # comment reworded, code kept + "// this one does its own thing\nint g() { return 0; }\n") +pr_set.write_text(json.dumps({"prs": [ + f"https://github.com/unslothai/llama.cpp/pull/1/commits/{sha7}"]})) +rc, rep, err = run(repo, pr_set) +check("a reworded comment does not fail the pin", rc == 0, err) +check("no symbol was harvested from the comment", + "LLM_ARCH_MISTRAL" not in json.dumps(rep["pins"][0]["symbols"]), rep) + +print() +print(f"{len(FAILS)} failure(s)" + (": " + ", ".join(FAILS) if FAILS else "")) +sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_pin_merge.py b/scripts/unsloth/test_pin_merge.py new file mode 100644 index 000000000000..40b227da5ce6 --- /dev/null +++ b/scripts/unsloth/test_pin_merge.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Tests for pin_merge.py. Run: python3 scripts/unsloth/test_pin_merge.py + +The first case is the real 08-27 collision: master had moved qwen4exp to 950f135b28 while the GLM-5-Next branch was replacing pin 118 with 125. +It was resolved by hand at the time; this asserts the script reproduces that answer. +""" +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "pin_merge.py" +FAILS = [] + +U = "https://github.com/unslothai/llama.cpp/pull" +BASE_PINS = [ + f"{U}/107/commits/74acc40c37ae2eb36031981feda392b793944f72", + f"{U}/108/commits/27278df7000ade4a638d044202dbe82975421df6", + f"{U}/70/commits/edfd4c1a3b7a653303a85257ddac2a1f3ce39a2f", + f"{U}/91/commits/c86ed269986f2dced6325c5c58bda966a2e2ead1", + f"{U}/95/commits/3db8cb5b2e9bf291057b9f19960e8601a162da81", + f"{U}/114/commits/c4ddc4805dbc12727897b354237bfd9225212b06", + f"{U}/118/commits/3766b41229c20249fd4d83d7ba297499d50e9b80", +] + + +def check(name, cond, extra=""): + print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def run(base, ours, theirs): + d = Path(tempfile.mkdtemp(prefix="pm_")) + paths = [] + for name, pins in (("base", base), ("ours", ours), ("theirs", theirs)): + p = d / f"{name}.json" + p.write_text(json.dumps({"prs": pins}, indent=2)) + paths.append(str(p)) + r = subprocess.run([sys.executable, str(SCRIPT), *paths, "--stdout"], + capture_output=True, text=True) + pins = json.loads(r.stdout)["prs"] if r.returncode == 0 else None + return r.returncode, pins, r.stderr.strip() + + +def sub(pins, i, sha): + out = list(pins) + out[i] = out[i].rsplit("/", 1)[0] + "/" + sha + return out + + +def replace(pins, i, url): + out = list(pins) + out[i] = url + return out + + +# 1. the real 08-27 collision +ours = replace(BASE_PINS, 6, f"{U}/125/commits/f48b99e1fd04628da2a3d4ea5acc335d9ea67f7a") +theirs = sub(BASE_PINS, 5, "950f135b28789057721a65d76de98fbbcd2f7dd6") +rc, pins, err = run(BASE_PINS, ours, theirs) +check("real 08-27 collision resolves", rc == 0, err) +if pins: + check("takes theirs for the pin only theirs moved", pins[5] == theirs[5], pins[5]) + check("takes ours for the pin only ours moved", pins[6] == ours[6], pins[6]) + check("leaves untouched pins alone", pins[:5] == BASE_PINS[:5]) + check("preserves pin order", [p.split("/pull/")[1].split("/")[0] for p in pins] + == ["107", "108", "70", "91", "95", "114", "125"]) + +# 2. both sides repin the same entry differently +rc, _, err = run(BASE_PINS, sub(BASE_PINS, 5, "a" * 40), sub(BASE_PINS, 5, "b" * 40)) +check("refuses a genuine two-sided repin", rc == 1, err) +check("says which pin was ambiguous", "pin 5" in err, err) + +# 3. a pin added on one side +rc, _, err = run(BASE_PINS, BASE_PINS + [f"{U}/999/commits/{'c' * 40}"], BASE_PINS) +check("refuses an added pin", rc == 1, err) + +# 4. a pin removed on one side +rc, _, err = run(BASE_PINS, BASE_PINS[:-1], BASE_PINS) +check("refuses a removed pin", rc == 1, err) + +# 5. both sides make the identical repin +same = sub(BASE_PINS, 5, "d" * 40) +rc, pins, err = run(BASE_PINS, same, same) +check("accepts an identical repin on both sides", rc == 0 and pins == same, err) + +# 6. neither side changed anything +rc, pins, err = run(BASE_PINS, BASE_PINS, BASE_PINS) +check("no-op merge is a no-op", rc == 0 and pins == BASE_PINS, err) + +# 7. object-form entries keep their other fields +objs = [{"url": u, "required": False} for u in BASE_PINS] +o = json.loads(json.dumps(objs)); o[5]["url"] = sub(BASE_PINS, 5, "e" * 40)[5] +d = Path(tempfile.mkdtemp(prefix="pm_")) +for name, pins_ in (("base", objs), ("ours", o), ("theirs", objs)): + (d / f"{name}.json").write_text(json.dumps({"prs": pins_}, indent=2)) +r = subprocess.run([sys.executable, str(SCRIPT), str(d / "base.json"), str(d / "ours.json"), + str(d / "theirs.json"), "--stdout"], capture_output=True, text=True) +ok = r.returncode == 0 and all(e.get("required") is False for e in json.loads(r.stdout)["prs"]) +check("object-form entries keep their other fields", ok, r.stdout[:200] + r.stderr) + + +def run_objs(base, ours, theirs): + d = Path(tempfile.mkdtemp(prefix="pm_")) + paths = [] + for name, entries in (("base", base), ("ours", ours), ("theirs", theirs)): + p = d / f"{name}.json" + p.write_text(json.dumps({"prs": entries}, indent=2)) + paths.append(str(p)) + r = subprocess.run([sys.executable, str(SCRIPT), *paths, "--stdout"], + capture_output=True, text=True) + return r.returncode, (json.loads(r.stdout)["prs"] if r.returncode == 0 else None), r.stderr.strip() + + +# 8. theirs flips `required` on one entry while ours repins a DIFFERENT one. +# Comparing only urls makes theirs' flip invisible, and the result is rebuilt from ours, so the flip is silently dropped by a merge that reports success. +objs = [{"url": u, "required": True} for u in BASE_PINS] +o = json.loads(json.dumps(objs)); o[5]["url"] = sub(BASE_PINS, 5, "e" * 40)[5] +t = json.loads(json.dumps(objs)); t[1]["required"] = False +rc, pins, err = run_objs(objs, o, t) +check("keeps theirs' non-url field change on an entry ours did not touch", + rc == 0 and pins is not None and pins[1]["required"] is False, err or json.dumps(pins)) +check("keeps ours' repin alongside theirs' field change", + rc == 0 and pins is not None and pins[5]["url"] == o[5]["url"], err) + +# 9. both sides touch the SAME entry, but different fields: still mergeable. +o = json.loads(json.dumps(objs)); o[3]["url"] = sub(BASE_PINS, 3, "f" * 40)[3] +t = json.loads(json.dumps(objs)); t[3]["required"] = False +rc, pins, err = run_objs(objs, o, t) +check("merges a repin and a field change on the same entry", + rc == 0 and pins is not None + and pins[3]["url"] == o[3]["url"] and pins[3]["required"] is False, err) + +# 10. both sides set the same field to different values: still refused. +o = json.loads(json.dumps(objs)); o[2]["required"] = False +t = json.loads(json.dumps(objs)); t[2]["required"] = "maybe" +rc, _, err = run_objs(objs, o, t) +check("refuses a two-sided change to the same field", rc == 1, err) + +def run_docs(base, ours, theirs): + """Like run(), but the caller supplies the whole document, not just pins.""" + d = Path(tempfile.mkdtemp(prefix="pm_")) + paths = [] + for name, doc in (("base", base), ("ours", ours), ("theirs", theirs)): + p = d / f"{name}.json" + p.write_text(json.dumps(doc, indent=2)) + paths.append(str(p)) + r = subprocess.run([sys.executable, str(SCRIPT), *paths, "--stdout"], + capture_output=True, text=True) + return r.returncode, (json.loads(r.stdout) if r.returncode == 0 else None), r.stderr.strip() + + +# 12. theirs edits a TOP-LEVEL field while ours repins an entry. +# Rebuilding the document from ours drops theirs' edit and still exits 0, and because a merge driver replaces git's text merge outright, nothing else ever sees the loss. +doc = {"_doc": ["old doc line"], "prs": list(BASE_PINS)} +o = json.loads(json.dumps(doc)); o["prs"] = sub(BASE_PINS, 5, "a" * 40) +t = json.loads(json.dumps(doc)); t["_doc"] = ["old doc line", "prune closed pins"] +rc, out, err = run_docs(doc, o, t) +check("keeps theirs' top-level field change alongside ours' repin", + rc == 0 and out is not None and out["_doc"] == t["_doc"], err or json.dumps(out)) +check("still takes ours' repin when theirs edited the document", + rc == 0 and out is not None and out["prs"] == o["prs"], err) +check("keeps .prs in its original key position", + rc == 0 and out is not None and list(out) == ["_doc", "prs"], json.dumps(list(out or {}))) + +# 13. theirs ADDS a top-level field ours has never seen: it has to survive. +t = json.loads(json.dumps(doc)); t["base_tag"] = "b10639" +rc, out, err = run_docs(doc, o, t) +check("keeps a top-level field only theirs added", + rc == 0 and out is not None and out.get("base_tag") == "b10639", err or json.dumps(out)) + +# 14. both sides set the same top-level field differently: refuse, never guess. +o2 = json.loads(json.dumps(doc)); o2["_doc"] = ["ours' rewrite"] +t2 = json.loads(json.dumps(doc)); t2["_doc"] = ["theirs' rewrite"] +rc, _, err = run_docs(doc, o2, t2) +check("refuses a two-sided change to the same top-level field", rc == 1, err) +check("names the clashing top-level field", "_doc" in err, err) + +# 15. a top-level field theirs deleted stays deleted. +t3 = json.loads(json.dumps(doc)); del t3["_doc"] +rc, out, err = run_docs(doc, o, t3) +check("honours a top-level field theirs deleted", + rc == 0 and out is not None and "_doc" not in out, err or json.dumps(out)) + +# 16. --help must not crash: argparse %-formats help strings, and the driver placeholders %O/%A/%B are literal percents that have to be escaped. +r = subprocess.run([sys.executable, str(SCRIPT), "--help"], capture_output=True, text=True) +check("--help does not crash on the %O/%A/%B placeholders", + r.returncode == 0 and "%O" in r.stdout, (r.stderr or r.stdout)[-200:]) + +# 17. one side REORDERS the pins while the other changes a field. +# Merging by position then combines fields belonging to different PRs: base [A(required), B(required)] with ours making A optional and theirs swapping the two produces B(required=false), so the release skips the wrong PR, and the driver exits 0 while doing it. +# A reorder must be refused instead. +two = [{"url": BASE_PINS[0], "required": True}, {"url": BASE_PINS[1], "required": True}] +o = json.loads(json.dumps(two)); o[0]["required"] = False +t = [two[1], two[0]] +rc, pins, err = run_objs(two, o, t) +check("refuses a reorder that would splice fields across PRs", rc == 1, + json.dumps(pins) if pins else err) +check("names the reordered position", "reorder" in err, err) +check("never emits a pin carrying another PR's field", + pins is None or pins[0]["required"] is not False, json.dumps(pins)) + +# 18. a reorder that also repins the moved entry still has to be refused: the url no longer matches, so only the PR number identifies the entry. +t = [dict(two[1]), dict(two[0])] +t[0]["url"] = sub(BASE_PINS, 1, "9" * 40)[1] +rc, _, err = run_objs(two, o, t) +check("refuses a reorder combined with a repin", rc == 1, err) + +# 19. a reorder on OUR side is refused too, not just on theirs. +o2 = [two[1], two[0]] +t2 = json.loads(json.dumps(two)); t2[0]["required"] = False +rc, _, err = run_objs(two, o2, t2) +check("refuses a reorder on ours", rc == 1, err) + +# 20. swapping a pin for a DIFFERENT PR at the same position is not a reorder and must keep merging, which is case 1's real 08-27 resolution. +rc, pins, err = run(BASE_PINS, + replace(BASE_PINS, 6, f"{U}/125/commits/{'a' * 40}"), + sub(BASE_PINS, 5, "b" * 40)) +check("a same-position swap to a new PR is not a reorder", rc == 0, err) + +# 21. a plain repin is not a reorder either, on either side. +rc, pins, err = run(BASE_PINS, sub(BASE_PINS, 0, "1" * 40), sub(BASE_PINS, 3, "2" * 40)) +check("two repins at different positions still merge", rc == 0, err) + +# 22. a same-position swap to a different PR while the OTHER side edits that entry's fields. +# Test 20's swap is safe only because nobody else touched the entry; here both sides did, so the field-wise merge runs and takes the url from one PR and `required` from another. +# Base A(required=true) with ours swapping in B and theirs making A optional yielded B(required=false) and exit 0, which makes the release skip a PR nobody made optional. +two = [{"url": BASE_PINS[0], "required": True}, {"url": BASE_PINS[1], "required": True}] +o = json.loads(json.dumps(two)); o[0]["url"] = f"{U}/999/commits/{'c' * 40}" +t = json.loads(json.dumps(two)); t[0]["required"] = False +rc, pins, err = run_objs(two, o, t) +check("refuses a same-position PR swap the other side also edited", rc == 1, + json.dumps(pins) if pins else err) +check("names both PRs in the refusal", "#999" in err and "#107" in err, err) +check("never emits the swapped-in PR carrying the other's field", + pins is None or pins[0].get("required") is not False, json.dumps(pins)) + +# 23. both sides swap position 0 to the SAME new PR but disagree on a field. +# Base still describes the PR that is gone, so every field comparison below is against settings that were never this PR's: refuse rather than pick one. +o = json.loads(json.dumps(two)); o[0]["url"] = f"{U}/999/commits/{'c' * 40}" +t = json.loads(json.dumps(two)) +t[0]["url"] = f"{U}/999/commits/{'c' * 40}"; t[0]["required"] = False +rc, _, err = run_objs(two, o, t) +check("refuses an agreed swap the sides disagree about", rc == 1, err) + +# 24. two entries pinning two commits of the SAME PR share one identity, so a swap between them looks like no change and the reorder guard never fires: a field theirs changed on the first entry then lands on the second. +dup = [{"url": f"{U}/107/commits/{'a' * 40}", "required": True}, + {"url": f"{U}/107/commits/{'b' * 40}", "required": True}] +o = [json.loads(json.dumps(dup[1])), json.loads(json.dumps(dup[0]))] +t = json.loads(json.dumps(dup)); t[0]["required"] = False +rc, pins, err = run_objs(dup, o, t) +check("refuses a pin set that names one PR twice", rc == 1, + json.dumps(pins) if pins else err) +check("names the duplicated PR", "#107" in err, err) + +# 25. neither side has a duplicate, but the merge makes one: ours puts a new PR at position 0 and theirs puts the SAME new PR at position 1. +# Base has it nowhere, so the reorder guard sees nothing, and the driver emitted [C, C]. +two_ab = [{"url": f"{U}/107/commits/{'a' * 40}"}, {"url": f"{U}/108/commits/{'b' * 40}"}] +c_entry = {"url": f"{U}/999/commits/{'c' * 40}"} +o = [json.loads(json.dumps(c_entry)), json.loads(json.dumps(two_ab[1]))] +t = [json.loads(json.dumps(two_ab[0])), json.loads(json.dumps(c_entry))] +rc, pins, err = run_objs(two_ab, o, t) +check("refuses a merge that would pin one PR twice", rc == 1, + json.dumps(pins) if pins else err) +check("says the duplicate is in the merged set", + "merged pin set" in err and "#999" in err, err) + +print() +print(f"{len(FAILS)} failure(s)" if FAILS else "all pin_merge tests passed") +sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_sync_deletes.py b/scripts/unsloth/test_sync_deletes.py new file mode 100644 index 000000000000..7be889a11c1a --- /dev/null +++ b/scripts/unsloth/test_sync_deletes.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Tests for sync_deletes.py. Run: python3 scripts/unsloth/test_sync_deletes.py + +Every case builds a real git merge, so the index stages are the ones git actually produces rather than a hand-written approximation. +""" +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "sync_deletes.py" +FAILS = [] + + +def check(name, cond, extra=""): + print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def git(repo, *args): + return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], + cwd=repo, capture_output=True, text=True) + + +def scenario(path, ours_deletes=True, upstream_modifies=True, upstream_adds=None): + """Base has `path`; upstream edits it; we delete it. Returns (repo, merge_base).""" + d = Path(tempfile.mkdtemp(prefix="sd_")) + git(d, "init", "-q", "-b", "main") + f = d / path + f.parent.mkdir(parents=True, exist_ok=True) + f.write_text("name: base\n") + (d / "src").mkdir(exist_ok=True) + (d / "src" / "model.cpp").write_text("int x;\n") + git(d, "add", "-A"); git(d, "commit", "-qm", "base") + base = git(d, "rev-parse", "HEAD").stdout.strip() + + git(d, "checkout", "-qb", "upstream") + if upstream_modifies: + f.write_text("name: upstream edit\n") + if upstream_adds: + na = d / upstream_adds + na.parent.mkdir(parents=True, exist_ok=True) + na.write_text("name: new upstream workflow\n") + git(d, "add", "-A"); git(d, "commit", "-qm", "upstream") + + git(d, "checkout", "-q", "main") + if ours_deletes: + git(d, "rm", "-q", str(f.relative_to(d))) + git(d, "commit", "-qm", "fork deletes it") + git(d, "-c", "merge.conflictStyle=diff3", "merge", "--no-ff", "--no-edit", "upstream") + return d, base + + +def run(repo, base=None): + args = [sys.executable, str(SCRIPT), "--repo", str(repo)] + if base: + args += ["--merge-base", base] + r = subprocess.run(args, capture_output=True, text=True) + return r.returncode, r.stdout + r.stderr + + +# 1. the 68-of-68 historical case +d, base = scenario(".github/workflows/build-apple.yml") +rc, out = run(d) +check("resolves an upstream-workflow modify/delete", rc == 0, out) +check("the file stays deleted", not (d / ".github/workflows/build-apple.yml").exists()) +check("no unmerged paths remain", git(d, "ls-files", "-u").stdout.strip() == "") + +# 2. a workflow we own must never be touched automatically +d, base = scenario(".github/workflows/unsloth-prebuilt.yml") +rc, out = run(d) +check("refuses a fork-owned workflow", rc == 1 and "we own this workflow" in out, out) + +# 3. a source file in the same shape must never be touched +d, base = scenario("src/model.cpp") +rc, out = run(d) +check("refuses a source file", rc == 1 and "not an upstream workflow path" in out, out) + +# 4. a workflow upstream added, which is not a conflict at all +d, base = scenario(".github/workflows/build-apple.yml", + upstream_adds=".github/workflows/build-wasm.yml") +rc, out = run(d, base) +check("drops a newly added upstream workflow", rc == 0, out) +check("the added workflow is gone", not (d / ".github/workflows/build-wasm.yml").exists()) + +# 5. an upstream composite ACTION must survive; only workflows are dropped +d, base = scenario(".github/workflows/build-apple.yml", + upstream_adds=".github/actions/ccache-buckets/action.yml") +rc, out = run(d, base) +check("keeps upstream composite actions", (d / ".github/actions/ccache-buckets/action.yml").exists(), out) + +# 6. source files are never removed by the added-workflow sweep +check("source file untouched throughout", (d / "src" / "model.cpp").exists()) + +# 7. an unusable --merge-base. +# git diff exits nonzero with empty stdout, which reads exactly like "upstream added nothing" if only stdout is looked at, so the run reported success and a sync would have carried every newly added upstream workflow in. +# The listing failing has to fail the script. +d, base = scenario(".github/workflows/build-apple.yml", + upstream_adds=".github/workflows/build-wasm.yml") +rc, out = run(d, "0000000000000000000000000000000000000000") +check("fails when the added-workflow listing cannot run", rc == 1, out) +check("says which rev it could not use", + "0000000000" in out and "could not list" in out, out) +check("and leaves the added workflow in place to be dealt with", + (d / ".github/workflows/build-wasm.yml").exists(), out) + +# 8. upstream RENAMES a workflow rather than adding one. +# Rename detection calls the new path R, not A, so --diff-filter=A saw nothing and the script exited 0 with the renamed upstream workflow left live in the fork. +def rename_scenario(): + d = Path(tempfile.mkdtemp(prefix="sd_")) + git(d, "init", "-q", "-b", "main") + wf = d / ".github" / "workflows" + wf.mkdir(parents=True) + # Long enough that git scores the move as a rename rather than add+delete. + (wf / "old.yml").write_text("".join(f"# line {i}\n" for i in range(40))) + (d / "src").mkdir() + (d / "src" / "model.cpp").write_text("int x;\n") + git(d, "add", "-A"); git(d, "commit", "-qm", "base") + base = git(d, "rev-parse", "HEAD").stdout.strip() + + git(d, "checkout", "-qb", "upstream") + git(d, "mv", ".github/workflows/old.yml", ".github/workflows/new.yml") + git(d, "commit", "-qm", "upstream renames it") + + git(d, "checkout", "-q", "main") + git(d, "merge", "--no-ff", "--no-edit", "upstream") + return d, base + + +d, base = rename_scenario() +rc, out = run(d, base) +check("drops an upstream workflow that arrived by rename", rc == 0, out) +check("the renamed workflow is gone", + not (d / ".github/workflows/new.yml").exists(), out) +check("source is still untouched", (d / "src" / "model.cpp").exists(), out) + +# 9. --repo names something that is not a git repository. ls-files fails, its +# empty stdout read as "no conflicts", and the script reported that it had +# resolved everything it was asked to. +notrepo = Path(tempfile.mkdtemp(prefix="sd_notrepo_")) +rc, out = run(notrepo) +check("fails when the unmerged listing cannot run", rc == 1, out) +check("says what could not be listed", "ls-files" in out, out) + +print() +print(f"{len(FAILS)} failure(s)" if FAILS else "all sync_deletes tests passed") +sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_upload_release_assets.sh b/scripts/unsloth/test_upload_release_assets.sh new file mode 100755 index 000000000000..35d7a725f125 --- /dev/null +++ b/scripts/unsloth/test_upload_release_assets.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# Tests for upload_release_assets.sh. Run: bash scripts/unsloth/test_upload_release_assets.sh +# +# Every case runs the real uploader against a stub `gh` that reproduces the +# failure modes seen against uploads.github.com: a PUT that wedges and never +# returns, a transient 5xx, an asset committed at the wrong size, an asset left +# in a non-uploaded state, and a `gh` that exits 0 without the asset landing. +# Budgets are shrunk so a stall is killed in ~1s instead of ~500s. +set -uo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +SCRIPT="$HERE/upload_release_assets.sh" +STUBDIR="$(mktemp -d)" +trap 'rm -rf "$STUBDIR"' EXIT +mkdir -p "$STUBDIR/bin" + +cat > "$STUBDIR/bin/gh" <<'STUB' +#!/usr/bin/env bash +# Stub gh: serves `release upload` and `release view` off a file registry. +set -uo pipefail +REG="${STUB_REG:?}"; mkdir -p "$REG/assets" "$REG/attempts" + +in_list() { case " ${2:-} " in *" $1 "*) return 0 ;; *) return 1 ;; esac; } + +if [ "$1" = "release" ] && [ "$2" = "upload" ]; then + file="${*: -1}"; size="$(stat -c %s "$file")" + # GitHub sanitises the asset name server-side; mirror that here so the test + # exercises the same name mapping the uploader has to verify against. + name="$(printf '%s' "$(basename "$file")" | tr -c 'A-Za-z0-9._-' '.')" + c="$REG/attempts/$name"; n=$(( $(cat "$c" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$c" + + if in_list "$name" "${STUB_STALL_ALWAYS:-}"; then sleep 300; exit 0; fi + if in_list "$name" "${STUB_STALL_ONCE:-}" && [ "$n" -le 1 ]; then sleep 300; exit 0; fi + if in_list "$name" "${STUB_FAIL_ALWAYS:-}"; then echo "stub: HTTP 502" >&2; exit 1; fi + if in_list "$name" "${STUB_FAIL_ONCE:-}" && [ "$n" -le 1 ]; then echo "stub: HTTP 502" >&2; exit 1; fi + # Below: exits 0, but leaves the release in a state the caller must catch. + if in_list "$name" "${STUB_WRONGSIZE:-}" && [ "$n" -le 1 ]; then + printf '%s\t%s\tuploaded\n' "$name" "$(( size - 1 ))" > "$REG/assets/$name"; exit 0 + fi + if in_list "$name" "${STUB_NOTUPLOADED:-}" && [ "$n" -le 1 ]; then + printf '%s\t%s\tstarter\n' "$name" "$size" > "$REG/assets/$name"; exit 0 + fi + if in_list "$name" "${STUB_SILENT_DROP:-}"; then exit 0; fi + printf '%s\t%s\tuploaded\n' "$name" "$size" > "$REG/assets/$name" + exit 0 +fi + +if [ "$1" = "release" ] && [ "$2" = "view" ]; then + if [ "${STUB_VIEW_FAILS:-}" = 1 ]; then echo "stub: HTTP 503" >&2; exit 1; fi + jqexpr="" + for ((i=1;i<=$#;i++)); do + if [ "${!i}" = "--jq" ]; then j=$((i+1)); jqexpr="${!j}"; fi + done + { echo '{"assets":[' + first=1 + for a in "$REG"/assets/*; do + [ -e "$a" ] || continue + IFS=$'\t' read -r n s st < "$a" + if [ "$first" = 1 ]; then first=0; else echo ','; fi + printf '{"name":"%s","size":%s,"state":"%s"}' "$n" "$s" "$st" + done + echo ']}'; } | jq -r "$jqexpr" + exit 0 +fi +echo "stub: unhandled: $*" >&2; exit 64 +STUB +chmod +x "$STUBDIR/bin/gh" +export PATH="$STUBDIR/bin:$PATH" + +export UPLOAD_GRACE_SECONDS=1 UPLOAD_MIN_RATE_MB_S=1000 UPLOAD_HEARTBEAT_SECONDS=3 +export UPLOAD_JOBS=4 UPLOAD_ATTEMPTS=3 + +FAILS=() + +run_case() { # name expected_rc [env ...] + local name="$1" want="$2"; shift 2 + local d; d="$(mktemp -d)"; mkdir -p "$d/dist" + local i + for i in 01 02 03 04 05 06; do head -c 1000000 /dev/zero > "$d/dist/bundle-$i.tar.gz"; done + # A name GitHub will rewrite, so the verify path's name mapping is covered. + head -c 1000 /dev/zero > "$d/dist/has space.json" + local out rc + out="$(STUB_REG="$d/reg" env "$@" bash "$SCRIPT" --tag T --repo o/r --dist "$d/dist" 2>&1)"; rc=$? + if [ "$rc" = "$want" ]; then + printf 'PASS %s\n' "$name" + else + printf 'FAIL %s :: rc=%s want=%s\n' "$name" "$rc" "$want" + printf '%s\n' "$out" | sed 's/^/ | /' + FAILS+=("$name") + fi + rm -rf "$d" +} + +run_case "happy path" 0 IGNORED=1 +run_case "stall, recovers on retry" 0 STUB_STALL_ONCE="bundle-02.tar.gz bundle-05.tar.gz" +run_case "transient 502, recovers" 0 STUB_FAIL_ONCE="bundle-03.tar.gz" +run_case "permanent stall aborts" 1 STUB_STALL_ALWAYS="bundle-04.tar.gz" +run_case "permanent 502 aborts" 1 STUB_FAIL_ALWAYS="bundle-01.tar.gz" +run_case "wrong size caught, re-uploaded" 0 STUB_WRONGSIZE="bundle-06.tar.gz" +run_case "non-uploaded state re-uploaded" 0 STUB_NOTUPLOADED="bundle-02.tar.gz" +run_case "gh exits 0, asset never lands" 1 STUB_SILENT_DROP="bundle-03.tar.gz" +run_case "phase deadline aborts" 1 UPLOAD_DEADLINE_MINUTES=0 STUB_STALL_ALWAYS="bundle-01.tar.gz" +# The verify read must fail the script, not just its subshell: a process +# substitution would swallow this and report every asset as verified. +run_case "verification API outage aborts" 1 STUB_VIEW_FAILS=1 + +echo +echo "${#FAILS[@]} failure(s)${FAILS[*]:+: ${FAILS[*]}}" +[ "${#FAILS[@]}" -eq 0 ] diff --git a/scripts/unsloth/test_verify_upstream_sync.py b/scripts/unsloth/test_verify_upstream_sync.py new file mode 100644 index 000000000000..a7ad460d7c68 --- /dev/null +++ b/scripts/unsloth/test_verify_upstream_sync.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Negative controls for verify_upstream_sync.py. + +A checker that only ever prints PASS is worthless, so this builds a throwaway repository +shaped like the real one (a base, an upstream that moves on, a fork that customises its own +files), then injects each failure mode one at a time and asserts the checker reports it. + +The four modes are the four ways a sync can actually eat our work: + + modify upstream reformats or reverts a file we own + delete our file is dropped in the merge + renumber a published GGML/LLAMA enum id shifts, which corrupts every GGUF that stores it + subtle a value inside our Python changes without the file being obviously touched + +Run: python3 scripts/unsloth/test_verify_upstream_sync.py +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +CHECKER = os.path.join(HERE, "verify_upstream_sync.py") + + +def sh(cwd: str, *args: str) -> str: + p = subprocess.run(args, cwd=cwd, capture_output=True, text=True) + if p.returncode: + raise SystemExit(f"{' '.join(args)} failed in {cwd}:\n{p.stderr}") + return p.stdout + + +def git(repo: str, *args: str) -> str: + return sh(repo, "git", "-c", "user.email=t@t", "-c", "user.name=t", *args) + + +def write(repo: str, path: str, text: str) -> None: + full = os.path.join(repo, path) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "w") as f: + f.write(text) + + +HEADER_BASE = """ + enum ggml_type { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_COUNT = 2, + }; +""" +HEADER_UPSTREAM = """ + enum ggml_type { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_COUNT = 3, + }; +""" +OURS_PY = 'OWNED = ("unslothai/", "danielhanchen/")\nLIMIT = 7\n\n\ndef pin():\n return OWNED\n' + + +def build_repo(tmp: str) -> str: + """base -> upstream advances; fork adds its own files and deletes an upstream one.""" + up = os.path.join(tmp, "upstream") + os.makedirs(up) + git(up, "init", "-q", "-b", "master") + write(up, "ggml/include/ggml.h", HEADER_BASE) + write(up, "src/model.cpp", "int main(){return 0;}\n") + write(up, ".github/workflows/ci.yml", "name: CI\n") + git(up, "add", "-A"); git(up, "commit", "-qm", "base") + + fork = os.path.join(tmp, "fork") + sh(tmp, "git", "clone", "-q", up, fork) + git(fork, "remote", "add", "upstream", up) + + # upstream moves on: a new type, so a new COUNT + write(up, "ggml/include/ggml.h", HEADER_UPSTREAM) + write(up, "src/model.cpp", "int main(){return 1;}\n") + git(up, "add", "-A"); git(up, "commit", "-qm", "upstream: add Q4_0") + + # the fork customises only its own tree, and drops upstream CI + write(fork, "scripts/unsloth/repin.py", OURS_PY) + write(fork, ".github/workflows/unsloth-prebuilt.yml", "name: prebuilt\n") + os.remove(os.path.join(fork, ".github/workflows/ci.yml")) + git(fork, "add", "-A"); git(fork, "commit", "-qm", "unsloth: our own CI and scripts") + git(fork, "branch", "-f", "forkmaster", "HEAD") + git(fork, "fetch", "-q", "upstream", "master") + return fork + + +def merge(fork: str, name: str) -> str: + git(fork, "checkout", "-q", "-B", name, "forkmaster") + p = subprocess.run(["git", "merge", "--no-commit", "--no-ff", "upstream/master"], + cwd=fork, capture_output=True, text=True) + # the fork's deletion of upstream CI conflicts as modify/delete; keep it deleted + conf = sh(fork, "git", "diff", "--name-only", "--diff-filter=U").split() + if conf: + git(fork, "rm", "-q", *conf) + git(fork, "commit", "-qm", f"merge {name}") + return sh(fork, "git", "rev-parse", "HEAD").strip() + + +def run_checker(fork: str, rev: str) -> tuple[int, str]: + p = subprocess.run([sys.executable, CHECKER, "--repo", fork, "--merge", rev, + "--fork", "forkmaster", "--upstream", "upstream/master"], + capture_output=True, text=True) + return p.returncode, p.stdout + p.stderr + + +def main() -> int: + fails = 0 + with tempfile.TemporaryDirectory() as tmp: + fork = build_repo(tmp) + + rc, out = run_checker(fork, merge(fork, "clean")) + if rc == 0 and "PASS: the sync is additive only" in out: + print("PASS clean merge is accepted") + else: + fails += 1 + print(f"FAIL clean merge was rejected (rc={rc})\n{out}") + + cases = [ + ("modify our file", "content", + lambda: write(fork, "scripts/unsloth/repin.py", OURS_PY + "# upstream reflow\n")), + ("delete our file", "content", + lambda: os.remove(os.path.join(fork, "scripts/unsloth/repin.py"))), + ("renumber a published id", "c_enums", + lambda: write(fork, "ggml/include/ggml.h", + HEADER_UPSTREAM.replace("GGML_TYPE_F16 = 1", "GGML_TYPE_F16 = 5"))), + ("subtle value change in our Python", "ast", + lambda: write(fork, "scripts/unsloth/repin.py", + OURS_PY.replace('"danielhanchen/"', '"someone-else/"'))), + ] + for i, (label, expect, mutate) in enumerate(cases): + rev = merge(fork, f"bad{i}") + mutate() + git(fork, "add", "-A") + git(fork, "commit", "-qm", f"negative control: {label}") + rev = sh(fork, "git", "rev-parse", "HEAD").strip() + rc, out = run_checker(fork, rev) + caught = rc == 1 and any(l.startswith("FAIL " + expect) for l in out.splitlines()) + if caught: + print(f"PASS caught: {label} (via {expect})") + else: + fails += 1 + print(f"FAIL MISSED: {label} (expected FAIL {expect}, rc={rc})\n{out}") + + print(f"\n{'all negative controls caught' if not fails else f'{fails} FAILURES'}") + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/upload_release_assets.sh b/scripts/unsloth/upload_release_assets.sh new file mode 100755 index 000000000000..b6f0e8adf26c --- /dev/null +++ b/scripts/unsloth/upload_release_assets.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# Upload a directory of files to a draft release, then check the release really +# holds them before the caller flips draft=false. +# Run: upload_release_assets.sh --tag TAG --repo OWNER/REPO --dist DIR +# +# `gh release create ... dist/*` uploads with a fixed 5-worker pool and no +# per-connection timeout, so a few wedged PUTs block everything behind them. In +# run 31335302864 six large bundles stalled at ~0.03 MB/s and held the pool for +# 3h45m, while the other 25 assets took 37s in total. The job finished 25 +# minutes short of its 350m cap. +# +# So: bound each attempt, bound the whole phase, and verify against the API +# before we publish. +set -euo pipefail + +# Defaults come from measured healthy throughput on the runners, 33-47 MB/s. +JOBS="${UPLOAD_JOBS:-4}" +ATTEMPTS="${UPLOAD_ATTEMPTS:-4}" +# Slowest rate we still call "progressing". ~17x below healthy, so a bad day +# retries nothing but a wedged PUT dies fast. +MIN_RATE_MB_S="${UPLOAD_MIN_RATE_MB_S:-2}" +# Per-attempt allowance for setup and server-side commit, which do not scale +# with file size. +GRACE_SECONDS="${UPLOAD_GRACE_SECONDS:-120}" +# Healthy is ~4 minutes for 7.2 GB, so only a real pathology trips this. +DEADLINE_MINUTES="${UPLOAD_DEADLINE_MINUTES:-90}" +HEARTBEAT_SECONDS="${UPLOAD_HEARTBEAT_SECONDS:-60}" + +log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; } +die() { printf '%s ERROR: %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; exit 1; } + +file_size() { stat -c %s "$1"; } + +# GitHub rewrites characters outside [A-Za-z0-9._-] to '.', so verify against +# the name the API will report. printf, not basename: basename's trailing +# newline is also outside the set and would become a phantom '.' on every name. +asset_name() { printf '%s' "${1##*/}" | tr -c 'A-Za-z0-9._-' '.'; } + +# Worker mode. The parent fans out with xargs by re-invoking this script, which +# is safer than `export -f`: an unexported function fails per file at runtime. +if [ "${1:-}" = "--upload-one" ]; then + f="$2" + : "${TAG:?}" "${REPO:?}" "${UPLOAD_DEADLINE_EPOCH:?}" + name="$(asset_name "$f")" + bytes="$(file_size "$f")" + mb=$(( bytes / 1000000 )) + budget=$(( GRACE_SECONDS + mb / MIN_RATE_MB_S )) + + for attempt in $(seq 1 "$ATTEMPTS"); do + now="$(date +%s)" + if [ "$now" -ge "$UPLOAD_DEADLINE_EPOCH" ]; then + die "deadline reached before uploading $name" + fi + # Keep one file's budget inside the phase deadline. Floor at 1: `timeout 0` + # means no timeout, which brings back the hang this script prevents. + remaining=$(( UPLOAD_DEADLINE_EPOCH - now )) + this_budget="$budget" + if [ "$this_budget" -gt "$remaining" ]; then this_budget="$remaining"; fi + if [ "$this_budget" -lt 1 ]; then this_budget=1; fi + + start="$now" + # --clobber keeps a retry after a killed upload idempotent, else GitHub + # 422s on the duplicate name. Take the status here, not from $? after an + # `if`: a false `if` with no else exits 0, so every stall would read clean. + rc=0 + timeout -k 30 "$this_budget" gh release upload "$TAG" --repo "$REPO" --clobber "$f" || rc=$? + elapsed=$(( $(date +%s) - start )) + if [ "$rc" -eq 0 ]; then + if [ "$elapsed" -lt 1 ]; then elapsed=1; fi + log "uploaded $name (${mb} MB in ${elapsed}s, $(( mb / elapsed )) MB/s, attempt ${attempt})" + exit 0 + fi + if [ "$rc" -ge 124 ]; then + log "STALLED $name: no completion in ${elapsed}s (budget ${this_budget}s, ${mb} MB); attempt ${attempt}/${ATTEMPTS}" + else + log "FAILED $name: gh exit ${rc} after ${elapsed}s; attempt ${attempt}/${ATTEMPTS}" + fi + if [ "$attempt" -eq "$ATTEMPTS" ]; then + die "gave up on $name after ${ATTEMPTS} attempts" + fi + sleep $(( attempt * 15 )) + done + exit 1 +fi + +# Parent mode. +TAG="" REPO="" DIST="" +while [ $# -gt 0 ]; do + case "$1" in + --tag) TAG="$2"; shift 2 ;; + --repo) REPO="$2"; shift 2 ;; + --dist) DIST="$2"; shift 2 ;; + *) die "unknown argument: $1" ;; + esac +done +[ -n "$TAG" ] || die "--tag is required" +[ -n "$REPO" ] || die "--repo is required" +[ -n "$DIST" ] || die "--dist is required" +[ -d "$DIST" ] || die "dist directory not found: $DIST" + +# NUL-delimited: a name with a space would otherwise split into two bad paths. +mapfile -d '' -t FILES < <(find "$DIST" -maxdepth 1 -type f -print0 | sort -z) +[ "${#FILES[@]}" -gt 0 ] || die "no files to upload in $DIST" + +total_bytes=0 +for f in "${FILES[@]}"; do total_bytes=$(( total_bytes + $(file_size "$f") )); done +log "uploading ${#FILES[@]} assets ($(( total_bytes / 1000000 )) MB) to draft $TAG with ${JOBS} workers" + +UPLOAD_DEADLINE_EPOCH=$(( $(date +%s) + DEADLINE_MINUTES * 60 )) +export TAG REPO UPLOAD_DEADLINE_EPOCH JOBS ATTEMPTS MIN_RATE_MB_S GRACE_SECONDS + +self="$(readlink -f "$0")" + +# The incident was 4 hours of silence, so report what the API has accepted. +heartbeat() { + while sleep "$HEARTBEAT_SECONDS"; do + n="$(gh release view "$TAG" --repo "$REPO" --json assets --jq '[.assets[]|select(.state=="uploaded")]|length' 2>/dev/null || echo '?')" + log "heartbeat: ${n}/${#FILES[@]} assets uploaded, $(( (UPLOAD_DEADLINE_EPOCH - $(date +%s)) / 60 ))m left in budget" + done +} +heartbeat & hb_pid=$! +trap 'kill "$hb_pid" 2>/dev/null || true' EXIT + +upload_pass() { + # Run through `bash`, so a lost exec bit cannot break the publish. + printf '%s\0' "$@" | xargs -0 -P "$JOBS" -n 1 bash "$self" --upload-one +} + +pass_rc=0 +upload_pass "${FILES[@]}" || pass_rc=$? +[ "$pass_rc" -eq 0 ] || log "upload pass reported failures (xargs exit ${pass_rc}); verification decides" + +# gh exiting 0 does not prove the asset is complete, so set BAD to every local +# file the release does not hold at the same size and state "uploaded". Read the +# API here, not inside `< <(...)`, where a failed read exits only the subshell +# and leaves BAD empty, i.e. publishes a release we never checked. +verify() { + local remote f + remote="$(gh release view "$TAG" --repo "$REPO" --json assets \ + --jq '.assets[] | select(.state=="uploaded") | "\(.name)\t\(.size)"')" \ + || die "could not read release assets for verification" + BAD=() + for f in "${FILES[@]}"; do + if ! grep -qxF "$(asset_name "$f") $(file_size "$f")" <<<"$remote"; then + BAD+=("$f") + fi + done +} + +verify +if [ "${#BAD[@]}" -gt 0 ]; then + log "verification found ${#BAD[@]} missing or mismatched assets; re-uploading" + for f in "${BAD[@]}"; do log " - $(asset_name "$f")"; done + upload_pass "${BAD[@]}" || true + verify +fi + +if [ "${#BAD[@]}" -gt 0 ]; then + for f in "${BAD[@]}"; do printf 'ERROR: asset never landed: %s\n' "$(asset_name "$f")" >&2; done + die "refusing to publish: ${#BAD[@]}/${#FILES[@]} assets missing after re-upload" +fi + +log "verified all ${#FILES[@]} assets present, sized and uploaded" diff --git a/scripts/unsloth/upstream-sync.json b/scripts/unsloth/upstream-sync.json new file mode 100644 index 000000000000..cc89fc2b09cb --- /dev/null +++ b/scripts/unsloth/upstream-sync.json @@ -0,0 +1,26 @@ +{ + "_doc": [ + "The upstream commit master was last synced to, and the invariant that keeps the sync cheap.", + "", + "Update BOTH fields in the same commit as the sync merge. unsloth-upstream-sync-guard.yml", + "reads this file and fails master if either invariant breaks:", + "", + " 1. `commit` must be an ancestor of master. This is the check that would have caught the", + " 08-07 sync (PR #80), which was squash-merged: its content landed but git never learned", + " upstream had been incorporated, so the merge base stayed at 2026-06-10 and every later", + " merge three-way merged against it. Merging b10632 conflicted in 539 files with that", + " base and in 21 with the true one. ALWAYS merge a sync PR with a merge commit.", + "", + " 2. The diff from `commit` to master must touch only .github/ and scripts/unsloth/. This", + " fork deliberately owns no llama.cpp source; that is what makes a sync provably additive", + " and lets scripts/unsloth/verify_upstream_sync.py check it exactly rather than by eye.", + " If this ever fails, the fork has acquired source divergence and syncs stop being cheap.", + "", + "Note for whoever runs the next sync: verify_upstream_sync.py derives its base from", + "merge-base(--fork, --upstream). Pass a --fork ref whose ancestry is already correct, or it", + "measures the stale set and reports failures that are artefacts of the bad base." + ], + "tag": "b10632", + "commit": "11cd98842874cc1b87ac274bd2d5cceb38102bb2", + "synced_at": "2026-08-26" +} diff --git a/scripts/unsloth/verify_upstream_sync.py b/scripts/unsloth/verify_upstream_sync.py new file mode 100755 index 000000000000..7fff2b605f5f --- /dev/null +++ b/scripts/unsloth/verify_upstream_sync.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Prove an upstream-sync merge is additive only: it may bring in upstream content, but it must +never modify, revert or delete anything this fork authored. + +The invariant, stated precisely. Let + + B = merge-base(fork_master, upstream_master) + F = fork_master (the fork before the sync) + U = upstream_master + M = the merge commit under test + +For every path P that the fork touched in B..F, the merge must satisfy M:P == F:P, byte for +byte. That is the whole rule, and it is checked exactly rather than approximated by reading a +diff. Three things can break it and each is reported separately: + + MODIFIED M:P exists but differs from F:P. Upstream edited a file we own, or a conflict + was resolved in upstream's favour. + DELETED P is in F and gone in M. Our work was dropped. + LOSTCOMMIT a commit reachable from F is not reachable from M. History was rewritten. + +Deletions of upstream files are legitimate and are checked in the other direction: every path +missing from M must ALSO have been missing from F. A file the fork already deleted staying +deleted preserves our state; a file the fork had that vanishes is a violation. + +Two extra layers beyond the byte comparison, because a byte-identical file can still be +semantically wrong if a neighbouring definition moved: + + * AST check on the Python surface (gguf-py and convert scripts). Every top-level class, + function and assignment name the fork defines must still be defined in the merged tree, + and every enum member the fork added must still carry the same value. A renumbered + GGML_TYPE_* would be caught here even if the file "looks" additive. + * enum-value check on the C headers, by regex over the id assignments that matter, since a + silently renumbered type id is the failure mode that would corrupt published GGUFs. + +Usage: + verify_upstream_sync.py --repo <path> --merge <rev> [--fork origin/master] + [--upstream upstream/master] [--json out.json] + +Exit 0 only if every check passes. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import os +import re +import subprocess +import sys +from collections import defaultdict + + +def git(repo: str, *args: str, ok_fail: bool = False) -> str: + p = subprocess.run(("git", "-C", repo) + args, capture_output=True, text=True) + if p.returncode and not ok_fail: + raise SystemExit(f"git {' '.join(args)} failed:\n{p.stderr.strip()}") + return p.stdout + +def lines(s: str) -> list[str]: + return [x for x in s.splitlines() if x.strip()] + +def blob(repo: str, rev: str, path: str) -> bytes | None: + """File content at a revision, or None if the path does not exist there.""" + p = subprocess.run(("git", "-C", repo, "show", f"{rev}:{path}"), + capture_output=True) + return p.stdout if p.returncode == 0 else None + + +# ---------------------------------------------------------------- AST surface + +def py_surface(src: bytes) -> dict[str, str]: + """Top-level names a Python file defines, plus every enum-ish member and its literal value. + + Keys are dotted so a member cannot collide across classes. Values are a repr of the + assigned constant where there is one, else the node type, so a renumber shows up as a + changed value rather than a missing name. + """ + try: + tree = ast.parse(src.decode("utf-8", "replace")) + except SyntaxError: + return {} + + out: dict[str, str] = {} + + def const(node: ast.AST) -> str: + """A value fingerprint that survives reformatting but not a real change. + + ast.unparse normalises whitespace, quote style and line breaks, so a reflow is + invisible while an edited element is not. Falling back to the node type name would + make every tuple, list and dict compare equal, which is how a dropped entry such as + OWNED = ("a", "b") -> ("a",) slips past. + """ + if isinstance(node, ast.Constant): + return repr(node.value) + try: + return ast.unparse(node) + except Exception: + return type(node).__name__ + + def walk(body: list[ast.stmt], prefix: str) -> None: + for n in body: + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)): + out[f"{prefix}{n.name}"] = "def" + elif isinstance(n, ast.ClassDef): + out[f"{prefix}{n.name}"] = "class" + walk(n.body, f"{prefix}{n.name}.") + elif isinstance(n, ast.Assign): + for t in n.targets: + if isinstance(t, ast.Name): + out[f"{prefix}{t.id}"] = const(n.value) + elif isinstance(n, ast.AnnAssign) and isinstance(n.target, ast.Name): + out[f"{prefix}{n.target.id}"] = const(n.value) if n.value else "ann" + walk(tree.body, "") + return out + + +# --------------------------------------------------------- C enum value check + +C_ENUM = re.compile( + rb"^\s*(GGML_TYPE_[A-Z0-9_]+|GGML_FTYPE_[A-Z0-9_]+|LLAMA_FTYPE_[A-Z0-9_]+)\s*=\s*(-?\d+)", + re.M) + +def c_enum_values(src: bytes) -> dict[str, int]: + return {m.group(1).decode(): int(m.group(2)) for m in C_ENUM.finditer(src)} + + +# --------------------------------------------------------------------- checks + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--repo", required=True) + ap.add_argument("--merge", required=True, help="the merge commit under test") + ap.add_argument("--fork", default="origin/master", help="fork state BEFORE the sync") + ap.add_argument("--upstream", default="upstream/master") + ap.add_argument("--json", help="write the full report here") + ap.add_argument("--show", type=int, default=15, help="max paths to print per category") + a = ap.parse_args() + + R = a.repo + if git(R, "rev-parse", "--is-shallow-repository").strip() == "true": + print("REFUSING: shallow clone, history checks would be meaningless. " + "Re-clone without --depth.", file=sys.stderr) + return 2 + + M = git(R, "rev-parse", a.merge).strip() + F = git(R, "rev-parse", a.fork).strip() + U = git(R, "rev-parse", a.upstream).strip() + B = git(R, "merge-base", a.fork, a.upstream).strip() + print(f"merge {M[:12]}\nfork {F[:12]} ({a.fork})\n" + f"upstream {U[:12]} ({a.upstream})\nmerge-base {B[:12]}\n") + + rep: dict = {"merge": M, "fork": F, "upstream": U, "base": B, "checks": {}} + fail = False + + # --- 1. no fork commit may be dropped ----------------------------------- + lost = lines(git(R, "rev-list", a.fork, f"^{M}")) + fork_commits = len(lines(git(R, "rev-list", f"{B}..{a.fork}"))) + rep["checks"]["history"] = {"fork_commits_since_base": fork_commits, + "lost": lost} + if lost: + fail = True + print(f"FAIL history: {len(lost)} fork commits are NOT ancestors of the merge") + for c in lost[:a.show]: + print(f" {c[:12]} {git(R, 'log', '-1', '--format=%s', c).strip()[:70]}") + else: + print(f"PASS history: all {fork_commits} fork commits since the base are ancestors " + f"of the merge, none dropped") + + # --- 2. every path the fork touched must survive byte-identical --------- + # --diff-filter with -M off: a rename upstream must not silently "move" our file. + touched = sorted(set(lines(git(R, "diff", "--name-only", "--no-renames", f"{B}..{a.fork}")))) + modified, deleted, ok = [], [], 0 + for p in touched: + fb = blob(R, a.fork, p) + mb = blob(R, M, p) + if fb is None: + # the fork itself deleted it; it must still be absent + if mb is not None: + modified.append((p, "fork deleted it, merge resurrected it")) + else: + ok += 1 + continue + if mb is None: + deleted.append(p) + elif mb != fb: + modified.append((p, f"{len(fb)} B -> {len(mb)} B")) + else: + ok += 1 + rep["checks"]["content"] = {"fork_touched_paths": len(touched), "identical": ok, + "modified": modified, "deleted": deleted} + if modified or deleted: + fail = True + print(f"FAIL content: of {len(touched)} fork-touched paths, " + f"{len(modified)} modified and {len(deleted)} deleted") + for p, why in modified[:a.show]: + print(f" MODIFIED {p} ({why})") + for p in deleted[:a.show]: + print(f" DELETED {p}") + else: + print(f"PASS content: all {len(touched)} paths the fork touched are byte-identical " + f"in the merge") + + # --- 3. nothing the fork had may vanish, even if it never touched it ----- + fork_tree = set(lines(git(R, "ls-tree", "-r", "--name-only", a.fork))) + merge_tree = set(lines(git(R, "ls-tree", "-r", "--name-only", M))) + vanished = sorted(fork_tree - merge_tree) + # a vanished path is only acceptable if upstream deleted it AND the fork never touched it + unexplained = [p for p in vanished if p in set(touched)] + rep["checks"]["tree"] = {"fork_files": len(fork_tree), "merge_files": len(merge_tree), + "vanished": vanished, "unexplained": unexplained} + if unexplained: + fail = True + print(f"FAIL tree: {len(unexplained)} fork-authored files vanished from the merge") + for p in unexplained[:a.show]: + print(f" {p}") + else: + print(f"PASS tree: {len(fork_tree)} fork files -> {len(merge_tree)} merged files, " + f"{len(vanished)} vanished and none of them fork-authored") + + # --- 4. every deletion must be upstream's own, never ours --------------- + # Upstream retires its own files and a sync has to carry that through, so a deletion is + # only a violation if the file is one WE own. Two independent tests, both must hold: + # the path is absent from upstream's tree (upstream really did delete it), and the fork + # never touched it. + gone = sorted(set(lines(git(R, "diff", "--name-only", "--diff-filter=D", + f"{a.fork}..{M}")))) + touched_set = set(touched) + ours, not_upstream = [], [] + for p in gone: + if p in touched_set: + ours.append(p) + elif blob(R, a.upstream, p) is not None: + not_upstream.append(p) # still exists upstream, so nobody asked us to drop it + rep["checks"]["deletions"] = {"deleted_vs_fork": gone, "fork_authored": ours, + "still_present_upstream": not_upstream} + if ours or not_upstream: + fail = True + print(f"FAIL deletions: {len(gone)} deletions, {len(ours)} fork-authored, " + f"{len(not_upstream)} not deleted upstream either") + for p in (ours + not_upstream)[:a.show]: + print(f" {p}") + else: + print(f"PASS deletions: all {len(gone)} deletions are upstream retiring its own " + f"files, none fork-authored") + + # --- 5. AST surface on Python the fork owns ----------------------------- + pyfiles = [p for p in touched if p.endswith(".py")] + lost_names: list[tuple[str, str, str, str]] = [] + for p in pyfiles: + fb, mb = blob(R, a.fork, p), blob(R, M, p) + if fb is None or mb is None: + continue + fs, ms = py_surface(fb), py_surface(mb) + for name, val in fs.items(): + if name not in ms: + lost_names.append((p, name, val, "MISSING")) + elif ms[name] != val: + lost_names.append((p, name, val, ms[name])) + rep["checks"]["ast"] = {"python_files": len(pyfiles), "regressions": lost_names} + if lost_names: + fail = True + print(f"FAIL ast: {len(lost_names)} Python definitions lost or changed value") + for p, n, was, now in lost_names[:a.show]: + print(f" {p}:{n} was {was} now {now}") + else: + print(f"PASS ast: every top-level definition and constant in the {len(pyfiles)} " + f"fork-touched Python files survives with the same value") + + # --- 6. C enum ids, the failure that would corrupt published GGUFs ------ + # Deliberately NOT limited to fork-touched headers. A published GGUF stores these ids, so + # an id that shifts or collides is a data-corruption bug no matter who moved it, and on a + # fork whose master carries no source changes the fork-touched set would be empty. + # The reference to compare against depends on who owns the header. For one the fork + # customises, our values must survive. For one the fork does not, the merged copy must + # match UPSTREAM exactly, and upstream growing an enum (a new type, so a new _COUNT) is + # the sync working, not a violation. Comparing an unowned header against the fork's stale + # copy would flag every legitimate upstream addition. + ID_HEADERS = ["ggml/include/ggml.h", "include/llama.h"] + hdrs = sorted(set([p for p in touched if p.endswith((".h", ".hpp"))]) | + {p for p in ID_HEADERS if blob(R, M, p) is not None}) + enum_bad: list[tuple[str, str, int, object]] = [] + dupes: list[tuple[str, int, list[str]]] = [] + owned = set(touched) + for p in hdrs: + ref = a.fork if p in owned else a.upstream + rb, mb = blob(R, ref, p), blob(R, M, p) + if rb is None or mb is None: + continue + fe, me = c_enum_values(rb), c_enum_values(mb) + for name, v in fe.items(): + now = me.get(name, "MISSING") + if now == v: + continue + # A _COUNT sentinel is not an id, it is one past the last one, so a branch that + # legitimately adds types must move it. Allow it to grow and require it to still + # bound every real id; anything else, including a shrink, is a violation. + if name.endswith("_COUNT") and isinstance(now, int) and now > v: + fam = name[:-len("_COUNT")] + real = [x for n2, x in me.items() if n2.startswith(fam) and n2 != name] + if real and now > max(real): + continue + enum_bad.append((f"{p} [vs {ref}]", name, v, now)) + # two names sharing one id in the same enum family is a collision + byfam: dict[str, dict[int, list[str]]] = defaultdict(lambda: defaultdict(list)) + for name, v in me.items(): + fam = name.split("_")[0] + ("_FTYPE" if "_FTYPE_" in name else "_TYPE") + byfam[fam][v].append(name) + for fam, vals in byfam.items(): + for v, names in vals.items(): + if len(names) > 1 and not any(n.endswith("_COUNT") for n in names): + dupes.append((f"{p}:{fam}", v, sorted(names))) + rep["checks"]["c_enums"] = {"headers": len(hdrs), "changed": enum_bad, "collisions": dupes} + if enum_bad or dupes: + fail = True + print(f"FAIL c_enums: {len(enum_bad)} ids changed, {len(dupes)} id collisions") + for p, n, was, now in enum_bad[:a.show]: + print(f" {p}:{n} was {was} now {now}") + for where, v, names in dupes[:a.show]: + print(f" COLLISION {where} = {v}: {', '.join(names)}") + else: + print(f"PASS c_enums: every GGML_TYPE/GGML_FTYPE/LLAMA_FTYPE id the fork defines " + f"keeps its value across {len(hdrs)} headers, no collisions") + + # --- context: what the merge actually brought in ------------------------ + added = len(lines(git(R, "diff", "--name-only", "--diff-filter=A", f"{a.fork}..{M}"))) + changed = len(lines(git(R, "diff", "--name-only", f"{a.fork}..{M}"))) + rep["summary"] = {"files_added_by_merge": added, "files_changed_by_merge": changed, + "pass": not fail} + print(f"\nmerge brings in {changed} changed files, {added} of them new") + + if a.json: + with open(a.json, "w") as f: + json.dump(rep, f, indent=1) + print(f"report: {a.json}") + + print("\n" + ("VIOLATION: the sync is not additive-only" if fail + else "PASS: the sync is additive only, nothing fork-authored was altered")) + return 1 if fail else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/llama-adapter.cpp b/src/llama-adapter.cpp index e6678a66d2a9..309840167d19 100644 --- a/src/llama-adapter.cpp +++ b/src/llama-adapter.cpp @@ -351,6 +351,21 @@ static void llama_adapter_lora_init_impl(llama_model & model, const char * path_ LLAMA_LOG_DEBUG("%s: lora for '%s' -> '%s'\n", __func__, model_tensor->name, ggml_backend_buft_name(buft)); + // [TAG_EXACT_CONCURRENCY] the adapter follows the weight it adapts, so a weight the mode + // leaves on the host puts the adapted matmul there too. token_embd is exempt from the + // context's weight check because get_rows is not offloaded by width, but its adapter is + // applied with a mul_mat, which is: see llm_graph_context::build_inp_embd(). + if (llama_exact_concurrency() && !llama_exact_buft_invariant(buft)) { + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set but the lora for '%s' would sit in a %s " + "buffer, which has no batch-invariant kernels: the adapted matmul's result would " + "depend on how many sequences share the step (move the tensor to the device that " + "holds the layers, for example --override-tensor %s=CUDA0, or serve this adapter " + "without the mode)\n", + __func__, model_tensor->name, ggml_backend_buft_name(buft), model_tensor->name); + + throw std::runtime_error("exact concurrency: a lora weight is not on the CUDA backend"); + } + ggml_context * dev_ctx = ctx_for_buft(buft); // validate tensor shape if (is_token_embd) { diff --git a/src/llama-batch.cpp b/src/llama-batch.cpp index 2b98a552f48f..5d52f5bac0b2 100644 --- a/src/llama-batch.cpp +++ b/src/llama-batch.cpp @@ -507,7 +507,54 @@ llama_ubatch llama_batch_allocr::split_simple(uint32_t n_ubatch) { return ubatch_add(idxs, idxs.size(), false); } -llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail) { +bool llama_batch_allocr::has_shared_tokens() const { + for (int32_t i = 0; i < batch.n_tokens; ++i) { + if (batch.n_seq_id[i] > 1) { + return true; + } + } + + return false; +} + +bool llama_batch_allocr::has_repeated_positions() const { + std::vector<size_t> n_per_seq(n_seq_max, 0); + + for (int32_t i = 0; i < batch.n_tokens; ++i) { + for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) { + n_per_seq[batch.seq_id[i][s]]++; + } + } + + for (uint32_t s = 0; s < n_seq_max; ++s) { + if (n_per_seq[s] > seq_pos[s].size()) { + return true; + } + } + + return false; +} + +bool llama_batch_allocr::has_seq_wider_than(uint32_t n_tokens) const { + std::vector<uint32_t> n_per_seq(n_seq_max, 0); + + for (int32_t i = 0; i < batch.n_tokens; ++i) { + // tokens already placed in an earlier ubatch do not make the rest of the batch a prompt + if (used[i]) { + continue; + } + + for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) { + if (++n_per_seq[batch.seq_id[i][s]] > n_tokens) { + return true; + } + } + } + + return false; +} + +llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail, uint32_t isolate_seqs_above) { if (sequential && has_cpl) { LLAMA_LOG_ERROR("%s: sequential split is not supported when there are coupled sequences in the input batch (you may need to use the -kvu flag)\n", __func__); @@ -518,6 +565,9 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential, llama_seq_id last_seq_id = -1; + // [TAG_EXACT_CONCURRENCY] tokens left in the first set taken, when isolating: only sets with the same count join it, so every set in the ubatch finishes in it + uint32_t n_left_first = 0; + // determine the non-overlapping sequence sets participating in this ubatch for (int32_t i = 0; i < batch.n_tokens; ++i) { if (used[i]) { @@ -540,6 +590,38 @@ llama_ubatch llama_batch_allocr::split_equal(uint32_t n_ubatch, bool sequential, } if (add) { + // [TAG_EXACT_CONCURRENCY] a set with more tokens left than a decode step carries is a prompt and gets a ubatch of its own; grouped sets need equal tokens left, or the expansion below changes their sum order + if (isolate_seqs_above > 0) { + uint32_t n_left = 0; + + for (const auto idx : seq_set_map[seq_set[i]]) { + if (!used[idx]) { + ++n_left; + } + } + + if (n_left > isolate_seqs_above) { + if (!cur_seq_set.empty()) { + // let the sets already taken have this ubatch; the prompt gets the next one + break; + } + + cur_seq_set.push_back(seq_set[i]); + + last_seq_id = batch.seq_id[i][0]; + + break; + } + + if (cur_seq_set.empty()) { + n_left_first = n_left; + } else if (n_left != n_left_first) { + continue; + } else if ((cur_seq_set.size() + 1) * n_left_first > n_ubatch) { + break; + } + } + cur_seq_set.push_back(seq_set[i]); last_seq_id = batch.seq_id[i][0]; diff --git a/src/llama-batch.h b/src/llama-batch.h index a3d1889d4a04..b70987864503 100644 --- a/src/llama-batch.h +++ b/src/llama-batch.h @@ -105,7 +105,16 @@ class llama_batch_allocr { // make ubatches of equal-length sequences sets // if sequential == true, the tokens in the ubatch will have increasing sequential sequence ids // n_keep_tail = minimum trailing tokens of a seq that must land in the same ubatch - llama_ubatch split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail); + // isolate_seqs_above = [TAG_EXACT_CONCURRENCY] when > 0, a sequence set with more than this many tokens left is a prompt and gets a ubatch of its own + llama_ubatch split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail, uint32_t isolate_seqs_above = 0); + + // [TAG_EXACT_CONCURRENCY] true if some sequence still has more than n_tokens left to place, i.e. what remains of the batch holds a prompt + bool has_seq_wider_than(uint32_t n_tokens) const; + + bool has_shared_tokens() const; + + // [TAG_EXACT_CONCURRENCY] true if a sequence has several tokens at one position, which the paged pool would give one cell + bool has_repeated_positions() const; // sequence-set-wise split - each ubatch contains a single sequence-set llama_ubatch split_seq(uint32_t n_ubatch); diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 21501574a911..8d19ae21bcca 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -13,6 +13,7 @@ #include "llama-sampler.h" #include "llama.h" +#include <atomic> #include <cinttypes> #include <cmath> #include <cstring> @@ -32,6 +33,61 @@ static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) { throw std::runtime_error("Unsupported ctx type"); } +// [TAG_EXACT_CONCURRENCY] the caches check where the KV lives; this checks where the weights that +// produce the tokens live. Every per-layer weight and the output head must be on a backend with +// the mode's kernels, otherwise a sequence's own matmuls change with the width of the step it +// shares, while the mode still reports itself as on. +// +// token_embd is deliberately not required to move: it feeds get_rows, a per-row copy, and +// GET_ROWS reports a batch size of 0 to the offload test, so it stays on the same backend at +// every width. A model that ties its head to the embedding uses that same tensor for the output +// matmul, and model.output points at it, so the head check below still covers that case. A lora +// that adapts it is applied with a mul_mat instead, and MUL_MAT reports the ubatch width, so +// llama_adapter_lora_init_impl() refuses one that inherits a host buffer. +static void llama_exact_check_weights(const llama_model & model) { + auto host_buft = [](const ggml_tensor * t) -> ggml_backend_buffer_type_t { + if (!t || !t->buffer) { + return nullptr; + } + + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(t->buffer); + + return llama_exact_buft_invariant(buft) ? nullptr : buft; + }; + + auto refuse = [&model](const char * name, ggml_backend_buffer_type_t buft, const char * what) { + const std::string tname = name; + + const char * fix = "pass -ngl to offload every layer, and no --override-tensor that keeps one on the host"; + + if (tname.find("_exps") != std::string::npos) { + fix = "do not pass --cpu-moe or --n-cpu-moe, and no --override-tensor that keeps an expert on the host"; + } else if (model.has_tensor_overrides()) { + fix = "drop the --override-tensor that placed it there, and pass -ngl to offload every layer"; + } + + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set but %s %s is in a %s buffer, which has no " + "batch-invariant kernels: its result would depend on how many sequences share the step (%s)\n", + __func__, what, tname.c_str(), ggml_backend_buft_name(buft), fix); + + throw std::runtime_error("exact concurrency: a weight is not on the CUDA backend"); + }; + + for (const auto & [name, t] : model.tensors_by_name) { + if (name.rfind("blk.", 0) != 0) { + continue; + } + + if (auto * buft = host_buft(t)) { + refuse(name.c_str(), buft, "layer weight"); + } + } + + if (auto * buft = host_buft(model.output)) { + refuse(ggml_get_name(model.output), buft, "the output head"); + } +} + struct llm_fused_op_probe { llm_fused_op op; const char * name; @@ -101,6 +157,17 @@ llama_context::llama_context( throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ)); } + // [TAG_EXACT_CONCURRENCY] the widest decode step this context can build, reported so a backend that splits columns covers it; reported at the end of the constructor + if (llama_exact_concurrency()) { + if (!llama_exact_check_n_seq(cparams.n_seq_max)) { + throw std::runtime_error("exact concurrency: the explicit column bound is below this context's decode width"); + } + + if (!hparams.vocab_only) { + llama_exact_check_weights(model); + } + } + cparams.n_rs_seq = params.n_rs_seq; if (cparams.n_rs_seq > 0 && !llm_arch_supports_rs_rollback(model.arch)) { LLAMA_LOG_DEBUG("%s: n_rs_seq=%u requested but model does not support recurrent partial rollback; clamping to 0\n", @@ -393,6 +460,12 @@ llama_context::llama_context( }; memory.reset(model.create_memory(params_mem, cparams)); + + // [TAG_EXACT_CONCURRENCY] the paged attention is causal, so a non-causal context with a cache would assert on its first graph + if (llama_exact_concurrency() && memory && !cparams.causal_attn) { + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set and this context has a KV cache, so it cannot be created with non-causal attention\n", __func__); + throw std::runtime_error("exact concurrency: non-causal attention is not supported with a KV cache"); + } } // init backends @@ -476,12 +549,24 @@ llama_context::llama_context( sampling.token_ids_full_vocab[i] = i; } } + + // [TAG_EXACT_CONCURRENCY] nothing above can fail now, so publish the width; a refusal here means the bound moved + if (llama_exact_concurrency() && !llama_exact_report_n_seq(cparams.n_seq_max)) { + throw std::runtime_error("exact concurrency: the explicit column bound is below this context's decode width"); + } } llama_context::~llama_context() { // wait for any pending asynchronous copies into the output buffers before they are freed synchronize(); + // a transfer still alive is drained first: synchronize() covers the graph backends, not the copy backend a transfer owns, and its KV buffers are about to go + state_seq_copies_drain(); + + for (auto & it : state_copy_fences) { + ggml_backend_event_free(it.second); + } + // when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation if (!model.hparams.no_alloc && !opt_ctx) { for (size_t i = 0; i < backend_ptrs.size(); ++i) { @@ -1197,6 +1282,11 @@ void llama_context::set_causal_attn(bool value) { return; } + if (!value && memory && llama_exact_concurrency()) { + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set and this context has a KV cache, so causal attention cannot be turned off; the change is refused\n", __func__); + return; + } + cparams.causal_attn = value; sched_need_reserve = true; @@ -1586,6 +1676,10 @@ int llama_context::encode(const llama_batch & batch_inp) { } } + if (!state_copy_fences.empty()) { + state_seq_copy_fence(); + } + return 0; } @@ -1707,6 +1801,12 @@ int llama_context::decode(const llama_batch & batch_inp) { return -1; } + // [TAG_EXACT_CONCURRENCY] an invalid batch, not a full cache: left to the memory it came back as 1, which callers retry + if (llama_exact_concurrency() && (balloc->has_shared_tokens() || balloc->has_repeated_positions())) { + LLAMA_LOG_ERROR("%s: exact concurrency needs every token at one sequence id and one position of its own\n", __func__); + return -1; + } + const uint32_t n_tokens_all = balloc->get_n_tokens(); const uint32_t n_outputs_all = balloc->get_n_outputs(); @@ -2033,6 +2133,10 @@ int llama_context::decode(const llama_batch & batch_inp) { // wait for the computation to finish (automatically done when obtaining the model output) //synchronize(); + if (!state_copy_fences.empty()) { + state_seq_copy_fence(); + } + return 0; } @@ -2575,16 +2679,132 @@ class llama_io_write_dummy : public llama_io_write_i { size_t size_written = 0; }; +// [TAG_STATE_COALESCE] one transfer per run of cells, not one per cell; the restore side asks for one per cell, and the transposed V layout repeats every run once per row +template <typename info_t> +static size_t llama_io_run_end(const std::vector<info_t> & infos, size_t i) { + size_t end = i + 1; + + while (end < infos.size() && + infos[end].tensor == infos[end - 1].tensor && + infos[end].offset == infos[end - 1].offset + infos[end - 1].size && + infos[end].ptr == infos[end - 1].ptr + infos[end - 1].size) { + end++; + } + + return end; +} + +template <typename info_t> +static size_t llama_io_run_size(const std::vector<info_t> & infos, size_t i, size_t end) { + size_t size = 0; + + for (size_t j = i; j < end; ++j) { + size += infos[j].size; + } + + return size; +} + +// [TAG_STATE_COALESCE] a comb of equal runs at a constant stride is one strided copy: sequences sharing a unified cache take their cells in turn +template <typename info_t, typename emit_t> +static void llama_io_emit(const std::vector<info_t> & infos, size_t first, size_t last, emit_t emit) { + std::vector<std::pair<size_t, size_t>> runs; + + for (size_t i = first; i < last; ) { + const size_t end = llama_io_run_end(infos, i); + + runs.emplace_back(i, end); + + i = end; + } + + for (size_t r = 0; r < runs.size(); ) { + const auto & head = infos[runs[r].first]; + + const size_t size = llama_io_run_size(infos, runs[r].first, runs[r].second); + + size_t n_copies = 1; + size_t stride_tensor = 0; + size_t stride_data = 0; + + if (r + 1 < runs.size()) { + const auto & next = infos[runs[r + 1].first]; + + if (next.tensor == head.tensor && next.offset > head.offset && next.ptr > head.ptr && + llama_io_run_size(infos, runs[r + 1].first, runs[r + 1].second) == size) { + stride_tensor = next.offset - head.offset; + stride_data = (size_t) (next.ptr - head.ptr); + + // a strided copy may not have its rows overlap, on either side + if (stride_tensor >= size && stride_data >= size) { + while (r + n_copies < runs.size()) { + const auto & cur = infos[runs[r + n_copies].first]; + + if (cur.tensor != head.tensor || + cur.offset != head.offset + n_copies * stride_tensor || + cur.ptr != head.ptr + n_copies * stride_data || + llama_io_run_size(infos, runs[r + n_copies].first, runs[r + n_copies].second) != size) { + break; + } + + n_copies++; + } + } + } + } + + emit(head.tensor, head.ptr, head.offset, size, n_copies, stride_tensor, stride_data); + + r += n_copies; + } +} + +// a null backend means the caller wants the copy to have happened by the time this returns +static void llama_io_get(ggml_backend_t backend, ggml_tensor * tensor, void * ptr, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + if (n_copies > 1) { + if (backend) { + ggml_backend_tensor_get_2d_async(backend, tensor, ptr, offset, size, n_copies, stride_tensor, stride_data); + } else { + ggml_backend_tensor_get_2d(tensor, ptr, offset, size, n_copies, stride_tensor, stride_data); + } + } else if (backend) { + ggml_backend_tensor_get_async(backend, tensor, ptr, offset, size); + } else { + ggml_backend_tensor_get(tensor, ptr, offset, size); + } +} + +static void llama_io_set(ggml_backend_t backend, ggml_tensor * tensor, const void * ptr, + size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) { + if (n_copies > 1) { + if (backend) { + ggml_backend_tensor_set_2d_async(backend, tensor, ptr, offset, size, n_copies, stride_tensor, stride_data); + } else { + ggml_backend_tensor_set_2d(tensor, ptr, offset, size, n_copies, stride_tensor, stride_data); + } + } else if (backend) { + ggml_backend_tensor_set_async(backend, tensor, ptr, offset, size); + } else { + ggml_backend_tensor_set(tensor, ptr, offset, size); + } +} + class llama_io_write_host : public llama_io_write_i { public: llama_io_write_host( uint8_t * p, size_t len) : ptr(p), buf_size(len) {} ~llama_io_write_host() { - // TODO: add backend support to batch tensor_get? or some other way to speed this up - for (const auto & winfo : winfos) { - ggml_backend_tensor_get(winfo.tensor, winfo.ptr, winfo.offset, winfo.size); + if (deferred) { + return; // [TAG_STATE_ASYNC] the derived class posts the copies itself } + + llama_io_emit(winfos, 0, winfos.size(), + [](ggml_tensor * tensor, uint8_t * ptr, size_t offset, size_t size, + size_t n_copies, size_t stride_tensor, size_t stride_data) { + llama_io_get(nullptr, tensor, ptr, offset, size, n_copies, stride_tensor, stride_data); + }); } void write(const void * src, size_t size) override { @@ -2614,10 +2834,8 @@ class llama_io_write_host : public llama_io_write_i { return size_written; } -private: - uint8_t * ptr; - size_t buf_size = 0; - size_t size_written = 0; +protected: + llama_io_write_host(uint8_t * p, size_t len, bool deferred) : ptr(p), buf_size(len), deferred(deferred) {} struct write_info { ggml_tensor * tensor; @@ -2626,6 +2844,12 @@ class llama_io_write_host : public llama_io_write_i { size_t offset; }; std::vector<write_info> winfos; + +private: + uint8_t * ptr; + size_t buf_size = 0; + size_t size_written = 0; + const bool deferred = false; }; class llama_io_read_host : public llama_io_read_i { @@ -2633,9 +2857,54 @@ class llama_io_read_host : public llama_io_read_i { llama_io_read_host(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {} ~llama_io_read_host() { + if (deferred) { + return; // [TAG_STATE_ASYNC] the derived class posts the copies itself + } + // flush the reads - for (const auto & rinfo : rinfos) { - ggml_backend_tensor_set(rinfo.tensor, rinfo.ptr, rinfo.offset, rinfo.size); + for (size_t i = 0; i < rinfos.size();) { + auto * tensor = rinfos[i].tensor; + size_t end = i + 1; + while (end < rinfos.size() && rinfos[end].tensor == tensor) { + end++; + } + // [TAG_STATE_COALESCE] the restore emits one fragment per cell, but the cost is the number of runs of adjacent cells, so count runs before falling back to staging + const size_t tensor_bytes = ggml_nbytes(tensor); + auto * buffer = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; + + const bool has_2d = ggml_backend_buffer_supports_2d(buffer); + + size_t n_runs = 0; + llama_io_emit(rinfos, i, end, + [&n_runs, has_2d](ggml_tensor *, const uint8_t *, size_t, size_t, size_t n_copies, size_t, size_t) { + n_runs += has_2d ? 1 : n_copies; + }); + if (n_runs >= 64 && tensor_bytes <= 64 * 1024 * 1024 && + !ggml_backend_buffer_is_host(buffer)) { + std::vector<uint8_t> staging; + try { + staging.resize(tensor_bytes); + } catch (const std::bad_alloc &) { + } + if (!staging.empty()) { + ggml_backend_tensor_get(tensor, staging.data(), 0, tensor_bytes); + for (size_t j = i; j < end; ++j) { + const auto & rinfo = rinfos[j]; + GGML_ASSERT(rinfo.offset <= tensor_bytes && rinfo.size <= tensor_bytes - rinfo.offset); + memcpy(staging.data() + rinfo.offset, rinfo.ptr, rinfo.size); + } + ggml_backend_tensor_set(tensor, staging.data(), 0, tensor_bytes); + i = end; + continue; + } + } + llama_io_emit(rinfos, i, end, + [](ggml_tensor * tensor, const uint8_t * ptr, size_t offset, size_t size, + size_t n_copies, size_t stride_tensor, size_t stride_data) { + llama_io_set(nullptr, tensor, ptr, offset, size, n_copies, stride_tensor, stride_data); + }); + + i = end; } } @@ -2666,10 +2935,8 @@ class llama_io_read_host : public llama_io_read_i { return size_read; } -private: - const uint8_t * ptr; - size_t buf_size = 0; - size_t size_read = 0; +protected: + llama_io_read_host(const uint8_t * p, size_t len, bool deferred) : ptr(p), buf_size(len), deferred(deferred) {} struct read_info { ggml_tensor * tensor; @@ -2678,6 +2945,12 @@ class llama_io_read_host : public llama_io_read_i { size_t offset; }; std::vector<read_info> rinfos; + +private: + const uint8_t * ptr; + size_t buf_size = 0; + size_t size_read = 0; + const bool deferred = false; }; class llama_io_write_file : public llama_io_write_i { @@ -3062,6 +3335,273 @@ size_t llama_context::state_set_data(const uint8_t * src, size_t size) { } } +// [TAG_STATE_ASYNC] a sequence state transfer that runs beside the decode instead of in it: the host buffer, one backend per device, each with its own stream, and one event per device +struct llama_state_seq_copy { + llama_context * ctx = nullptr; + + struct dev_copy { + ggml_backend_ptr backend; + ggml_backend_event_t event = nullptr; + bool pending = false; + }; + + std::map<ggml_backend_dev_t, dev_copy> devs; + + ggml_backend_buffer_ptr host_buf; + + bool counted = false; // held in the context's count of live transfers + + uint8_t * data = nullptr; + size_t size = 0; // bytes the current transfer covers + size_t capacity = 0; // bytes actually held, kept across transfers + bool pinned = false; + bool can_pin = false; + + size_t n_copies = 0; + int64_t t_sync_us = 0; + + ~llama_state_seq_copy() { + if (counted) { + ctx->state_seq_copy_release(this); + } + + wait(); + + for (auto & it : devs) { + if (it.second.event) { + ggml_backend_event_free(it.second.event); + } + } + } + + // the stream this tensor is copied on, or null when it needs none: a host tensor is a memcpy, and a split buffer fails every backend's async copy assert + ggml_backend_t backend_for(const ggml_tensor * t) { + ggml_backend_buffer_t buf = t->view_src ? t->view_src->buffer : t->buffer; + + if (!buf || ggml_backend_buffer_is_host(buf)) { + return nullptr; + } + + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf); + + ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft); + + if (!dev || buft != ggml_backend_dev_buffer_type(dev)) { + return nullptr; + } + + auto it = devs.find(dev); + + if (it == devs.end()) { + return nullptr; + } + + it->second.pending = true; + + return it->second.backend.get(); + } + + void record() { + + for (auto & it : devs) { + if (it.second.pending) { + ggml_backend_event_record(it.second.event, it.second.backend.get()); + } + } + } + + // order the copies behind the compute already queued on each device: the copy stream waits for the context's fence, recorded at the end of every decode + void order_after(const std::map<ggml_backend_dev_t, ggml_backend_event_t> & fences) { + for (auto & it : devs) { + const auto fence = fences.find(it.first); + + if (fence != fences.end()) { + ggml_backend_event_wait(it.second.backend.get(), fence->second); + } + } + } + + // order the context's compute behind the copies just recorded, for a restore only: its copies write KV cells while other sequences read every cell up to n_kv + void order_before(const std::vector<ggml_backend_ptr> & compute) { + for (auto & it : devs) { + if (!it.second.pending) { + continue; + } + + for (const auto & backend : compute) { + if (ggml_backend_get_device(backend.get()) == it.first) { + ggml_backend_event_wait(backend.get(), it.second.event); + } + } + } + } + + bool done() { + bool res = true; + + for (auto & it : devs) { + if (!it.second.pending) { + continue; + } + + if (ggml_backend_event_query(it.second.event)) { + it.second.pending = false; + } else { + res = false; + } + } + + return res; + } + + void wait() { + for (auto & it : devs) { + if (!it.second.pending) { + continue; + } + + ggml_backend_event_synchronize(it.second.event); + + it.second.pending = false; + } + } + + // grow-only: pinning host memory costs about as long as the copy it is for, and a caller parking the same sequence asks for a slightly different size each time + uint8_t * buf_resize(size_t size_new) { + if (size_new <= capacity) { + size = size_new; + + return size_new == 0 ? nullptr : data; + } + + // never move memory a copy could still be reading or writing + wait(); + + host_buf.reset(); + + data = nullptr; + size = 0; + capacity = 0; + pinned = false; + + ggml_backend_buffer_type_t host_buft = host_buffer_type(); + + ggml_backend_buffer_t buf = ggml_backend_buft_alloc_buffer(host_buft, size_new); + + if (!buf) { + return nullptr; + } + + uint8_t * base = (uint8_t *) ggml_backend_buffer_get_base(buf); + + if (!base) { + ggml_backend_buffer_free(buf); + return nullptr; + } + + host_buf.reset(buf); + + data = base; + size = size_new; + capacity = size_new; + // a host buffer type may quietly hand back ordinary memory when pinning is off, so believe the buffer that came back rather than the type + pinned = can_pin && ggml_backend_buffer_get_type(buf) == host_buft; + + return data; + } + + void buf_free() { + wait(); + + host_buf.reset(); + + data = nullptr; + size = 0; + capacity = 0; + pinned = false; + } + + ggml_backend_buffer_type_t host_buffer_type() { + for (auto & it : devs) { + ggml_backend_buffer_type_t buft = ggml_backend_dev_host_buffer_type(it.first); + + if (buft) { + return buft; + } + } + + return ggml_backend_cpu_buffer_type(); + } +}; + +// [TAG_STATE_ASYNC] the buffer walk of llama_io_write_host, with the copies posted on the transfer's stream instead of made here +class llama_io_write_host_async : public llama_io_write_host { +public: + llama_io_write_host_async(uint8_t * p, size_t len, llama_state_seq_copy & cpy) : + llama_io_write_host(p, len, true), cpy(cpy) {} + + // posted from the destructor, and only once serialisation reached the end: a caller told of a partial failure by a zero return is free to reuse the buffer at once + void commit() { + committed = true; + } + + ~llama_io_write_host_async() { + if (!committed) { + return; + } + + llama_io_emit(winfos, 0, winfos.size(), + [this](ggml_tensor * tensor, uint8_t * ptr, size_t offset, size_t size, + size_t n_copies, size_t stride_tensor, size_t stride_data) { + llama_io_get(cpy.backend_for(tensor), tensor, ptr, offset, size, + n_copies, stride_tensor, stride_data); + + cpy.n_copies++; + }); + + cpy.record(); + } + +private: + llama_state_seq_copy & cpy; + + bool committed = false; +}; + +// [TAG_STATE_ASYNC] the read half of the same, without llama_io_read_host's whole-tensor staging: a write-back would undo whatever the sequences sharing the tensor wrote while these copies ran +class llama_io_read_host_async : public llama_io_read_host { +public: + llama_io_read_host_async(const uint8_t * p, size_t len, llama_state_seq_copy & cpy) : + llama_io_read_host(p, len, true), cpy(cpy) {} + + // see llama_io_write_host_async::commit(): a restore that failed part way has dropped the sequence, and copies posted for it would write cells that are no longer its own + void commit() { + committed = true; + } + + ~llama_io_read_host_async() { + if (!committed) { + return; + } + + llama_io_emit(rinfos, 0, rinfos.size(), + [this](ggml_tensor * tensor, const uint8_t * ptr, size_t offset, size_t size, + size_t n_copies, size_t stride_tensor, size_t stride_data) { + llama_io_set(cpy.backend_for(tensor), tensor, ptr, offset, size, + n_copies, stride_tensor, stride_data); + + cpy.n_copies++; + }); + + cpy.record(); + } + +private: + llama_state_seq_copy & cpy; + + bool committed = false; +}; + static constexpr uint32_t io_magic = 0xaf143cd8; size_t llama_context::state_seq_get_size(llama_seq_id seq_id, llama_state_seq_flags flags) { @@ -3135,6 +3675,237 @@ size_t llama_context::state_seq_set_data(llama_seq_id seq_id, const uint8_t * sr } } +// [TAG_STATE_ASYNC] + +void llama_context::state_seq_copies_drain() { + for (auto * cpy : state_copies) { + cpy->wait(); + cpy->ctx = nullptr; + cpy->counted = false; + } + + state_copies.clear(); +} + +void llama_context::state_seq_copy_release(llama_state_seq_copy * cpy) { + GGML_ASSERT(state_copies.erase(cpy) == 1); + + if (state_copies.empty()) { + for (auto & it : state_copy_fences) { + ggml_backend_event_free(it.second); + } + + state_copy_fences.clear(); + } +} + +void llama_context::state_seq_copy_fence() { + for (const auto & it : state_copy_fences) { + for (const auto & backend : backends) { + if (ggml_backend_get_device(backend.get()) == it.first) { + ggml_backend_event_record(it.second, backend.get()); + } + } + } +} + +llama_state_seq_copy * llama_context::state_seq_copy_init() { + std::unique_ptr<llama_state_seq_copy> cpy(new llama_state_seq_copy()); + + cpy->ctx = this; + + for (auto & backend : backends) { + ggml_backend_dev_t dev = ggml_backend_get_device(backend.get()); + + if (!dev || cpy->devs.find(dev) != cpy->devs.end()) { + continue; + } + + ggml_backend_dev_props props; + ggml_backend_dev_get_props(dev, &props); + + if (!props.caps.async || !props.caps.events) { + continue; + } + + // a device that advertises events but does not implement event_query makes the first poll wait for the whole copy, so leave it out and let state_seq_copy_init() return NULL + if (!ggml_backend_dev_supports_event_query(dev)) { + static std::atomic<bool> warned(false); + + if (!warned.exchange(true)) { + LLAMA_LOG_INFO("%s: %s cannot test an event without waiting for it, so sequence " + "states are copied synchronously\n", __func__, ggml_backend_dev_name(dev)); + } + + continue; + } + + // a backend of its own, not the one the graphs are computed on: that one moves its copies to whichever stream it is using, so a transfer could end up ordered behind a graph + ggml_backend_t backend_cpy = ggml_backend_dev_init(dev, nullptr); + + if (!backend_cpy) { + continue; + } + + ggml_backend_event_t event = ggml_backend_event_new(dev); + + if (!event) { + ggml_backend_free(backend_cpy); + continue; + } + + auto & dc = cpy->devs[dev]; + + dc.backend.reset(backend_cpy); + dc.event = event; + } + + if (cpy->devs.empty()) { + return nullptr; + } + + // the devices above are the ones the graphs run on, not the ones the state lives on: with most layers on the CPU every copy takes the synchronous branch of backend_for() + if (memory) { + bool on_device = false; + + for (const auto & [buft, size] : memory->memory_breakdown()) { + if (size == 0) { + continue; + } + + ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft); + + if (ggml_backend_buft_is_host(buft) || !dev || buft != ggml_backend_dev_buffer_type(dev) || + cpy->devs.find(dev) == cpy->devs.end()) { + LLAMA_LOG_INFO("%s: the sequence state is not all in device memory (%s), so it is copied synchronously\n", + __func__, ggml_backend_buft_name(buft)); + return nullptr; + } + + on_device = true; + } + + if (!on_device) { + return nullptr; + } + } + + cpy->can_pin = cpy->host_buffer_type() != ggml_backend_cpu_buffer_type(); + + // one fence per device, shared by every transfer and recorded after every decode; installed only after the checks above, so a refused transfer leaves nothing behind + std::vector<ggml_backend_dev_t> fences_new; + + for (const auto & it : cpy->devs) { + if (state_copy_fences.find(it.first) != state_copy_fences.end()) { + continue; + } + + ggml_backend_event_t fence = ggml_backend_event_new(it.first); + + if (!fence) { + for (auto dev : fences_new) { + ggml_backend_event_free(state_copy_fences[dev]); + state_copy_fences.erase(dev); + } + + return nullptr; + } + + state_copy_fences[it.first] = fence; + fences_new.push_back(it.first); + } + + state_seq_copy_fence(); + + state_copies.insert(cpy.get()); + cpy->counted = true; + + return cpy.release(); +} + +size_t llama_context::state_seq_copy_get(llama_state_seq_copy & cpy, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags) { + // the library owns this buffer, so the extent can be checked instead of believed: every bounds check validates against it, so an oversized one agrees and the copy overruns + if (!cpy.data || size == 0 || size > cpy.size) { + LLAMA_LOG_ERROR("%s: cannot cover %zu bytes, the transfer's buffer holds %zu\n", __func__, size, cpy.size); + return 0; + } + + // LLAMA_STATE_SEQ_FLAGS_ON_DEVICE has nowhere to leave the data here, and get_size_ext() with that flag reports a metadata-sized state, so the two cannot be paired + if (flags & LLAMA_STATE_SEQ_FLAGS_ON_DEVICE) { + LLAMA_LOG_ERROR("%s: LLAMA_STATE_SEQ_FLAGS_ON_DEVICE is not supported here, the copies go through host memory\n", __func__); + return 0; + } + + const int64_t t_sync = ggml_time_us(); + cpy.order_after(state_copy_fences); + cpy.t_sync_us = ggml_time_us() - t_sync; + + cpy.n_copies = 0; + + llama_io_write_host_async io(cpy.data, size, cpy); + + try { + io.write(&io_magic, sizeof(io_magic)); + io.write(&seq_id, sizeof(seq_id)); + + const size_t n = state_seq_write_data(io, seq_id, flags); + + io.commit(); + + return n; + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what()); + return 0; + } +} + +size_t llama_context::state_seq_copy_set(llama_state_seq_copy & cpy, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags) { + if (!cpy.data || size == 0 || size > cpy.size) { + LLAMA_LOG_ERROR("%s: cannot cover %zu bytes, the transfer's buffer holds %zu\n", __func__, size, cpy.size); + return 0; + } + + if (flags & LLAMA_STATE_SEQ_FLAGS_ON_DEVICE) { + LLAMA_LOG_ERROR("%s: LLAMA_STATE_SEQ_FLAGS_ON_DEVICE is not supported here, the copies go through host memory\n", __func__); + return 0; + } + + // the cells this restore was given may still be read, masked, by a graph in flight, so the copy stream waits for the compute stream on the device, see order_after() + const int64_t t_sync = ggml_time_us(); + cpy.order_after(state_copy_fences); + cpy.t_sync_us = ggml_time_us() - t_sync; + + cpy.n_copies = 0; + + size_t n = 0; + + { + llama_io_read_host_async io(cpy.data, size, cpy); + + try { + uint32_t magic_read; + io.read(&magic_read, sizeof(magic_read)); + if (io_magic != magic_read) { + throw std::runtime_error("wrong sequence state magic"); + } + + llama_seq_id seq_id_read; + io.read(&seq_id_read, sizeof(seq_id_read)); + + n = state_seq_read_data(io, seq_id, flags); + + io.commit(); + } catch (const std::exception & err) { + LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what()); + return 0; + } + } + + cpy.order_before(backends); + + return n; +} + bool llama_context::state_load_file(const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { llama_file file(filepath, "rb"); @@ -3290,6 +4061,11 @@ size_t llama_context::state_write_data(llama_io_write_i & io) { } size_t llama_context::state_read_data(llama_io_read_i & io) { + // [TAG_EXACT_CONCURRENCY] a whole-context restore writes cells at their recorded physical index, which the paged pool owns; refused before anything is parsed + if (memory && memory->alloc_granularity() > 1) { + throw std::runtime_error("whole-context restore is not supported with LLAMA_EXACT_CONCURRENCY, restore per sequence"); + } + LLAMA_LOG_DEBUG("%s: reading state\n", __func__); // read model info @@ -4114,6 +4890,22 @@ bool llama_memory_can_shift(llama_memory_t mem) { return mem->get_can_shift(); } +uint32_t llama_memory_alloc_granularity(llama_memory_t mem) { + if (!mem) { + return 1; + } + + return mem->alloc_granularity(); +} + +bool llama_memory_update(llama_context * ctx) { + if (!ctx) { + return false; + } + + return ctx->memory_update(false); +} + // llama state API // deprecated @@ -4209,6 +5001,66 @@ size_t llama_state_seq_set_data_ext(llama_context * ctx, const uint8_t * src, si return ctx->state_seq_set_data(seq_id, src, size, flags); } +llama_state_seq_copy * llama_state_seq_copy_init(llama_context * ctx) { + return ctx->state_seq_copy_init(); +} + +void llama_state_seq_copy_free(llama_state_seq_copy * cpy) { + delete cpy; // waits for anything still in flight +} + +uint8_t * llama_state_seq_copy_buf_resize(llama_state_seq_copy * cpy, size_t size) { + return cpy->buf_resize(size); +} + +uint8_t * llama_state_seq_copy_buf(llama_state_seq_copy * cpy) { + return cpy->data; +} + +size_t llama_state_seq_copy_buf_size(llama_state_seq_copy * cpy) { + return cpy->size; +} + +size_t llama_state_seq_copy_buf_capacity(llama_state_seq_copy * cpy) { + return cpy->capacity; +} + +size_t llama_state_seq_copy_n_copies(llama_state_seq_copy * cpy) { + return cpy->n_copies; +} + +int64_t llama_state_seq_copy_sync_us(llama_state_seq_copy * cpy) { + return cpy->t_sync_us; +} + +void llama_state_seq_copy_buf_free(llama_state_seq_copy * cpy) { + cpy->buf_free(); +} + +bool llama_state_seq_copy_buf_is_pinned(llama_state_seq_copy * cpy) { + return cpy->pinned; +} + +bool llama_state_seq_copy_buf_can_pin(llama_state_seq_copy * cpy) { + return cpy->can_pin; +} + +size_t llama_state_seq_copy_get(llama_state_seq_copy * cpy, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags) { + return cpy->ctx->state_seq_copy_get(*cpy, size, seq_id, flags); +} + +size_t llama_state_seq_copy_set(llama_state_seq_copy * cpy, size_t size, llama_seq_id dest_seq_id, llama_state_seq_flags flags) { + return cpy->ctx->state_seq_copy_set(*cpy, size, dest_seq_id, flags); +} + +bool llama_state_seq_copy_done(llama_state_seq_copy * cpy) { + return cpy->done(); +} + +void llama_state_seq_copy_wait(llama_state_seq_copy * cpy) { + cpy->wait(); +} + size_t llama_state_seq_save_file(llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) { ctx->synchronize(); diff --git a/src/llama-context.h b/src/llama-context.h index bf91daa8b562..f44f505a05f3 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -12,6 +12,7 @@ #include "ggml-opt.h" #include <map> +#include <set> #include <vector> struct llama_model; @@ -39,6 +40,8 @@ struct llama_memory_buffer { using llama_memory_buffers = std::map<ggml_backend_buffer_type_t, llama_memory_buffer>; +struct llama_state_seq_copy; + struct llama_context { // init scheduler and compute buffers, reserve worst-case graphs llama_context( @@ -156,6 +159,19 @@ struct llama_context { size_t state_seq_get_data(llama_seq_id seq_id, uint8_t * dst, size_t size, llama_state_seq_flags flags); size_t state_seq_set_data(llama_seq_id seq_id, const uint8_t * src, size_t size, llama_state_seq_flags flags); + // [TAG_STATE_ASYNC] the same two transfers, issued on a stream of their own and left running + llama_state_seq_copy * state_seq_copy_init(); + + size_t state_seq_copy_get(llama_state_seq_copy & cpy, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags); + size_t state_seq_copy_set(llama_state_seq_copy & cpy, size_t size, llama_seq_id dest_seq_id, llama_state_seq_flags flags); + + // [TAG_STATE_ASYNC] mark the point the compute streams have reached, for the copies to wait for; recorded after every decode and encode once a transfer exists + void state_seq_copy_fence(); + + void state_seq_copy_release(llama_state_seq_copy * cpy); + + void state_seq_copies_drain(); + bool state_load_file( const char * filepath, llama_token * tokens_out, @@ -348,6 +364,12 @@ struct llama_context { ggml_backend_t backend_cpu = nullptr; std::vector<ggml_backend_ptr> backends; + // [TAG_STATE_ASYNC] one event per device that copies asynchronously, recorded on the compute stream at the end of every decode; see state_seq_copy_fence() + std::map<ggml_backend_dev_t, ggml_backend_event_t> state_copy_fences; + + // transfers alive on this context; the fences go when the last one does, and a context freed with transfers still alive drains them and lets them go first + std::set<llama_state_seq_copy *> state_copies; + // training ggml_opt_context_t opt_ctx = nullptr; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 5855393ef7cc..01d1d35d3818 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -21,11 +21,25 @@ #include <cstring> #include <numeric> #include <sstream> +#include <stdexcept> #include <string> #include <unordered_set> // dedup helpers +// [TAG_EXACT_CONCURRENCY] the page table is wired into llm_graph_input_attn_kv only, so a V-less layout would attend in physical order with the mode reporting itself on +static void llm_graph_reject_exact_concurrency(const char * layout) { + if (!llama_exact_concurrency()) { + return; + } + + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set, but this model uses the %s attention " + "layout, which carries no page table and would attend in physical cell order\n", + __func__, layout); + + throw std::runtime_error("exact concurrency: unsupported attention layout"); +} + static ggml_tensor * build_attn_inp_kq_mask( ggml_context * ctx, const llama_kv_cache_context * mctx, @@ -468,6 +482,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) { } void llm_graph_input_attn_kv::set_input(const llama_ubatch * ubatch) { + if (self_pages && self_pages->buffer) { mctx->set_input_pages(self_pages, ubatch); } mctx->set_input_k_idxs(self_k_idxs, ubatch); mctx->set_input_v_idxs(self_v_idxs, ubatch); @@ -1087,6 +1102,7 @@ void llm_graph_input_attn_cross::set_input(const llama_ubatch * ubatch) { } void llm_graph_input_mem_hybrid::set_input(const llama_ubatch * ubatch) { + if (inp_attn->self_pages) { mctx->get_attn()->set_input_pages(inp_attn->self_pages, ubatch); } mctx->get_attn()->set_input_k_idxs(inp_attn->self_k_idxs, ubatch); mctx->get_attn()->set_input_v_idxs(inp_attn->self_v_idxs, ubatch); @@ -2598,7 +2614,8 @@ ggml_tensor * llm_graph_context::build_attn_mha( ggml_tensor * v_mla, int64_t n_kv_max, float kq_scale, - int il) const { + int il, + ggml_tensor * pages) const { const bool v_trans = v->nb[1] > v->nb[2]; // split the batch into streams if needed @@ -2631,6 +2648,7 @@ ggml_tensor * llm_graph_context::build_attn_mha( cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias, hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f); + cur->src[5] = pages; res->add_fused_node({LLM_FUSED_OP_FLASH_ATTN, cur, il}); ggml_flash_attn_ext_add_sinks(cur, sinks); @@ -2822,6 +2840,8 @@ static std::unique_ptr<llm_graph_input_attn_kv> build_attn_inp_kv_impl( inp->self_kq_mask_cnv = inp->self_kq_mask; } + inp->self_pages = mctx_cur->build_input_pages(ctx0, ubatch); + GGML_ASSERT(!inp->self_pages || (cparams.flash_attn && cparams.causal_attn)); inp->self_k_rot = mctx_cur->build_input_k_rot(ctx0); inp->self_v_rot = mctx_cur->build_input_v_rot(ctx0); @@ -2884,7 +2904,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_tensor * k = mctx_cur->get_k(ctx0, il); ggml_tensor * v = mctx_cur->get_v(ctx0, il); - ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il); + ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il, inp->self_pages); cb(cur, "kqv_out", il); if (inp->self_v_rot) { @@ -2918,6 +2938,8 @@ static std::unique_ptr<llm_graph_input_attn_k> build_attn_inp_k_impl( const llama_cparams & cparams, const llama_kv_cache_context * mctx_cur) { + llm_graph_reject_exact_concurrency("V-less KV (attn_k)"); + auto inp = std::make_unique<llm_graph_input_attn_k>(hparams, cparams, mctx_cur); { @@ -3294,6 +3316,8 @@ static std::unique_ptr<llm_graph_input_attn_k_dsa> build_attn_inp_k_dsa_impl( const llama_cparams & cparams, const llama_kv_cache_dsa_context * mctx_cur) { + llm_graph_reject_exact_concurrency("sparse V-less KV (attn_k_dsa)"); + auto inp = std::make_unique<llm_graph_input_attn_k_dsa>(hparams, cparams, mctx_cur); { @@ -3411,6 +3435,8 @@ llm_graph_input_attn_kv_iswa * llm_graph_context::build_attn_inp_kv_iswa() const llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { const auto * mctx_cur = static_cast<const llama_kv_cache_iswa_context *>(mctx); + llm_graph_reject_exact_concurrency("V-less sliding window KV (attn_k_iswa)"); + auto inp = std::make_unique<llm_graph_input_attn_k_iswa>(hparams, cparams, mctx_cur); { diff --git a/src/llama-graph.h b/src/llama-graph.h index cc4110639d4b..0c1e139b3529 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -320,6 +320,7 @@ class llm_graph_input_attn_no_cache : public llm_graph_input_i { class llm_graph_input_attn_kv : public llm_graph_input_i { public: + ggml_tensor * self_pages = nullptr; // I32 [1 + physical pages, n_tokens] llm_graph_input_attn_kv( const llama_hparams & hparams, const llama_cparams & cparams, @@ -1187,7 +1188,8 @@ struct llm_graph_context { ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v] int64_t n_kv_max, float kq_scale, - int il) const; + int il, + ggml_tensor * pages = nullptr) const; llm_graph_input_attn_no_cache * build_attn_inp_no_cache() const; diff --git a/src/llama-impl.cpp b/src/llama-impl.cpp index b3a94b946d28..c6e6d356655a 100644 --- a/src/llama-impl.cpp +++ b/src/llama-impl.cpp @@ -1,11 +1,15 @@ #include "llama-impl.h" +#include "ggml-backend.h" #include "gguf.h" #include "llama.h" #include <cinttypes> #include <climits> +#include <atomic> +#include <mutex> #include <cstdarg> +#include <cstdlib> #include <cstring> #include <vector> #include <sstream> @@ -169,3 +173,162 @@ std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) { return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0); } } + +// [TAG_EXACT_CONCURRENCY] +bool llama_exact_backend_name(const char * reg_name) { + return reg_name && (strcmp(reg_name, "CUDA") == 0 || strcmp(reg_name, "ROCm") == 0 || strcmp(reg_name, "MUSA") == 0); +} + +// [TAG_EXACT_CONCURRENCY] a host buffer is the interesting case: the scheduler runs an operation on +// the backend holding its weight, and moves a host weight's operation to the GPU only once the batch +// is wide enough (ggml_backend_cuda_device_offload_op), while the CPU matmul picks between its SGEMM +// and its vector dot by the batch width too. +bool llama_exact_buft_invariant(ggml_backend_buffer_type_t buft) { + if (!buft || ggml_backend_buft_is_host(buft)) { + return false; + } + + ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + + return reg && llama_exact_backend_name(ggml_backend_reg_name(reg)); +} + +bool llama_exact_concurrency() { + static const bool enabled = []() { + const char * val = getenv("LLAMA_EXACT_CONCURRENCY"); + return val && atoi(val) != 0; + }(); + + return enabled; +} + +// [TAG_EXACT_CONCURRENCY] tokens one sequence contributes to a decode step, see llama.h +static std::atomic<uint32_t> g_exact_decode_tokens{1}; + +// one lock for the token figure, the sequence count and the width: a report interleaved with a change of figure could leave the backend with a width that covers neither +static std::recursive_mutex g_exact_mutex; + +// the most sequences any context was created with; the tokens figure is process wide, so raising it re-reports every context's width +static std::atomic<uint32_t> g_exact_max_n_seq{0}; + +static bool llama_exact_width_within_explicit_bound(uint32_t n_cols); + +static bool llama_exact_width_of(uint32_t n_seq, uint32_t n_tokens, uint32_t & n_cols) { + const uint64_t w = (uint64_t) n_seq * (uint64_t) n_tokens; + + if (w > (uint64_t) INT32_MAX) { + LLAMA_LOG_ERROR("%s: a decode step of %u sequences with %u tokens each is too wide to report\n", __func__, n_seq, n_tokens); + return false; + } + + n_cols = (uint32_t) w; + + return true; +} + +bool llama_exact_check_n_seq(uint32_t n_seq) { + std::lock_guard<std::recursive_mutex> lock(g_exact_mutex); + + const uint32_t n_seq_max = std::max(n_seq, g_exact_max_n_seq.load(std::memory_order_relaxed)); + + uint32_t n_cols = 0; + + return llama_exact_width_of(n_seq_max, llama_exact_decode_tokens(), n_cols) && llama_exact_width_within_explicit_bound(n_cols); +} + +bool llama_exact_report_n_seq(uint32_t n_seq) { + std::lock_guard<std::recursive_mutex> lock(g_exact_mutex); + + const uint32_t n_seq_max = std::max(n_seq, g_exact_max_n_seq.load(std::memory_order_relaxed)); + + uint32_t n_cols = 0; + + if (!llama_exact_width_of(n_seq_max, llama_exact_decode_tokens(), n_cols) || !llama_set_exact_decode_width(n_cols)) { + return false; + } + + uint32_t cur = g_exact_max_n_seq.load(std::memory_order_relaxed); + + while (n_seq > cur && !g_exact_max_n_seq.compare_exchange_weak(cur, n_seq, std::memory_order_relaxed)) { + } + + return true; +} + +bool llama_set_exact_decode_tokens(uint32_t n_tokens) { + n_tokens = n_tokens > 0 ? n_tokens : 1; + + std::lock_guard<std::recursive_mutex> lock(g_exact_mutex); + + // never lowered: a narrower context set up later would turn an existing speculative context's verify steps into prompts + if (n_tokens <= g_exact_decode_tokens.load(std::memory_order_relaxed)) { + return true; + } + + // every context widens with the figure, so report the width first; one the explicit bound cannot cover leaves the old figure in place + const uint32_t n_seq = g_exact_max_n_seq.load(std::memory_order_relaxed); + + uint32_t n_cols = 0; + + if (n_seq > 0 && (!llama_exact_width_of(n_seq, n_tokens, n_cols) || !llama_set_exact_decode_width(n_cols))) { + return false; + } + + g_exact_decode_tokens.store(n_tokens, std::memory_order_relaxed); + + return true; +} + +uint32_t llama_exact_decode_tokens(void) { + return g_exact_decode_tokens.load(std::memory_order_relaxed); +} + +// [TAG_EXACT_CONCURRENCY] the widest decode ubatch reported so far, see llama.h; reached through the registry so an absent or late-loaded backend costs nothing +static std::atomic<uint32_t> g_exact_decode_width{0}; + +// an explicit column bound wins in the CUDA backend, so a width above it would leave decodes batched past the bound +static bool llama_exact_width_within_explicit_bound(uint32_t n_cols) { + static const int explicit_cols = []() { + const char * val = getenv("GGML_CUDA_BATCH_INVARIANT_MAX_COLS"); + return val ? atoi(val) : -1; + }(); + + if (explicit_cols > 0 && (uint32_t) explicit_cols < n_cols) { + LLAMA_LOG_ERROR("%s: GGML_CUDA_BATCH_INVARIANT_MAX_COLS is %d but LLAMA_EXACT_CONCURRENCY needs at least %u columns for the decode step just requested; raise it, set it to 0 for no bound, or unset it\n", + __func__, explicit_cols, n_cols); + return false; + } + + return true; +} + +bool llama_set_exact_decode_width(uint32_t n_cols) { + if (!llama_exact_width_within_explicit_bound(n_cols)) { + return false; + } + + std::lock_guard<std::recursive_mutex> lock(g_exact_mutex); + + uint32_t cur = g_exact_decode_width.load(std::memory_order_relaxed); + + while (n_cols > cur && !g_exact_decode_width.compare_exchange_weak(cur, n_cols, std::memory_order_relaxed)) { + } + + const uint32_t widest = g_exact_decode_width.load(std::memory_order_relaxed); + + for (size_t i = 0; i < ggml_backend_reg_count(); ++i) { + ggml_backend_reg_t reg = ggml_backend_reg_get(i); + + auto * fn = (void (*)(int)) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cuda_set_exact_decode_width"); + if (fn) { + fn((int) widest); + } + } + + return true; +} + +uint32_t llama_exact_decode_width(void) { + return g_exact_decode_width.load(std::memory_order_relaxed); +} diff --git a/src/llama-impl.h b/src/llama-impl.h index 4988b06d2ca0..dc4e21eb8114 100644 --- a/src/llama-impl.h +++ b/src/llama-impl.h @@ -1,6 +1,7 @@ #pragma once #include "ggml.h" // for ggml_log_level +#include "ggml-backend.h" #include <string> #include <type_traits> @@ -103,3 +104,17 @@ std::string llama_format_tensor_shape(const std::vector<int64_t> & ne); std::string llama_format_tensor_shape(const struct ggml_tensor * t); std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i); + +// [TAG_EXACT_CONCURRENCY] opt-in mode under which a sequence's attention depends only on its own cells, so its output does not change when others share the KV cache +bool llama_exact_concurrency(); + +// [TAG_EXACT_CONCURRENCY] whether a backend registry carries the mode's batch-invariant kernels +bool llama_exact_backend_name(const char * reg_name); + +// [TAG_EXACT_CONCURRENCY] whether a tensor placed in this buffer type is computed by such a backend +bool llama_exact_buft_invariant(ggml_backend_buffer_type_t buft); + +// [TAG_EXACT_CONCURRENCY] a context reports how many sequences it was created with, so the backend knows the width every context needs +bool llama_exact_report_n_seq(uint32_t n_seq); + +bool llama_exact_check_n_seq(uint32_t n_seq); diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index a342ee1191d4..c75aa6633a4c 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -11,6 +11,7 @@ #include <cstring> #include <limits> #include <map> +#include <set> #include <stdexcept> #include <unordered_map> @@ -62,6 +63,68 @@ static void ggml_gen_hadamard(ggml_tensor * tensor) { // llama_kv_cache // +// [TAG_EXACT_CONCURRENCY] the paged specialization lives in the CUDA sources; every other backend ignores src[5] and walks the pool in physical cell order +static bool llama_dev_has_paged_attn(ggml_backend_dev_t dev) { + if (!dev) { + return false; + } + + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); + if (!reg) { + return false; + } + + return llama_exact_backend_name(ggml_backend_reg_name(reg)); +} + +// [TAG_EXACT_CONCURRENCY] whether the device can actually run the paged attention op for a layer of this shape: the registry name only says which backends carry the kernels +static bool llama_dev_supports_paged_attn( + ggml_backend_dev_t dev, + ggml_type type_k, ggml_type type_v, + uint32_t n_embd_head_k, uint32_t n_embd_head_v, + uint32_t n_head, uint32_t n_head_kv, + uint32_t n_cells, uint32_t page_size) { + if (!llama_dev_has_paged_attn(dev)) { + return false; + } + + ggml_init_params ip = { + /*.mem_size =*/ ggml_tensor_overhead()*16 + ggml_graph_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + + ggml_context * ctx = ggml_init(ip); + if (!ctx) { + return false; + } + + bool res = true; + + const int64_t n_kv = page_size; + + for (const int64_t n_tokens : { (int64_t) 1, (int64_t) 4, (int64_t) 16, (int64_t) 512 }) { + ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_embd_head_k, n_tokens, n_head, 1); + ggml_tensor * k = ggml_new_tensor_4d(ctx, type_k, n_embd_head_k, n_kv, n_head_kv, 1); + ggml_tensor * v = ggml_new_tensor_4d(ctx, type_v, n_embd_head_v, n_kv, n_head_kv, 1); + ggml_tensor * m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, n_kv, n_tokens, 1, 1); + + ggml_tensor * op = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf((float) n_embd_head_k), 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(op, GGML_PREC_F32); + + op->src[5] = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, 1 + n_cells/page_size, n_tokens); + + if (!ggml_backend_dev_supports_op(dev, op)) { + res = false; + break; + } + } + + ggml_free(ctx); + + return res; +} + llama_kv_cache::llama_kv_cache( const llama_model & model, const llama_hparams & hparams, @@ -86,6 +149,9 @@ llama_kv_cache::llama_kv_cache( v_cells_impl(other ? other->v_cells_impl : std::make_shared<llama_kv_cells_vec>()), v_cells(*v_cells_impl) { + // [TAG_EXACT_CONCURRENCY] read the knob through the same cached reader the graph and the CUDA dispatcher use, so a mid-process change cannot leave them disagreeing + exact_pages = llama_exact_concurrency(); + // shared cells view the source cache's K/V tensors, so the cell count // follows the source allocation: a fitted target can be smaller than the // draft default and oversized views would overflow the source tensors @@ -99,6 +165,27 @@ llama_kv_cache::llama_kv_cache( GGML_ASSERT(kv_size % n_pad == 0); + if (exact_pages) { + const char * unsupported = nullptr; + + if (!unified) { + unsupported = "it needs a unified KV cache (pass --kv-unified)"; + } else if (v_trans) { + unsupported = "it needs a non-transposed V cache (pass --flash-attn on)"; + } else if (n_swa != 0) { + unsupported = "the paged pool does not support sliding window attention"; + } else if (type_k != GGML_TYPE_F16 || type_v != GGML_TYPE_F16) { + unsupported = "it needs an F16 KV cache (do not pass --cache-type-k or --cache-type-v)"; + } else if (kv_size % exact_page_size != 0) { + unsupported = "the context size must be a multiple of 256 (pass -c as a multiple of 256)"; + } + + if (unsupported) { + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set but %s\n", __func__, unsupported); + throw std::runtime_error("exact concurrency: unsupported KV cache configuration"); + } + } + const uint32_t n_layer = hparams.n_layer_all; // define a comparator for the buft -> ctx map to ensure that the order is well-defined: @@ -222,6 +309,39 @@ llama_kv_cache::llama_kv_cache( LLAMA_LOG_DEBUG("%s: layer %3d: dev = %s\n", __func__, il, dev_name); + // [TAG_EXACT_CONCURRENCY] the paged kernel handles 256-wide K and V heads only; any other width would run unpaged while the mode reports itself as on + if (exact_pages && (hparams.n_embd_head_k(il) != 256 || (!is_mla && hparams.n_embd_head_v(il) != 256) || is_mla)) { + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set but layer %d has %u-wide K heads and %u-wide V heads%s, " + "and the paged attention kernel supports 256-wide K and V heads only\n", + __func__, il, hparams.n_embd_head_k(il), hparams.n_embd_head_v(il), is_mla ? " (MLA)" : ""); + throw std::runtime_error("exact concurrency: unsupported attention head size"); + } + + if (exact_pages && hparams.attn_soft_cap) { + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set but this model soft-caps its attention logits (%.1f), " + "which the paged attention kernel does not apply\n", __func__, hparams.f_attn_logit_softcapping); + throw std::runtime_error("exact concurrency: attention soft cap is not supported"); + } + + if (exact_pages && !(offload && llama_dev_has_paged_attn(model.dev_layer(il)))) { + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set but layer %d keeps its KV cache on %s, " + "which has no paged attention: every layer must be offloaded to the CUDA backend " + "(pass -ngl to offload all layers and do not pass --no-kv-offload)\n", + __func__, il, dev_name); + throw std::runtime_error("exact concurrency: KV cache layer is not on the CUDA backend"); + } + + // [TAG_EXACT_CONCURRENCY] right backend; ask whether this layer's attention, with the page table attached, lands on one of its kernels at all + if (exact_pages && !llama_dev_supports_paged_attn(model.dev_layer(il), type_k, type_v, + hparams.n_embd_head_k(il), hparams.n_embd_head_v(il), + hparams.n_head(il), hparams.n_head_kv(il), kv_size, exact_page_size)) { + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set but %s cannot run the paged attention for layer %d " + "(K %s, V %s, %u-wide heads): the build or the device has no flash attention kernel for it, " + "and the op would fall to the CPU, which ignores the page table\n", + __func__, dev_name, il, ggml_type_name(type_k), ggml_type_name(type_v), hparams.n_embd_head_k(il)); + throw std::runtime_error("exact concurrency: the device cannot run the paged attention"); + } + ggml_context * ctx = ctx_for_buft(buft); if (!ctx) { throw std::runtime_error("failed to create ggml context for kv cache"); @@ -366,7 +486,99 @@ llama_kv_cache::llama_kv_cache( debug = LLAMA_KV_CACHE_DEBUG ? atoi(LLAMA_KV_CACHE_DEBUG) : 0; } +void llama_kv_cache::exact_pages_rebuild() const { + const auto & cells = v_cells[0]; + + exact_page_owner.assign(cells.size()/exact_page_size, exact_page{}); + exact_page_live .assign(cells.size()/exact_page_size, 0); + + for (uint32_t i = 0; i < cells.size(); ++i) { + if (cells.is_empty(i)) { + continue; + } + + GGML_ASSERT(cells.seq_count(i) == 1); + + const auto pos = cells.pos_get(i); + + GGML_ASSERT(pos >= 0 && uint32_t(pos)%exact_page_size == i%exact_page_size); + + const exact_page cur { cells.seq_get(i), llama_pos(pos/(llama_pos) exact_page_size) }; + + auto & owner = exact_page_owner[i/exact_page_size]; + + GGML_ASSERT(owner.seq < 0 || (owner.seq == cur.seq && owner.lpg == cur.lpg)); + + owner = cur; + + ++exact_page_live[i/exact_page_size]; + } + + exact_page_owner_dirty = false; +} + +void llama_kv_cache::exact_pages_sync() const { + if (exact_page_owner_dirty) { + exact_pages_rebuild(); + + return; + } + + if (debug > 0) { + // what was maintained has to say what the cells say + const auto kept = exact_page_owner; + const auto kept_live = exact_page_live; + + exact_pages_rebuild(); + + GGML_ASSERT(kept.size() == exact_page_owner.size()); + + for (size_t p = 0; p < kept.size(); ++p) { + GGML_ASSERT(kept[p].seq == exact_page_owner[p].seq && kept[p].lpg == exact_page_owner[p].lpg); + GGML_ASSERT(kept_live[p] == exact_page_live[p]); + } + } +} + +void llama_kv_cache::exact_pages_claim(uint32_t idx, llama_seq_id seq, llama_pos pos) { + if (exact_page_owner_dirty || exact_page_owner.empty()) { + return; + } + + const exact_page cur { seq, llama_pos(pos/(llama_pos) exact_page_size) }; + + auto & owner = exact_page_owner[idx/exact_page_size]; + + GGML_ASSERT(owner.seq < 0 || (owner.seq == cur.seq && owner.lpg == cur.lpg)); + + owner = cur; + + ++exact_page_live[idx/exact_page_size]; +} + +// [TAG_EXACT_CONCURRENCY] a page stays with its sequence for as long as one of its cells is live, +// so a removal frees it only when it takes the last one. Counting per page is what keeps a removal +// that empties nothing, such as the rejected tail of every accepted speculative step, from costing +// a rescan of the pool. +void llama_kv_cache::exact_pages_release(uint32_t idx) { + ++exact_page_n_release; + + if (exact_page_owner_dirty || exact_page_owner.empty()) { + return; + } + + const uint32_t page = idx/exact_page_size; + + GGML_ASSERT(exact_page_live[page] > 0); + + if (--exact_page_live[page] == 0) { + exact_page_owner[page] = exact_page{}; + } +} + void llama_kv_cache::clear(bool data) { + exact_page_owner_dirty = true; + for (uint32_t s = 0; s < n_stream; ++s) { v_cells[s].reset(); v_heads[s] = 0; @@ -408,6 +620,11 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { } if (cells.seq_has(i, seq_id) && cells.seq_rm(i, seq_id)) { + // [TAG_EXACT_CONCURRENCY] the cell is gone; the page goes with the last of them + if (exact_pages) { + exact_pages_release(i); + } + if (new_head == cells.size()) { new_head = i; } @@ -431,6 +648,10 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { continue; } + if (exact_pages) { + exact_pages_release(i); + } + cells.rm(i); if (new_head == cells.size()) { @@ -454,6 +675,14 @@ void llama_kv_cache::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, ll return; } + // [TAG_EXACT_CONCURRENCY] a page belongs to one sequence, so refuse a copy that would share cells rather than abort. After the shared-cells return, so a draft cache is unaffected. + if (exact_pages && seq_id_src != seq_id_dst) { + LLAMA_LOG_ERROR("%s: exact concurrency does not support copying cells between " + "sequences (%d -> %d); ignoring the copy\n", + __func__, seq_id_src, seq_id_dst); + return; + } + GGML_ASSERT(seq_id_src >= 0 && (size_t) seq_id_src < seq_to_stream.size()); GGML_ASSERT(seq_id_dst >= 0 && (size_t) seq_id_dst < seq_to_stream.size()); @@ -555,6 +784,11 @@ void llama_kv_cache::seq_keep(llama_seq_id seq_id) { for (uint32_t i = 0; i < cells.size(); ++i) { if (cells.seq_keep(i, seq_id)) { + // [TAG_EXACT_CONCURRENCY] as in seq_rm, the cell emptied here + if (exact_pages) { + exact_pages_release(i); + } + if (new_head == cells.size()) { new_head = i; } @@ -573,6 +807,14 @@ void llama_kv_cache::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, ll return; } + // [TAG_EXACT_CONCURRENCY] a cell's offset in its page is its position modulo the page size, so shifting positions would misplace every cell + if (exact_pages && shift != 0) { + LLAMA_LOG_ERROR("%s: exact concurrency does not support shifting positions " + "(seq %d, shift %d); ignoring the shift\n", + __func__, seq_id, shift); + return; + } + GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); GGML_ASSERT(hparams.n_pos_per_embd() == 1 && "seq_add() is only supported for n_pos_per_embd() == 1"); @@ -623,6 +865,13 @@ void llama_kv_cache::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, in return; } + if (exact_pages && d != 1) { + LLAMA_LOG_ERROR("%s: exact concurrency does not support dividing positions " + "(seq %d, d %d); ignoring the division\n", + __func__, seq_id, d); + return; + } + GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size()); GGML_ASSERT(hparams.n_pos_per_embd() == 1 && "seq_div() is only supported for n_pos_per_embd() == 1"); @@ -706,11 +955,23 @@ llama_memory_context_ptr llama_kv_cache::init_batch( GGML_UNUSED(embd_all); do { + // [TAG_EXACT_CONCURRENCY] a token shared by several sequences would be one cell in a page that belongs to one sequence, so refuse it here rather than assert at placement + if (exact_pages && balloc.has_shared_tokens()) { + LLAMA_LOG_ERROR("%s: exact concurrency does not support tokens shared by several sequence ids; " + "give every token exactly one sequence id\n", __func__); + break; + } + balloc.split_reset(); std::vector<llama_ubatch> ubatches; while (true) { - auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0); + // [TAG_EXACT_CONCURRENCY] split_simple packs every sequence's prompt into one ubatch, so a prefill would run at a width its solo run never sees; the set split gives each its own + const uint32_t isolate = llama_exact_concurrency() && balloc.has_seq_wider_than(llama_exact_decode_tokens()) ? llama_exact_decode_tokens() : 0; + + auto ubatch = n_stream == 1 && !isolate + ? balloc.split_simple(n_ubatch) + : balloc.split_equal(n_ubatch, n_stream > 1, 0, isolate); if (ubatch.n_tokens == 0) { break; @@ -757,11 +1018,19 @@ llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector<llama_ std::vector<uint32_t> v_heads_old; // old positions of the heads, before placing the ubatch std::vector<llama_kv_cells> v_cells; // copy of the old cells, before placing the ubatch + + // [TAG_EXACT_CONCURRENCY] page ownership and occupancy before the ubatch, so undoing a speculative placement does not force a rebuild from every cell + std::vector<exact_page> exact_page_owner_old; + std::vector<uint32_t> exact_page_live_old; }; // remember the old state of the cells so we can restore it in the end std::vector<state_t> states; + // [TAG_EXACT_CONCURRENCY] a placement can purge positions outside the cells it restores below, + // and those are not undone; count removals to notice + const uint64_t n_release_before = exact_page_n_release; + bool success = true; for (const auto & ubatch : ubatches) { @@ -777,7 +1046,7 @@ llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector<llama_ // store the old state of the cells in the recovery stack { - state_t state = { sinfo_new, v_heads, {} }; + state_t state = { sinfo_new, v_heads, {}, exact_page_owner, exact_page_live }; for (uint32_t s = 0; s < sinfo_new.n_stream(); ++s) { auto & cells = v_cells[sinfo_new.strm[s]]; @@ -794,6 +1063,10 @@ llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector<llama_ GGML_ASSERT(!states.empty() || !success); + // [TAG_EXACT_CONCURRENCY] what the allocator knew is the answer unless the placement also + // removed cells, in which case only the cells can say what is left + const bool exact_rebuild = exact_page_owner_dirty || exact_page_n_release != n_release_before; + // iterate backwards and restore the cells to their original state for (auto it = states.rbegin(); it != states.rend(); ++it) { const auto & sinfo = it->sinfo; @@ -805,6 +1078,16 @@ llama_kv_cache::slot_info_vec_t llama_kv_cache::prepare(const std::vector<llama_ cells.set(sinfo.idxs[s], it->v_cells[s]); head = it->v_heads_old[s]; } + + // [TAG_EXACT_CONCURRENCY] put back what the allocator knew, unless the placement also removed cells, when only the cells can say what is left + if (!exact_rebuild) { + exact_page_owner = it->exact_page_owner_old; + exact_page_live = it->exact_page_live_old; + } + } + + if (exact_rebuild) { + exact_page_owner_dirty = true; } if (!success) { @@ -963,6 +1246,48 @@ llama_kv_cache::slot_info llama_kv_cache::find_slot(const llama_ubatch & ubatch, } } + if (exact_pages) { + const auto & cells = v_cells[0]; + + exact_pages_sync(); + + using page_key = std::pair<llama_seq_id, llama_pos>; + + exact_page_owner_tmp = exact_page_owner; + + auto & owner = exact_page_owner_tmp; + + std::map<page_key, uint32_t> pages; + + for (uint32_t p = 0; p < owner.size(); ++p) { + if (owner[p].seq >= 0) { + pages.emplace(page_key {owner[p].seq, owner[p].lpg}, p); + } + } + + std::set<uint32_t> assigned; + slot_info res {0, 0, {0}, {{}}}; + for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { + GGML_ASSERT(ubatch.n_seq_id[i] == 1 && ubatch.pos[i] >= 0); + const page_key key {ubatch.seq_id[i][0], ubatch.pos[i]/exact_page_size}; + auto it = pages.find(key); + if (it == pages.end()) { + uint32_t page = v_heads[0]/exact_page_size; + uint32_t tested = 0; + while (tested < owner.size() && owner[page%owner.size()].seq >= 0) { ++page; ++tested; } + if (tested == owner.size()) { return {}; } + page %= owner.size(); + owner[page] = exact_page {key.first, key.second}; + it = pages.emplace(key, page).first; + } + const uint32_t idx = it->second*exact_page_size + ubatch.pos[i]%exact_page_size; + if (!cells.is_empty(idx) || !assigned.insert(idx).second) { return {}; } + res.idxs[0].push_back(idx); + } + if (cont && !res.is_contiguous()) { return {}; } + return res; + } + uint32_t n_tokens = ubatch.n_tokens; uint32_t n_seqs = 1; @@ -1125,6 +1450,10 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & seq_pos_max_rm[seq_id] = std::max(seq_pos_max_rm[seq_id], pos); + if (exact_pages) { + exact_pages_release(idx); + } + cells.rm(idx); } @@ -1154,6 +1483,12 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & for (int32_t s = 0; s < ubatch.n_seq_id[i]; s++) { cells.seq_add(idx, ubatch.seq_id[i][s]); } + + if (exact_pages) { + GGML_ASSERT(ubatch.n_seq_id[i] == 1); + + exact_pages_claim(idx, ubatch.seq_id[i][0], ubatch.pos[i]); + } } } @@ -1185,7 +1520,16 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch & } } +uint32_t llama_kv_cache::alloc_granularity() const { + // [TAG_EXACT_CONCURRENCY] a page is given to one (sequence, position / page) pair, so n tokens hold round_up(n, exact_page_size) cells: the tail page is charged in full + return exact_pages ? exact_page_size : 1; +} + bool llama_kv_cache::get_can_shift() const { + // [TAG_EXACT_CONCURRENCY] a cell's offset in its page is its position modulo 256, so the pool cannot shift positions; reporting it disables --context-shift and --cache-reuse at load + if (exact_pages) { + return false; + } // Step35 uses per-layer RoPE dims; K-shift assumes a single global n_rot. if (model.arch == LLM_ARCH_STEP35) { return false; @@ -1247,7 +1591,50 @@ const llama_kv_cells & llama_kv_cache::get_cells(llama_seq_id seq_id) const { return v_cells[seq_to_stream[seq_id]]; } +ggml_tensor * llama_kv_cache::build_input_pages(ggml_context * ctx, const llama_ubatch & ubatch) const { + if (!exact_pages) { return nullptr; } + auto * pages = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, 1 + get_size()/exact_page_size, ubatch.n_tokens); + ggml_set_input(pages); + ggml_set_name(pages, "attn_logical_pages"); + return pages; +} + +void llama_kv_cache::set_input_pages(ggml_tensor * dst, const llama_ubatch * ubatch) const { + GGML_ASSERT(exact_pages && dst->ne[1] == ubatch->n_tokens); + + exact_pages_sync(); + + std::map<llama_seq_id, std::map<llama_pos, uint32_t>> pages; + for (uint32_t p = 0; p < exact_page_owner.size(); ++p) { + const auto & owner = exact_page_owner[p]; + if (owner.seq >= 0) { + pages[owner.seq][owner.lpg] = p; + } + } + std::vector<int32_t> data(ggml_nelements(dst), -1); + for (uint32_t i = 0; i < ubatch->n_tokens; ++i) { + GGML_ASSERT(ubatch->n_seq_id[i] == 1); + auto * row = data.data() + i*dst->ne[0]; + row[0] = 0; + for (const auto & page : pages[ubatch->seq_id[i][0]]) { + if (page.first*exact_page_size > uint32_t(ubatch->pos[i])) { break; } + row[++row[0]] = page.second; + } + } + ggml_backend_tensor_set(dst, data.data(), 0, data.size()*sizeof(int32_t)); +} + +ggml_tensor * llama_kv_cache_context::build_input_pages(ggml_context * ctx, const llama_ubatch & ubatch) const { + return kv->build_input_pages(ctx, ubatch); +} + +void llama_kv_cache_context::set_input_pages(ggml_tensor * dst, const llama_ubatch * ubatch) const { + kv->set_input_pages(dst, ubatch); +} + uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { + // the per-query page map is the only loop bound for exact attention, so neighbours cannot extend it + if (exact_pages) { return get_size(); } uint32_t result = 0; // pad the n_kv value so that the graph remains constant across batches and can be reused @@ -2130,6 +2517,12 @@ void llama_kv_cache::state_read_sinfo( llama_state_seq_flags flags, slot_info_vec_t * sinfos_out, const slot_info_vec_t * sinfos_in) { + // [TAG_EXACT_CONCURRENCY] a whole-cache restore writes cells at their recorded physical index, which the paged pool owns; refused before a byte is read + if (exact_pages && seq_id == -1) { + LLAMA_LOG_ERROR("%s: LLAMA_EXACT_CONCURRENCY is set, which supports per-sequence state restore only\n", __func__); + throw std::runtime_error("whole-cache restore is not supported with LLAMA_EXACT_CONCURRENCY"); + } + // TODO: refactor [TAG_KV_CACHE_SHARE_CELLS] if (other) { return; @@ -2449,6 +2842,8 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32 } else { // whole KV cache restore + GGML_ASSERT(!exact_pages); + if (cell_count > cells.size()) { LLAMA_LOG_ERROR("%s: not enough cells in kv cache\n", __func__); return false; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index c4d8699def12..d11f9f764c28 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -133,6 +133,9 @@ class llama_kv_cache : public llama_memory_i { bool get_can_shift() const override; + // [TAG_EXACT_CONCURRENCY] the page size under exact mode, 1 otherwise + uint32_t alloc_granularity() const override; + void clear(bool data) override; bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) override; @@ -184,6 +187,8 @@ class llama_kv_cache : public llama_memory_i { // uint32_t get_n_kv(const slot_info & sinfo) const; + ggml_tensor * build_input_pages(ggml_context * ctx, const llama_ubatch & ubatch) const; + void set_input_pages(ggml_tensor * dst, const llama_ubatch * ubatch) const; // get views of the current state of the cache ggml_tensor * get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; @@ -259,6 +264,38 @@ class llama_kv_cache : public llama_memory_i { std::vector<ggml_tensor *> v_stream; }; + static constexpr uint32_t exact_page_size = 256; + bool exact_pages = false; + + // [TAG_EXACT_CONCURRENCY] which (sequence, logical page) owns each physical page; seq < 0 means free, and it is kept current as cells are placed and dirtied by removals + struct exact_page { + llama_seq_id seq = -1; + llama_pos lpg = -1; + }; + + mutable std::vector<exact_page> exact_page_owner; + mutable bool exact_page_owner_dirty = true; + + // live cells in each physical page, so a removal can free the page it emptied without rescanning the pool + mutable std::vector<uint32_t> exact_page_live; + + // with LLAMA_KV_CACHE_DEBUG set this also rebuilds, to check what was maintained + void exact_pages_sync() const; + + void exact_pages_rebuild() const; + + void exact_pages_claim(uint32_t idx, llama_seq_id seq, llama_pos pos); + + // record that the cell at physical index idx has just become empty + void exact_pages_release(uint32_t idx); + + // how many cells have been released, so prepare() can tell whether a placement removed cells + // it is not going to restore + uint64_t exact_page_n_release = 0; + + // scratch for find_slot(), which must not touch the ownership it reads + mutable std::vector<exact_page> exact_page_owner_tmp; + bool v_trans = true; // the value tensor is transposed const uint32_t n_seq_max = 1; @@ -390,6 +427,8 @@ class llama_kv_cache_context : public llama_memory_context_i { // uint32_t get_n_kv() const; + ggml_tensor * build_input_pages(ggml_context * ctx, const llama_ubatch & ubatch) const; + void set_input_pages(ggml_tensor * dst, const llama_ubatch * ubatch) const; ggml_type type_k() const; ggml_type type_v() const; diff --git a/src/llama-memory-hybrid.cpp b/src/llama-memory-hybrid.cpp index 42c7381a9e6f..a596f35bd8d5 100644 --- a/src/llama-memory-hybrid.cpp +++ b/src/llama-memory-hybrid.cpp @@ -66,6 +66,13 @@ llama_memory_hybrid::llama_memory_hybrid( llama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) { do { + // [TAG_EXACT_CONCURRENCY] refused before the attention half asserts on it, see llama_kv_cache::init_batch + if (llama_exact_concurrency() && balloc.has_shared_tokens()) { + LLAMA_LOG_ERROR("%s: exact concurrency does not support tokens shared by several sequence ids; " + "give every token exactly one sequence id\n", __func__); + break; + } + balloc.split_reset(); // follow the recurrent pattern for creating the ubatch splits @@ -86,7 +93,10 @@ llama_memory_context_ptr llama_memory_hybrid::init_batch(llama_batch_allocr & ba // so that the rollback snapshots remain valid const uint32_t n_rs_seq = mem_recr->n_rs_seq; - ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0); + // [TAG_EXACT_CONCURRENCY] the recurrent half is not invariant to the ubatch shape, so a prompt gets a ubatch of its own + const uint32_t isolate = llama_exact_concurrency() ? llama_exact_decode_tokens() : 0; + + ubatch = balloc.split_equal(n_ubatch, !unified, n_rs_seq > 0 ? n_rs_seq + 1 : 0, isolate); } if (ubatch.n_tokens == 0) { @@ -135,6 +145,11 @@ bool llama_memory_hybrid::get_can_shift() const { return mem_attn->get_can_shift(); } +uint32_t llama_memory_hybrid::alloc_granularity() const { + // the recurrent half holds one state per sequence, so the attention half is the one whose cells a caller is planning capacity for + return mem_attn->alloc_granularity(); +} + void llama_memory_hybrid::clear(bool data) { mem_attn->clear(data); mem_recr->clear(data); @@ -150,6 +165,13 @@ bool llama_memory_hybrid::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1 } void llama_memory_hybrid::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { + // [TAG_EXACT_CONCURRENCY] the attention half refuses this, so refuse before either half is touched or the two could end up describing different states + if (llama_exact_concurrency() && seq_id_src != seq_id_dst) { + LLAMA_LOG_ERROR("%s: exact concurrency does not support copying cells between sequences (%d -> %d); ignoring the copy\n", + __func__, seq_id_src, seq_id_dst); + return; + } + mem_attn->seq_cp(seq_id_src, seq_id_dst, p0, p1); mem_recr->seq_cp(seq_id_src, seq_id_dst, p0, p1); } @@ -160,11 +182,23 @@ void llama_memory_hybrid::seq_keep(llama_seq_id seq_id) { } void llama_memory_hybrid::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos shift) { + if (llama_exact_concurrency() && shift != 0) { + LLAMA_LOG_ERROR("%s: exact concurrency does not support shifting positions (seq %d, shift %d); ignoring the shift\n", + __func__, seq_id, shift); + return; + } + mem_attn->seq_add(seq_id, p0, p1, shift); mem_recr->seq_add(seq_id, p0, p1, shift); } void llama_memory_hybrid::seq_div(llama_seq_id seq_id, llama_pos p0, llama_pos p1, int d) { + if (llama_exact_concurrency() && d != 1) { + LLAMA_LOG_ERROR("%s: exact concurrency does not support dividing positions (seq %d, d %d); ignoring the division\n", + __func__, seq_id, d); + return; + } + mem_attn->seq_div(seq_id, p0, p1, d); mem_recr->seq_div(seq_id, p0, p1, d); } diff --git a/src/llama-memory-hybrid.h b/src/llama-memory-hybrid.h index 484eafb74991..70ba19ca3239 100644 --- a/src/llama-memory-hybrid.h +++ b/src/llama-memory-hybrid.h @@ -58,6 +58,8 @@ class llama_memory_hybrid : public llama_memory_i { bool get_can_shift() const override; + uint32_t alloc_granularity() const override; + void clear(bool data) override; bool seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) override; diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp index 57919accf095..88b14faa8e4b 100644 --- a/src/llama-memory-recurrent.cpp +++ b/src/llama-memory-recurrent.cpp @@ -442,7 +442,10 @@ llama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr & // [TAG_RECURRENT_ROLLBACK_SPLITS] // the trailing (1 + n_rs_seq) tokens of each seq must stay in the same ubatch // so that the rollback snapshots remain valid - ubatch = balloc.split_equal(n_ubatch, true, n_rs_seq > 0 ? n_rs_seq + 1 : 0); + // [TAG_EXACT_CONCURRENCY] same rule as the hybrid memory: the state a prompt leaves behind depends on what shared its ubatch, so isolate prompts + const uint32_t isolate = llama_exact_concurrency() ? llama_exact_decode_tokens() : 0; + + ubatch = balloc.split_equal(n_ubatch, true, n_rs_seq > 0 ? n_rs_seq + 1 : 0, isolate); } if (ubatch.n_tokens == 0) { diff --git a/src/llama-memory.h b/src/llama-memory.h index db825396645e..61cd348f2dd9 100644 --- a/src/llama-memory.h +++ b/src/llama-memory.h @@ -100,6 +100,9 @@ struct llama_memory_i { // getters virtual bool get_can_shift() const = 0; + // [TAG_EXACT_CONCURRENCY] cells this module hands out in one indivisible unit: 1 unless a mode allocates in larger blocks, when n tokens occupy round_up(n, granularity) cells. Not pure, so old modules inherit 1. + virtual uint32_t alloc_granularity() const { return 1; } + // // ops // diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5531c4ce3ce9..292b7da9412e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -325,6 +325,15 @@ llama_build_and_test(test-backend-sampler.cpp LABEL "model") llama_build_and_test(test-state-restore-fragmented.cpp LABEL "model" ARGS -m "${MODEL_DEST}") set_tests_properties(test-state-restore-fragmented PROPERTIES FIXTURES_REQUIRED test-download-model) +# Guards on the asynchronous per-sequence state transfer +# Skips itself on a backend that cannot copy asynchronously +llama_build_and_test(test-state-seq-copy.cpp LABEL "model" ARGS -m "${MODEL_DEST}") +set_tests_properties(test-state-seq-copy PROPERTIES FIXTURES_REQUIRED test-download-model) + +# [TAG_EXACT_CONCURRENCY] page bookkeeping of the paged KV pool; skips itself where the mode cannot run +llama_build_and_test(test-exact-pages.cpp LABEL "model" ARGS -m "${MODEL_DEST}") +set_tests_properties(test-exact-pages PROPERTIES FIXTURES_REQUIRED test-download-model) + if (APPLE) llama_build(test-rset-release.cpp) endif() @@ -347,6 +356,17 @@ unset(LLAMA_TEST_NAME) llama_build_and_test(test-mtmd-impl.cpp) target_link_libraries(test-mtmd-impl PRIVATE mtmd) +# [TAG_EXACT_CONCURRENCY] the batch shape and the buffer types the mode requires, checked without a model +llama_build_and_test(test-exact-geometry.cpp) +llama_build_and_test(test-exact-buft.cpp) + +# server helpers that need no model +if (LLAMA_BUILD_TOOLS) + llama_build_and_test(test-server-tokens.cpp) + target_link_libraries(test-server-tokens PRIVATE server-context mtmd) + target_include_directories(test-server-tokens PRIVATE ${PROJECT_SOURCE_DIR}/tools/server ${PROJECT_SOURCE_DIR}/tools/mtmd) +endif() + # GGUF model data fetcher library for tests that need real model metadata # Only compile when cpp-httplib has SSL support (CPPHTTPLIB_OPENSSL_SUPPORT) if (TARGET cpp-httplib) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 8030186fb496..388bca0ea280 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7715,6 +7715,40 @@ struct test_flash_attn_ext : public test_case { } }; +// same attention as the CPU mask reference, but visiting nonadjacent pages in a different order +struct test_flash_attn_ext_pages : public test_flash_attn_ext { + test_flash_attn_ext_pages(int64_t batch) : + test_flash_attn_ext(256, 256, 2, {8, 1}, 1024, batch) {} + + std::string vars() override { return test_flash_attn_ext::vars() + ",exact_pages=1"; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + auto * out = test_flash_attn_ext::build_graph(ctx); + out->src[5] = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, 5, nb); + ggml_set_name(out->src[5], "pages"); + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + test_flash_attn_ext::initialize_tensors(ctx); + auto * pages = ggml_get_tensor(ctx, "pages"); + auto * mask = ggml_get_tensor(ctx, "m"); + std::vector<int32_t> ids(5*nb, -1); + std::vector<ggml_fp16_t> values(1024*nb, ggml_fp32_to_fp16(-INFINITY)); + for (int64_t q = 0; q < nb; ++q) { + ids[5*q] = q%2 ? 1 : 2; + ids[5*q + 1] = 2; + ids[5*q + 2] = 0; + for (int j = 0; j < 256; ++j) { values[1024*q + 512 + j] = ggml_fp32_to_fp16(0.0f); } + if (q%2 == 0) { + for (int j = 0; j < 17; ++j) { values[1024*q + j] = ggml_fp32_to_fp16(0.0f); } + } + } + ggml_backend_tensor_set(pages, ids.data(), 0, ids.size()*sizeof(int32_t)); + ggml_backend_tensor_set(mask, values.data(), 0, values.size()*sizeof(ggml_fp16_t)); + } +}; + // GGML_OP_CROSS_ENTROPY_LOSS struct test_cross_entropy_loss : public test_case { const ggml_type type; @@ -9770,6 +9804,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, m, 2, 1024, { 1, 1 }, { 1, 1 })); } + for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_Q8_0}) { + for (int n : {1, 17, 307}) { + test_cases.emplace_back(new test_mul_mat(type, GGML_TYPE_F32, 64, n, 256, {1, 1}, {4, 1})); + test_cases.emplace_back(new test_mul_mat(type, GGML_TYPE_F32, 64, n, 256, {1, 1}, {1, 4})); + } + } + + // MoE projections at the token counts a decode ubatch forms; 17 tokens is past the width exact concurrency pins + for (ggml_type type_a : {GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_Q8_0, GGML_TYPE_F16}) { + for (int n : {1, 2, 4, 8, 17}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 16, 8, true, 512, n, 2048)); + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 16, 8, false, 2048, n, 512)); + } + } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); @@ -10586,6 +10635,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() { } // mixed quant and Q1_0 test cases + for (int64_t batch : {1, 4, 12}) { + test_cases.emplace_back(new test_flash_attn_ext_pages(batch)); + } test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(72, 72, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0)); diff --git a/tests/test-exact-buft.cpp b/tests/test-exact-buft.cpp new file mode 100644 index 000000000000..c7f93c857d7a --- /dev/null +++ b/tests/test-exact-buft.cpp @@ -0,0 +1,60 @@ +// [TAG_EXACT_CONCURRENCY] which buffer types the mode accepts a weight in. A host buffer is not one +// of them: the scheduler runs an operation on the backend holding its weight, and moves a host +// weight's operation to the GPU only once the batch is wide enough, so its result would depend on +// how many sequences share the step. This is the predicate behind both the context's weight check +// and the refusal of a lora that would inherit such a buffer. + +#include "ggml-backend.h" + +#include "../src/llama-impl.h" + +#include <cstdio> + +#undef NDEBUG +#include <cassert> + +int main() { + ggml_backend_load_all(); + + // nothing placed anywhere is nothing to trust + assert(!llama_exact_buft_invariant(nullptr)); + + // the plain CPU buffer, and the pinned host buffer a GPU backend offers, are both host memory + assert(!llama_exact_buft_invariant(ggml_backend_cpu_buffer_type())); + + bool checked_gpu = false; + + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + + if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) { + continue; + } + + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); + + const bool invariant = reg && llama_exact_backend_name(ggml_backend_reg_name(reg)); + + // a device's own buffer follows its backend, and its host buffer never does + assert(llama_exact_buft_invariant(ggml_backend_dev_buffer_type(dev)) == invariant); + + if (auto * host = ggml_backend_dev_host_buffer_type(dev)) { + assert(!llama_exact_buft_invariant(host)); + } + + checked_gpu = checked_gpu || invariant; + } + + // the registry names the mode trusts, whatever this build has + assert(llama_exact_backend_name("CUDA")); + assert(llama_exact_backend_name("ROCm")); + assert(llama_exact_backend_name("MUSA")); + assert(!llama_exact_backend_name("CPU")); + assert(!llama_exact_backend_name("BLAS")); + assert(!llama_exact_backend_name(nullptr)); + + printf("%s: all tests passed%s\n", __func__, + checked_gpu ? "" : " (no batch-invariant device here, the positive case was not exercised)"); + + return 0; +} diff --git a/tests/test-exact-geometry.cpp b/tests/test-exact-geometry.cpp new file mode 100644 index 000000000000..7e6e80e07f6b --- /dev/null +++ b/tests/test-exact-geometry.cpp @@ -0,0 +1,55 @@ +// [TAG_EXACT_CONCURRENCY] the batch shape a prefill needs to be split into the ubatches it would +// get alone: the server adds a prompt in whole ubatches, so the batch has to hold one of those +// beside a decode step of every slot, or the prompt is left the shorter remainder. + +#include "common.h" + +#include <cstdio> + +#undef NDEBUG +#include <cassert> + +int main() { + int n_min = 0; + + // the reported minimum is the ubatch plus the decode step, whether or not the batch reaches it + assert(common_exact_batch_geometry(2048, 512, 4, &n_min)); + assert(n_min == 516); + + // the case that used to warn and carry on: one decoder beside the prompt leaves it 511 tokens + assert(!common_exact_batch_geometry(512, 512, 1, &n_min)); + assert(n_min == 513); + + assert(!common_exact_batch_geometry(512, 512, 2, &n_min)); + assert(n_min == 514); + + // exactly enough, and one short of it + assert(common_exact_batch_geometry(514, 512, 2, &n_min) && n_min == 514); + assert(!common_exact_batch_geometry(513, 512, 2, &n_min) && n_min == 514); + + // a single slot with no draft still needs room for its own decoded token + assert(!common_exact_batch_geometry(512, 512, 1)); + assert(common_exact_batch_geometry(1024, 512, 1)); + + // an unset ubatch is the whole batch, which then cannot hold a decode step as well + assert(!common_exact_batch_geometry(2048, 0, 4, &n_min)); + assert(n_min == 2052); + + // a ubatch larger than the batch is clamped to it, so it cannot pass either + assert(!common_exact_batch_geometry(512, 4096, 1, &n_min)); + assert(n_min == 513); + + // the shape a context settles on when its size clamps the batch: n_batch becomes min(n_ctx, -b) + // and n_ubatch min(n_batch, -ub), so a context of 256 cells leaves the two equal and no column + // for a decode step, whatever -b and -ub asked for + assert(!common_exact_batch_geometry(256, 256, 2, &n_min)); + assert(n_min == 258); + + // no slot decoding at all: the prompt has the batch to itself + assert(common_exact_batch_geometry(512, 512, 0, &n_min)); + assert(n_min == 512); + + printf("%s: all tests passed\n", __func__); + + return 0; +} diff --git a/tests/test-exact-pages.cpp b/tests/test-exact-pages.cpp new file mode 100644 index 000000000000..8ea77e7ea3d9 --- /dev/null +++ b/tests/test-exact-pages.cpp @@ -0,0 +1,152 @@ +// [TAG_EXACT_CONCURRENCY] page bookkeeping of the paged KV pool: a removal that empties nothing, +// a removal that leaves holes, and the pages those holes keep reserved. +// +// LLAMA_KV_CACHE_DEBUG=1 makes the pool rebuild its page ownership from the live cells on every +// ubatch and assert that it says what the incrementally maintained one says, so this test drives +// the removal paths and lets that oracle check them. +// +// The mode needs a CUDA (or ROCm/MUSA) build, 256-wide K and V heads and a fully offloaded F16 KV +// cache. Where the context cannot be created the test reports what it skipped and passes: it has +// nothing to say about a build without those. + +#include "arg.h" +#include "common.h" +#include "llama.h" + +#include <cstdio> +#include <cstdlib> +#include <vector> + +static const uint32_t PAGE = 256; + +static bool decode_range(llama_context * ctx, llama_seq_id seq, llama_pos first, llama_pos last) { + llama_batch batch = llama_batch_init(64, 0, 1); + + bool ok = true; + + for (llama_pos p = first; p <= last && ok; ) { + common_batch_clear(batch); + + for (int i = 0; i < 64 && p <= last; ++i, ++p) { + common_batch_add(batch, 1, p, {seq}, false); + } + + // every decode asks for one set of logits, so none of them is a batch with no output + batch.logits[batch.n_tokens - 1] = true; + + ok = llama_decode(ctx, batch) == 0; + } + + llama_batch_free(batch); + + return ok; +} + +int main(int argc, char ** argv) { + // read before the model is loaded: both are latched on first use + setenv("LLAMA_EXACT_CONCURRENCY", "1", 0); + setenv("LLAMA_KV_CACHE_DEBUG", "1", 0); + + common_params params; + + params.sampling.seed = 1234; + params.kv_unified = true; + params.n_parallel = 2; + params.n_ctx = 2*4*PAGE; + params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; + + common_init(); + + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + // after the parser, which requires the default here + params.n_gpu_layers = 999; + + ggml_backend_load_all(); + + common_init_result_ptr llama_init = common_init_from_params(params); + + llama_context * ctx = llama_init->context(); + + if (llama_init->model() == nullptr || ctx == nullptr) { + printf("%s : skipped, this build and model cannot run exact concurrency\n", __func__); + return 0; + } + + llama_memory_t mem = llama_get_memory(ctx); + + const uint32_t gran = llama_memory_alloc_granularity(mem); + if (gran != PAGE) { + fprintf(stderr, "%s : allocation granularity is %u, expected %u\n", __func__, gran, PAGE); + return 1; + } + + // positions 0..599 of sequence 0: three pages, the last one part full + if (!decode_range(ctx, 0, 0, 599)) { + fprintf(stderr, "%s : failed to fill sequence 0\n", __func__); + return 1; + } + + // the removal every accepted speculative step makes: a rejected tail that is not there. It + // must leave the pool alone, ownership included + if (!llama_memory_seq_rm(mem, 0, 600, -1) || llama_memory_seq_pos_max(mem, 0) != 599) { + fprintf(stderr, "%s : a removal past the tail changed the sequence, its end is %d\n", + __func__, llama_memory_seq_pos_max(mem, 0)); + return 1; + } + + if (!decode_range(ctx, 0, 600, 655)) { + fprintf(stderr, "%s : failed to continue sequence 0 after a removal that removed nothing\n", __func__); + return 1; + } + + // holes: positions 1 to 510 go, 0 and 511 to 655 stay, so the first two pages each keep a live + // cell and neither is free for another sequence. A hybrid memory refuses to remove the middle + // of a sequence, and then there is nothing to check here + const bool holes = llama_memory_seq_rm(mem, 0, 1, 511); + + if (holes && llama_memory_seq_pos_max(mem, 0) != 655) { + fprintf(stderr, "%s : a partial removal changed the end of the sequence: %d\n", __func__, + llama_memory_seq_pos_max(mem, 0)); + return 1; + } + + printf("%s : interior removal %s\n", __func__, holes ? "left holes" : "was refused, skipping the hole case"); + + // sequence 1 fills what is left of the pool. The pool holds 8 pages and sequence 0 holds 3 of + // them, holes and a part full tail page included, so 5 remain + if (!decode_range(ctx, 1, 0, 5*PAGE - 1)) { + fprintf(stderr, "%s : failed to fill the pages sequence 0 does not hold\n", __func__); + return 1; + } + + // one page more than the pool has left: it has to refuse rather than take a page that still + // has a live cell in it + if (decode_range(ctx, 1, 5*PAGE, 5*PAGE)) { + fprintf(stderr, "%s : the pool allocated a page that sequence 0 still holds\n", __func__); + return 1; + } + + // a whole sequence goes back to the pool as whole pages, holes included + if (!llama_memory_seq_rm(mem, 0, -1, -1) || llama_memory_seq_pos_max(mem, 0) != -1) { + fprintf(stderr, "%s : sequence 0 is still in the pool after a full removal\n", __func__); + return 1; + } + + if (!decode_range(ctx, 1, 5*PAGE, 8*PAGE - 1)) { + fprintf(stderr, "%s : the three pages of the removed sequence were not reusable\n", __func__); + return 1; + } + + // the pool is full again + if (decode_range(ctx, 1, 8*PAGE, 8*PAGE)) { + fprintf(stderr, "%s : the pool allocated a ninth page\n", __func__); + return 1; + } + + printf("%s : ok, page ownership survived a no-op removal, holes and a full removal\n", __func__); + + return 0; +} diff --git a/tests/test-server-tokens.cpp b/tests/test-server-tokens.cpp new file mode 100644 index 000000000000..051e01a15f0d --- /dev/null +++ b/tests/test-server-tokens.cpp @@ -0,0 +1,184 @@ +// [TAG_PREEMPT] the server converts between a KV position and a token count when it rewinds a slot to +// what the cache holds. With M-RoPE media the two differ, so the conversion is exercised here on a +// hand-built image chunk, without a model. + +#include "server-common.h" + +#include "mtmd.h" + +#include <cstdint> +#include <cstdio> +#include <cstring> +#include <stdexcept> +#include <string> +#include <vector> + +#undef NDEBUG +#include <cassert> + +// the wire format of mtmd_input_chunk_save(), which needs a context to produce a chunk; written here so +// that a chunk of a known shape can be loaded without one +struct chunk_writer { + std::vector<char> buf; + + template <typename T> void put(T v) { + const char * p = reinterpret_cast<const char *>(&v); + buf.insert(buf.end(), p, p + sizeof(T)); + } + + void put_str(const std::string & s) { + put<uint64_t>(s.size()); + buf.insert(buf.end(), s.begin(), s.end()); + } +}; + +// nx*ny tokens of one image, max(nx, ny) positions under M-RoPE +static mtmd::input_chunk_ptr make_image_chunk(uint32_t nx, uint32_t ny) { + chunk_writer w; + + w.put<uint64_t>(2); // MTMD_SERIALIZATION_VERSION + w.put<uint32_t>(MTMD_INPUT_CHUNK_TYPE_IMAGE); + w.put<uint64_t>(0); // tokens_text + w.put<uint8_t>(1); // tokens_image follows + w.put<uint32_t>(nx); + w.put<uint32_t>(ny); + w.put<uint32_t>(1); // MTMD_POS_TYPE_MROPE + w.put<uint32_t>(0); // image_idx + w.put<uint32_t>(1); // n_temporal_merge + w.put_str("test-image"); // id + w.put<uint8_t>(0); // batch_f32.is_audio + w.put<uint64_t>(1); // one entry + w.put<uint8_t>(0); // entry.add_viewsep + w.put<uint8_t>(0); // entry.add_newline + w.put<int32_t>(0); // entry.lead_pad + w.put<int32_t>(1); // entry.nx + w.put<int32_t>(1); // entry.ny + w.put<uint8_t>(0); // no tokens_audio + + mtmd::input_chunk_ptr chunk(mtmd_input_chunk_load(w.buf.data(), w.buf.size())); + + assert(chunk && "the serialized image chunk was rejected"); + + return chunk; +} + +// 10 text tokens, an image of 256 tokens and 16 positions, 20 text tokens, then n_gen generated tokens +static server_tokens make_prompt(size_t n_gen, const mtmd_input_chunk * chunk) { + server_tokens res; + + res.has_mtmd = true; + + for (size_t i = 0; i < 10; ++i) { + res.push_back((llama_token) (100 + i)); + } + + res.push_back(chunk); + + for (size_t i = 0; i < 20; ++i) { + res.push_back((llama_token) (200 + i)); + } + + for (size_t i = 0; i < n_gen; ++i) { + res.push_back((llama_token) (300 + i)); + } + + return res; +} + +int main() { + const mtmd::input_chunk_ptr chunk = make_image_chunk(16, 16); + + assert(mtmd_input_chunk_get_n_tokens(chunk.get()) == 256); + assert(mtmd_input_chunk_get_n_pos (chunk.get()) == 16); + + // a cut in the generated tail: the cache reports 86 positions, which is 326 tokens + { + server_tokens prompt = make_prompt(40, chunk.get()); + + assert(prompt.size() == 326); + assert(prompt.pos_next() == 86); + + const llama_pos pos_cached = 86; + const size_t n_cached = prompt.size_up_to_pos(pos_cached); + + assert(n_cached == 326); + assert(prompt.pos_next(n_cached) == pos_cached); + + // the same number taken for a token count falls inside the image + bool threw = false; + + try { + prompt.keep_first((size_t) pos_cached); + } catch (const std::exception &) { + threw = true; + } + + assert(threw && "a position used as a token count cuts the image in half"); + } + + // the same prompt with a longer tail, cut inside the generated tokens + { + server_tokens prompt = make_prompt(300, chunk.get()); + + assert(prompt.size() == 586); + assert(prompt.pos_next() == 346); + + const llama_pos pos_cached = 106; // 10 text + 16 image + 20 text + 60 generated + const size_t n_cached = prompt.size_up_to_pos(pos_cached); + + assert(n_cached == 346); + assert(prompt.pos_next(n_cached) == pos_cached); + + prompt.keep_first(n_cached); + + assert(prompt.size() == 346); + assert(prompt.pos_next() == pos_cached); + } + + // a cut before the image, and one at its first token: both are token boundaries + { + server_tokens prompt = make_prompt(0, chunk.get()); + + assert(prompt.size_up_to_pos(10) == 10); + assert(prompt.pos_next(10) == 10); + + // the image ends at position 26 and token 266 + assert(prompt.size_up_to_pos(26) == 266); + assert(prompt.pos_next(266) == 26); + } + + // a cut inside the image: the conversion cannot land there, and stepping back reaches the chunk's first token + { + server_tokens prompt = make_prompt(0, chunk.get()); + + const llama_pos pos_cached = 20; // inside the image, which spans positions 10..25 + + size_t n_cached = prompt.size_up_to_pos(pos_cached); + + assert(n_cached == 266); // rounded up to the whole chunk + + while (n_cached > 0 && prompt.pos_next(n_cached) > pos_cached) { + n_cached--; + } + + assert(n_cached == 10); + assert(prompt.pos_next(n_cached) == 10); + + prompt.keep_first(n_cached); // would throw if it cut the image in half + assert(prompt.size() == 10); + } + + // an empty cache has to be handled by the caller: the walk always consumes its first token + { + server_tokens prompt = make_prompt(4, chunk.get()); + + const llama_pos pos_cached = 0; + + assert(prompt.size_up_to_pos(pos_cached) == 1); + assert((pos_cached > 0 ? prompt.size_up_to_pos(pos_cached) : 0) == 0); + } + + printf("%s: all tests passed\n", __func__); + + return 0; +} diff --git a/tests/test-state-restore-fragmented.cpp b/tests/test-state-restore-fragmented.cpp index d5548afba179..5a1502f747fd 100644 --- a/tests/test-state-restore-fragmented.cpp +++ b/tests/test-state-restore-fragmented.cpp @@ -73,6 +73,13 @@ int main(int argc, char ** argv) { } fprintf(stderr, "%s : saved seq 1 state, %zu bytes\n", __func__, ncopy); + // a fragmented restore may stage a whole device tensor, so check every sequence byte-for-byte, neighbours included + std::vector<std::vector<uint8_t>> before(params.n_parallel); + for (int s = 0; s < params.n_parallel; ++s) { + before[s].resize(llama_state_seq_get_size(ctx, s)); + GGML_ASSERT(llama_state_seq_get_data(ctx, before[s].data(), before[s].size(), s) == before[s].size()); + } + // clear seq 1 to create a "hole" in the KV cache (fragmentation) // 0.20.20.20.2.... llama_memory_t mem = llama_get_memory(ctx); @@ -96,6 +103,13 @@ int main(int argc, char ** argv) { } fprintf(stderr, "%s : restored state into seq 1, %zu bytes\n", __func__, nset); + for (int s = 0; s < params.n_parallel; ++s) { + std::vector<uint8_t> after(llama_state_seq_get_size(ctx, s)); + GGML_ASSERT(llama_state_seq_get_data(ctx, after.data(), after.size(), s) == after.size()); + GGML_ASSERT(before[s] == after); + } + fprintf(stderr, "%s : all %d sequence snapshots are byte-identical after restore\n", __func__, params.n_parallel); + // Verify we can decode with the restored state // Generate one token to verify the restored state is usable auto sparams = llama_sampler_chain_default_params(); diff --git a/tests/test-state-seq-copy.cpp b/tests/test-state-seq-copy.cpp new file mode 100644 index 000000000000..bd3cd9cc30a2 --- /dev/null +++ b/tests/test-state-seq-copy.cpp @@ -0,0 +1,140 @@ +// [TAG_STATE_ASYNC] guards on the asynchronous state transfer: the buffer belongs to the transfer, so an oversized size is refused, and ON_DEVICE is refused as these go via the host + +#include "arg.h" +#include "common.h" +#include "llama.h" + +#include <cstdio> +#include <cstring> +#include <vector> + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "%s : FAILED at line %d: %s\n", __func__, \ + __LINE__, #cond); \ + return 1; \ + } \ + } while (0) + +int main(int argc, char ** argv) { + common_params params; + + params.sampling.seed = 1234; + params.kv_unified = true; + params.n_parallel = 2; + params.n_ctx = 256; + + common_init(); + + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + ggml_backend_load_all(); + + common_init_result_ptr llama_init = common_init_from_params(params); + + llama_context * ctx = llama_init->context(); + + if (llama_init->model() == nullptr || ctx == nullptr) { + fprintf(stderr, "%s : failed to init\n", __func__); + return 1; + } + + // two sequences interleaved, so the cells of each are a comb rather than one block, which is what the transfer is built for + std::vector<llama_token> tokens(60, 1); + + llama_batch batch = llama_batch_init(params.n_parallel*tokens.size(), 0, 1); + for (size_t i = 0; i < tokens.size(); i++) { + for (int s = 0; s < params.n_parallel; ++s) { + common_batch_add(batch, tokens[i], i, {s}, false); + } + } + batch.logits[batch.n_tokens - 1] = true; + + if (llama_decode(ctx, batch)) { + fprintf(stderr, "%s : failed to decode\n", __func__); + llama_batch_free(batch); + return 1; + } + + llama_batch_free(batch); + + llama_state_seq_copy * cpy = llama_state_seq_copy_init(ctx); + + if (cpy == nullptr) { + fprintf(stderr, "%s : this backend cannot copy sequence states asynchronously, skipping\n", __func__); + return 0; + } + + const int seq_id = 1; + const size_t size = llama_state_seq_get_size_ext(ctx, seq_id, LLAMA_STATE_SEQ_FLAGS_NONE); + + CHECK(size > 0); + + // nothing is allocated yet, so nothing is page-locked yet, whatever the backend offers + CHECK(llama_state_seq_copy_buf_is_pinned(cpy) == false); + CHECK(llama_state_seq_copy_buf(cpy) == nullptr); + + CHECK(llama_state_seq_copy_buf_resize(cpy, size) != nullptr); + CHECK(llama_state_seq_copy_buf_size(cpy) == size); + + fprintf(stderr, "%s : seq %d state is %zu bytes, %s host memory (backend offers %s)\n", + __func__, seq_id, size, + llama_state_seq_copy_buf_is_pinned(cpy) ? "pinned" : "pageable", + llama_state_seq_copy_buf_can_pin(cpy) ? "pinned" : "pageable"); + + CHECK(llama_state_seq_copy_get(cpy, size + 1, seq_id, LLAMA_STATE_SEQ_FLAGS_NONE) == 0); + CHECK(llama_state_seq_copy_set(cpy, size + 1, seq_id, LLAMA_STATE_SEQ_FLAGS_NONE) == 0); + + CHECK(llama_state_seq_copy_get(cpy, 0, seq_id, LLAMA_STATE_SEQ_FLAGS_NONE) == 0); + CHECK(llama_state_seq_copy_set(cpy, 0, seq_id, LLAMA_STATE_SEQ_FLAGS_NONE) == 0); + + CHECK(llama_state_seq_copy_get(cpy, size, seq_id, LLAMA_STATE_SEQ_FLAGS_ON_DEVICE) == 0); + CHECK(llama_state_seq_copy_set(cpy, size, seq_id, LLAMA_STATE_SEQ_FLAGS_ON_DEVICE) == 0); + + CHECK(llama_state_seq_copy_done(cpy)); + + fprintf(stderr, "%s : oversized, empty and ON_DEVICE transfers are all refused\n", __func__); + + // a transfer that fails part way must post nothing: the caller is told it failed and is free to reuse the buffer at once + CHECK(llama_state_seq_copy_get(cpy, size - 1, seq_id, LLAMA_STATE_SEQ_FLAGS_NONE) == 0); + CHECK(llama_state_seq_copy_n_copies(cpy) == 0); + CHECK(llama_state_seq_copy_done(cpy)); + + fprintf(stderr, "%s : a transfer one byte short is refused and posts no copies\n", __func__); + + std::vector<uint8_t> before(llama_state_seq_get_size(ctx, seq_id)); + CHECK(llama_state_seq_get_data(ctx, before.data(), before.size(), seq_id) == before.size()); + + CHECK(llama_state_seq_copy_get(cpy, size, seq_id, LLAMA_STATE_SEQ_FLAGS_NONE) == size); + llama_state_seq_copy_wait(cpy); + + llama_memory_seq_rm(llama_get_memory(ctx), seq_id, -1, -1); + + CHECK(llama_state_seq_copy_set(cpy, size - 1, seq_id, LLAMA_STATE_SEQ_FLAGS_NONE) == 0); + CHECK(llama_state_seq_copy_n_copies(cpy) == 0); + CHECK(llama_state_seq_copy_done(cpy)); + + CHECK(llama_state_seq_copy_set(cpy, size, seq_id, LLAMA_STATE_SEQ_FLAGS_NONE) == size); + llama_state_seq_copy_wait(cpy); + + std::vector<uint8_t> after(llama_state_seq_get_size(ctx, seq_id)); + CHECK(after.size() == before.size()); + CHECK(llama_state_seq_get_data(ctx, after.data(), after.size(), seq_id) == after.size()); + CHECK(before == after); + + fprintf(stderr, "%s : a transfer at the buffer's own size round-trips seq %d byte-for-byte\n", + __func__, seq_id); + + llama_state_seq_copy_buf_free(cpy); + CHECK(llama_state_seq_copy_buf_is_pinned(cpy) == false); + CHECK(llama_state_seq_copy_buf_capacity(cpy) == 0); + + llama_state_seq_copy_free(cpy); + + fprintf(stderr, "%s : SUCCESS\n", __func__); + + return 0; +} diff --git a/tools/server/README.md b/tools/server/README.md index 71ebb95434e4..036cf1be006e 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -168,6 +168,8 @@ For the full list of features, please refer to [server's changelog](https://gith | `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)<br/>(env: LLAMA_ARG_CTX_CHECKPOINTS) | | `-cms, --checkpoint-min-step N` | minimum spacing between context checkpoints in tokens (default: 8192, 0 = no minimum)<br/>(env: LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT) | | `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) | +| `--preempt-ram N` | with a unified KV cache, park a slot in host RAM instead of failing every slot when the cache fills; N is the maximum host RAM for parked sequences in MiB (default: 8192, -1 - no limit, 0 - disable)<br/>(env: LLAMA_ARG_PREEMPT_RAM) | +| `--preempt-async`, `--no-preempt-async` | copy a parked sequence out of and back into the KV cache on a stream of its own: the copy out overlaps with the slots that keep decoding, while a copy back in, and a kv-full retry behind a copy out that has not landed, wait for it (default: enabled, needs a backend that can copy asynchronously, otherwise the copies are synchronous as before)<br/>(env: LLAMA_ARG_PREEMPT_ASYNC) | | `-kvu, --kv-unified, -no-kvu, --no-kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)<br/>(env: LLAMA_ARG_KV_UNIFIED) | | `--cache-idle-slots, --no-cache-idle-slots` | save idle slots to the prompt cache on new task, and clear them when using unified KV (default: enabled, requires cache-ram)<br/>(env: LLAMA_ARG_CACHE_IDLE_SLOTS) | | `--context-shift, --no-context-shift` | whether to use context shift on infinite text generation (default: disabled)<br/>(env: LLAMA_ARG_CONTEXT_SHIFT) | @@ -675,6 +677,18 @@ These words will not be included in the completion, so make sure to add them to - `tokens_cached`: Number of tokens from the prompt which could be re-used from previous completion - `tokens_evaluated`: Number of tokens evaluated in total from the prompt - `truncated`: Boolean indicating if the context size was exceeded during generation, i.e. the number of tokens provided in the prompt (`tokens_evaluated`) plus tokens generated (`tokens predicted`) exceeded the context size (`n_ctx`) +- `preempt`: How the request was served while the unified KV cache was full (see `--preempt-ram`). `parks` is how often the request was parked to make room for another, and `recomputes` is how many of those parks dropped the sequence's cells because `--preempt-ram` was spent, so that the resume re-prefilled its tokens instead of restoring the bytes that were saved. A re-prefilled sequence continues from the same tokens, but its numerics are not guaranteed identical to the sequence that left, `LLAMA_EXACT_CONCURRENCY` included: raise `--preempt-ram` until `recomputes` stays 0 where that matters. Both fields are present in the final response of a streamed completion as well. + +While a request is streaming, the server sends SSE comment lines that a client reading raw lines can act on and every SSE event consumer ignores: + +- `: preempted` - the slot was parked and the stream is silent until it comes back. A parked stream is kept alive with the same comment about every two seconds. +- `: resumed` - the slot is running again. +- `: recomputed` - sent right after `: resumed` when that resume re-prefilled the sequence rather than restoring its saved bytes, i.e. what follows is the continuation `preempt.recomputes` counts. +- `: preempt-keepalive` - sent while the slot stays parked, at most every 2 seconds, or at the request's `sse_ping_interval` when that is shorter. + +A park can happen while the prompt is still being processed, before the request has produced a token. The notice is not held back for the first chunk in that case: the response headers and the `: preempted` line go out at the moment the slot is parked, on every streaming surface (`/completion`, `/v1/chat/completions`, `/v1/responses`, `/v1/messages`), so a client never has to tell that silence from a stall. `: resumed`, and `: recomputed` where it applies, follow when the slot runs again. + +With more than one prompt in the request, the index of the prompt follows the word, for example `: resumed 1`. ### POST `/tokenize`: Tokenize a given text @@ -980,6 +994,8 @@ This endpoint is enabled by default and can be disabled with `--no-slots`. It ca If query param `?fail_on_no_slot=1` is set, this endpoint will respond with status code 503 if there is no available slots. +Every entry also reports how its request has been served under preemption (see `--preempt-ram`): `is_preempted` and `is_transferring` say whether the slot's cells have been released or a copy is in flight, `n_preempt` counts the parks of the current task, and `n_recompute` counts how many of those dropped the cells, so that the resume re-prefilled the sequence instead of restoring its saved bytes. + **Response format** <details> @@ -1147,6 +1163,11 @@ In *router mode* the query param `?model={model_id}` has to be set. This endpoin | `llamacpp:spec_decode_num_accepted_tokens_total` | Counter | Total draft tokens accepted by the target model (0 when spec-decode is off). | | `llamacpp:spec_decode_num_drafts_total` | Counter | Total speculative decoding verification steps (0 when spec-decode is off). | | `llamacpp:spec_decode_num_accepted_tokens_per_pos_total` | Counter | Accepted tokens per draft position (labeled `position="N"`; absent when spec-decode is off or before the first completed speculative request). | +| `llamacpp:n_preempt_total` | Counter | Slots parked to make room in the unified KV cache (0 unless `--kv-unified` with more than one slot). | +| `llamacpp:n_resume_total` | Counter | Parked slots put back. | +| `llamacpp:preempt_recompute_total` | Counter | Parks that dropped their cells because `--preempt-ram` was spent, so the resume re-prefills the sequence instead of restoring its saved bytes. | +| `llamacpp:requests_preempted` | Gauge | Requests currently parked, waiting for room in the unified KV cache. | +| `llamacpp:preempt_ram_bytes` | Gauge | Host RAM held by parked sequences. | ### POST `/slots/{id_slot}?action=save`: Save the prompt cache of the specified slot to a file. diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 2ac98b6fddbc..76e348dab1cc 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -470,6 +470,12 @@ const mtmd::input_chunk_ptr & server_tokens::find_chunk(size_t idx) const { throw std::runtime_error("Chunk not found"); } +size_t server_tokens::chunk_n_tokens_at(size_t idx) const { + auto it = map_idx_to_media.find(idx); + + return it == map_idx_to_media.end() ? 0 : mtmd_input_chunk_get_n_tokens(it->second.get()); +} + std::pair<const mtmd::input_chunk_ptr *, size_t> server_tokens::find_next_media_chunk(size_t idx) const { auto it = map_idx_to_media.upper_bound(idx); if (it != map_idx_to_media.end()) { diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 6c681a2cf56d..2c3e4a26352b 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -186,6 +186,9 @@ struct server_tokens { const mtmd::input_chunk_ptr & find_chunk(size_t idx) const; + // tokens of the media chunk that starts at idx, 0 if none starts there + size_t chunk_n_tokens_at(size_t idx) const; + // find next media chunk after idx // returns a pair of pointer to the chunk (nullptr if not found) and its start index in tokens std::pair<const mtmd::input_chunk_ptr *, size_t> find_next_media_chunk(size_t idx) const; @@ -474,6 +477,12 @@ struct server_metrics { uint64_t n_decode = 0; uint64_t n_busy_slots = 0; + uint64_t n_preempt = 0; + uint64_t n_resume = 0; + + // [TAG_PREEMPT] parks that dropped their cells: those resumes re-prefill, and a re-prefill is not bit-for-bit the state that left + uint64_t n_preempt_recompute = 0; + uint64_t n_draft_tokens = 0; // Total draft tokens generated uint64_t n_draft_accepted = 0; // Draft tokens actually accepted uint64_t n_draft_verif_steps = 0; // Total draft token verification steps by the target model diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index fe068d3e9104..55cfef08eaf4 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -18,7 +18,9 @@ #include "mtmd-helper.h" #include <algorithm> +#include <set> #include <cstddef> +#include <cstring> #include <cinttypes> #include <exception> #include <memory> @@ -104,8 +106,79 @@ enum slot_state { SLOT_STATE_PROCESSING_PROMPT, SLOT_STATE_DONE_PROMPT, SLOT_STATE_GENERATING, + SLOT_STATE_PREEMPTED, // [TAG_PREEMPT] cells released, everything needed to resume is in host RAM + SLOT_STATE_PREEMPTING, // [TAG_PREEMPT_ASYNC] the copy out is running; the cells are still this slot's + SLOT_STATE_RESTORING, // [TAG_PREEMPT_ASYNC] the copy back in is running; the cells are allocated but not yet filled }; +// [TAG_PREEMPT] server-side request preemption: instead of ending every conversation in flight with a context error, one slot's sequence is copied to host RAM and back when there is room +constexpr int32_t PREEMPT_N_MARGIN = 8; // cells left spare on top of the reservation +constexpr int64_t PREEMPT_KEEPALIVE_MS = 2000; // SSE keepalive period while a streaming slot is parked +constexpr int32_t PREEMPT_N_STARVED = 3; // preemptions after which a slot is protected + +static std::string preempt_notice_comment(const server_task_result_preempt_notice & notice) { + const std::string suffix = (notice.index > 0 ? " " + std::to_string(notice.index) : "") + "\n\n"; + + std::string res = (notice.parked ? ": preempted" : ": resumed") + suffix; + + // [TAG_PREEMPT] the resume rebuilt the sequence from its tokens, so what follows is not the continuation the saved bytes would have given + if (!notice.parked && notice.recomputed) { + res += ": recomputed" + suffix; + } + + return res; +} +constexpr int32_t PREEMPT_N_FAIL_MAX = 8; // failed restores before the slot is given up on +constexpr int64_t PREEMPT_FAIL_US = 60ll * 1000 * 1000; // ... and only after this long parked +constexpr int64_t PREEMPT_ROTATE_US = 2ll * 1000 * 1000; // a resident cycling through context shifts gives way to a parked head that has waited this long +constexpr int64_t PREEMPT_ROTATE_RECOMPUTE_US = 30ll * 1000 * 1000; // ... and after this long it gives way even where that costs the resident a re-prefill + +// [TAG_PREEMPT_ASYNC] an asynchronous park only releases its cells when its copy lands, so it must fire this many decode steps before the pool would run out +constexpr int32_t PREEMPT_N_ASYNC_STEPS = 8; + +// [TAG_PREEMPT] test knob: LLAMA_SERVER_PREEMPT_FAIL_SAVE=N fails the host allocation of the Nth park, which no budget check can rule out, so that the fall back to a recompute park is exercised +static bool preempt_fail_save() { + static int32_t n_left = []() { + const char * val = getenv("LLAMA_SERVER_PREEMPT_FAIL_SAVE"); + + return val ? std::max(0, atoi(val)) : 0; + }(); + + return n_left > 0 && --n_left == 0; +} + +using llama_state_seq_copy_ptr = std::shared_ptr<llama_state_seq_copy>; + +static llama_state_seq_copy_ptr llama_state_seq_copy_make(llama_context * ctx) { + llama_state_seq_copy * cpy = ctx ? llama_state_seq_copy_init(ctx) : nullptr; + + return cpy ? llama_state_seq_copy_ptr(cpy, llama_state_seq_copy_free) : llama_state_seq_copy_ptr(); +} + +// [TAG_EXACT_CONCURRENCY] the planner counts cells, not tokens: a page belongs to one sequence, so a token count sees room find_slot cannot find and nobody is ever parked + +static constexpr int32_t preempt_n_cells_g(int32_t n_tokens, int32_t g) { + return (g <= 1 || n_tokens <= 0) ? n_tokens : ((n_tokens + g - 1) / g) * g; +} + +static constexpr int32_t preempt_n_cells_step_g(int32_t n_tokens, int32_t n_step, int32_t g) { + return preempt_n_cells_g(n_tokens + n_step, g) - preempt_n_cells_g(n_tokens, g); +} + +static_assert(preempt_n_cells_g(0, 1) == 0 && preempt_n_cells_g(1, 1) == 1 && + preempt_n_cells_g(8191, 1) == 8191 && preempt_n_cells_g(-3, 1) == -3, + "at a granularity of 1 a run of n tokens has to cost exactly n cells"); +static_assert(preempt_n_cells_step_g(0, 1, 1) == 1 && preempt_n_cells_step_g(8191, 1, 1) == 1 && + preempt_n_cells_step_g(1000, 512, 1) == 512, + "at a granularity of 1 a step of n tokens has to cost exactly n cells"); + +static_assert(preempt_n_cells_g(1, 256) == 256 && preempt_n_cells_g(256, 256) == 256 && + preempt_n_cells_g(257, 256) == 512, + "a tail page is charged in full"); +static_assert(preempt_n_cells_step_g(255, 1, 256) == 0 && preempt_n_cells_step_g(256, 1, 256) == 256 && + preempt_n_cells_step_g(256, 257, 256) == 512, + "a step is free until it crosses a page boundary and costs whole pages when it does"); + struct server_slot; // forward declaration struct server_batch { @@ -339,6 +412,380 @@ struct server_slot { prompt.clear(); } + slot_state state_before_preempt = SLOT_STATE_IDLE; + std::vector<uint8_t> preempt_state_tgt; + std::vector<uint8_t> preempt_state_dft; + + // [TAG_PREEMPT_ASYNC] the two transfers this slot parks and resumes through; they own the pinned host buffers, and are shared_ptr only so a slot survives the vector's reallocation + llama_state_seq_copy_ptr preempt_cpy_tgt; + llama_state_seq_copy_ptr preempt_cpy_dft; + + bool preempt_is_async() const { + return (bool) preempt_cpy_tgt; + } + + template <typename F> + auto preempt_sum(F f) const -> decltype(f(preempt_cpy_tgt.get())) { + if (!preempt_is_async()) { + return 0; + } + + return f(preempt_cpy_tgt.get()) + (preempt_cpy_dft ? f(preempt_cpy_dft.get()) : 0); + } + + template <typename F> + void preempt_each(F f) const { + if (preempt_cpy_tgt) { + f(preempt_cpy_tgt.get()); + } + + if (preempt_cpy_dft) { + f(preempt_cpy_dft.get()); + } + } + + int64_t preempt_sync_us() const { + return preempt_sum(llama_state_seq_copy_sync_us); + } + + size_t preempt_n_copies() const { + return preempt_sum(llama_state_seq_copy_n_copies); + } + + // [TAG_PREEMPT_ASYNC] a copy is running: the slot must not be scheduled but still owns cells, so it is neither running nor parked + bool preempt_in_flight() const { + return state == SLOT_STATE_PREEMPTING || state == SLOT_STATE_RESTORING; + } + + bool preempt_is_out() const { + return state == SLOT_STATE_PREEMPTED || preempt_in_flight(); + } + int32_t n_preempt = 0; // times the CURRENT task has been preempted + int32_t n_recompute = 0; // ... of which parked by dropping the cells, so the resume re-prefilled + int32_t n_ctx_shift = 0; // context shifts it has made: it is at the pool's limit and cycling + int32_t n_preempt_fail = 0; // consecutive failed restores + int64_t t_preempt_us = 0; // when it was parked + int64_t t_preempt_copy_us = 0; // [TAG_PREEMPT_ASYNC] when the current copy was issued + bool preempt_rotation_refused = false; // this park has logged a rotation refused for budget + + // [TAG_PREEMPT] a park with no room left under --preempt-ram: the cells are dropped instead of copied out, and the resume re-prefills the tokens + bool preempt_recompute = false; // parked by dropping its cells + bool preempt_reprefill = false; // putting back, as a prompt, what such a park dropped + server_tokens preempt_tokens; // what the re-prefill decodes: the prompt and everything generated so far + + size_t preempt_state_size() const { + // for a transfer the capacity, not the live size: the pinned buffers are kept between parks, so --preempt-ram has to bound what is held + return preempt_is_async() ? preempt_sum(llama_state_seq_copy_buf_capacity) + : preempt_state_tgt.size() + preempt_state_dft.size(); + } + + void preempt_state_free() { + // waits for anything in flight first: release() is reached with a copy possibly still using the buffer + preempt_each(llama_state_seq_copy_buf_free); + + preempt_state_tgt.clear(); + preempt_state_tgt.shrink_to_fit(); + preempt_state_dft.clear(); + preempt_state_dft.shrink_to_fit(); + } + + void preempt_copy_wait() { + preempt_each(llama_state_seq_copy_wait); + } + + size_t preempt_state_required() const { + return llama_state_seq_get_size_ext(ctx_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE) + + (ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0); + } + + // take the slot out of the step that is about to be built; the draft is a prediction, not a result, so it goes with the cells + void preempt_detach() { + spec_draft.clear(); + spec_i_batch.clear(); + spec_ckpt.clear(); + spec_is_replay = false; + + i_batch = -1; + } + + bool preempt_copy_done() { + return llama_state_seq_copy_done(preempt_cpy_tgt.get()) && + (!preempt_cpy_dft || llama_state_seq_copy_done(preempt_cpy_dft.get())); + } + + bool preempt_resumed() { + n_preempt_fail = 0; + + state = state_before_preempt; + + if (state == SLOT_STATE_GENERATING && can_speculate()) { + common_speculative_begin(spec, id, prompt.tokens.get_text_tokens()); + } + + return true; + } + + bool preempt_save_poll() { + if (!preempt_copy_done()) { + return false; + } + + mem.seq_rm(id, -1, -1); + + state = SLOT_STATE_PREEMPTED; + + return true; + } + + bool preempt_restore_poll() { + if (!preempt_copy_done()) { + return false; + } + + llama_state_seq_copy_buf_resize(preempt_cpy_tgt.get(), 0); + + if (preempt_cpy_dft) { + llama_state_seq_copy_buf_resize(preempt_cpy_dft.get(), 0); + } + + return preempt_resumed(); + } + + // the tokens the prompt step works through: its own list while re-prefilling, the request's otherwise + const server_tokens & preempt_input() const { + return preempt_tokens.empty() ? task->tokens : preempt_tokens; + } + + int32_t preempt_n_input() const { + return preempt_tokens.empty() ? (task ? task->n_tokens() : 0) : (int32_t) preempt_tokens.size(); + } + + // [TAG_PREEMPT] park with the host budget spent: drop the cells, keep the tokens, re-prefill them on resume. The sampler and the counters are untouched, so the stream carries on from the same token; the resume is bit-exact only as far as prefill numerics match decode numerics. A media chunk comes back the way it went in: the prompt step reads the chunk's data off the task and reserves its cells whole, so the placeholder the re-prefill list carries is all it needs + bool preempt_save_recompute() { + preempt_state_free(); + preempt_detach(); + + if (state == SLOT_STATE_GENERATING) { + preempt_tokens = std::move(prompt.tokens); + prompt.tokens = server_tokens(); + + prompt.tokens.has_mtmd = preempt_tokens.has_mtmd; // the re-prefill pushes the chunk's placeholder back into it + } + + prompt_clear(); + + state_before_preempt = state; + state = SLOT_STATE_PREEMPTED; + t_preempt_us = ggml_time_us(); + preempt_recompute = true; + preempt_rotation_refused = false; + + n_preempt++; + n_recompute++; + + return true; + } + + bool preempt_restore_recompute() { + preempt_recompute = false; + n_preempt_fail = 0; + + if (preempt_tokens.empty()) { + state = state_before_preempt; // its prompt had not been processed yet, so it is processed again from the start + + return true; + } + + preempt_reprefill = true; + state = SLOT_STATE_PROCESSING_PROMPT; + + return true; + } + + // every token is back in the cache: the slot goes on generating from the token it had already sampled + void preempt_reprefill_done() { + preempt_reprefill = false; + preempt_tokens.clear(); + + i_batch = -1; + state = SLOT_STATE_GENERATING; + + if (can_speculate()) { + common_speculative_begin(spec, id, prompt.tokens.get_text_tokens()); + } + } + + // [TAG_PREEMPT_ASYNC] copy the sequence out and release its cells; with a transfer this returns once the copy is issued and the cells stay the slot's until preempt_save_poll() sees it land + bool preempt_save() { + if (preempt_fail_save()) { + SLT_ERR(*this, "%s", "failed to allocate the host memory for the preemption state (test knob)\n"); + preempt_state_free(); + return false; + } + + const size_t size_tgt = llama_state_seq_get_size_ext(ctx_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE); + const size_t size_dft = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0; + + if (preempt_is_async()) { + if (!llama_state_seq_copy_buf_resize(preempt_cpy_tgt.get(), size_tgt) || + (size_dft > 0 && (!preempt_cpy_dft || + !llama_state_seq_copy_buf_resize(preempt_cpy_dft.get(), size_dft)))) { + SLT_ERR(*this, "failed to allocate %.3f MiB of pinned host memory for the preemption state\n", + (size_tgt + size_dft) / (1024.0 * 1024.0)); + preempt_state_free(); + return false; + } + + // [TAG_PREEMPT_ASYNC] the load-time probe saw pinned memory, but a larger buffer can still come back pageable, and a copy into pageable memory blocks; such a slot parks synchronously from now on + const bool pageable = !llama_state_seq_copy_buf_is_pinned(preempt_cpy_tgt.get()) || + (size_dft > 0 && !llama_state_seq_copy_buf_is_pinned(preempt_cpy_dft.get())); + + if (pageable) { + SLT_WRN(*this, "the host memory for a %.3f MiB park is pageable, so this slot parks synchronously from now on\n", + (size_tgt + size_dft) / (1024.0 * 1024.0)); + + preempt_cpy_tgt.reset(); + preempt_cpy_dft.reset(); + } else { + if (llama_state_seq_copy_get(preempt_cpy_tgt.get(), size_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE) != size_tgt) { + SLT_ERR(*this, "%s", "failed to issue the copy of the target sequence out of the KV cache\n"); + preempt_state_free(); + return false; + } + + if (size_dft > 0 && + llama_state_seq_copy_get(preempt_cpy_dft.get(), size_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) != size_dft) { + SLT_ERR(*this, "%s", "failed to issue the copy of the draft sequence out of the KV cache\n"); + preempt_state_free(); + return false; + } + + preempt_detach(); + + // note: no mem.seq_rm() here. The copy is still reading these cells; preempt_save_poll() releases them. + state_before_preempt = state; + state = SLOT_STATE_PREEMPTING; + t_preempt_us = ggml_time_us(); + + n_preempt++; + + return true; + } + } + + try { + preempt_state_tgt.resize(size_tgt); + preempt_state_dft.resize(size_dft); + } catch (const std::bad_alloc & e) { + SLT_ERR(*this, "failed to allocate %.3f MiB for the preemption state: %s\n", + (size_tgt + size_dft) / (1024.0 * 1024.0), e.what()); + preempt_state_free(); + return false; + } + + if (llama_state_seq_get_data_ext(ctx_tgt, preempt_state_tgt.data(), size_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE) != size_tgt) { + SLT_ERR(*this, "%s", "failed to copy the target sequence out of the KV cache\n"); + preempt_state_free(); + return false; + } + + if (size_dft > 0 && + llama_state_seq_get_data_ext(ctx_dft, preempt_state_dft.data(), size_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) != size_dft) { + SLT_ERR(*this, "%s", "failed to copy the draft sequence out of the KV cache\n"); + preempt_state_free(); + return false; + } + + preempt_detach(); + + // note: prompt.tokens is deliberately kept - the resume sizes its request from it + mem.seq_rm(id, -1, -1); + + state_before_preempt = state; + state = SLOT_STATE_PREEMPTED; + t_preempt_us = ggml_time_us(); + preempt_rotation_refused = false; + + n_preempt++; + + return true; + } + + // [TAG_PREEMPT_ASYNC] put the sequence back; with a transfer this returns once the copy is issued, leaving the slot RESTORING: it owns the cells, but they hold no state until the copy lands + bool preempt_restore() { + if (preempt_recompute) { + return preempt_restore_recompute(); + } + + if (preempt_is_async()) { + const size_t size_tgt = llama_state_seq_copy_buf_size(preempt_cpy_tgt.get()); + const size_t size_dft = preempt_cpy_dft ? llama_state_seq_copy_buf_size(preempt_cpy_dft.get()) : 0; + + if (llama_state_seq_copy_set(preempt_cpy_tgt.get(), size_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE) != size_tgt || + (size_dft > 0 && + llama_state_seq_copy_set(preempt_cpy_dft.get(), size_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) != size_dft)) { + // no room after all: let what was issued finish before the half-written sequence is dropped, or cells go while a copy still writes them + preempt_copy_wait(); + mem.seq_rm(id, -1, -1); + n_preempt_fail++; + return false; + } + + state = SLOT_STATE_RESTORING; + + return true; + } + + const size_t size_tgt = preempt_state_tgt.size(); + const size_t size_dft = preempt_state_dft.size(); + + if (llama_state_seq_set_data_ext(ctx_tgt, preempt_state_tgt.data(), size_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE) != size_tgt || + (size_dft > 0 && + llama_state_seq_set_data_ext(ctx_dft, preempt_state_dft.data(), size_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) != size_dft)) { + mem.seq_rm(id, -1, -1); + n_preempt_fail++; + return false; + } + + preempt_state_free(); + + return preempt_resumed(); + } + + // [TAG_PREEMPT] bring prompt.tokens back to what the cache holds, for a batch given up after it was built: never-decoded tokens and the draft come off, `sampled` is kept + void rewind_to_cache() { + // the memory counts positions, and with M-RoPE media a position is not a token, so convert before truncating + const llama_pos pos_cached = llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), id) + 1; + + size_t n_cached = pos_cached > 0 ? prompt.tokens.size_up_to_pos(pos_cached) : 0; + + // a cut inside a media chunk is not a token boundary: keep what precedes the chunk, and the cells of its head go too + bool split_chunk = false; + + while (n_cached > 0 && prompt.tokens.pos_next(n_cached) > pos_cached) { + n_cached--; + split_chunk = true; + } + + if (n_cached < (size_t) prompt.n_tokens()) { + prompt.tokens.keep_first(n_cached); + } + + if (split_chunk) { + mem.seq_rm(id, prompt.tokens.pos_next(), -1); + } + + // what is kept ends where the cache does, or the next decode is positioned from the wrong count + GGML_ASSERT(prompt.tokens.pos_next() <= pos_cached); + + // the last chunk was marked done when it was built but never ran, so it is not done + if (state == SLOT_STATE_DONE_PROMPT && task && prompt.n_tokens() < preempt_n_input()) { + state = SLOT_STATE_PROCESSING_PROMPT; + } + + preempt_detach(); + } + std::vector<common_adapter_lora_info> lora; int32_t alora_invocation_start = -1; @@ -397,6 +844,17 @@ struct server_slot { n_predict_max = -1; + preempt_state_free(); + preempt_tokens.clear(); + preempt_recompute = false; + preempt_reprefill = false; + state_before_preempt = SLOT_STATE_IDLE; + n_preempt = 0; + n_recompute = 0; + n_preempt_fail = 0; + n_ctx_shift = 0; + t_preempt_us = 0; + llama_set_sampler(ctx_tgt, id, nullptr); // clear alora start @@ -549,6 +1007,13 @@ struct server_slot { t_last_used = ggml_time_us(); + // [TAG_PREEMPT] [TAG_PREEMPT_ASYNC] a parked slot's cells are already gone, so the mirror must not outlive them or the next task prefix-matches an empty cache; wait for any copy first, its buffer and its cells are about to be handed on + if (preempt_is_out()) { + preempt_copy_wait(); + preempt_state_free(); + prompt_clear(); + } + state = SLOT_STATE_IDLE; // do not keep context of the child slots - the parent's context is enough @@ -687,10 +1152,15 @@ struct server_slot { json res; res = { - {"id", id}, - {"n_ctx", n_ctx}, - {"speculative", can_speculate()}, - {"is_processing", is_processing()}, + {"id", id}, + {"n_ctx", n_ctx}, + {"speculative", can_speculate()}, + {"is_processing", is_processing()}, + // [TAG_PREEMPT] parked means the cells are gone; a copy out still owns them and a restore has already taken them back, so a scraper counting residency has to keep counting those two + {"is_preempted", state == SLOT_STATE_PREEMPTED}, + {"is_transferring", preempt_in_flight()}, + {"n_preempt", n_preempt}, + {"n_recompute", n_recompute}, }; const auto & ptask = task ? task : task_prev; @@ -872,6 +1342,30 @@ struct server_context_impl { metrics.reset_bucket(); } + // [TAG_PREEMPT] the first prompt of a request that cannot be served, with the error response it gets; false when every one of them passes. + // A park notice opens the stream of the member it belongs to, so a member rejected after that could only be told inside a stream that has already answered 200. + bool tasks_prompt_rejected(const std::vector<server_task> & tasks, json & error) const { + std::string msg; + error_type type = ERROR_TYPE_SERVER; + + for (const auto & task : tasks) { + if (!task_prompt_rejected(task, msg, type)) { + continue; + } + + error = format_error_response(msg, type); + + if (type == ERROR_TYPE_EXCEED_CONTEXT_SIZE) { + error["n_prompt_tokens"] = task.n_tokens(); + error["n_ctx"] = n_ctx_slot(); + } + + return true; + } + + return false; + } + private: // note: accessing these fields outside of this class is not thread-safe // use server_context methods instead @@ -936,6 +1430,17 @@ struct server_context_impl { int64_t t_last_load_progress_ms = 0; void destroy() { + // [TAG_PREEMPT_ASYNC] the slots outlive this call and may hold a copy reading or writing KV tensors of the contexts about to be freed; release() makes the same wait for one slot + for (auto & slot : slots) { + slot.preempt_copy_wait(); + + slot.preempt_cpy_tgt.reset(); + slot.preempt_cpy_dft.reset(); + } + + preempt_ram_kind_logged = false; + preempt_recompute_logged = false; + spec.reset(); spec_init.reset(); @@ -1310,6 +1815,20 @@ struct server_context_impl { } }; + // [TAG_PREEMPT_ASYNC] one transfer per context, reused for every park and resume, because each owns a backend and installs the fences the context records after every decode + if (preempt_async_possible()) { + slot.preempt_cpy_tgt = llama_state_seq_copy_make(ctx_tgt); + + if (slot.preempt_cpy_tgt && ctx_dft) { + slot.preempt_cpy_dft = llama_state_seq_copy_make(ctx_dft); + + if (!slot.preempt_cpy_dft) { + // a draft that cannot go asynchronously would have to be waited for mid-park, so the whole slot stays synchronous + slot.preempt_cpy_tgt.reset(); + } + } + } + slot.reset(); } @@ -1331,6 +1850,110 @@ struct server_context_impl { } } + { + preempt_async_ok = !slots.empty(); + + for (const auto & slot : slots) { + preempt_async_ok = preempt_async_ok && slot.preempt_is_async(); + } + + if (preempt_async_possible()) { + if (preempt_async_ok) { + // a copy into pageable memory is staged by the driver and blocks the thread that issued it, and a host buffer type is free to hand back pageable memory rather than fail + bool pinned = llama_state_seq_copy_buf_can_pin(slots[0].preempt_cpy_tgt.get()); + + if (pinned) { + auto * cpy = slots[0].preempt_cpy_tgt.get(); + + pinned = llama_state_seq_copy_buf_resize(cpy, 1u << 20) != nullptr && + llama_state_seq_copy_buf_is_pinned(cpy); + + llama_state_seq_copy_buf_free(cpy); + } + + if (pinned) { + SRV_INF("%s", "preemption: parking and resuming asynchronously through pinned host memory\n"); + } else { + SRV_WRN("%s", "preemption: the host memory on offer is pageable, so a copy would block the decode; parking and resuming synchronously\n"); + preempt_async_ok = false; + } + } else { + SRV_WRN("%s", "preemption: this backend cannot copy asynchronously, parking and resuming synchronously\n"); + } + } else if (params_base.preempt_async && !llama_model_is_recurrent(model_tgt) && preempt_state_relocates()) { + SRV_WRN("%s", "preemption: a recurrent state does not stay in one row, so a copy running beside the decode could read another sequence; parking and resuming synchronously\n"); + } + + if (!preempt_async_ok) { + for (auto & slot : slots) { + slot.preempt_cpy_tgt.reset(); + slot.preempt_cpy_dft.reset(); + } + } + } + + { + preempt_alloc_granularity = (int32_t) std::max(1u, llama_memory_alloc_granularity(llama_get_memory(ctx_tgt))); + + // test knob: the paged kernel needs a head size of 256, so a harness model cannot reach the paged arithmetic otherwise + const char * LLAMA_SERVER_PREEMPT_GRANULARITY = getenv("LLAMA_SERVER_PREEMPT_GRANULARITY"); + + if (LLAMA_SERVER_PREEMPT_GRANULARITY) { + preempt_alloc_granularity = std::max(1, atoi(LLAMA_SERVER_PREEMPT_GRANULARITY)); + + SRV_WRN("LLAMA_SERVER_PREEMPT_GRANULARITY = %d (test knob: planning the kv pool in blocks of %d cells)\n", + preempt_alloc_granularity, preempt_alloc_granularity); + } else if (preempt_alloc_granularity > 1) { + SRV_INF("preemption: the kv pool allocates %d cells at a time, planning in pages\n", + preempt_alloc_granularity); + } + } + + { + preempt_resume_head = true; + + const char * LLAMA_SERVER_PREEMPT_RESUME = getenv("LLAMA_SERVER_PREEMPT_RESUME"); + if (LLAMA_SERVER_PREEMPT_RESUME && strcmp(LLAMA_SERVER_PREEMPT_RESUME, "head") != 0) { + if (strcmp(LLAMA_SERVER_PREEMPT_RESUME, "pass") != 0) { + SRV_ERR("LLAMA_SERVER_PREEMPT_RESUME = %s is not a resume order; use head (the default) or pass\n", + LLAMA_SERVER_PREEMPT_RESUME); + return false; + } + preempt_resume_head = false; + SRV_WRN("%s", "LLAMA_SERVER_PREEMPT_RESUME = pass (parked slots come back most-preempted first, and a smaller slot may pass a head that does not fit)\n"); + } + + const char * LLAMA_SERVER_PREEMPT_EVERY = getenv("LLAMA_SERVER_PREEMPT_EVERY"); + preempt_test_every = LLAMA_SERVER_PREEMPT_EVERY ? atoi(LLAMA_SERVER_PREEMPT_EVERY) : 0; + + // LLAMA_SERVER_PREEMPT_POLICY: which non-leader the planner parks; smallest (default), largest, youngest, oldest + const char * LLAMA_SERVER_PREEMPT_POLICY = getenv("LLAMA_SERVER_PREEMPT_POLICY"); + preempt_test_policy = LLAMA_SERVER_PREEMPT_POLICY ? LLAMA_SERVER_PREEMPT_POLICY : "smallest"; + + if (preempt_test_policy != "smallest") { + SRV_WRN("LLAMA_SERVER_PREEMPT_POLICY = %s (test knob: victim choice for comparison only)\n", preempt_test_policy.c_str()); + } + + if (preempt_test_every > 0) { + SRV_WRN("LLAMA_SERVER_PREEMPT_EVERY = %d (test knob: preempting every slot every %d tokens)\n", + preempt_test_every, preempt_test_every); + } + + const char * LLAMA_SERVER_PREEMPT_PLANNER = getenv("LLAMA_SERVER_PREEMPT_PLANNER"); + preempt_planner_off = LLAMA_SERVER_PREEMPT_PLANNER && strcmp(LLAMA_SERVER_PREEMPT_PLANNER, "off") == 0; + + if (preempt_planner_off) { + SRV_WRN("%s", "LLAMA_SERVER_PREEMPT_PLANNER = off (test knob: nothing is parked ahead of the decode, only as a last resort)\n"); + } + + // assigned, not only set: a context reloaded with an attention model after a recurrent one gets preemption back + preempt_recurrent = llama_model_is_recurrent(model_tgt); + + if (preempt_recurrent) { + SRV_WRN("%s", "preemption: off, the recurrent cache holds one state per sequence whatever its length, so there is no cell pool to run out of\n"); + } + } + { const char * LLAMA_SERVER_SLOTS_N_DIFF = getenv("LLAMA_SERVER_SLOTS_N_DIFF"); slots_n_diff = LLAMA_SERVER_SLOTS_N_DIFF ? atoi(LLAMA_SERVER_SLOTS_N_DIFF) : 0; @@ -2043,8 +2666,25 @@ struct server_context_impl { queue_results.send(std::move(res)); } - void send_partial_response(server_slot & slot, const completion_token_output & tkn, bool is_progress, bool is_begin = false) { - auto res = std::make_unique<server_task_result_cmpl_partial>(); + void send_preempt_notice(server_slot & slot, bool parked, bool recomputed = false) { + if (!slot.task || !slot.task->params.stream) { + return; + } + + auto res = std::make_unique<server_task_result_preempt_notice>(); + + res->id = slot.task->id; + res->index = slot.task->index; + res->id_slot = slot.id; + res->parked = parked; + res->recomputed = recomputed; + res->n_preempt = slot.n_preempt; + + queue_results.send(std::move(res)); + } + + void send_partial_response(server_slot & slot, const completion_token_output & tkn, bool is_progress, bool is_begin = false) { + auto res = std::make_unique<server_task_result_cmpl_partial>(); res->id = slot.task->id; res->index = slot.task->index; @@ -2120,6 +2760,8 @@ struct server_context_impl { res->stopping_word = slot.stopping_word; res->stop = slot.stop; res->post_sampling_probs = slot.task->params.post_sampling_probs; + res->n_preempt = slot.n_preempt; + res->n_recompute = slot.n_recompute; res->verbose = slot.task->params.verbose; res->stream = slot.task->params.stream; @@ -2501,11 +3143,15 @@ struct server_context_impl { case SERVER_TASK_TYPE_METRICS: { int n_processing_slots = 0; + int n_preempted_slots = 0; for (server_slot & slot : slots) { if (slot.is_processing()) { n_processing_slots++; } + if (slot.preempt_is_out()) { + n_preempted_slots++; + } } SRV_DBG("n_processing_slots = %d\n", n_processing_slots); @@ -2513,6 +3159,8 @@ struct server_context_impl { res->id = task.id; res->n_processing_slots = n_processing_slots; res->n_tasks_deferred = queue_tasks.queue_tasks_deferred_size(); + res->n_preempted_slots = n_preempted_slots; + res->preempt_ram_bytes = preempt_ram_used(); res->metrics = metrics; if (task.metrics_reset_bucket) { @@ -2724,71 +3372,1068 @@ struct server_context_impl { } break; } - return true; - } + return true; + } + + void iterate(std::vector<server_slot> & slots, std::function<void(server_slot &)> callback) { + for (auto & slot : slots) { + try { + callback(slot); + } catch (const std::exception & e) { + SLT_ERR(slot, "got exception: %s\n", e.what()); + send_error(slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER); + slot.release(); + } + } + } + + void iterate(std::vector<server_slot *> & slots, std::function<void(server_slot &)> callback) { + for (auto & slot : slots) { + try { + callback(*slot); + } catch (const std::exception & e) { + SLT_ERR(*slot, "got exception: %s\n", e.what()); + send_error(*slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER); + slot->release(); + } + } + } + + void abort_all_slots(const std::string & reason) { + for (auto & slot : slots) { + // [TAG_PREEMPT] a parked slot, or one with a copy in flight, took no part in what failed and comes back when there is room + if (slot.is_processing() && !slot.preempt_is_out()) { + send_error(slot, reason, ERROR_TYPE_SERVER); + slot.release(); + } + } + } + + // @ngxson : for debugging only + int64_t t_pre_decode = 0; + int64_t t_decode = 0; + int64_t t_post_decode = 0; + int64_t t_sampl = 0; + int64_t n_pre_decode = 0; + int64_t n_decode = 0; + int64_t n_post_decode = 0; + int64_t n_sampl = 0; +// #define DEBUG_TIMINGS +#ifdef DEBUG_TIMINGS + struct scoped_timer { + int64_t & t; + int64_t & n; + int64_t t_start; + scoped_timer(int64_t & t_, int64_t & n_) : t(t_), n(n_) { + t_start = ggml_time_us(); + } + ~scoped_timer() { + t += ggml_time_us() - t_start; + n++; + } + }; +#else + struct scoped_timer { + scoped_timer(int64_t &, int64_t &) {} + ~scoped_timer() {} + }; +#endif + + + // LLAMA_SERVER_PREEMPT_EVERY=N: preempt every generating slot every N tokens, pressure or not, so the determinism test can blame any difference on the preemption + int32_t preempt_test_every = 0; + std::string preempt_test_policy = "smallest"; // LLAMA_SERVER_PREEMPT_POLICY, see load_model + + // [TAG_PREEMPT_ASYNC] whether parks go through a transfer; false with --no-preempt-async or a backend that cannot copy asynchronously + bool preempt_async_ok = false; + + // [TAG_EXACT_CONCURRENCY] cells the pool hands out at a time, read once at load: 1 ordinarily, the page size under exact concurrency. LLAMA_SERVER_PREEMPT_GRANULARITY overrides it. + int32_t preempt_alloc_granularity = 1; + + int32_t preempt_n_cells(int32_t n_tokens) const { + return preempt_n_cells_g(n_tokens, preempt_alloc_granularity); + } + + int32_t preempt_n_cells_step(int32_t n_tokens, int32_t n_step) const { + return preempt_n_cells_step_g(n_tokens, n_step, preempt_alloc_granularity); + } + + // LLAMA_SERVER_PREEMPT_PLANNER=off (test knob): no parking ahead of the decode, leaving only the KV-full retry ladder + bool preempt_planner_off = false; + + // LLAMA_SERVER_PREEMPT_RESUME=head or pass, read at load, per context + bool preempt_resume_head = true; + + bool preempt_recurrent = false; + + bool preempt_batch_abandoned = false; + + // [TAG_PREEMPT_ASYNC] a context shift was recorded this round; it is applied in place inside the next llama_decode + bool preempt_shift_pending = false; + + int32_t preempt_n_spec_max() const { + return spec ? std::max(0, common_speculative_n_max(¶ms_base.speculative)) : 0; + } + + bool preempt_ram_kind_logged = false; + + // [TAG_PREEMPT] the fall back to a recompute park is logged once, not per park + bool preempt_recompute_logged = false; + + void preempt_log_ram_kind(const server_slot & slot) { + if (preempt_ram_kind_logged || !slot.preempt_is_async()) { + return; + } + + if (llama_state_seq_copy_buf_capacity(slot.preempt_cpy_tgt.get()) == 0) { + return; // nothing held, so nothing to report yet + } + + preempt_ram_kind_logged = true; + + SRV_INF("preemption: parking into %s host memory\n", + llama_state_seq_copy_buf_is_pinned(slot.preempt_cpy_tgt.get()) ? "pinned" : "pageable"); + } + + int32_t preempt_n_spec(const server_slot & slot) const { + int32_t res = preempt_n_spec_max(); + + if (res == 0 || !slot.task || !slot.can_speculate()) { + return 0; + } + + // a recompute park moved the prompt out of the slot, so the tokens it comes back with bound the draft, not the empty prompt: read as empty, a 2000-token sequence in a 2048-cell pool was charged a whole draft and failed as impossible + const int32_t n_tokens = std::max(slot.prompt.n_tokens(), slot.preempt_n_input()); + + res = std::min(res, slot.n_ctx - n_tokens - 2); + + if (slot.n_remaining() > 0) { + res = std::min(res, slot.n_remaining() - 1); + } + + return std::max(0, res); + } + + size_t preempt_ram_used() const { + size_t res = 0; + + for (const auto & slot : slots) { + res += slot.preempt_state_size(); + } + + return res; + } + + // [TAG_PREEMPT_ASYNC] a restored slot keeps its pinned buffer, so that idle capacity is given back largest first when a park does not fit under --preempt-ram + void preempt_reclaim_idle_ram(size_t budget, size_t extra, const server_slot & keep) { + for (;;) { + if (preempt_ram_used() + extra <= budget) { + return; + } + + server_slot * best = nullptr; + + for (auto & other : slots) { + if (&other == &keep) { + continue; + } + + if (other.state == SLOT_STATE_PREEMPTED || other.state == SLOT_STATE_PREEMPTING || other.state == SLOT_STATE_RESTORING) { + continue; + } + + if (other.preempt_state_size() == 0) { + continue; + } + + if (!best || other.preempt_state_size() > best->preempt_state_size()) { + best = &other; + } + } + + if (!best) { + return; + } + + SLT_INF(*best, "%.1f MiB of idle parked RAM returned so that another slot can park\n", + best->preempt_state_size() / (1024.0 * 1024.0)); + + best->preempt_state_free(); + } + } + + size_t preempt_ram_budget() const { + return params_base.preempt_ram_mib < 0 ? SIZE_MAX : (size_t) params_base.preempt_ram_mib * 1024 * 1024; + } + + bool preempt_fits_budget(const server_slot & slot) { + const size_t budget = preempt_ram_budget(); + + // what this slot already holds is counted by preempt_ram_used() and reused, so a park costs only the rest + const size_t held = slot.preempt_state_size(); + const size_t need = slot.preempt_state_required(); + const size_t extra = need > held ? need - held : 0; + + preempt_reclaim_idle_ram(budget, extra, slot); + + return preempt_ram_used() + extra <= budget; + } + + void preempt_trim_ram(server_slot & slot) { + if (preempt_ram_used() > preempt_ram_budget() && slot.preempt_state_size() > 0) { + SLT_INF(slot, "%.1f MiB of parked RAM returned: the pool is over its budget\n", slot.preempt_state_size() / (1024.0 * 1024.0)); + slot.preempt_state_free(); + } + } + + size_t preempt_n_keep(const server_slot & slot) const { + if (!slot.task->params.cache_prompt) { + return 0; + } + + size_t n_keep = slot.prompt.tokens.get_common_prefix(slot.task->tokens); + + if (slot.alora_invocation_start > 0) { + n_keep = std::min(n_keep, (size_t) (slot.alora_invocation_start - 1)); + } + + return n_keep; + } + + // a slot just given a task still mirrors the previous request's prompt, so what it holds and what it asks for both count from preempt_n_keep() + int32_t preempt_n_retained(const server_slot & slot) const { + if (slot.preempt_recompute) { + return (int32_t) slot.preempt_tokens.size(); // parked by dropping its cells: it comes back needing all of them at once + } + + if (slot.state == SLOT_STATE_STARTED && slot.task) { + return (int32_t) preempt_n_keep(slot); + } + + return slot.prompt.n_tokens(); + } + + // [TAG_PREEMPT] the cells the media chunks pending at n_have take: pre_decode() runs a whole chunk through llama_decode() calls of its own, which no kv-full retry covers, so the planner reserves the lot before it is decoded + int32_t preempt_n_mtmd_pending(const server_slot & slot, int32_t n_have) const { + if (!slot.task || !slot.task->tokens.has_mtmd) { + return 0; + } + + const auto & tokens = slot.task->tokens; + + int32_t res = 0; + + for (int32_t i = n_have; i >= 0 && i < (int32_t) tokens.size(); ) { + const int32_t n = (int32_t) tokens.chunk_n_tokens_at(i); + + if (n <= 0) { + break; + } + + res += n; + i += n; + } + + return res; + } + + int32_t preempt_n_need(const server_slot & slot) const { + int32_t res = preempt_n_retained(slot); + + if (slot.state_before_preempt == SLOT_STATE_GENERATING) { + res += 1 + preempt_n_spec(slot); + } else { + const int32_t n_mtmd = preempt_n_mtmd_pending(slot, res); + const int32_t n_left = slot.preempt_n_input() - res; + + res += n_mtmd > 0 ? n_mtmd : std::max(1, std::min((int32_t) llama_n_batch(ctx_tgt), n_left)); + } + + // [TAG_EXACT_CONCURRENCY] a restore takes fresh pages and its tail page is charged in full; undercounting admits a resume find_slot cannot satisfy + return preempt_n_cells(res); + } + + int32_t preempt_kv_used() const { + int32_t res = 0; + + // n_cmpl > 1: a family shares the prompt's cells through seq_cp, so it is charged once, to the first resident member + std::vector<int> charged; + + for (const auto & slot : slots) { + if (slot.state == SLOT_STATE_PREEMPTED) { + continue; // parked: its cells are in host RAM, not in the pool + } + + // [TAG_PREEMPT_ASYNC] deliberately not skipped: a slot with a copy in flight holds cells either way, and skipping it would hand the same cells out twice + + if (slot.state == SLOT_STATE_WAIT_OTHER) { + res += preempt_n_cells(slot.prompt.n_tokens()); + continue; + } + + if (slot.task && (slot.task->is_parent() || slot.task->is_child())) { + const int family = slot.task->is_parent() ? slot.task->id : slot.task->id_parent; + + if (std::find(charged.begin(), charged.end(), family) != charged.end()) { + res += preempt_n_cells(std::max(0, slot.prompt.n_tokens() - slot.task->n_tokens())); + continue; + } + + charged.push_back(family); + } + + res += preempt_n_cells(slot.prompt.n_tokens()); + } + + return res; + } + + // [TAG_PREEMPT_ASYNC] the room the pool is kept clear of, so everything still decoding has somewhere to put its tokens until a park lands; a resume candidate is charged the same runway + int32_t preempt_n_margin(int32_t n_additional_running = 0) const { + if (!preempt_async_ok) { + // [TAG_EXACT_CONCURRENCY] a margin of eight cells is no margin where a step can cost a whole page + return preempt_n_cells(PREEMPT_N_MARGIN); + } + + int32_t n_running = n_additional_running; + + for (const auto & slot : slots) { + if (slot.is_processing() && (!slot.preempt_is_out() || slot.state == SLOT_STATE_RESTORING)) { + n_running++; + } + } + + // [TAG_EXACT_CONCURRENCY] round the runway up to a page, once and not per slot, which would keep a page per slot out of the users' reach + return preempt_n_cells( + PREEMPT_N_MARGIN + n_running * (1 + preempt_n_spec_max()) * PREEMPT_N_ASYNC_STEPS); + } + + int32_t preempt_kv_reserve() const { + const int32_t n_batch = llama_n_batch(ctx_tgt); + + int32_t res = 0; + int32_t res_pmt = 0; + int32_t res_mm = 0; + int32_t n_pmt = 0; + + // [TAG_EXACT_CONCURRENCY] reserve the cells the next step ADDS, not its tokens: the used figure already rounds every tail page up, and only a page crossing can empty the pool + for (const auto & slot : slots) { + const int32_t n_cur = slot.prompt.n_tokens(); + + // [TAG_PREEMPT_ASYNC] a restoring slot decodes as soon as its copy lands, so it is charged the step of the state it goes back to, or that first step preempts somebody else + const slot_state state = slot.state == SLOT_STATE_RESTORING ? slot.state_before_preempt : slot.state; + + switch (state) { + case SLOT_STATE_GENERATING: + case SLOT_STATE_DONE_PROMPT: + { + res += preempt_n_cells_step(n_cur, 1 + preempt_n_spec(slot)); + } break; + case SLOT_STATE_STARTED: + case SLOT_STATE_PROCESSING_PROMPT: + { + const int32_t n_have = preempt_n_retained(slot); + const int32_t n_mtmd = preempt_n_mtmd_pending(slot, n_have); + + // a media chunk is decoded whole, past the batch cap below and past the kv-full retry + if (n_mtmd > 0) { + res_mm += preempt_n_cells_step(n_have, n_mtmd); + break; + } + + const int32_t n_left = slot.preempt_n_input() - n_have; + + res_pmt += preempt_n_cells_step(n_have, std::max(1, std::min(n_batch, n_left))); + n_pmt++; + } break; + default: + break; + } + } + + return res + res_mm + std::min(res_pmt, preempt_n_cells(n_batch) + std::max(0, n_pmt - 1) * (preempt_alloc_granularity - 1)); + } + + // [TAG_PREEMPT] trim a just-started slot to the prefix it keeps first, or it is copied out, charged and sized by the previous request's prompt + bool preempt_normalize_started_all() { + bool res = false; + + for (auto & slot : slots) { + const int32_t before = slot.prompt.n_tokens(); + + preempt_normalize_started(slot); + + if (slot.prompt.n_tokens() < before) { + SLT_INF(slot, "trimmed to the %d cells its request keeps ahead of the batch builder, %d released\n", + slot.prompt.n_tokens(), before - slot.prompt.n_tokens()); + res = true; + } + } + + return res; + } + + void preempt_normalize_started(server_slot & slot) { + if (slot.state != SLOT_STATE_STARTED || !slot.task) { + return; + } + + const size_t n_keep = preempt_n_keep(slot); + + if (n_keep >= slot.prompt.tokens.size()) { + return; + } + + // a memory that cannot remove part of a sequence aborts on a partial removal, so drop the whole stale sequence + const bool partial_ok = ctx_tgt_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_PART && + (!ctx_dft || ctx_dft_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_PART); + + if (!partial_ok) { + slot.prompt.tokens.clear(); + slot.mem.seq_rm(slot.id, -1, -1); + return; + } + + slot.prompt.tokens.keep_first(n_keep); + slot.mem.seq_rm(slot.id, slot.prompt.tokens.pos_next(), -1); + } + + // `recompute` asks for a fallback: with no victim the host budget can hold, the same policy picks again and the park drops the cells instead of copying them + server_slot * preempt_pick_victim(bool * recompute = nullptr) { + server_slot * leader = nullptr; + int32_t n_running = 0; + + for (auto & slot : slots) { + preempt_normalize_started(slot); + } + + for (auto & slot : slots) { + if (slot.is_processing() && !slot.preempt_is_out()) { + n_running++; + + if (!leader || slot.prompt.n_tokens() > leader->prompt.n_tokens()) { + leader = &slot; + } + } + } + + if (n_running < 2) { + // one conversation that does not fit alone is a real overflow, not a scheduling problem + return nullptr; + } + + server_slot * victim = preempt_pick_victim_pass(leader, false); + + if (!victim && recompute) { + victim = preempt_pick_victim_pass(leader, true); + + *recompute = victim != nullptr; + } + + return victim; + } + + server_slot * preempt_pick_victim_pass(const server_slot * leader, bool recompute) { + server_slot * victim = nullptr; + + for (auto & slot : slots) { + // before the batch is built every slot is at a token boundary; one holding no cells is still worth parking + if (slot.state != SLOT_STATE_GENERATING && + slot.state != SLOT_STATE_PROCESSING_PROMPT && + slot.state != SLOT_STATE_STARTED) { + continue; + } + + if (&slot == leader) { + continue; + } + + if (slot.task && (slot.task->is_parent() || slot.task->is_child())) { + continue; // n_cmpl > 1 slots share one sequence, out of scope here + } + + // a started slot the STARTED block is about to reject gets its error on its own pass: a park notice would open the stream and turn that 4xx into 200 plus an in-stream error + if (slot.state == SLOT_STATE_STARTED) { + std::string msg; + error_type type = ERROR_TYPE_SERVER; + + if (slot_prompt_rejected(slot, msg, type)) { + continue; + } + } + + if (!recompute && !preempt_fits_budget(slot)) { + continue; + } + + const bool starved = slot.n_preempt >= PREEMPT_N_STARVED; + const bool starved_cur = victim && victim->n_preempt >= PREEMPT_N_STARVED; + + if (!victim || + (starved_cur && !starved) || + (starved_cur == starved && preempt_better_victim(slot, *victim))) { + victim = &slot; + } + } + + return victim; + } + + bool preempt_better_victim(const server_slot & a, const server_slot & b) const { + if (preempt_test_policy == "largest") { + return a.prompt.n_tokens() > b.prompt.n_tokens(); + } + + if (preempt_test_policy == "youngest") { + return a.task->id > b.task->id; + } + + if (preempt_test_policy == "oldest") { + return a.task->id < b.task->id; + } + + return a.prompt.n_tokens() < b.prompt.n_tokens(); + } + + // [TAG_PREEMPT] park a slot: a synchronous park is finished here, an asynchronous one only issued, and update_preempt_copies() counts it when its copy lands. The notice goes with the save, not the cell release: preempt_save() has already detached the slot, so a release-time notice would leave the copy's silence unexplained. + bool preempt_park(server_slot & slot, int64_t t_start, bool recompute = false) { + slot.t_preempt_copy_us = t_start; + + // [TAG_PREEMPT] a budget check grants permission to allocate, not a successful allocation: preempt_save() unwinds and waits for whatever it issued, so the same victim can still be parked by dropping its cells + if (!recompute && !slot.preempt_save()) { + SLT_WRN(slot, "%s", "the park could not take the host memory the budget allowed, so it drops its cells instead and the resume re-prefills its tokens\n"); + + recompute = true; + } + + if (recompute) { + if (!slot.preempt_save_recompute()) { + return false; + } + + if (!preempt_recompute_logged) { + preempt_recompute_logged = true; + + // [TAG_EXACT_CONCURRENCY] a state that comes back from host memory is the state that left; one rebuilt by re-prefilling is the same on CPU and differs in the last bits on CUDA, where a prefill of a token and a decode of it take different kernels + if (common_exact_concurrency()) { + SRV_WRN("%s", "exact concurrency: a re-prefilled sequence is not guaranteed byte-identical to one that was never parked; raise --preempt-ram until every parked sequence fits it\n"); + } + + SRV_WRN("preemption: --preempt-ram %d MiB holds no further parked sequence, so a park drops its cells and the resume re-prefills its tokens\n", + params_base.preempt_ram_mib); + } + } + + preempt_log_ram_kind(slot); + + if (slot.state == SLOT_STATE_PREEMPTED) { + metrics.n_preempt++; + } + + if (slot.preempt_recompute) { + metrics.n_preempt_recompute++; + } + + send_preempt_notice(slot, true); + + return true; + } + + void preempt_parked(server_slot & slot, const char * note) { + metrics.n_preempt++; + + SLT_WRN(slot, "park completed after %.2f ms%s: %d cells released, %.1f MiB parked, kv %d/%d\n", + (ggml_time_us() - slot.t_preempt_copy_us) / 1e3, note, + slot.prompt.n_tokens(), + slot.preempt_state_size() / (1024.0 * 1024.0), + preempt_kv_used(), n_ctx); + } + + // [TAG_PREEMPT] a resume whose copy has landed; announced here rather than where the restore was issued, this being the first moment the slot can be scheduled again + void preempt_restored(server_slot & slot, const char * note) { + metrics.n_resume++; + + preempt_trim_ram(slot); + + send_preempt_notice(slot, false); + + SLT_WRN(slot, "restore completed after %.2f ms%s: %d tokens back in the cache, kv %d/%d, preemptions %d\n", + (ggml_time_us() - slot.t_preempt_copy_us) / 1e3, note, + slot.prompt.n_tokens(), + preempt_kv_used(), n_ctx, + slot.n_preempt); + } + + void update_preempt_copies() { + for (auto & slot : slots) { + if (slot.state == SLOT_STATE_PREEMPTING) { + if (slot.preempt_save_poll()) { + preempt_parked(slot, ""); + } + } else if (slot.state == SLOT_STATE_RESTORING) { + if (slot.preempt_restore_poll()) { + preempt_restored(slot, ""); + } + } + } + } + + // [TAG_PREEMPT_ASYNC] wait for every copy in flight before a shift: the shift is one in-place graph over the whole K cache, so a copy beside it reads or writes half-shifted cells + void preempt_wait_for_shift() { + if (!preempt_shift_pending) { + return; + } + + preempt_shift_pending = false; + + while (preempt_wait_in_flight()) { + } + + for (auto & slot : slots) { + if (slot.state != SLOT_STATE_RESTORING) { + continue; + } + + slot.preempt_copy_wait(); + + if (slot.preempt_restore_poll()) { + preempt_restored(slot, " (waited for, a context shift is due)"); + } + } + } + + // [TAG_PREEMPT] run the recorded shift here rather than leave it to the next llama_decode(): a park in between would serialize the positions the shift has already moved together with the K values it has not, and removing the sequence would drop the pending deltas with it + void preempt_apply_shift() { + if (!preempt_shift_pending) { + return; + } + + preempt_wait_for_shift(); + + llama_memory_update(ctx_tgt); + + if (ctx_dft) { + llama_memory_update(ctx_dft); + } + } + + bool preempt_copies_in_flight() const { + for (const auto & slot : slots) { + if (slot.state == SLOT_STATE_PREEMPTING) { + return true; + } + } + + return false; + } + + bool preempt_wait_in_flight() { + for (auto & slot : slots) { + if (slot.state != SLOT_STATE_PREEMPTING) { + continue; + } + + slot.preempt_copy_wait(); + + if (!slot.preempt_save_poll()) { + continue; + } + + preempt_parked(slot, " (waited for)"); + + return true; + } + + return false; + } + + void update_preemption() { + if (!params_base.kv_unified || slots.size() < 2) { + return; // with a cache per slot, no slot can take another one's cells + } + + if (!llama_get_memory(ctx_tgt)) { + return; // no cache at all (an embedding model): nothing to run out of, nothing to park + } + + update_preempt_copies(); + + if (params_base.preempt_ram_mib == 0 || preempt_recurrent) { + return; // --preempt-ram 0, or a recurrent cache: the KV-full retry ladder, as before + } + + const int32_t n_cells = n_ctx; + + const bool head_of_line = preempt_resume_head; + + for (;;) { + std::vector<server_slot *> parked; + + for (auto & slot : slots) { + if (slot.state == SLOT_STATE_PREEMPTED) { + parked.push_back(&slot); + } + } + + if (parked.empty()) { + break; + } + + std::sort(parked.begin(), parked.end(), [head_of_line](const server_slot * a, const server_slot * b) { + if (!head_of_line && a->n_preempt != b->n_preempt) { + return a->n_preempt > b->n_preempt; + } + + return a->t_preempt_us < b->t_preempt_us; + }); + + if (head_of_line) { + parked.resize(1); + } + + server_slot * best = nullptr; + + const auto impossible = std::find_if(parked.begin(), parked.end(), + [this, n_cells](const server_slot * slot) { return preempt_n_need(*slot) > n_cells; }); + + if (impossible != parked.end()) { + SLT_WRN(**impossible, "parked sequence of %d tokens cannot fit the pool of %d cells even alone, failing it\n", + preempt_n_need(**impossible), n_cells); + send_error(**impossible, "Context size has been exceeded."); + (*impossible)->release(); + continue; + } + + // room for the sequence and for the next step of everything running, the candidate included, or a resume immediately preempts somebody; with nobody resident an exact fit is let in + for (;;) { + const int32_t occupied = preempt_kv_used() + preempt_kv_reserve(); + const int32_t margin = occupied == 0 ? 0 : preempt_n_margin(1); + + for (auto * slot : parked) { + if (occupied + preempt_n_need(*slot) + margin <= n_cells) { + best = slot; + break; + } + } + + if (best) { + break; + } + + if (preempt_normalize_started_all()) { + continue; + } + + if (!try_clear_idle_slots()) { + break; + } + } + + // nothing fits: a resident cycling through context shifts holds the room for as long as it generates, so it is parked once the head has waited its turn + if (!best) { + server_slot * head = parked.front(); + + // [TAG_PREEMPT_ASYNC] a park still copying holds its cells, so a rotation now would only park another resident on top + if (!preempt_copies_in_flight() && ggml_time_us() - head->t_preempt_us >= PREEMPT_ROTATE_US) { + const int32_t occupied = preempt_kv_used() + preempt_kv_reserve(); + const int32_t need = preempt_n_need(*head) + preempt_n_margin(1); + + server_slot * pick = nullptr; + bool pick_enough = false; + bool budget_refused = false; + bool recompute = false; + + auto rotate_pick = [&](bool with_recompute) { + for (auto & slot : slots) { + if (slot.state != SLOT_STATE_GENERATING || slot.n_ctx_shift == 0) { + continue; + } + + if (slot.task && (slot.task->is_parent() || slot.task->is_child())) { + continue; + } + + // the head's own bytes are not credited as leaving: the resident is parked before the head is restored and freed, so both states are held at once + if (!with_recompute && !preempt_fits_budget(slot)) { + budget_refused = true; + continue; + } + + const bool enough = occupied - preempt_n_cells(slot.prompt.n_tokens()) + need <= n_cells; + + if (!pick || + (enough && !pick_enough) || + (enough == pick_enough && (enough ? slot.prompt.n_tokens() < pick->prompt.n_tokens() + : slot.prompt.n_tokens() > pick->prompt.n_tokens()))) { + pick = &slot; + pick_enough = enough; + } + } + }; + + rotate_pick(false); + + // [TAG_PREEMPT] a resident the budget cannot swap out is rotated by dropping its cells, as ordinary victim selection does: waiting instead has no bound, a resident that keeps shifting need never finish. The head waits longer for this than for a swap: the rotated resident pays a whole re-prefill, and under exact concurrency it stops being the sequence that was parked + if (!pick && budget_refused && ggml_time_us() - head->t_preempt_us >= PREEMPT_ROTATE_RECOMPUTE_US) { + rotate_pick(true); + + recompute = pick != nullptr; + } + + const int64_t t_start = ggml_time_us(); + const int32_t n_rotated = pick ? pick->prompt.n_tokens() : 0; + + if (!pick && budget_refused && !head->preempt_rotation_refused) { + head->preempt_rotation_refused = true; + + SLT_WRN(*head, "no rotation: --preempt-ram %d MiB does not hold this parked state and a resident's at once, and the two are held together while the resident is parked and the head restored; the head waits for a resident to finish, or %.0f s for one to be rotated out by dropping its cells\n", + params_base.preempt_ram_mib, PREEMPT_ROTATE_RECOMPUTE_US / 1e6); + } + + if (pick && preempt_park(*pick, t_start, recompute)) { + server_slot & slot = *pick; + + if (slot.preempt_recompute) { + SLT_WRN(slot, "rotated out after %d context shifts: %d cells dropped, %d tokens to re-prefill on resume, a head parked %.1f s takes its turn%s, preemptions %d\n", + slot.n_ctx_shift, n_rotated, + slot.preempt_n_input(), + (ggml_time_us() - head->t_preempt_us) / 1e6, + pick_enough ? "" : " (not enough room by itself)", + slot.n_preempt); + } else { + SLT_WRN(slot, "rotated out after %d context shifts: %d cells released, %.1f MiB parked, a head parked %.1f s takes its turn%s, preemptions %d\n", + slot.n_ctx_shift, slot.prompt.n_tokens(), + slot.preempt_state_size() / (1024.0 * 1024.0), + (ggml_time_us() - head->t_preempt_us) / 1e6, + pick_enough ? "" : " (not enough room by itself)", + slot.n_preempt); + } + + // [TAG_PREEMPT_ASYNC] a synchronous park has released its cells, so the head is re-examined now; an asynchronous one on the pass that sees the copy land + if (slot.state == SLOT_STATE_PREEMPTED) { + best = head; + } + } + } + + if (best) { + continue; + } + + break; + } + + const int64_t t_start = ggml_time_us(); + + const bool recompute = best->preempt_recompute; + + best->t_preempt_copy_us = t_start; + + if (!best->preempt_restore()) { + if (best->n_preempt_fail % 64 == 1) { + SLT_WRN(*best, "resume failed (%d in a row, parked %.1f s), staying preempted\n", + best->n_preempt_fail, (ggml_time_us() - best->t_preempt_us) / 1e6); + } + + if (best->n_preempt_fail >= PREEMPT_N_FAIL_MAX && + ggml_time_us() - best->t_preempt_us > PREEMPT_FAIL_US) { + send_error(*best, "failed to restore the preempted sequence"); + best->release(); + } + + break; + } + + if (best->state == SLOT_STATE_RESTORING) { + SLT_WRN(*best, "resumed after %.2f s: %d tokens, restore issued in %.2f ms (%zu transfers, %.2f ms sync), kv %d/%d, preemptions %d\n", + (ggml_time_us() - best->t_preempt_us) / 1e6, + best->prompt.n_tokens(), + (ggml_time_us() - t_start) / 1e3, + best->preempt_n_copies(), best->preempt_sync_us() / 1e3, + preempt_kv_used(), n_cells, + best->n_preempt); + + continue; + } + + metrics.n_resume++; + + // [TAG_PREEMPT] the synchronous restore returns with the slot already back in its old state, so issue and landing are the same moment here + send_preempt_notice(*best, false, recompute); + + if (recompute) { + SLT_WRN(*best, "resumed after %.2f s: %d tokens to re-prefill, kv %d/%d, preemptions %d\n", + (ggml_time_us() - best->t_preempt_us) / 1e6, + best->preempt_n_input(), + preempt_kv_used(), n_cells, + best->n_preempt); + } else { + SLT_WRN(*best, "resumed after %.2f s: %d tokens back in the cache in %.2f ms, kv %d/%d, preemptions %d\n", + (ggml_time_us() - best->t_preempt_us) / 1e6, + best->prompt.n_tokens(), + (ggml_time_us() - t_start) / 1e3, + preempt_kv_used(), n_cells, + best->n_preempt); + } + } + + if (preempt_test_every > 0) { + for (auto & slot : slots) { + if (slot.state != SLOT_STATE_GENERATING || + (int32_t) slot.stats.n_gen < (slot.n_preempt + 1) * preempt_test_every) { + continue; + } - void iterate(std::vector<server_slot> & slots, std::function<void(server_slot &)> callback) { - for (auto & slot : slots) { - try { - callback(slot); - } catch (const std::exception & e) { - SLT_ERR(slot, "got exception: %s\n", e.what()); - send_error(slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER); - slot.release(); + // the budget refusing is the recompute park's case, so the knob reaches it too + const bool recompute = !preempt_fits_budget(slot); + + if (preempt_park(slot, ggml_time_us(), recompute)) { + SLT_WRN(slot, "preempted on request after %d generated tokens, %.1f MiB parked\n", + (int32_t) slot.stats.n_gen, slot.preempt_state_size() / (1024.0 * 1024.0)); + } } } - } - void iterate(std::vector<server_slot *> & slots, std::function<void(server_slot &)> callback) { - for (auto & slot : slots) { - try { - callback(*slot); - } catch (const std::exception & e) { - SLT_ERR(*slot, "got exception: %s\n", e.what()); - send_error(*slot, std::string("got exception: ") + e.what(), ERROR_TYPE_SERVER); - slot->release(); + if (preempt_planner_off) { + return; // test knob: leave the pool to the retry ladder and its last resort + } + + for (;;) { + const int32_t n_used = preempt_kv_used() + preempt_kv_reserve(); + + if (n_used + preempt_n_margin() <= n_cells) { + break; + } + + if (try_clear_idle_slots()) { + continue; + } + + // [TAG_PREEMPT_ASYNC] a park issued and not landed holds cells that are already spoken for, so waiting for it is quicker than parking somebody else + if (preempt_copies_in_flight()) { + if (n_used > n_cells) { + if (preempt_wait_in_flight()) { + continue; + } + } else { + break; + } + } + + bool recompute = false; + + server_slot * victim = preempt_pick_victim(&recompute); + + if (!victim) { + SRV_DBG("the kv pool needs %d of %d cells and nothing can be preempted (parked %.1f MiB of the %d MiB --preempt-ram budget)\n", + n_used, n_cells, preempt_ram_used() / (1024.0 * 1024.0), params_base.preempt_ram_mib); + break; + } + + const int32_t n_tokens = victim->prompt.n_tokens(); + const int64_t t_start = ggml_time_us(); + + if (!preempt_park(*victim, t_start, recompute)) { + break; // could not park it; the existing retry ladder is still behind us + } + + if (victim->state == SLOT_STATE_PREEMPTING) { + SLT_WRN(*victim, "preempted: %d cells, park issued in %.2f ms (%zu transfers, %.2f ms sync), %.1f MiB parked, kv %d/%d (wanted %d), preemptions %d\n", + n_tokens, + (ggml_time_us() - t_start) / 1e3, + victim->preempt_n_copies(), victim->preempt_sync_us() / 1e3, + victim->preempt_state_size() / (1024.0 * 1024.0), + preempt_kv_used(), n_cells, n_used, + victim->n_preempt); + + // [TAG_PREEMPT_ASYNC] short of the lookahead only, the step still fits and leaving is the point; out of room for it the cells are held until the copy lands, so the retry ladder ends every request instead of waiting + if (n_used + preempt_n_margin() > n_cells) { + continue; + } + + break; + } + + if (victim->preempt_recompute) { + SLT_WRN(*victim, "preempted: %d cells dropped in %.2f ms, %d tokens to re-prefill on resume, kv %d/%d (wanted %d), preemptions %d\n", + n_tokens, + (ggml_time_us() - t_start) / 1e3, + victim->preempt_n_input(), + preempt_kv_used(), n_cells, n_used, + victim->n_preempt); + } else { + SLT_WRN(*victim, "preempted: %d cells released in %.2f ms, %.1f MiB parked, kv %d/%d (wanted %d), preemptions %d\n", + n_tokens, + (ggml_time_us() - t_start) / 1e3, + victim->preempt_state_size() / (1024.0 * 1024.0), + preempt_kv_used(), n_cells, n_used, + victim->n_preempt); } } } - void abort_all_slots(const std::string & reason) { - for (auto & slot : slots) { - if (slot.is_processing()) { - send_error(slot, reason, ERROR_TYPE_SERVER); - slot.release(); + // the checks a request has to pass before its prompt is processed; true when it is rejected. An empty prompt is not here: it is a final response, not an error. + bool task_prompt_rejected(const server_task & task, std::string & msg, error_type & type) const { + // TODO: support memory-less logits computation + if (task.need_logits() && !llama_get_memory(ctx_tgt)) { + msg = "the current context does not logits computation. skipping"; + type = ERROR_TYPE_SERVER; + return true; + } + + // as launch_slot_with_task(), ahead of it: a sibling parked behind a running one used to fail inside a stream that had already opened 200 + if (!task.tokens.validate(ctx_tgt)) { + msg = "Prompt contains invalid tokens"; + type = ERROR_TYPE_INVALID_REQUEST; + return true; + } + + // as server_slot::can_split(), from the task alone + const bool can_split = + !task.need_embd() || + (llama_get_memory(ctx_tgt) && llama_pooling_type(ctx_tgt) == LLAMA_POOLING_TYPE_LAST); + + if (!can_split) { + const int32_t n_ubatch = llama_n_ubatch(ctx_tgt); + + if (task.n_tokens() > n_ubatch) { + msg = string_format( + "input (%d tokens) is too large to process. increase the physical batch " + "size (current batch size: %d)", + task.n_tokens(), n_ubatch); + type = ERROR_TYPE_SERVER; + return true; + } + + if (task.n_tokens() > n_ctx_slot()) { + msg = string_format( + "input (%d tokens) is larger than the max context size (%d tokens). skipping", + task.n_tokens(), n_ctx_slot()); + type = ERROR_TYPE_EXCEED_CONTEXT_SIZE; + return true; } + + return false; } - } - // @ngxson : for debugging only - int64_t t_pre_decode = 0; - int64_t t_decode = 0; - int64_t t_post_decode = 0; - int64_t t_sampl = 0; - int64_t n_pre_decode = 0; - int64_t n_decode = 0; - int64_t n_post_decode = 0; - int64_t n_sampl = 0; -// #define DEBUG_TIMINGS -#ifdef DEBUG_TIMINGS - struct scoped_timer { - int64_t & t; - int64_t & n; - int64_t t_start; - scoped_timer(int64_t & t_, int64_t & n_) : t(t_), n(n_) { - t_start = ggml_time_us(); + if (task.n_tokens() >= n_ctx_slot()) { + msg = string_format( + "request (%d tokens) exceeds the available context size (%d tokens), try increasing it", + task.n_tokens(), n_ctx_slot()); + type = ERROR_TYPE_EXCEED_CONTEXT_SIZE; + return true; } - ~scoped_timer() { - t += ggml_time_us() - t_start; - n++; + + return false; + } + + bool slot_prompt_rejected(const server_slot & slot, std::string & msg, error_type & type) const { + if (!slot.task) { + return false; } - }; -#else - struct scoped_timer { - scoped_timer(int64_t &, int64_t &) {} - ~scoped_timer() {} - }; -#endif + + return task_prompt_rejected(*slot.task, msg, type); + } void update_slots() { #ifdef DEBUG_TIMINGS @@ -2804,7 +4449,6 @@ struct server_context_impl { } #endif - // check if all slots are idle { bool all_idle = true; @@ -2832,6 +4476,14 @@ struct server_context_impl { } try { + // [TAG_PREEMPT] make the pool fit the step about to be built, measured after any context shift; inside the guard because a shift or a park can throw + pre_decode_shift(); + + // before update_preemption(), and not only before the decode: a slot must never be parked with a shift still pending on its cells + preempt_apply_shift(); + + update_preemption(); + scoped_timer t(t_pre_decode, n_pre_decode); pre_decode(); batch.render(); @@ -2867,6 +4519,10 @@ struct server_context_impl { llama_batch batch_view; int32_t off_next = 0; int32_t n_batch = llama_n_batch(ctx_tgt); + + // [TAG_PREEMPT_ASYNC] and once more here: a shift --cache-reuse asks for is found inside pre_decode(), after the wait above + preempt_wait_for_shift(); + for (int32_t off = 0; off < batch.size(); off = off_next) { const int32_t n_tokens = std::min(n_batch, batch.size() - off); try { @@ -2879,6 +4535,11 @@ struct server_context_impl { llama_synchronize(ctx_tgt); #endif + if (preempt_batch_abandoned) { + preempt_batch_abandoned = false; + break; + } + if (ok) { // move the head of the batch forward with the number of tokens we just processed off_next = off + n_tokens; @@ -2906,9 +4567,9 @@ struct server_context_impl { } } - void pre_decode() { - // apply context-shift if needed - // TODO: simplify and improve + // apply context-shift if needed + // TODO: simplify and improve + void pre_decode_shift() { iterate(slots, [&](server_slot & slot) { if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) { if (!params_base.ctx_shift) { @@ -2948,6 +4609,9 @@ struct server_context_impl { SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard); + slot.n_ctx_shift++; + preempt_shift_pending = true; + slot.mem.seq_rm (slot.id, n_keep , n_keep + n_discard); slot.mem.seq_add(slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard); @@ -2970,7 +4634,9 @@ struct server_context_impl { slot.truncated = true; } }); + } + void pre_decode() { // start populating the batch for this iteration batch.clear(); @@ -3116,7 +4782,8 @@ struct server_context_impl { return; // batch is full, skip remaining slots } - if (!slot.is_processing()) { + // [TAG_PREEMPT] a parked slot is processing but has nothing in the cache to batch until it is restored + if (!slot.is_processing() || slot.preempt_is_out()) { return; } @@ -3133,7 +4800,10 @@ struct server_context_impl { // this slot still has a prompt to be processed if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_STARTED) { - const auto & input_tokens = slot.task->tokens; + const auto & input_tokens = slot.preempt_input(); + + // [TAG_PREEMPT] what the prompt step works towards: the re-prefill list of a park that dropped its cells, the request otherwise + const int32_t n_input_tokens = slot.preempt_n_input(); // used to determine the number of tokens added to the batch for the current slot const auto n_tokens_prev = batch.size(); @@ -3174,46 +4844,18 @@ struct server_context_impl { return; } - // TODO: support memory-less logits computation - if (slot.task->need_logits() && !llama_get_memory(ctx_tgt)) { - send_error(slot, "the current context does not logits computation. skipping", ERROR_TYPE_SERVER); - slot.release(); - return; - } - - if (!slot.can_split()) { - if (slot.task->n_tokens() > n_ubatch) { - send_error(slot, - string_format( - "input (%d tokens) is too large to process. increase the physical batch " - "size (current batch size: %d)", - slot.task->n_tokens(), n_ubatch), - ERROR_TYPE_SERVER); - slot.release(); - return; - } + { + std::string msg; + error_type type = ERROR_TYPE_SERVER; - if (slot.task->n_tokens() > slot.n_ctx) { - send_error( - slot, - string_format( - "input (%d tokens) is larger than the max context size (%d tokens). skipping", - slot.task->n_tokens(), slot.n_ctx), - ERROR_TYPE_EXCEED_CONTEXT_SIZE); - slot.release(); - return; - } - } else { - if (slot.task->n_tokens() >= slot.n_ctx) { - send_error(slot, - string_format("request (%d tokens) exceeds the available context size (%d " - "tokens), try increasing it", - slot.task->n_tokens(), slot.n_ctx), - ERROR_TYPE_EXCEED_CONTEXT_SIZE); + if (slot_prompt_rejected(slot, msg, type)) { + send_error(slot, msg, type); slot.release(); return; } + } + if (slot.can_split()) { if (slot.task->params.cache_prompt) { // reuse any previously computed tokens that are common with the new prompt n_past = slot.prompt.tokens.get_common_prefix(input_tokens); @@ -3269,6 +4911,8 @@ struct server_context_impl { slot.mem.seq_rm (slot.id, head_p, head_c); slot.mem.seq_add(slot.id, head_c, head_c + n_match, kv_shift); + preempt_shift_pending = true; + for (size_t i = 0; i < n_match; i++) { slot.prompt.tokens.set_token(head_p + i, slot.prompt.tokens[head_c + i]); n_past++; @@ -3427,11 +5071,32 @@ struct server_context_impl { if (!slot.can_split()) { // cannot fit the prompt in the current batch - will try next iter - if (batch.size() + slot.task->n_tokens() > n_batch) { + if (batch.size() + n_input_tokens > n_batch) { return; } } + // [TAG_EXACT_CONCURRENCY] a prompt is isolated into ubatches of its own, so it gets the shapes it would get alone only if what it adds here is a whole number of ubatches: otherwise a neighbour's decoded token shortens the last one, and 512,512,512,509 is not 512,512,512,512 + int32_t n_batch_cur = n_batch; + + if (common_exact_concurrency() && slot.can_split() && !slot.prompt.tokens.has_mtmd) { + const int32_t n_avail = n_batch - (int32_t) batch.size(); + const int32_t n_left = n_input_tokens - slot.prompt.n_tokens(); + + if (n_left > n_avail) { + const int32_t n_take = n_avail - n_avail % n_ubatch; + + if (n_take > 0) { + n_batch_cur = (int32_t) batch.size() + n_take; + } else { + // the waiting ends: common_exact_batch_geometry() refused a batch that cannot hold a whole ubatch beside a decode step of every slot, and the prompts ahead of this one in the same batch are finite + SLT_DBG(slot, "exact concurrency: %d of %d batch tokens left, short of a %d-token ubatch: the prefill waits\n", + n_avail, n_batch, n_ubatch); + return; + } + } + } + // note: the prompt timing is advanced in post_decode(), so it does not cover // the tokens added to the batch below slot.print_timings_pp(); @@ -3462,6 +5127,9 @@ struct server_context_impl { // make checkpoints only for completion tasks do_checkpoint = do_checkpoint && slot.task->type == SERVER_TASK_TYPE_COMPLETION; + // a re-prefill walks its own list, which the request's message spans do not index + do_checkpoint = do_checkpoint && !slot.preempt_reprefill; + // make a checkpoint of the parts of the memory that cannot be rolled back. // checkpoints are created only if: // - the model does not support partial sequence removal @@ -3478,12 +5146,16 @@ struct server_context_impl { while (true) { auto cur_token_idx = slot.prompt.n_tokens(); if ( - cur_token_idx >= slot.task->n_tokens() || + cur_token_idx >= n_input_tokens || input_tokens[cur_token_idx] != LLAMA_TOKEN_NULL // encountered a text token ) { break; } + // [TAG_PREEMPT_ASYNC] the chunk decodes whole, past the kv-full retry, so a park the planner issued for it has to land first + while (preempt_wait_in_flight()) { + } + // process the mtmd chunk // note: it submits its own decode, potentially be async // so the timing is queued and flushed on the next sync @@ -3521,7 +5193,7 @@ struct server_context_impl { const auto last_user_pos = spans.last_user_message_pos(); // add prompt tokens for processing in the current batch - while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.size() < n_batch) { + while (slot.prompt.n_tokens() < n_input_tokens && batch.size() < n_batch_cur) { // get next token to process llama_token cur_tok = input_tokens[slot.prompt.n_tokens()]; if (cur_tok == LLAMA_TOKEN_NULL) { @@ -3546,6 +5218,11 @@ struct server_context_impl { /* is_prompt = */ true); slot.prompt.tokens.push_back(cur_tok); + // [TAG_EXACT_CONCURRENCY] a token that was decoded goes back through the arithmetic that decoded it: one per step, in the narrow set beside the other decodes. Re-prefilled wide it went through batched arithmetic, and the output diverged at the second park + if (slot.preempt_reprefill && common_exact_concurrency() && slot.prompt.n_tokens() >= slot.task->n_tokens()) { + break; + } + // break at the last user message, or at user messages at least min step past the last checkpoint if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) { const auto pos = slot.prompt.n_tokens(); @@ -3567,7 +5244,7 @@ struct server_context_impl { bool should_break = false; for (int offset : checkpoint_offsets) { const int n_last = std::min(n_batch, offset); - if (slot.task->n_tokens() == slot.prompt.n_tokens() + n_last) { + if (n_input_tokens == slot.prompt.n_tokens() + n_last) { should_break = true; break; } @@ -3583,13 +5260,13 @@ struct server_context_impl { const auto n_tokens_start = slot.prompt.n_tokens() - n_tokens_cur; - const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch; + const bool near_prompt_end = n_input_tokens < slot.prompt.n_tokens() + n_ubatch; const bool is_user_start = spans.is_user_start(n_tokens_start); const bool is_last_user_message = n_tokens_start == last_user_pos; // entire prompt has been processed - if (slot.prompt.n_tokens() == slot.task->n_tokens()) { + if (slot.prompt.n_tokens() == n_input_tokens) { slot.state = SLOT_STATE_DONE_PROMPT; GGML_ASSERT(batch.size() > 0); @@ -3597,10 +5274,14 @@ struct server_context_impl { // extract the logits only for the last token batch.set_output(batch.size() - 1, true); - slot.stats.n_gen = 0; - slot.i_batch = batch.size() - 1; + slot.i_batch = batch.size() - 1; - slot.init_sampler(); + // [TAG_PREEMPT] a re-prefill only puts back what the park dropped: the sampler and the counters carry on from where the park found them + if (!slot.preempt_reprefill) { + slot.stats.n_gen = 0; + + slot.init_sampler(); + } } else { // skip ordinary mid-prompt checkpoints, unless the batch starts a user // message or we are near the end of the prompt @@ -3642,6 +5323,123 @@ struct server_context_impl { } } + // [TAG_PREEMPT_ASYNC] a recurrent memory keeps no fixed row per sequence: find_slot() gathers the active ones into contiguous rows, so a decode beside a park moves or overwrites the row the copy is still reading. A hybrid carries that half too, and so can the draft. + static bool preempt_state_relocates(const llama_model * model) { + return model && (llama_model_is_recurrent(model) || llama_model_is_hybrid(model)); + } + + bool preempt_state_relocates() const { + return preempt_state_relocates(model_tgt) || preempt_state_relocates(model_dft); + } + + // [TAG_PREEMPT_ASYNC] whether a park can happen at all and go asynchronously + bool preempt_async_possible() const { + return params_base.preempt_async && params_base.kv_unified && params_base.preempt_ram_mib != 0 && + slots.size() >= 2 && llama_get_memory(ctx_tgt) && !preempt_state_relocates(); + } + + bool preempt_last_resort_possible() const { + return params_base.kv_unified && params_base.preempt_ram_mib != 0 && !preempt_recurrent && slots.size() >= 2 && llama_get_memory(ctx_tgt); + } + + // [TAG_PREEMPT] the retry ladder ran out: give the batch up, rewind every resident to the token boundary the cache is at and park the smallest. A media chunk mid-prompt keeps the old path. + bool preempt_last_resort(int32_t off) { + if (!preempt_last_resort_possible()) { + return false; + } + + int32_t n_running = 0; + + for (auto & slot : slots) { + if (!slot.is_processing() || slot.state == SLOT_STATE_PREEMPTED || slot.preempt_in_flight()) { + continue; + } + + // a media chunk decodes whole through calls of its own, so a resident still inside its prompt cannot be rewound to a token boundary; one that is generating can + if (slot.prompt.tokens.has_mtmd && slot.state != SLOT_STATE_GENERATING) { + return false; + } + + n_running++; + } + + if (n_running < 2) { + return false; // one conversation that does not fit alone is a real overflow + } + + for (auto & slot : slots) { + if (slot.is_processing() && slot.state != SLOT_STATE_PREEMPTED && slot.state != SLOT_STATE_WAIT_OTHER && + !slot.preempt_in_flight()) { + slot.rewind_to_cache(); + } + } + + const int32_t n_cells = n_ctx; + int32_t n_parked = 0; + + for (;;) { + const int32_t n_used = preempt_kv_used() + preempt_kv_reserve(); + + if (n_parked > 0 && n_used + preempt_n_margin() <= n_cells) { + break; + } + + bool recompute = false; + + server_slot * victim = preempt_pick_victim(&recompute); + + if (!victim) { + break; + } + + const int32_t n_tokens = victim->prompt.n_tokens(); + const int64_t t_start = ggml_time_us(); + + if (!preempt_park(*victim, t_start, recompute)) { + break; + } + + n_parked++; + + // [TAG_PREEMPT_ASYNC] the cells are wanted now, not next iteration: wait for the copy, which releases them + if (victim->state == SLOT_STATE_PREEMPTING) { + while (preempt_wait_in_flight()) { + } + + SLT_WRN(*victim, "preempted as a last resort: %d cells released, kv %d/%d (wanted %d), preemptions %d\n", + n_tokens, preempt_kv_used(), n_cells, n_used, victim->n_preempt); + continue; + } + + SLT_WRN(*victim, "preempted as a last resort%s: %d cells released in %.2f ms, %.1f MiB parked, kv %d/%d (wanted %d), preemptions %d\n", + victim->preempt_recompute ? " by dropping its cells" : "", + n_tokens, + (ggml_time_us() - t_start) / 1e3, + victim->preempt_state_size() / (1024.0 * 1024.0), + preempt_kv_used(), n_cells, n_used, + victim->n_preempt); + } + + if (n_parked == 0) { + return false; // nothing could be parked: the error path clears what the rewind left + } + + SRV_WRN("last resort: batch given up at off = %d, %d slot(s) parked, kv %d/%d resident\n", + off, n_parked, preempt_kv_used(), n_cells); + + return true; + } + + bool batch_has_spec_groups() const { + for (const auto & slot : slots) { + if (!slot.spec_i_batch.empty()) { + return true; + } + } + + return false; + } + // returns true = success ; false = retry with smaller batch size // throw std::runtime_error on fatal error bool decode(int32_t & n_batch, int32_t off, llama_batch & batch_view) { @@ -3686,10 +5484,31 @@ struct server_context_impl { }); if (ret != 0) { + // [TAG_PREEMPT_ASYNC] halving the batch returns no cells, so wait for an issued park first, or the ladder runs down to n_batch == 1 and ends every request + if (ret == 1 && preempt_wait_in_flight()) { + SRV_WRN("%s", "waited for an in-flight park before retrying the decode\n"); + return false; // retry at the same batch size, with the cells it freed + } + { std::string err; + // [TAG_PREEMPT] a slot's sampled token and its draft have to stay in one view, so halving would split the group and make the verify step throw + if (ret == 1 && n_batch > 1 && preempt_last_resort_possible() && batch_has_spec_groups()) { + if (try_clear_idle_slots()) { + SRV_WRN("%s", "failed to find free space in the KV cache, retrying after purging an idle slot\n"); + return false; // retry at the same width + } + + n_batch = 1; + } + if (n_batch == 1 && ret == 1) { + if (preempt_last_resort(off)) { + preempt_batch_abandoned = true; + return true; + } + // TODO: try to terminate only the largest active slot/sequence and continue with the rest // need to remove the tokens from the current batch too err = "Context size has been exceeded."; @@ -3710,7 +5529,7 @@ struct server_context_impl { SRV_ERR("%s off = %d, n_batch = %d, ret = %d\n", err.c_str(), off, n_batch, ret); for (auto & slot : slots) { - if (slot.is_processing()) { + if (slot.is_processing() && slot.state != SLOT_STATE_PREEMPTED && !slot.preempt_in_flight()) { send_error(slot, err); slot.release(); @@ -3805,7 +5624,7 @@ struct server_context_impl { iterate(slots, [&](server_slot & slot) { // optionally send prompt processing progress if (slot.state == SLOT_STATE_PROCESSING_PROMPT || slot.state == SLOT_STATE_DONE_PROMPT) { - if (slot.task->params.stream && slot.task->params.return_progress) { + if (slot.task->params.stream && slot.task->params.return_progress && !slot.preempt_reprefill) { send_partial_response(slot, {}, true); } } @@ -3816,6 +5635,12 @@ struct server_context_impl { } if (slot.state == SLOT_STATE_DONE_PROMPT) { + // [TAG_PREEMPT] the re-prefill is back in the cache; the token this slot had already sampled is decoded next, so nothing is sampled here + if (slot.preempt_reprefill) { + slot.preempt_reprefill_done(); + return; + } + if (slot.task->type == SERVER_TASK_TYPE_EMBEDDING) { // prompt evaluated for embedding send_embedding(slot, batch_view); @@ -4071,7 +5896,7 @@ struct server_context_impl { void metrics_post_decode(int32_t off, int32_t n_tokens, bool has_output) { metrics.n_decode++; for (const auto & slot : slots) { - if (slot.is_processing()) { + if (slot.is_processing() && !slot.preempt_is_out()) { metrics.n_busy_slots++; } metrics.n_tokens_max = std::max(metrics.n_tokens_max, (uint64_t) slot.prompt.n_tokens()); @@ -4328,6 +6153,14 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl( task.params.oaicompat_cmpl_id = completion_id; task.params.oaicompat_model = meta->model_name; + // [TAG_EXACT_CONCURRENCY] exact mode gives a page to a single sequence, so refuse an n_cmpl > 1 child here, where it becomes a 400 rather than at seq_cp + if (task.params.n_cmpl > 1 && common_exact_concurrency()) { + throw std::runtime_error( + "n > 1 is not supported while LLAMA_EXACT_CONCURRENCY is set: each " + "completion needs its own sequence, and in exact mode a KV page belongs " + "to a single sequence. Send n separate requests, or unset the variable."); + } + // prepare child tasks if (task.params.n_cmpl > 1) { int n_children = task.params.n_cmpl - 1; @@ -4339,6 +6172,16 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl( tasks.push_back(std::move(task)); } + // [TAG_PREEMPT] every prompt of the request, before any of them is queued: one member can be parked, and its notice opens the stream, before another member is rejected + { + json error; + + if (ctx_server.tasks_prompt_rejected(tasks, error)) { + res->error(error); + return res; + } + } + rd.post_tasks(std::move(tasks)); } catch (const std::exception & e) { res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST)); @@ -4381,37 +6224,51 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl( // in streaming mode, the first error must be treated as non-stream response // this is to match the OAI API behavior // ref: https://github.com/ggml-org/llama.cpp/pull/16486#discussion_r2419657309 + // [TAG_PREEMPT] a slot can be parked before any token exists, so those notices are kept and sent in front of the first real result + std::string preempt_prefix; + std::set<size_t> parked_idx; // prompts of this request that are parked right now auto first_result = rd.next(req.should_stop); - if (first_result == nullptr) { - GGML_ASSERT(req.should_stop()); - return res; // connection is closed - } + if (first_result != nullptr && dynamic_cast<server_task_result_preempt_notice*>(first_result.get()) != nullptr) { + const auto * notice = static_cast<server_task_result_preempt_notice*>(first_result.get()); + preempt_prefix = preempt_notice_comment(*notice); + if (notice->parked) { + parked_idx.insert(notice->index); + } else { + parked_idx.erase(notice->index); + } + first_result.reset(); + } else { + if (first_result == nullptr) { + GGML_ASSERT(req.should_stop()); + return res; // connection is closed + } - if (first_result->is_error()) { - res->error(first_result->to_json()); - return res; - } + if (first_result->is_error()) { + res->error(first_result->to_json()); + return res; + } - GGML_ASSERT( - dynamic_cast<server_task_result_cmpl_partial*>(first_result.get()) != nullptr || - dynamic_cast<server_task_result_cmpl_final*> (first_result.get()) != nullptr - ); + GGML_ASSERT( + dynamic_cast<server_task_result_cmpl_partial*>(first_result.get()) != nullptr || + dynamic_cast<server_task_result_cmpl_final*> (first_result.get()) != nullptr + ); + } - // next responses are streamed - // to be sent immediately - json first_result_json = first_result->to_json(); + json first_result_json = first_result ? first_result->to_json() : json(nullptr); if (first_result_json == nullptr) { - res->data = ""; // simply send HTTP headers and status code + res->data = preempt_prefix; // simply send HTTP headers and status code } else if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { - res->data = format_anthropic_sse(first_result_json); + res->data = preempt_prefix + format_anthropic_sse(first_result_json); } else if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) { - res->data = format_oai_resp_sse(first_result_json); + res->data = preempt_prefix + format_oai_resp_sse(first_result_json); } else { - res->data = format_oai_sse(first_result_json); + res->data = preempt_prefix + format_oai_sse(first_result_json); } res->status = 200; res->content_type = "text/event-stream"; - res->set_next([res_this = res.get(), res_type, sse_ping_interval](std::string & output) -> bool { + res->set_next([res_this = res.get(), res_type, sse_ping_interval, parked_idx](std::string & output) mutable -> bool { + const bool parked = !parked_idx.empty(); + static auto format_error = [](task_response_type res_type, const json & res_json) { if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { return format_anthropic_sse({ @@ -4462,10 +6319,13 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl( // receive subsequent results bool timeout = false; int64_t start_time = ggml_time_ms(); - auto result = rd.next([&timeout, &start_time, sse_ping_interval, &effective_should_stop]() { + // [TAG_PREEMPT] a parked slot produces nothing, so ping at least every 2 s whether or not --sse-ping asked for one, and name it; a shorter interval asked for is kept + const int64_t ping_cfg = sse_ping_interval > 0 ? (int64_t) sse_ping_interval * 1000 : -1; + const int64_t ping_ms = parked ? (ping_cfg > 0 ? std::min(ping_cfg, PREEMPT_KEEPALIVE_MS) : PREEMPT_KEEPALIVE_MS) : ping_cfg; + auto result = rd.next([&timeout, &start_time, ping_ms, &effective_should_stop]() { if (effective_should_stop()) { return true; // should_stop condition met - } else if (sse_ping_interval > 0 && ggml_time_ms() - start_time > (int64_t)sse_ping_interval * 1000) { + } else if (ping_ms > 0 && ggml_time_ms() - start_time > ping_ms) { timeout = true; return true; // timeout } @@ -4475,7 +6335,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl( if (timeout) { // some clients may time out (e.g. undici) will time out if no data is received for a while, so we need to send a ping to keep the connection alive SRV_DBG("%s", "sending SSE ping\n"); - output = ":\n\n"; + output = parked ? ": preempt-keepalive\n\n" : ":\n\n"; return true; } @@ -4491,12 +6351,23 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl( output = format_error(res_type, res_json); SRV_DBG("%s", "error received during streaming, terminating stream\n"); return false; // terminate on error + } else if (const auto * notice = dynamic_cast<server_task_result_preempt_notice*>(result.get())) { + if (notice->parked) { + parked_idx.insert(notice->index); + } else { + parked_idx.erase(notice->index); + } + output = preempt_notice_comment(*notice); } else { GGML_ASSERT( dynamic_cast<server_task_result_cmpl_partial*>(result.get()) != nullptr || dynamic_cast<server_task_result_cmpl_final*>(result.get()) != nullptr ); json res_json = result->to_json(); + if (res_json.is_null()) { + // [TAG_PREEMPT] the empty signal a prompt sends before its first token has nothing to add once a notice has opened the stream + return true; + } if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { output = format_anthropic_sse(res_json); } else if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) { @@ -4624,6 +6495,8 @@ static json get_res_props(const server_context_meta & meta, const common_params { "endpoint_slots", params.endpoint_slots }, { "endpoint_props", params.endpoint_props }, { "endpoint_metrics", params.endpoint_metrics }, + // [TAG_EXACT_CONCURRENCY] a client that asked for the mode reads here whether this process runs it: a build that ignores the variable starts all the same + { "exact_concurrency", common_exact_concurrency() }, { "ui", params.ui }, { "ui_settings", meta.json_ui_settings }, { "chat_template", tmpl_default }, diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp index 78169e9a5d86..b5c8ab4a8ace 100644 --- a/tools/server/server-queue.cpp +++ b/tools/server/server-queue.cpp @@ -448,6 +448,9 @@ server_task_result_ptr server_response::recv(const std::unordered_set<int> & id_ } server_task_result_ptr server_response::recv_with_timeout(const std::unordered_set<int> & id_tasks, int timeout) { + // [TAG_PREEMPT] the timeout is a deadline, not a per-wait duration: send() notify_all()s for every result of every task, and with wait_for() each wakeup restarted the wait + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout); + while (true) { std::unique_lock<std::mutex> lock(mutex_results); @@ -459,7 +462,7 @@ server_task_result_ptr server_response::recv_with_timeout(const std::unordered_s } } - std::cv_status cr_res = condition_results.wait_for(lock, std::chrono::seconds(timeout)); + std::cv_status cr_res = condition_results.wait_until(lock, deadline); if (!running) { RES_DBG("%s : queue result stop\n", __func__); std::terminate(); // we cannot return here since the caller is HTTP code diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 0d3beb313cea..e9fc854c7961 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -355,6 +355,7 @@ json server_task_result_cmpl_final::to_json_non_oaicompat() { {"stopping_word", stopping_word}, {"tokens_cached", n_tokens_cached}, {"timings", stats.to_json()}, + {"preempt", preempt_to_json()}, }; if (!stream && !probs_output.empty()) { res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs); @@ -362,6 +363,14 @@ json server_task_result_cmpl_final::to_json_non_oaicompat() { return response_fields.empty() ? res : json_get_nested_values(response_fields, res); } +// [TAG_PREEMPT] how the request was served: a recompute resume re-prefilled its tokens, so its continuation is not the bytes that were parked +json server_task_result_cmpl_final::preempt_to_json() const { + return json { + {"parks", n_preempt}, + {"recomputes", n_recompute}, + }; +} + json server_task_result_cmpl_final::usage_json_oaicompat() { return json { {"completion_tokens", n_decoded}, @@ -406,6 +415,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() { } if (stats.is_set()) { res["timings"] = stats.to_json(); + res["preempt"] = preempt_to_json(); } return res; @@ -454,6 +464,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() { } if (stats.is_set()) { res["timings"] = stats.to_json(); + res["preempt"] = preempt_to_json(); } return res; @@ -515,6 +526,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() { if (stats.is_set()) { deltas.back()["timings"] = stats.to_json(); + deltas.back()["preempt"] = preempt_to_json(); } // extra fields for debugging purposes @@ -591,6 +603,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() { {"total_tokens", n_decoded + n_prompt_tokens}, {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }}, }}, + {"preempt", preempt_to_json()}, }; return res; @@ -708,6 +721,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() { if (stats.is_set()) { server_sent_events.back().at("data")["timings"] = stats.to_json(); + server_sent_events.back().at("data")["preempt"] = preempt_to_json(); } return server_sent_events; @@ -724,6 +738,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_asr() { {"total_tokens", n_decoded + n_prompt_tokens}, {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }}, }}, + {"preempt", preempt_to_json()}, }; return event; } @@ -788,7 +803,8 @@ json server_task_result_cmpl_final::to_json_anthropic() { {"cache_read_input_tokens", n_prompt_tokens_cache}, {"input_tokens", n_prompt_tokens - n_prompt_tokens_cache}, {"output_tokens", n_decoded} - }} + }}, + {"preempt", preempt_to_json()} }; return res; @@ -968,7 +984,8 @@ json server_task_result_cmpl_final::to_json_anthropic_stream() { }}, {"usage", { {"output_tokens", n_decoded} - }} + }}, + {"preempt", preempt_to_json()} }} }); @@ -1023,6 +1040,14 @@ void server_task_result_cmpl_partial::update(task_result_state & state) { } } +json server_task_result_preempt_notice::to_json() { + return json { + {"preempted", parked}, + {"recomputed", recomputed}, + {"n_preempt", n_preempt}, + }; +} + json server_task_result_cmpl_partial::to_json() { GGML_ASSERT(is_updated && "update() must be called before to_json()"); if (is_begin) { @@ -1562,6 +1587,18 @@ std::string server_task_result_metrics::to_metrics() { "spec_decode_num_drafts_total", "Speculative: Total speculative decoding verification steps", (double) metrics.n_draft_verif_steps + }, { + "n_preempt_total", + "Preemption: Total slots parked to make room in the unified KV cache", + (double) metrics.n_preempt + }, { + "n_resume_total", + "Preemption: Total parked slots put back", + (double) metrics.n_resume + }, { + "preempt_recompute_total", + "Preemption: Total parks that dropped their cells, whose resume re-prefills instead of restoring the saved bytes", + (double) metrics.n_preempt_recompute }, }; @@ -1586,6 +1623,14 @@ std::string server_task_result_metrics::to_metrics() { "n_busy_slots_per_decode", "Average number of busy slots per llama_decode() call", (double) metrics.n_busy_slots / std::max((double) metrics.n_decode, 1.0) + }, { + "requests_preempted", + "Preemption: Number of requests currently parked, waiting for room in the unified KV cache", + (double) n_preempted_slots + }, { + "preempt_ram_bytes", + "Preemption: Host RAM held by parked sequences", + (double) preempt_ram_bytes }, }; diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 9c99143f8e19..0d852757eca7 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -339,6 +339,12 @@ struct server_task_result_cmpl_final : server_task_result { std::vector<completion_token_output> probs_output; std::vector<std::string> response_fields; + // [TAG_PREEMPT] how the request was served: how often it was parked, and how many of those parks re-prefilled instead of restoring saved bytes + int32_t n_preempt = 0; + int32_t n_recompute = 0; + + json preempt_to_json() const; + task_params generation_params; // response formatting @@ -392,6 +398,18 @@ struct server_task_result_cmpl_final : server_task_result { json to_json_anthropic_stream(); }; +// [TAG_PREEMPT] out-of-band notice for a streaming task whose slot was parked or restored, sent as an SSE comment (": preempted", ": resumed") every existing client ignores +struct server_task_result_preempt_notice : server_task_result { + bool parked = false; // true when the slot was just parked, false when restored + bool recomputed = false; // this resume re-prefilled its tokens instead of restoring saved bytes + int32_t n_preempt = 0; // how many times this task has been parked so far + + virtual bool is_stop() override { + return false; + } + virtual json to_json() override; +}; + struct server_task_result_cmpl_partial : server_task_result { std::string content; llama_tokens tokens; @@ -494,6 +512,8 @@ struct server_task_result_metrics : server_task_result { // these are immediate stats, not accumulated (server_metrics is cumulative) int n_processing_slots = 0; int n_tasks_deferred = 0; + int n_preempted_slots = 0; // [TAG_PREEMPT] processing slots currently parked + size_t preempt_ram_bytes = 0; // [TAG_PREEMPT] host RAM their parked sequences hold server_metrics metrics; diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py new file mode 100644 index 000000000000..06776d0bbf25 --- /dev/null +++ b/tools/server/tests/unit/test_preempt.py @@ -0,0 +1,887 @@ +import base64 +import json +import os +import re +import struct +import subprocess +import threading +import time +import tempfile +import pytest +import requests +from utils import * + +# Preemption on a unified KV pool: one slot is parked, its sequence copied to host RAM and its cells released, instead of every slot being terminated. Needs --kv-unified. + +server = ServerPreset.tinyllama2() + +_ASYNC_BANNER = "parking and resuming asynchronously" + +_PROMPT_A = "Once upon a time there was a brave knight who" +_PROMPT_B = "The quick brown fox jumps over the lazy dog and" +_PROMPT_C = "In a small village by the sea there lived a fisherman who" + + +@pytest.fixture(autouse=True) +def create_server(): + global server + server = ServerPreset.tinyllama2() + server.n_slots = 2 + server.kv_unified = True + server.server_slots = True + server.server_metrics = True + server.temperature = 0.0 + server.seed = 42 + fd, server.log_path = tempfile.mkstemp(suffix=".log") + os.close(fd) + yield + for name in ("LLAMA_SERVER_PREEMPT_EVERY", "LLAMA_SERVER_PREEMPT_GRANULARITY", + "LLAMA_SERVER_PREEMPT_PLANNER", "LLAMA_ARG_PREEMPT_RAM", "LLAMA_ARG_PREEMPT_ASYNC", + "LLAMA_SERVER_PREEMPT_FAIL_SAVE", "LLAMA_ARG_SPEC_DRAFT_P_MIN", "LLAMA_ARG_LOG_VERBOSITY", + "LLAMA_BATCH_DEBUG", "LLAMA_ARG_CTX_CHECKPOINTS", + "LLAMA_MEDIA_MARKER", "LLAMA_EXACT_CONCURRENCY"): + os.environ.pop(name, None) + + +def _start(**kwargs): + """Start the server with these settings; its log starts empty again on every start.""" + for key, value in kwargs.items(): + setattr(server, key, value) + server.start() + + +def _start_async(**kwargs): + """As _start, on the asynchronous park path; a backend that cannot copy off-thread skips the test.""" + os.environ["LLAMA_ARG_PREEMPT_ASYNC"] = "1" + _start(n_gpu_layer=99, **kwargs) + _require_async(_log()) + + +def _log() -> str: + return open(server.log_path).read() + + +def _require_async(text: str): + if _ASYNC_BANNER not in text: + pytest.skip("this backend cannot copy asynchronously, the async park path is not exercised") + + +def _complete(n_predict: int, prompt="Hi how are you", id_slot: int = -1, delay: float = 0.0, after_slot_busy=None): + time.sleep(delay) + if after_slot_busy is not None: + # sent once that slot is processing, so the request queues behind it whatever the host's speed + for _ in range(200): + slots = server.make_request("GET", "/slots").body + if any(s["id"] == after_slot_busy and s["is_processing"] for s in slots): + break + time.sleep(0.02) + return server.make_request("POST", "/completion", data={ + "n_predict": n_predict, "prompt": prompt, "id_slot": id_slot, + "ignore_eos": True, "return_tokens": True, "temperature": 0.0, "seed": 42, + }) + + +def _complete_all(n_predict: int, prompts=(_PROMPT_A, _PROMPT_B)): + return parallel_function_calls([(_complete, (n_predict, prompt)) for prompt in prompts]) + + +def _prompt_of(n_tokens: int, text: str) -> list: + """A prompt of exactly n_tokens tokens, as ids: no BOS is added to one of those.""" + base = server.make_request("POST", "/tokenize", data={"content": text}).body["tokens"] + assert base + return (base * (n_tokens // len(base) + 1))[:n_tokens] + + +def _assert_completed(results, n_predict: int): + for res in results: + assert res.status_code == 200, res.body + assert res.body["timings"]["predicted_n"] == n_predict + + +def _assert_recovered(text: str, parked: str = "preempted:"): + """Nothing was ended for want of cells: a slot was parked and came back.""" + assert "Context size has been exceeded" not in text + assert parked in text + assert "resumed after" in text + + +def _metrics() -> dict: + res = server.make_request("GET", "/metrics") + assert res.status_code == 200 + return { + name[len("llamacpp:"):]: float(value) + for name, value in (line.split(" ", 1) for line in res.body.splitlines() if line.startswith("llamacpp:")) + } + + +@pytest.mark.parametrize("mode", ["sync", "async", "no-async"]) +def test_forced_parks_do_not_change_the_output(mode): + if mode != "sync": + server.n_gpu_layer = 99 + os.environ["LLAMA_ARG_PREEMPT_ASYNC"] = "1" if mode == "async" else "0" + _start(n_ctx=512) + if mode == "async": + _require_async(_log()) + reference = _complete(64) + assert reference.status_code == 200 + server.stop() + + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + _start() + preempted = _complete(64) + assert preempted.status_code == 200 + assert preempted.body["timings"]["predicted_n"] == 64 + assert preempted.body["content"] == reference.body["content"] + assert preempted.body["tokens"] == reference.body["tokens"] + + text = _log() + assert text.count("preempted on request") >= 6 + assert text.count("resumed after") >= 6 + if mode == "async": + assert "park completed after" in text + assert "restore issued in" in text + assert "restore completed after" in text + if mode == "no-async": + assert _ASYNC_BANNER not in text + assert "park issued in" not in text + + +@pytest.mark.parametrize("knob", ["planner", "pages", "async", "last-resort", "last-resort-unlimited"]) +def test_two_generations_that_do_not_fit_together_both_finish(knob): + # each request fits the pool alone (168 of 256 cells) but not together; without preemption both end with "Context size has been exceeded" + if knob == "pages": + # a block allocator gives a whole block to one sequence, so the planner has to count cells: counting tokens it sees room the allocator cannot find + os.environ["LLAMA_SERVER_PREEMPT_GRANULARITY"] = "64" + if knob.startswith("last-resort"): + os.environ["LLAMA_SERVER_PREEMPT_PLANNER"] = "off" + if knob == "last-resort-unlimited": + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "-1" + (_start_async if knob == "async" else _start)(n_ctx=256) + + n_predict = 160 + results = _complete_all(n_predict) + text = _log() + _assert_recovered(text, "preempted as a last resort" if knob.startswith("last-resort") else "preempted:") + _assert_completed(results, n_predict) + for res in results: + assert res.body["truncated"] is False + assert len(res.body["tokens"]) == n_predict + + if knob == "pages": + held = [int(n) for n in re.findall(r"kv (\d+)/256", text)] + wanted = [int(n) for n in re.findall(r"\(wanted (\d+)\)", text)] + assert held and wanted, f"the planner logged no figures:\n{text}" + assert all(n % 64 == 0 for n in held + wanted), f"not whole blocks: {held} {wanted}" + if knob.startswith("last-resort"): + assert "preempted:" not in text, "the planner was off, nothing may be parked ahead of the decode" + assert "last resort: batch given up" in text + if knob == "planner": + metrics = _metrics() + assert metrics["n_preempt_total"] >= 1 + assert metrics["n_resume_total"] == metrics["n_preempt_total"] + assert metrics["requests_preempted"] == 0 + assert metrics["preempt_ram_bytes"] == 0 + + +@pytest.mark.parametrize("knob", ["ram-0", "family"]) +def test_a_request_that_cannot_be_helped_gets_the_context_error_and_the_server_lives(knob): + if knob == "ram-0": + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "0" + else: + # a family member is not a victim for the other, so a two-completion request gets the error it would get alone + os.environ["LLAMA_SERVER_PREEMPT_PLANNER"] = "off" + _start(n_ctx=256) + + if knob == "ram-0": + assert any(res.status_code != 200 for res in _complete_all(160)) + else: + res = server.make_request("POST", "/completion", data={ + "n_predict": 160, "n_cmpl": 2, "prompt": _PROMPT_A, + "ignore_eos": True, "temperature": 0.0, "seed": 42, + }) + assert res.status_code == 500 + assert "Context size has been exceeded" in res.body["error"]["message"] + + text = _log() + assert "Context size has been exceeded" in text + assert "preempted" not in text, "nothing could be parked here" + assert "GGML_ASSERT" not in text + after = _complete(8) + assert after.status_code == 200 + assert after.body["timings"]["predicted_n"] == 8 + + +@pytest.mark.parametrize("planner", ["on", "off"]) +def test_a_late_prompt_and_a_generating_slot_both_finish(planner): + if planner == "off": + os.environ["LLAMA_SERVER_PREEMPT_PLANNER"] = "off" + _start(n_ctx=256) + + n_b = 150 + n_predict_a = 230 + n_predict_b = 90 + assert 8 + n_predict_a + n_b + n_predict_b > 256 + results = parallel_function_calls([ + (_complete, (n_predict_a, "Hi how are you")), + (_complete, (n_predict_b, _prompt_of(n_b, _PROMPT_C), -1, 0.02)), + ]) + + text = _log() + assert "Context size has been exceeded" not in text + assert ("preempted as a last resort" if planner == "off" else "preempted:") in text + for res, n_predict in zip(results, (n_predict_a, n_predict_b)): + assert res.status_code == 200, res.body + assert res.body["timings"]["predicted_n"] == n_predict + # the chunk in the batch given up is processed once after the rewind, never twice + assert results[1].body["timings"]["prompt_n"] == n_b + + +def test_a_prompt_parked_before_its_first_token_is_issued_whole(): + # both prompts are too close to n_ctx to leave the usual margin, so the second is parked before it takes a cell and has to come back once the first has finished + _start(n_ctx=256, n_batch=256) + + n_prompt = 240 + n_predict = 4 + long_prompt = _prompt_of(n_prompt, "Once upon a time there was a little girl") + together = _complete_all(n_predict, [long_prompt, long_prompt]) + + assert "cannot fit the pool" not in _log() + _assert_completed(together, n_predict) + for res in together: + assert res.body["timings"]["prompt_n"] == n_prompt, "the prompt was not issued once and whole" + + +def test_a_resident_cycling_through_context_shifts_is_rotated_out_for_a_parked_head(): + # with context shift on a resident would hold its cells for as long as it generates, so once the head has waited its turn the resident is parked and the two take turns + _start(n_slots=3, n_ctx=384, enable_ctx_shift=True) + + n_predict = 9000 + _assert_completed(_complete_all(n_predict, (_PROMPT_A, _PROMPT_B, _PROMPT_C)), n_predict) + + text = _log() + _assert_recovered(text, "rotated out after") + assert "slot context shift" in text + + +def test_a_park_whose_host_allocation_fails_is_parked_by_recompute(): + # the budget grants permission to allocate, not a successful allocation: a failed save used to stop the planner and leave the pool to overflow, although the same victim could be parked by dropping its cells + os.environ["LLAMA_SERVER_PREEMPT_FAIL_SAVE"] = "1" + _start(n_ctx=256) + + n_predict = 160 + results = _complete_all(n_predict) + + text = _log() + assert "could not take the host memory" in text, "the injected allocation failure never fired" + assert "tokens to re-prefill" in text, "the failed save did not fall back to recompute" + assert "Context size has been exceeded" not in text + _assert_completed(results, n_predict) + + +def test_a_resident_that_cannot_be_swapped_out_is_rotated_by_recompute(): + # 1 MiB holds no snapshot, so rotation refused every resident and only logged that the head waits: a resident that keeps context-shifting need never finish, and the head waited behind it for good + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "1" + _start(n_ctx=2048, n_slots=2, n_batch=2048, enable_ctx_shift=True) + + # a small n_discard keeps the resident near the end of the pool, so its state never fits the budget + def unending_request(): + return server.make_request("POST", "/completion", data={ + "n_predict": 100000, "prompt": _PROMPT_A, "n_keep": 1, "n_discard": 64, + "ignore_eos": True, "temperature": 0.0, "seed": 42, + }, timeout=600) + + unending = [] + t = threading.Thread(target=lambda: unending.append(unending_request())) + t.start() + + try: + # the resident has to be at the pool's limit and cycling before a second prompt cannot fit beside it + for _ in range(3000): + if "slot context shift" in _log(): + break + time.sleep(0.05) + else: + pytest.fail("the resident never reached the end of the pool") + + waiting = server.make_request("POST", "/completion", data={ + "n_predict": 8, "prompt": _prompt_of(1800, _PROMPT_C), + "ignore_eos": True, "temperature": 0.0, "seed": 42, + }, timeout=300) + + assert waiting.status_code == 200, waiting.body + assert waiting.body["timings"]["predicted_n"] == 8, "the second request never made progress" + assert not unending, "the first request ended before the second made progress" + finally: + server.stop() + t.join(60) + + text = _log() + assert "slot context shift" in text + assert re.search(r"rotated out after .* cells dropped", text), "the rotation did not fall back to recompute" + assert "Context size has been exceeded" not in text + + +def test_cancel_while_a_copy_is_in_flight_frees_the_slot(): + # a cancelled request can reach release() with a park or a resume still running, where the host buffer is freed and the cells handed on, so both have to wait for the copy + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + _start_async(n_ctx=512) + + for i in range(4): + try: + server.make_request("POST", "/completion", data={ + "n_predict": 96, "prompt": _PROMPT_A, "ignore_eos": True, "temperature": 0.0, "seed": 42, + }, timeout=0.05 + 0.1 * i) + except Exception: + pass # the point is the drop, not the response + + for _ in range(600): + res = server.make_request("GET", "/slots") + assert res.status_code == 200 + if all(not slot["is_processing"] for slot in res.body): + break + time.sleep(0.2) + else: + pytest.fail("a slot never came back after a cancel during a copy") + for slot in res.body: + assert slot["is_preempted"] is False + assert slot["is_transferring"] is False + assert _metrics()["preempt_ram_bytes"] == 0, "a cancelled slot kept its parked memory" + + res = _complete(16) + assert res.status_code == 200 + assert res.body["timings"]["predicted_n"] == 16 + + +def test_a_started_slot_is_counted_by_the_cells_it_holds_not_by_the_prompt_it_keeps(): + # the last request waits for slot 0 and is started on it holding the first request's cells; counted by the prompt it keeps instead, the pool looks free and a parked slot is restored into cells that are still taken + _start(n_ctx=256, n_slots=3) + + results = parallel_function_calls([ + (_complete, (60, _prompt_of(115, _PROMPT_C), 0)), + (_complete, (100, _PROMPT_A, 1)), + (_complete, (100, _PROMPT_B, 2)), + (_complete, (8, _PROMPT_C, 0, 0.0, 0)), + ]) + + text = _log() + assert "trimmed to the" in text, "the started slot kept the cells of the request before it" + assert "resume failed" not in text + assert "Context size has been exceeded" not in text + for res, n_predict in zip(results, (60, 100, 100, 8)): + assert res.status_code == 200, res.body + assert res.body["timings"]["predicted_n"] == n_predict + + +def test_a_recurrent_model_is_served_without_preemption(): + server.model_file = os.environ.get("LLAMA_SERVER_TEST_RECURRENT_MODEL") + if server.model_file: + server.model_hf_repo = server.model_hf_file = None + else: + server.model_hf_repo = "Felladrin/gguf-mamba-130m-hf" + server.model_hf_file = "mamba-130m-hf.Q2_K.gguf" + server.offline = False + server.n_ctx = 1024 + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + server.start(timeout_seconds=300) + + # not "Once upon a time": what this Q2_K model decodes from it on CUDA carries bytes the content parser refuses, master included, which is not what this test measures + results = _complete_all(64, ["The quick brown fox", "Hello world"]) + _assert_completed(results, 64) + + text = _log() + assert "preemption: off, the recurrent cache holds one state per sequence" in text + assert "preempted" not in text + assert "Context size has been exceeded" not in text + + +def _stream_completion(n_predict: int, prompt: str) -> tuple[list[str], dict]: + """One streaming completion: its SSE comment lines and the last response object.""" + url = f"http://{server.server_host}:{server.server_port}/completion" + res = requests.post(url, json={ + "prompt": prompt, "n_predict": n_predict, "ignore_eos": True, + "temperature": 0.0, "seed": 42, "stream": True, + }, stream=True, timeout=600) + assert res.status_code == 200, res.text + comments, datas = [], [] + for raw in res.iter_lines(): + line = raw.decode("utf-8") + if line.startswith(":"): + comments.append(line) + elif line.startswith("data: ") and line[6:] != "[DONE]": + datas.append(json.loads(line[6:])) + return comments, datas[-1] + + +@pytest.mark.parametrize("planner", ["on", "off"]) +def test_a_budget_that_holds_no_sequence_parks_by_dropping_the_cells(planner): + # 1 MiB holds neither sequence, so no victim fits the budget: the park drops the cells and the resume re-prefills the tokens, instead of the pool overflowing and ending both; the last resort falls back the same way + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "1" + if planner == "off": + os.environ["LLAMA_SERVER_PREEMPT_PLANNER"] = "off" + # one batch for the re-prefill: a prompt of this length in 32-token batches hangs the CUDA build with graphs on, master included, so that is not what this test measures + _start(n_ctx=3840, n_batch=2048) + + n_predict = 2000 + if planner == "on": + results = parallel_function_calls([(_stream_completion, (n_predict, p)) for p in (_PROMPT_A, _PROMPT_B)]) + for comments, final in results: + assert "error" not in final, final + assert final["tokens_predicted"] == n_predict + comments = [c for cs, _ in results for c in cs] + assert ": preempted" in comments and ": resumed" in comments, comments + else: + _assert_completed(_complete_all(n_predict), n_predict) + + text = _log() + assert "Context size has been exceeded" not in text + assert "tokens to re-prefill" in text, "no park fell back to recompute" + if planner == "off": + assert "preempted as a last resort by dropping its cells" in text + + +_IMG_URL = "https://huggingface.co/ggml-org/tinygemma3-GGUF/resolve/main/test/11_truck.png" + + +def test_a_media_chunk_is_reserved_whole_before_it_is_decoded(): + # a chunk is decoded whole inside one iteration, through decodes the kv-full retry does not cover: unless the planner reserves every cell it takes, the second of two image requests that each fit alone fails part way through its chunk + os.environ["LLAMA_MEDIA_MARKER"] = "<__media__>" + server.model_hf_repo = "ggml-org/tinygemma3-GGUF:Q8_0" + server.model_hf_file = None + server.model_alias = "tinygemma3" + _start(n_ctx=400, n_batch=64, n_ubatch=64) + + image = base64.b64encode(requests.get(_IMG_URL, timeout=60).content).decode() + prompt = {"prompt_string": "<__media__>\nWhat is in this image?", "multimodal_data": [image]} + results = parallel_function_calls([ + (server.make_request, ("POST", "/completion", { + "prompt": prompt, "n_predict": 4, "temperature": 0.0, "seed": 42, + })) for _ in range(2) + ]) + + text = _log() + assert "failed to process mtmd chunk" not in text + assert "preempted:" in text, "nothing was parked to make room for a chunk" + for res in results: + assert res.status_code == 200, res.body + assert res.body["timings"]["prompt_n"] > 64, "the chunk fits one batch, so it never spans several decodes" + + +def test_an_mtp_draft_stays_inside_the_reservation_it_was_priced_for(): + # near the end of the pool the planner prices one draft token and one sampled token, but the draft loop stopped against the configured window and could attempt positions past the reservation + path = os.environ.get("LLAMA_SERVER_TEST_MTP_MODEL") + if not path: + pytest.skip("set LLAMA_SERVER_TEST_MTP_MODEL to a gguf carrying an MTP head") + server.model_file = path + server.model_hf_repo = server.model_hf_file = None + server.spec_type = "draft-mtp" + os.environ["LLAMA_ARG_SPEC_DRAFT_P_MIN"] = "0.0" # nothing but the bounds stops the draft loop + os.environ["LLAMA_ARG_LOG_VERBOSITY"] = "5" # the wrapper says so when it truncates what an implementation returned + _start(n_ctx=2048, n_slots=2, n_batch=2048, n_gpu_layer=99, spec_draft_n_max=128, spec_draft_n_min=1) + + n_prompt = 2045 + res = server.make_request("POST", "/completion", data={ + "prompt": _prompt_of(n_prompt, _PROMPT_C), "n_predict": 2, "ignore_eos": True, + "temperature": 0.0, "seed": 42, "cache_prompt": False, + }, timeout=600) + + assert res.status_code == 200, res.body + assert res.body["timings"]["prompt_n"] == n_prompt + assert res.body["tokens_predicted"] == 2 + + text = _log() + assert "truncating draft to" not in text, "the draft was scheduled past the tokens the planner reserved" + assert "llama_decode[" not in text + assert "Context size has been exceeded" not in text + + +def test_a_hybrid_model_parks_synchronously(): + # the recurrent half of a hybrid gathers the active sequences into contiguous rows on every batch, so a copy running beside the decode could read a row another sequence has been moved into + path = os.environ.get("LLAMA_SERVER_TEST_HYBRID_MODEL") + if not path: + pytest.skip("set LLAMA_SERVER_TEST_HYBRID_MODEL to a hybrid attention/recurrent gguf") + server.model_file = path + server.model_hf_repo = server.model_hf_file = None + os.environ["LLAMA_ARG_PREEMPT_ASYNC"] = "1" + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + _start(n_ctx=1024, n_gpu_layer=99) + + res = _complete(24, "Once upon a time") + assert res.status_code == 200, res.body + assert res.body["timings"]["predicted_n"] == 24 + + text = _log() + assert "a recurrent state does not stay in one row" in text + assert _ASYNC_BANNER not in text + assert "park issued in" not in text + _assert_recovered(text, "preempted on request") + + +# [TAG_EXACT_CONCURRENCY] the paged pool places a cell from the sequence and the position alone, so a layout that gives several tokens one position cannot be served + +def _mrope_model() -> str: + path = os.environ.get("LLAMA_SERVER_TEST_MROPE_MODEL") + if not path: + pytest.skip("set LLAMA_SERVER_TEST_MROPE_MODEL to an M-RoPE gguf") + return path + + +def test_exact_concurrency_refuses_an_mrope_model_with_a_projector(): + # every token of one image shares a temporal position under M-RoPE, so the pool would give the second one the first one's cell + path = _mrope_model() + with tempfile.TemporaryDirectory() as tmp: + mmproj = os.path.join(tmp, "mmproj.gguf") + with open(mmproj, "wb") as f: + f.write(b"GGUF" + struct.pack("<IQQ", 3, 0, 0)) # header only: never read, the refusal comes first + proc = subprocess.run([ + os.environ.get("LLAMA_SERVER_BIN_PATH", "../../../build/bin/llama-server"), + "--model", path, "--mmproj", mmproj, "--host", "127.0.0.1", "--port", str(server.server_port), + "-c", "512", "--parallel", "2", "--kv-unified", "-fa", "on", "-ngl", "99", "--no-warmup", "--no-webui", + ], env={**os.environ, "LLAMA_EXACT_CONCURRENCY": "1"}, capture_output=True, text=True, timeout=900) + out = proc.stdout + proc.stderr + assert proc.returncode != 0, out + assert "does not support M-RoPE together with a projector" in out, out + + +def test_exact_concurrency_refuses_a_batch_that_cannot_hold_a_whole_ubatch(): + # a prompt is added in whole ubatches, so -b 512 -ub 512 -np 2 would leave a prefill 511 tokens beside one decoder: the geometry is refused at startup rather than reported as exact and served short + path = os.environ.get("LLAMA_SERVER_TEST_EXACT_MODEL") + if not path: + pytest.skip("set LLAMA_SERVER_TEST_EXACT_MODEL to a gguf exact concurrency accepts") + proc = subprocess.run([ + os.environ.get("LLAMA_SERVER_BIN_PATH", "../../../build/bin/llama-server"), + "--model", path, "--host", "127.0.0.1", "--port", str(server.server_port), + "-c", "2048", "-b", "512", "-ub", "512", "--parallel", "2", "--kv-unified", + "-fa", "on", "-ngl", "99", "--no-warmup", "--no-webui", + ], env={**os.environ, "LLAMA_EXACT_CONCURRENCY": "1"}, capture_output=True, text=True, timeout=900) + out = proc.stdout + proc.stderr + assert proc.returncode != 0, out + assert "needs a batch of at least 514 tokens for a 512-token ubatch" in out, out + assert "Raise -b to 514" in out, out + + +def test_exact_concurrency_serves_an_mrope_model_without_a_projector(): + # the refusal is about images, not the rope layout: a text prompt gives every token its own position + server.model_file = _mrope_model() + server.model_hf_repo = server.model_hf_file = None + os.environ["LLAMA_EXACT_CONCURRENCY"] = "1" + # the mode needs a batch that holds a whole ubatch beside a decode step of every slot + _start(n_ctx=512, n_slots=2, n_batch=512, n_ubatch=128, fa="on", n_gpu_layer=99) + + res = _complete(16, "Once upon a time") + assert res.status_code == 200, res.body + assert res.body["timings"]["predicted_n"] == 16 + + text = _log() + assert "does not support M-RoPE" not in text + assert "the kv pool allocates 256 cells at a time" in text + + +def _ubatch_widths(text: str) -> list: + """The tokens of every ubatch a split produced, in order; needs LLAMA_BATCH_DEBUG.""" + res = [] + pending = False + for line in text.splitlines(): + if "added ubatch to split" in line: + pending = True + elif pending and "n_tokens" in line: + res.append(int(line.split("=")[-1])) + pending = False + return res + + +def test_exact_concurrency_prefills_a_prompt_in_the_ubatches_it_would_get_alone(): + # generated tokens enter the batch first and a prompt took what was left, so its ubatches were 512,512,512,509 beside three decoders and 512,512,512,512 alone: isolating the sequences does not make the shapes equal by itself + # geometry: -b 2048 -ub 512 -np 4, so the batch holds a whole ubatch beside a decode step of every slot (516 tokens), which is what common_exact_batch_geometry() requires of a start + path = os.environ.get("LLAMA_SERVER_TEST_EXACT_MODEL") + if not path: + pytest.skip("set LLAMA_SERVER_TEST_EXACT_MODEL to a gguf exact concurrency accepts") + server.model_file = path + server.model_hf_repo = server.model_hf_file = None + os.environ["LLAMA_EXACT_CONCURRENCY"] = "1" + os.environ["LLAMA_BATCH_DEBUG"] = "1" + os.environ["LLAMA_ARG_LOG_VERBOSITY"] = "5" + os.environ["LLAMA_ARG_CTX_CHECKPOINTS"] = "0" + _start(n_ctx=16384, n_slots=4, n_batch=2048, n_ubatch=512, fa="on", n_gpu_layer=99, cache_ram=0) + + def prefill(first_token: int) -> list: + mark = len(open(server.log_path, errors="replace").read()) + res = server.make_request("POST", "/completion", data={ + "prompt": list(range(first_token, first_token + 3500)), "n_predict": 1, + "cache_prompt": False, "temperature": 0.0, "seed": 42, + }, timeout=600) + assert res.status_code == 200, res.body + assert res.body["timings"]["prompt_n"] == 3500 + text = open(server.log_path, errors="replace").read()[mark:] + # a decode step is one token per slot, so the prompt's own ubatches are the wide ones + return [w for w in _ubatch_widths(text) if w > 3] + + alone = prefill(1000) + assert alone, "no ubatch was recorded, LLAMA_BATCH_DEBUG did not reach the log" + + def decoder(i): + server.make_request("POST", "/completion", data={ + "prompt": list(range(20000 + 100 * i, 20000 + 100 * i + 8)), "n_predict": 100000, + "cache_prompt": False, "ignore_eos": True, "temperature": 0.0, "seed": 42, + }, timeout=600) + + threads = [threading.Thread(target=decoder, args=(i,), daemon=True) for i in range(3)] + for t in threads: + t.start() + + try: + for _ in range(600): + slots = server.make_request("GET", "/slots").body + if sum(1 for s in slots if s["is_processing"] and s["n_prompt_tokens"] > 0) >= 3: + break + time.sleep(0.1) + else: + pytest.fail("the decoders never started") + time.sleep(1.0) + + beside = prefill(50000) + finally: + server.stop() + for t in threads: + t.join(30) + + assert beside == alone, f"alone {alone}, beside three decoders {beside}" + + +def test_slots_reports_a_transferring_slot_apart_from_a_parked_one(): + # a copy out still owns its cells and a restore has already taken them back, so a reader counting residency has to keep counting both; only a fully parked slot holds nothing + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "1" + _start_async(n_ctx=256) + + done = [] + t = threading.Thread(target=lambda: done.extend(_complete_all(900))) + t.start() + seen_parked = seen_transferring = False + try: + deadline = time.time() + 120 + while time.time() < deadline and not (seen_parked and seen_transferring): + res = server.make_request("GET", "/slots") + assert res.status_code == 200 + for slot in res.body: + assert not (slot["is_preempted"] and slot["is_transferring"]), slot + if slot["is_transferring"]: + seen_transferring = True + assert slot["n_prompt_tokens"] > 0, "a slot with a copy in flight still holds its cells" + seen_parked = seen_parked or slot["is_preempted"] + finally: + t.join(180) + + assert len(done) == 2, done + for res in done: + assert res.status_code == 200, res.body + assert res.body["tokens_predicted"] > 0 + assert seen_parked, "no parked slot was ever reported" + assert seen_transferring, "no slot with a copy in flight was ever reported" + + +def _shift_completion(n_predict: int): + """A completion whose context shifts, on a token prompt so its length is exact.""" + return server.make_request("POST", "/completion", data={ + "prompt": [1] + list(range(10, 70)), "n_predict": n_predict, "n_keep": 16, "n_discard": 64, + "ignore_eos": True, "return_tokens": True, "cache_prompt": False, "temperature": 0.0, "seed": 42, + }) + + +def test_a_park_right_after_a_context_shift_does_not_change_the_output(): + # the shift moves the positions and leaves the K transformation for the next decode, so a park in between used to save the new positions with the old K + os.environ["LLAMA_ARG_PREEMPT_ASYNC"] = "0" + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "0" + _start(n_ctx=256, n_batch=32, n_ubatch=32, enable_ctx_shift=True, cache_ram=0) + + n_predict = 320 + reference = _shift_completion(n_predict) + assert reference.status_code == 200, reference.body + assert reference.body["timings"]["predicted_n"] == n_predict + n_prompt = reference.body["timings"]["prompt_n"] + server.stop() + + # park on the step the shift lands on: the pool holds n_ctx cells, so the first shift is that many tokens in + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "8192" + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = str(256 - n_prompt) + _start() + + parked = _shift_completion(n_predict) + assert parked.status_code == 200, parked.body + assert parked.body["timings"]["predicted_n"] == n_predict + + text = _log() + assert "slot context shift" in text + _assert_recovered(text, "preempted on request") + first_diff = next((i for i, (a, b) in enumerate(zip(reference.body["tokens"], parked.body["tokens"])) if a != b), None) + assert first_diff is None, f"the parked run diverged at token {first_diff}" + + +def test_a_sibling_prompt_with_an_invalid_token_is_refused_before_anything_streams(): + # validated with the others ahead of posting: parked behind a running sibling, it used to fail inside a stream that had already opened 200 + _start(n_ctx=256, n_slots=2, n_batch=256) + + res = server.make_request("POST", "/completion", data={ + "prompt": [[1] * 240, [1] * 240, [9999999]], "n_predict": 4, "temperature": 0.0, "seed": 42, + }) + assert res.status_code == 400, res.body + assert "invalid tokens" in str(res.body) + + text = _log() + assert "preempted" not in text + + +def test_a_recompute_park_bounds_its_draft_by_the_tokens_it_comes_back_with(): + # a recompute park moves the prompt out of the slot, and the draft was bounded by the empty prompt: 2000 tokens and a whole draft could not fit a 2048-cell pool "even alone", failing a request that fits + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "1" + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "1" + server.spec_type = "ngram-mod" + _start(n_ctx=2048, n_slots=2, n_batch=2048, n_ubatch=512, spec_ngram_mod_n_max=128, spec_ngram_mod_n_min=1) + + prompt = [1] + list(range(10, 110)) * 19 + list(range(10, 109)) + assert len(prompt) == 2000 + res = server.make_request("POST", "/completion", data={ + "prompt": prompt, "n_predict": 24, "ignore_eos": True, "temperature": 0.0, "seed": 42, "cache_prompt": False, + }) + assert res.status_code == 200, res.body + assert res.body["tokens_predicted"] == 24 + + text = _log() + assert "tokens to re-prefill" in text + assert "cannot fit the pool" not in text + assert "Context size has been exceeded" not in text + + +def test_props_says_whether_exact_concurrency_is_running(): + # a client that asked for the mode reads the answer here: a build that ignores the variable starts all the same + _start(n_ctx=256) + res = server.make_request("GET", "/props") + assert res.status_code == 200 + assert res.body["exact_concurrency"] is False + + +def test_props_reports_exact_concurrency_on(): + server.model_file = _mrope_model() + server.model_hf_repo = server.model_hf_file = None + os.environ["LLAMA_EXACT_CONCURRENCY"] = "1" + _start(n_ctx=512, n_slots=2, n_batch=512, n_ubatch=128, fa="on", n_gpu_layer=99) + res = server.make_request("GET", "/props") + assert res.status_code == 200 + assert res.body["exact_concurrency"] is True + + +def test_a_recompute_park_under_exact_concurrency_says_it_is_not_byte_identical(): + # a state that comes back from host memory is the state that left; one rebuilt by re-prefilling differs in the last bits on CUDA, so the mode says so the first time it happens + server.model_file = _mrope_model() + server.model_hf_repo = server.model_hf_file = None + os.environ["LLAMA_EXACT_CONCURRENCY"] = "1" + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "1" + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "4" + _start(n_ctx=512, n_slots=2, n_batch=512, n_ubatch=128, fa="on", n_gpu_layer=99) + + res = _complete(16, "Once upon a time") + assert res.status_code == 200, res.body + text = _log() + assert "tokens to re-prefill" in text + assert "not guaranteed byte-identical" in text + + +def test_a_recompute_park_is_reported_to_the_client_and_to_metrics(): + # a recompute resume re-prefills instead of restoring the saved bytes, so it is not the continuation the parked state would have given: the request, /slots and /metrics all say how often that happened + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "1" + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + # a sequence this long holds more than the 1 MiB budget, so every park drops its cells + _start(n_ctx=2048, n_batch=2048) + + n_predict = 24 + prompt = _prompt_of(1950, _PROMPT_C) + comments, final = _stream_completion(n_predict, prompt) + + assert "error" not in final, final + assert final["tokens_predicted"] == n_predict + assert final["preempt"]["parks"] >= 2, final["preempt"] + assert final["preempt"]["recomputes"] == final["preempt"]["parks"], final["preempt"] + + # the notice comes right after the resume it belongs to + resumed = [i for i, c in enumerate(comments) if c.startswith(": resumed")] + assert len(resumed) == final["preempt"]["recomputes"], comments + for i in resumed: + assert comments[i + 1].startswith(": recomputed"), comments + + metrics = _metrics() + assert metrics["preempt_recompute_total"] == final["preempt"]["recomputes"] + assert metrics["preempt_recompute_total"] == metrics["n_preempt_total"] + + # a request that is never parked says so + plain = server.make_request("POST", "/completion", data={ + "n_predict": 4, "prompt": _PROMPT_B, "temperature": 0.0, "seed": 42, + }) + assert plain.status_code == 200, plain.body + assert plain.body["preempt"] == {"parks": 0, "recomputes": 0} + + +def test_slots_reports_the_recomputes_of_the_current_task(): + # a reader watching the slots sees the same count the request is given at the end + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "1" + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + _start(n_ctx=2048, n_batch=2048) + + done = [] + prompt = _prompt_of(1950, _PROMPT_C) + t = threading.Thread(target=lambda: done.append(_complete(64, prompt))) + t.start() + + seen = 0 + try: + deadline = time.time() + 180 + while time.time() < deadline and seen == 0 and t.is_alive(): + res = server.make_request("GET", "/slots") + assert res.status_code == 200 + for slot in res.body: + assert slot["n_recompute"] <= slot["n_preempt"] + seen = max(seen, slot["n_recompute"]) + time.sleep(0.02) + finally: + t.join(180) + + assert seen > 0, "no slot ever reported a recompute park" + assert done and done[0].status_code == 200, done + assert done[0].body["preempt"]["recomputes"] >= seen + + +def test_a_swap_park_is_not_reported_as_a_recompute(): + # the same park with room for its bytes keeps the sequence it saved, and the client is told so + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + _start(n_ctx=512) + + comments, final = _stream_completion(24, _PROMPT_A) + + assert final["preempt"]["parks"] >= 1, final["preempt"] + assert final["preempt"]["recomputes"] == 0, final["preempt"] + assert ": resumed" in comments and not any(c.startswith(": recomputed") for c in comments), comments + assert _metrics()["preempt_recompute_total"] == 0 + + +def test_two_image_chats_that_outgrow_the_parking_budget_both_finish(): + # a media chunk could not be parked by recompute, so with the host budget spent nothing could be parked at all and the pool overflowing ended both chats. The chunk comes back the way it went in: re-encoded off the task, its cells reserved whole + os.environ["LLAMA_MEDIA_MARKER"] = "<__media__>" + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "1" + server.model_hf_repo = "ggml-org/tinygemma3-GGUF:Q8_0" + server.model_hf_file = None + server.model_alias = "tinygemma3" + _start(n_ctx=1024, n_slots=2, n_batch=64, n_ubatch=64) + + image = base64.b64encode(requests.get(_IMG_URL, timeout=60).content).decode() + prompt = {"prompt_string": "<__media__>\nWhat is in this image?", "multimodal_data": [image]} + n_predict = 700 + results = parallel_function_calls([ + (server.make_request, ("POST", "/completion", { + "prompt": prompt, "n_predict": n_predict, "ignore_eos": True, "temperature": 0.0, "seed": 42, + })) for _ in range(2) + ]) + + text = _log() + assert "Context size has been exceeded" not in text + assert "failed to process mtmd chunk" not in text + assert "tokens to re-prefill" in text, "no park fell back to recompute" + for res in results: + assert res.status_code == 200, res.body + assert res.body["tokens_predicted"] == n_predict diff --git a/tools/server/tests/unit/test_preempt_notify.py b/tools/server/tests/unit/test_preempt_notify.py new file mode 100644 index 000000000000..84c9983259fd --- /dev/null +++ b/tools/server/tests/unit/test_preempt_notify.py @@ -0,0 +1,267 @@ +import json +import os +import tempfile +import threading +import pytest +import requests +from utils import * + +# [TAG_PREEMPT] a streaming client is told when its slot is parked and restored, as SSE comments every existing client ignores; a keepalive every 2 s keeps proxies from giving up + +server = ServerPreset.tinyllama2() + +_PROMPT_A = "Once upon a time there was a brave knight who" +_PROMPT_B = "The quick brown fox jumps over the lazy dog and" +_PROMPT_C = "In a small village by the sea there lived a fisherman who" + + +@pytest.fixture(autouse=True) +def create_server(): + global server + server = ServerPreset.tinyllama2() + server.n_slots = 2 + server.kv_unified = True + server.temperature = 0.0 + server.seed = 42 + fd, server.log_path = tempfile.mkstemp(suffix=".log") + os.close(fd) + yield + for name in ("LLAMA_SERVER_PREEMPT_EVERY", "LLAMA_ARG_PREEMPT_RAM"): + os.environ.pop(name, None) + + +def _start(**kwargs): + for key, value in kwargs.items(): + setattr(server, key, value) + server.start() + + +def _completion_payload(n_predict: int, prompt: str = "Hi how are you", **extra) -> dict: + return {"n_predict": n_predict, "prompt": prompt, "ignore_eos": True, + "temperature": 0.0, "seed": 42, "stream": True, **extra} + + +def _chat_payload(n_predict: int) -> dict: + return {"max_tokens": n_predict, "messages": [{"role": "user", "content": "Hi how are you"}], + "temperature": 0.0, "seed": 42, "stream": True} + + +def _post(path: str, data: dict): + return requests.post(f"http://{server.server_host}:{server.server_port}{path}", json=data, stream=True) + + +def _stream_raw(path: str, data: dict) -> tuple[list[str], list[str]]: + """The SSE lines of one streaming request: (comment lines, data lines).""" + res = _post(path, data) + assert res.status_code == 200 + comments, datas = [], [] + for raw in res.iter_lines(): + line = raw.decode("utf-8") + if line.startswith(":"): + comments.append(line) + elif line.startswith("data: "): + datas.append(line[6:]) + return comments, datas + + +def _stream_all(n_predict: int, prompts, **extra): + return parallel_function_calls([ + (_stream_raw, ("/completion", _completion_payload(n_predict, prompt, **extra))) for prompt in prompts + ]) + + +def _behind_a_resident(payload: dict) -> tuple[int, str, list[str]]: + """Run this request behind a resident holding the pool: its status, its body, and its SSE lines.""" + started = threading.Event() + + def _resident(): + res = _post("/completion", _completion_payload(390, " ".join([_PROMPT_A] * 6))) + assert res.status_code == 200 + for raw in res.iter_lines(): + if raw.decode("utf-8").startswith("data: "): + started.set() + + t = threading.Thread(target=_resident) + t.start() + try: + assert started.wait(60) + res = _post("/completion", payload) + if res.status_code != 200: + return res.status_code, res.text, [] + lines, alive = [], None + for raw in res.iter_lines(): + line = raw.decode("utf-8") + if line: + alive = t.is_alive() if alive is None else alive + lines.append(line) + assert alive, "the resident had finished before this request was told anything" + return 200, "", lines + finally: + t.join(120) + + +def _content(datas: list[str]) -> str: + out = "" + for d in datas: + if d == "[DONE]": + break + j = json.loads(d) + out += j.get("content") or "" + for ch in j.get("choices", []) or []: + out += (ch.get("delta") or {}).get("content") or "" + return out + + +def _final(datas: list[str]) -> dict: + """The last response object of a finished stream, past the [DONE] marker.""" + return json.loads([d for d in datas if d != "[DONE]"][-1]) + + +def _notices(comments: list[str]) -> list[str]: + return [c for c in comments if c in (": preempted", ": resumed")] + + +@pytest.mark.parametrize("path,payload", [ + ("/completion", _completion_payload(64)), + ("/v1/chat/completions", _chat_payload(64)), +]) +def test_every_park_in_a_stream_is_announced_paired_with_a_resume_and_changes_nothing(path, payload): + _start(n_ctx=512) + ref_comments, ref_datas = _stream_raw(path, payload) + assert _notices(ref_comments) == [] + assert _content(ref_datas) + server.stop() + + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + server.start() + comments, datas = _stream_raw(path, payload) + seq = _notices(comments) + assert len(seq) >= 12, comments + assert seq == [": preempted", ": resumed"] * (len(seq) // 2), seq + assert _content(datas) == _content(ref_datas) + + +def _prefill_payload(path: str, prompt: str, n_predict: int) -> dict: + """The same request on each streaming surface.""" + if path == "/completion": + return {"prompt": prompt, "n_predict": n_predict, "ignore_eos": True, + "temperature": 0.0, "seed": 42, "stream": True} + if path == "/v1/responses": + return {"model": "test", "input": prompt, "max_output_tokens": n_predict, + "temperature": 0.0, "stream": True} + return {"model": "test", "messages": [{"role": "user", "content": prompt}], + "max_tokens": n_predict, "temperature": 0.0, "stream": True} + + +@pytest.mark.parametrize("path", ["/completion", "/v1/chat/completions", "/v1/responses", "/v1/messages"]) +def test_a_park_during_prompt_processing_opens_the_stream_with_the_notice(path): + # a park before the first token is the case a client cannot tell from a stall, so the notice goes out with the response headers rather than waiting for a chunk that is not coming + import time + + server.server_slots = True + _start(n_ctx=2048, n_batch=256) + + def _resident(): + res = _post("/completion", _completion_payload(1900, _PROMPT_A)) + for _ in res.iter_lines(): + pass + + t = threading.Thread(target=_resident, daemon=True) + t.start() + + # the pool has to be nearly full before the second prompt starts, so that its prefill is what runs out of cells + for _ in range(600): + slots = requests.get(f"http://{server.server_host}:{server.server_port}/slots").json() + if any(slot.get("n_prompt_tokens", 0) >= 1400 for slot in slots): + break + time.sleep(0.02) + else: + pytest.fail("the resident never grew into the pool") + + res = _post(path, _prefill_payload(path, " ".join([_PROMPT_B] * 31), 8)) + assert res.status_code == 200 + + t0 = time.time() + seen = [] + for raw in res.iter_lines(): + line = raw.decode("utf-8") + if line: + seen.append((time.time() - t0, line)) + t.join(120) + + text = open(server.log_path).read() + assert "preempted:" in text, "nothing was parked while the prompt was being processed" + + comments = [(at, line) for at, line in seen if line.startswith(":")] + datas = [(at, line) for at, line in seen if line.startswith("data:")] + + assert comments and comments[0][1] == ": preempted", [line for _, line in seen[:4]] + assert datas, "the request never produced a chunk" + + # sent when the slot was parked, not batched with the chunk that came later + assert comments[0][0] + 0.05 < datas[0][0], [(round(at, 3), line[:24]) for at, line in seen[:4]] + assert any(line == ": resumed" for _, line in comments), [line for _, line in comments[:4]] + + +def test_a_stream_parked_before_its_first_token_starts_with_the_notice(): + # n_batch: the whole prompt in one batch, so the planner sees its size at once + _start(n_ctx=512, n_batch=512) + + status, _, lines = _behind_a_resident(_completion_payload(32, " ".join([_PROMPT_B] * 14))) + assert status == 200 + events = [l for l in lines if l in (": preempted", ": resumed") or l.startswith("data: ")] + assert events[:2] == [": preempted", ": resumed"], events[:3] + assert events[2].startswith("data: "), events[:3] + datas = [l[6:] for l in lines if l.startswith("data: ")] + assert _content(datas) + assert _final(datas)["tokens_predicted"] == 32 + + +def test_an_oversized_prompt_is_errored_instead_of_parked(): + # a slot just given a task has not passed the prompt checks yet, and a notice opens the stream, so parking it would turn a plain error response into 200 plus an in-stream one + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + _start(n_ctx=512, n_batch=512) + + status, body, _ = _behind_a_resident(_completion_payload(16, " ".join([_PROMPT_B] * 80))) + assert status != 200, body + assert not body.lstrip().startswith(":"), body + assert "error" in json.loads(body), body + + +def test_a_rotation_tells_both_streams_and_a_head_parked_past_the_budget_is_kept_alive(): + # --preempt-ram 2 MiB holds one parked state but not a resident's and the head's at once, so that rotation is refused and the head waits parked for longer than the 2 s keepalive + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "2" + _start(n_slots=3, n_ctx=2048, enable_ctx_shift=True) + + n_predict = 12000 + # the default parked keepalive is 2 s, which is also when a resident is rotated out for the head: + # a park that ends with that rotation could beat its own keepalive. Ask for a 1 s ping instead, so + # any park that outlasts one rotation window is still required to say so + results = _stream_all(n_predict, (_PROMPT_A, _PROMPT_B, _PROMPT_C), sse_ping_interval=1) + n_parked = n_keepalive = 0 + for comments, datas in results: + assert _final(datas)["tokens_predicted"] == n_predict + seq = _notices(comments) + assert seq == [": preempted", ": resumed"] * (len(seq) // 2), seq + n_parked += len(seq) // 2 + n_keepalive += comments.count(": preempt-keepalive") + assert n_parked >= 2, [r[0] for r in results] + assert n_keepalive >= 1, "a parked stream was left silent past its keepalive interval" + + text = open(server.log_path).read() + assert "rotated out after" in text + assert "no rotation: --preempt-ram 2 MiB" in text + assert "resumed after" in text + assert "Context size has been exceeded" not in text + + +def test_an_oversized_sibling_prompt_is_errored_before_a_valid_one_is_parked(): + # a request can carry several prompts; a valid one can be parked and its notice opens the stream, so the sibling that does not fit has to be found before any of them is queued + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "8192" + _start(n_ctx=256, n_slots=3, n_batch=512) + + status, body, _ = _behind_a_resident(_completion_payload(8) | {"prompt": [[1] * 120, [1] * 300]}) + assert status == 400, (status, body) + assert not body.lstrip().startswith(":"), body + assert "error" in json.loads(body), body + From 62cf502303c970f4782f822592d2d72f031df5ae Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Wed, 9 Sep 2026 14:26:37 +0000 Subject: [PATCH 02/22] Drop the fork's own workflow and script files from the pin tree; the nightly composes them from master --- .github/actions/prebuilt-alert/action.yml | 119 -- .github/workflows/unsloth-pin-preflight.yml | 276 ---- .github/workflows/unsloth-pr-set-lint.yml | 164 -- .github/workflows/unsloth-prebuilt-cpu.yml | 376 ----- .../unsloth-prebuilt-cuda-windows.yml | 310 ---- .github/workflows/unsloth-prebuilt-cuda.yml | 290 ---- .../workflows/unsloth-prebuilt-deadman.yml | 81 - .github/workflows/unsloth-prebuilt-macos.yml | 211 --- .github/workflows/unsloth-prebuilt-retry.yml | 119 -- .github/workflows/unsloth-prebuilt-rocm.yml | 878 ----------- .github/workflows/unsloth-prebuilt-vulkan.yml | 409 ----- .github/workflows/unsloth-prebuilt.yml | 1317 ----------------- .github/workflows/unsloth-repin-bot.yml | 209 --- .../workflows/unsloth-upstream-sync-guard.yml | 79 - scripts/unsloth/additive_merge.py | 265 ---- scripts/unsloth/assemble_metadata.py | 533 ------- scripts/unsloth/assert_macho_minos.sh | 55 - scripts/unsloth/carry_vintage.py | 160 -- scripts/unsloth/check_workflow_scalars.py | 97 -- scripts/unsloth/feature-checks.json | 104 -- scripts/unsloth/feature_matrix.py | 200 --- scripts/unsloth/merge_checks.py | 314 ---- scripts/unsloth/package_bundle.py | 381 ----- scripts/unsloth/pin_contract.py | 361 ----- scripts/unsloth/pin_merge.py | 220 --- scripts/unsloth/pr-set.json | 35 - scripts/unsloth/repin.py | 272 ---- scripts/unsloth/sync_deletes.py | 142 -- scripts/unsloth/test_additive_merge.py | 206 --- scripts/unsloth/test_carry_vintage.py | 378 ----- scripts/unsloth/test_feature_matrix.py | 153 -- scripts/unsloth/test_merge_checks.py | 370 ----- scripts/unsloth/test_pin_contract.py | 187 --- scripts/unsloth/test_pin_merge.py | 274 ---- scripts/unsloth/test_sync_deletes.py | 150 -- scripts/unsloth/test_upload_release_assets.sh | 111 -- scripts/unsloth/test_verify_upstream_sync.py | 158 -- scripts/unsloth/upload_release_assets.sh | 165 --- scripts/unsloth/upstream-sync.json | 26 - scripts/unsloth/verify_upstream_sync.py | 345 ----- 40 files changed, 10500 deletions(-) delete mode 100644 .github/actions/prebuilt-alert/action.yml delete mode 100644 .github/workflows/unsloth-pin-preflight.yml delete mode 100644 .github/workflows/unsloth-pr-set-lint.yml delete mode 100644 .github/workflows/unsloth-prebuilt-cpu.yml delete mode 100644 .github/workflows/unsloth-prebuilt-cuda-windows.yml delete mode 100644 .github/workflows/unsloth-prebuilt-cuda.yml delete mode 100644 .github/workflows/unsloth-prebuilt-deadman.yml delete mode 100644 .github/workflows/unsloth-prebuilt-macos.yml delete mode 100644 .github/workflows/unsloth-prebuilt-retry.yml delete mode 100644 .github/workflows/unsloth-prebuilt-rocm.yml delete mode 100644 .github/workflows/unsloth-prebuilt-vulkan.yml delete mode 100644 .github/workflows/unsloth-prebuilt.yml delete mode 100644 .github/workflows/unsloth-repin-bot.yml delete mode 100644 .github/workflows/unsloth-upstream-sync-guard.yml delete mode 100644 scripts/unsloth/additive_merge.py delete mode 100644 scripts/unsloth/assemble_metadata.py delete mode 100755 scripts/unsloth/assert_macho_minos.sh delete mode 100755 scripts/unsloth/carry_vintage.py delete mode 100644 scripts/unsloth/check_workflow_scalars.py delete mode 100644 scripts/unsloth/feature-checks.json delete mode 100644 scripts/unsloth/feature_matrix.py delete mode 100755 scripts/unsloth/merge_checks.py delete mode 100644 scripts/unsloth/package_bundle.py delete mode 100644 scripts/unsloth/pin_contract.py delete mode 100755 scripts/unsloth/pin_merge.py delete mode 100644 scripts/unsloth/pr-set.json delete mode 100644 scripts/unsloth/repin.py delete mode 100755 scripts/unsloth/sync_deletes.py delete mode 100644 scripts/unsloth/test_additive_merge.py delete mode 100644 scripts/unsloth/test_carry_vintage.py delete mode 100644 scripts/unsloth/test_feature_matrix.py delete mode 100644 scripts/unsloth/test_merge_checks.py delete mode 100644 scripts/unsloth/test_pin_contract.py delete mode 100644 scripts/unsloth/test_pin_merge.py delete mode 100644 scripts/unsloth/test_sync_deletes.py delete mode 100755 scripts/unsloth/test_upload_release_assets.sh delete mode 100644 scripts/unsloth/test_verify_upstream_sync.py delete mode 100755 scripts/unsloth/upload_release_assets.sh delete mode 100644 scripts/unsloth/upstream-sync.json delete mode 100755 scripts/unsloth/verify_upstream_sync.py diff --git a/.github/actions/prebuilt-alert/action.yml b/.github/actions/prebuilt-alert/action.yml deleted file mode 100644 index f7fc2b4be039..000000000000 --- a/.github/actions/prebuilt-alert/action.yml +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: Prebuilt failure alert -description: Open, update or close a deduplicated issue tracking prebuilt pipeline health. - -inputs: - status: - description: 'failure or success' - required: true - key: - description: 'Dedup key; all alerts sharing it collapse onto one issue' - required: true - title: - description: 'Issue title' - required: true - details: - description: 'Markdown describing what broke' - required: false - default: '' - label: - description: 'Label applied to the issue' - required: false - default: 'prebuilt-failure' - token: - description: 'Token with issues:write' - required: true - -runs: - using: composite - steps: - - name: Open, update or close the tracking issue - shell: bash - env: - GH_TOKEN: ${{ inputs.token }} - STATUS: ${{ inputs.status }} - KEY: ${{ inputs.key }} - TITLE: ${{ inputs.title }} - DETAILS: ${{ inputs.details }} - LABEL: ${{ inputs.label }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPO: ${{ github.repository }} - run: | - # GitHub runs `shell: bash` with -e, which `set -uo pipefail` does not - # undo. Alerting must never fail the run it reports on, so turn it off - # explicitly; every gh call below is already individually tolerated. - set +e - set -uo pipefail - - case "$STATUS" in - failure|success) ;; - *) echo "::warning::prebuilt-alert: unknown status '$STATUS'"; exit 0 ;; - esac - - MARKER="<!-- prebuilt-alert:${KEY} -->" - - # Scan open labelled issues rather than the search index, which lags by - # minutes and would double-open on back-to-back runs. - EXISTING="" - while read -r num; do - [ -n "$num" ] || continue - if gh issue view "$num" --repo "$REPO" --json body --jq .body 2>/dev/null | grep -qF "$MARKER"; then - EXISTING="$num"; break - fi - done < <(gh issue list --repo "$REPO" --label "$LABEL" --state open \ - --limit 100 --json number --jq '.[].number' 2>/dev/null || true) - - if [ "$STATUS" = "success" ]; then - if [ -n "$EXISTING" ]; then - gh issue comment "$EXISTING" --repo "$REPO" \ - --body "Recovered: [\`${GITHUB_WORKFLOW}\` run](${RUN_URL}) succeeded. Closing." >/dev/null 2>&1 || true - gh issue close "$EXISTING" --repo "$REPO" >/dev/null 2>&1 \ - || echo "::warning::prebuilt-alert: could not close #${EXISTING}" - echo "closed #${EXISTING} ($KEY)" - fi - exit 0 - fi - - BODY="$(printf '%s\n\n%s\n\n**Run:** %s\n\n%s\n' \ - "$MARKER" \ - "\`${GITHUB_WORKFLOW}\` failed on \`${GITHUB_EVENT_NAME}\`." \ - "$RUN_URL" \ - "$DETAILS")" - - # The run summary always works: no token, no permission, no repo - # setting. Issues can be disabled (they are on a fresh fork), and an - # alert nobody can read is the failure this whole pipeline keeps - # having, so write the details somewhere visible before trying. - { - echo "## ${TITLE}" - echo - echo "$DETAILS" - } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" - - if [ -n "$EXISTING" ]; then - # Comment rather than open a second issue, so an outage is one thread. - gh issue comment "$EXISTING" --repo "$REPO" --body "$BODY" >/dev/null 2>&1 \ - || echo "::warning::prebuilt-alert: could not comment on #${EXISTING}" - echo "updated #${EXISTING} ($KEY)" - else - # gh refuses to create with a label that does not exist yet. - gh label create "$LABEL" --repo "$REPO" --color B60205 \ - --description "Prebuilt release pipeline is failing" >/dev/null 2>&1 || true - NEW="$(gh issue create --repo "$REPO" --title "$TITLE" --label "$LABEL" \ - --body "$BODY" 2>&1 | tail -1)" - case "$NEW" in - https://*) echo "opened $NEW ($KEY)" ;; - *) - # The one case worth shouting about: a real failure went - # unreported. Name the usual cause so it is actionable. - if [ "$(gh api "repos/${REPO}" --jq .has_issues 2>/dev/null)" = "false" ]; then - echo "::error::prebuilt-alert: issues are disabled on ${REPO}, so ${KEY} cannot be tracked; see the run summary for details" - else - echo "::error::prebuilt-alert: could not open an issue for ${KEY} (${NEW}); the failure at ${RUN_URL} is unreported" - fi - ;; - esac - fi - exit 0 diff --git a/.github/workflows/unsloth-pin-preflight.yml b/.github/workflows/unsloth-pin-preflight.yml deleted file mode 100644 index f390f381e6fc..000000000000 --- a/.github/workflows/unsloth-pin-preflight.yml +++ /dev/null @@ -1,276 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: Unsloth pin preflight - -# Runs the nightly's merge a few hours early, on the same base tag it will -# pick, and files the conflict before the schedule burns 39 build jobs on it. -# -# unsloth-pr-set-lint.yml checks that pins are well formed and belong to their -# PR, which catches a bad edit. It cannot catch the failure that actually -# recurs: a pin that was fine yesterday and stops merging today because the -# base tag moved under it. That is what killed 08-02, and four nights in a -# week between the two causes. - -on: - schedule: - - cron: '47 16 * * *' - push: - paths: - - scripts/unsloth/pr-set.json - workflow_dispatch: - -permissions: - # write, not read: the mirror step pushes refs/pins/<sha>. With read it - # failed every time with "Permission to unslothai/llama.cpp.git denied to - # github-actions[bot]" (403), and the only refs that existed were ones - # pushed by hand. - contents: write - issues: write - -# Two runs of the same ref probe the same pins against the same base, so the -# second adds nothing and just competes for runners. On 08-04 a dispatch and the -# schedule sat queued together for an hour. Newest wins: it sees the newest -# pr-set.json. -# -# Per ref, though, not globally. This file also runs on any push that touches -# pr-set.json, so with one shared group a push to a second branch cancelled the -# first branch's run: observed on 09-03, where the run that would have said -# whether a repin fixed the nightly was cancelled by an unrelated branch, and -# the PR was left showing the failure from before the fix. -concurrency: - group: unsloth-pin-preflight-${{ github.ref }} - cancel-in-progress: true - -jobs: - preflight: - name: Dry-run the pin merges - runs-on: ubuntu-24.04 - env: - GH_TOKEN: ${{ github.token }} - REPIN_TOKEN: ${{ secrets.REPIN_TOKEN }} - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - - name: Resolve base and dry-run the merges - id: p - run: | - set -uo pipefail - # Everything below reports through `status`/`details`, so a death - # anywhere else leaves both empty and the alert blank: a red X on a - # scheduled run nobody opens. Report the abort through the same - # channel as a finding, so the repin bot sees a failure either way. - trap 'rc=$?; if [ "$rc" != 0 ]; then { - echo "status=failure" - echo "details<<ALERT_EOF" - echo "The preflight script exited ${rc} before it finished probing, so the pins were not fully checked. See the run log for the last command it reached." - echo "ALERT_EOF"; } >> "$GITHUB_OUTPUT"; fi' EXIT - AGE_H="${UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS:-6}" - CUTOFF="$(date -u -d "-${AGE_H} hours" +%s)" - # Same base tag the nightly resolves: newest aged b#### build. - # Upstream marks those prerelease since 08-21, so match the tag shape. - BASE="$(gh api 'repos/ggml-org/llama.cpp/releases?per_page=100' \ - | jq -r --argjson cutoff "$CUTOFF" '[.[] | select(.draft==false) | select(.tag_name|test("^b[0-9]+$")) | select((.published_at|fromdateiso8601) <= $cutoff)] | max_by(.published_at|fromdateiso8601) | .tag_name')" - if [ -z "$BASE" ] || [ "$BASE" = "null" ]; then - echo "::warning::no aged upstream release found; skipping" - echo "status=skip" >> "$GITHUB_OUTPUT"; exit 0 - fi - echo "base $BASE" - - git clone -q --filter=blob:none https://github.com/ggml-org/llama.cpp.git scratch - cd scratch - # GITHUB_TOKEN mirrors most pins, but GitHub refuses any ref that ADDS - # a workflow file the repo does not already have. Observed on 08-03, - # three pins mirrored and the fourth rejected: - # ! [remote rejected] ... -> refs/pins/c3fb9724... - # (refusing to allow a GitHub App to create or update workflow - # `.github/workflows/build-self-hosted.yml` without `workflows` - # permission) - # The check applies to any ref, not just branches. REPIN_TOKEN has - # workflow scope and covers those; without it we still mirror what we - # can rather than nothing, and warn about the rest. - MIRROR_TOKEN="${REPIN_TOKEN:-$GH_TOKEN}" - MIRROR="https://x-access-token:${MIRROR_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git fetch -q --no-tags origin "refs/tags/${BASE}:refs/tags/${BASE}" - git checkout -q --detach "refs/tags/${BASE}" - - # Mirror every pin to refs/pins/<sha> before probing anything. The - # 07-31 outage was a reviewed commit pruned out of its PR by a force - # push, which no amount of checking can recover from after the fact. - # A ref of our own keeps the object alive; resolve falls back to it. - # All these repos are in one fork network, so this transfers nothing. - # Done first, and past failures, so a conflict in an early pin does - # not leave the later ones unmirrored. - while read -r url; do - SRC="$(sed -E 's|https://github.com/([^/]+)/llama.cpp/pull/.*|\1|' <<<"$url")/llama.cpp" - SHA="$(sed -E 's|.*/commits/([0-9a-f]{40})/?$|\1|' <<<"$url")" - if git ls-remote --exit-code "$MIRROR" "refs/pins/${SHA}" >/dev/null 2>&1; then - echo "mirrored already: ${SHA:0:10}" - continue - fi - if ! git fetch -q --no-tags "https://github.com/${SRC}.git" "$SHA" 2>/dev/null; then - echo "::warning::cannot mirror ${SHA:0:10}; it is already unfetchable from ${SRC}" - continue - fi - # Report git's own error. Guessing the cause hid a 403 behind a - # workflow-scope message for a week. - if ERR="$(git push -q "$MIRROR" "${SHA}:refs/pins/${SHA}" 2>&1)"; then - echo "mirrored ${SHA:0:10}" - else - echo "::warning::could not mirror refs/pins/${SHA:0:10}: $(sed "s|${MIRROR_TOKEN}|***|g" <<<"$ERR" | tr '\n' ' '). A ref adding a workflow file needs REPIN_TOKEN (workflow scope); that pin stays deletable by a force-push." - fi - done < <(jq -r '.prs[] | if type == "string" then . else .url end' \ - ../scripts/unsloth/pr-set.json) - - PROBLEMS="" - MERGED="" - while read -r url REQUIRED; do - SRC="$(sed -E 's|https://github.com/([^/]+)/llama.cpp/pull/.*|\1|' <<<"$url")/llama.cpp" - NUM="$(sed -E 's|.*/pull/([0-9]+)/commits/.*|\1|' <<<"$url")" - SHA="$(sed -E 's|.*/commits/([0-9a-f]{40})/?$|\1|' <<<"$url")" - - STATE="$(gh api "repos/${SRC}/pulls/${NUM}" --jq .state 2>/dev/null || echo unknown)" - if [ "$STATE" != "open" ]; then - # Non-open required pins are still merged by the nightly, so keep - # probing them here rather than reporting them as a problem; an - # optional one is skipped there, so skip it here too. - if [ "$REQUIRED" = "false" ]; then - continue - fi - echo "note: ${SRC}#${NUM} is ${STATE}; still probing because required pins are merged regardless of state" - fi - if ! git fetch -q --no-tags "https://github.com/${SRC}.git" "$SHA" 2>/dev/null; then - PROBLEMS="${PROBLEMS}- \`${SRC}#${NUM}\` pinned commit \`${SHA:0:10}\` cannot be fetched; it was probably force-pushed away.\n" - continue - fi - if git -c user.name=preflight -c user.email=preflight@local \ - -c merge.conflictStyle=diff3 \ - merge --no-ff --no-edit -m "probe ${SRC}#${NUM}" "$SHA" >/dev/null 2>&1; then - echo "ok ${SRC}#${NUM}" - MERGED=1 - continue - fi - # Mirror resolve: a pure add/add is what the nightly will merge - # automatically, so reporting it as a conflict here is a false - # alarm. Anything additive_merge.py refuses is still a conflict. - if python3 ../scripts/unsloth/additive_merge.py >/dev/null 2>&1 \ - && [ -z "$(git diff --name-only --diff-filter=U)" ]; then - git -c user.name=preflight -c user.email=preflight@local commit -q --no-edit - echo "ok ${SRC}#${NUM} (additive resolve)" - MERGED=1 - continue - fi - FILES="$(git diff --name-only --diff-filter=U | sed 's/^/ /')" - # `|| true` is load-bearing, not tidying. GitHub runs a `run:` block - # under `bash -e` whatever this script's own `set` line says, `head` - # closes the pipe after 20 lines, and pipefail then makes the whole - # assignment fail. So on 09-03 the step died right here, on the first - # real conflict, with `grep: write error: Broken pipe` and no alert: - # the one path this job exists to report was the one it could not - # survive. It only fires when the conflict diff is bigger than the - # 64 KiB pipe buffer, since a smaller one is written before `head` - # ever closes it, which is why most conflicts got reported fine. - HUNKS="$(git diff --diff-filter=U -U0 2>/dev/null | grep -E '^\+|^-' | grep -vE '^(\+\+\+|---)' | head -20 || true)" - git merge --abort 2>/dev/null - PROBLEMS="${PROBLEMS}- \`${SRC}#${NUM}\` (\`${SHA:0:10}\`) does not merge onto \`${BASE}\` + the pins before it.\n\n Conflicting files:\n\n\`\`\`\n${FILES}\n\`\`\`\n\n <details><summary>conflict hunks</summary>\n\n\`\`\`diff\n${HUNKS}\n\`\`\`\n\n </details>\n" - # Stop here, like resolve does. Probing later pins against a tree - # missing this one reports conflicts that are consequences of it. - PROBLEMS="${PROBLEMS}\nLater pins were not probed; fix this one first.\n" - break - done < <(jq -r '.prs[] | if type == "string" then {url: ., required: true} else . end - | "\(.url)\t\(if .required == null then true else .required end)"' \ - ../scripts/unsloth/pr-set.json | tr '\t' ' ') - - # Only when a pin actually merged. With no pins, or only optional closed ones, this tree is pristine upstream, and a finding there is not a pin problem to alert on. The nightly gates the same check on MERGED_PINS. - if [ -z "$PROBLEMS" ] && [ -n "$MERGED" ]; then - # Merging cleanly is not the same as merging correctly. Two mistakes - # made on 08-27 compiled fine and would have shipped: a tensor-map key - # defined twice, which Python resolves silently by keeping the last, - # and an arch arm made unreachable by the same arch appearing in an - # earlier fallthrough condition. Both are checked here, on the tree the - # pins just produced, because this is the first point it exists. - if ! python3 ../scripts/unsloth/merge_checks.py --root . ; then - PROBLEMS="${PROBLEMS}- the merged tree builds, but \`scripts/unsloth/merge_checks.py\` found a resolution that is silently wrong. See the run log for file and line.\n" - fi - - # The other half of that question. merge_checks.py asks whether the - # tree contains something wrong; this asks whether it still contains - # what each pin carries. A pin that has rotted into a no-op, or an - # arch registration a resolution quietly dropped, is invisible to - # every other check here and to the compiler. - if ! python3 ../scripts/unsloth/pin_contract.py --root . --base "$BASE" \ - --pr-set ../scripts/unsloth/pr-set.json --report "${RUNNER_TEMP}/pin_contract.json" ; then - PROBLEMS="${PROBLEMS}- the merged tree is missing code a pin carries. See the run log for the pin and file.\n" - fi - NOTES="$(jq -r '.notices[]?' "${RUNNER_TEMP}/pin_contract.json" 2>/dev/null || true)" - if [ -n "$NOTES" ]; then - PROBLEMS="${PROBLEMS}- pins upstream has taken over, safe to delete from \`pr-set.json\`:\n\n\`\`\`\n${NOTES}\n\`\`\`\n" - fi - - # A clean merge is not a compiling tree. On 09-03 ggml-org#27754 - # merged with no conflicts at all and did not compile: upstream had - # added a parameter to build_attn_mha and the pin's new - # build_attn_sparse still called the old signature. Nothing above - # can see that. CPU only and the `llama` target only, which is where - # that translation unit lives; 59s cold at -j4 with no ccache. - GATE_OK=1 - if ! cmake -B "${RUNNER_TEMP}/gate" -DCMAKE_BUILD_TYPE=Release \ - -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=ON -DLLAMA_BUILD_SERVER=OFF \ - -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_CURL=OFF > /dev/null \ - || ! cmake --build "${RUNNER_TEMP}/gate" -j "$(nproc)" \ - --target llama test-llama-archs test-backend-ops test-mtmd-impl ; then - GATE_OK= - PROBLEMS="${PROBLEMS}- the pins merge cleanly and the merged tree does not compile. See the run log for the file and line; this is the failure that only shows up in the CUDA leg once the nightly has fanned out.\n" - fi - - # The last question, and the only one that needs a binary: does each - # feature we ship still work. Everything above is about the source. - # CPU only, because no runner in this pipeline has a GPU -- see the - # note in feature_matrix.py about what that does and does not prove. - if [ -n "$GATE_OK" ]; then - if ! python3 ../scripts/unsloth/feature_matrix.py \ - --build-dir "${RUNNER_TEMP}/gate" \ - --feature-checks ../scripts/unsloth/feature-checks.json \ - --report "${RUNNER_TEMP}/feature_matrix.json" ; then - PROBLEMS="${PROBLEMS}- the merged tree compiles and a feature we ship could not be shown to work. See the run log for which feature and which probe.\n" - fi - fi - fi - - if [ -z "$PROBLEMS" ]; then - echo "all pins merge cleanly onto ${BASE}" - echo "status=success" >> "$GITHUB_OUTPUT" - echo "details=" >> "$GITHUB_OUTPUT" - exit 0 - fi - echo "status=failure" >> "$GITHUB_OUTPUT" - { - echo 'details<<ALERT_EOF' - echo "Tonight's nightly will fail on base \`${BASE}\` unless \`scripts/unsloth/pr-set.json\` is repinned." - echo - printf '%b' "$PROBLEMS" - echo - echo "For a pin on a branch we control, merge \`${BASE}\` into it and repin. For a third-party PR, wait for the author to merge master or drop the pin." - echo 'ALERT_EOF' - } >> "$GITHUB_OUTPUT" - - - name: Alert - if: ${{ steps.p.outputs.status != 'skip' }} - uses: ./.github/actions/prebuilt-alert - with: - status: ${{ steps.p.outputs.status }} - key: llama-pin-preflight - title: 'Pinned PRs no longer merge onto the current base tag' - details: ${{ steps.p.outputs.details }} - token: ${{ github.token }} - - # The probe reports through its output, so without this the run still - # ends green and unsloth-repin-bot.yml, which waits for a failed - # workflow_run, never fires. On 08-05 the Inkling pin stopped merging - # onto b10280, the alert said so, the run said success, and the bot - # skipped. Fails last so the alert is always posted first. - - name: Fail the run when a pin does not merge - if: ${{ steps.p.outputs.status == 'failure' }} - run: | - echo "::error::pins do not merge onto the current base tag; see the alert above" - exit 1 diff --git a/.github/workflows/unsloth-pr-set-lint.yml b/.github/workflows/unsloth-pr-set-lint.yml deleted file mode 100644 index 8a898539f022..000000000000 --- a/.github/workflows/unsloth-pr-set-lint.yml +++ /dev/null @@ -1,164 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: "Unsloth: lint pr-set.json" - -# Tripwire for edits to the mix-build PR set: validates the file the moment a -# push or PR touches it (the team usually edits it by pushing straight to -# master, so the push trigger is the main hook). A bad entry can never build -- -# the nightly's resolve job re-runs the same checks, including that each -# pinned commit actually belongs to the PR it is listed under -- but a red -# lint does not stop the schedule; this just surfaces the mistake on the -# commit within seconds instead of failing the 3 AM build. - -on: - push: - paths: - - scripts/unsloth/pr-set.json - - scripts/unsloth/additive_merge.py - - scripts/unsloth/pin_merge.py - - scripts/unsloth/merge_checks.py - - scripts/unsloth/carry_vintage.py - - scripts/unsloth/sync_deletes.py - - scripts/unsloth/test_*.py - - scripts/unsloth/check_workflow_scalars.py - # Every workflow, not just this one: the size guard only guards a file if editing it runs the guard. - - .github/workflows/*.yml - - .github/workflows/*.yaml - pull_request: - paths: - - scripts/unsloth/pr-set.json - - scripts/unsloth/additive_merge.py - - scripts/unsloth/pin_merge.py - - scripts/unsloth/merge_checks.py - - scripts/unsloth/carry_vintage.py - - scripts/unsloth/sync_deletes.py - - scripts/unsloth/test_*.py - - scripts/unsloth/check_workflow_scalars.py - # Every workflow, not just this one: the size guard only guards a file if editing it runs the guard. - - .github/workflows/*.yml - - .github/workflows/*.yaml - -permissions: - contents: read - -jobs: - lint: - name: Validate pr-set.json - runs-on: ubuntu-24.04 - env: - GH_TOKEN: ${{ github.token }} - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - run: | - set -euo pipefail - FILE=scripts/unsloth/pr-set.json - # Mirrors the resolve step's schema: a bare url string (required), - # or {"url": ..., "required": false} for a pin the release may ship - # without. - jq -e '.prs | type == "array" and all(.[]; - type == "string" - or (type == "object" and (.url | type == "string") - and ((if .required == null then true else .required end) | type == "boolean")))' "$FILE" >/dev/null \ - || { echo "::error file=$FILE::.prs must be an array of PR url strings, or {url, required} objects" >&2; exit 1; } - URL_RE='^https://github\.com/(ggml-org|unslothai)/llama\.cpp/pull/([0-9]+)/commits/([0-9a-f]{40})/?$' - fail=0 - while read -r url REQUIRED; do - if ! [[ "$url" =~ $URL_RE ]]; then - echo "::error file=$FILE::malformed entry '$url' (expected https://github.com/{ggml-org,unslothai}/llama.cpp/pull/<n>/commits/<40-hex-sha>)" - fail=1 - continue - fi - SRC="${BASH_REMATCH[1]}/llama.cpp"; NUM="${BASH_REMATCH[2]}"; PIN="${BASH_REMATCH[3]}" - # Don't let a 404 under set -e kill the collect-all-errors loop. - if ! PR_JSON="$(gh api "repos/${SRC}/pulls/${NUM}" --jq '{state: .state, commits: .commits, head: .head.sha}')"; then - echo "::error file=$FILE::could not fetch ${SRC}#${NUM} (nonexistent PR number in '$url', or a transient API failure)" - fail=1 - continue - fi - STATE="$(jq -r .state <<<"$PR_JSON")" - COMMITS="$(jq -r .commits <<<"$PR_JSON")" - HEAD="$(jq -r .head <<<"$PR_JSON")" - if [ "$STATE" != "open" ]; then - if [ "$REQUIRED" != "false" ]; then - echo "::warning file=$FILE::${SRC}#${NUM} is ${STATE}; the nightly will keep merging its pinned commit (a no-op once the base tag contains it), so drop the entry when you no longer want that code" - else - echo "::warning file=$FILE::${SRC}#${NUM} is ${STATE}; the nightly will skip this optional entry" - fi - fi - # The commits listing is capped at 250 by the API; past that the - # membership check cannot be trusted, so skip it rather than - # false-fail a legitimate giant PR. - if [ "$COMMITS" -gt 250 ]; then - echo "::notice::${SRC}#${NUM} has ${COMMITS} commits (over the API listing cap); skipping pin membership check" - elif ! gh api "repos/${SRC}/pulls/${NUM}/commits" --paginate --jq '.[].sha' | grep -qx "$PIN"; then - echo "::error file=$FILE::pinned commit ${PIN} is not a commit of ${SRC}#${NUM}" - fail=1 - continue - fi - [ "$PIN" = "$HEAD" ] || echo "::notice::${SRC}#${NUM} pin ${PIN} is behind its head ${HEAD}" - echo "OK: ${SRC}#${NUM} @ ${PIN} (${STATE}, required=${REQUIRED})" - done < <(jq -r '.prs[] | if type == "string" then {url: ., required: true} else . end - | "\(.url)\t\(if .required == null then true else .required end)"' "$FILE" | tr '\t' ' ') - exit "$fail" - - resolver-tests: - # These tests existed for additive_merge.py and ran nowhere, so a change to a merge resolver was covered by nothing at all. - # Every resolver and check under scripts/unsloth/ runs here. - name: Resolver and merge-check tests - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: { fetch-depth: 1 } - - - name: Run the resolver tests - run: | - set -euo pipefail - fail=0 - for t in scripts/unsloth/test_additive_merge.py \ - scripts/unsloth/test_pin_merge.py \ - scripts/unsloth/test_merge_checks.py \ - scripts/unsloth/test_sync_deletes.py \ - scripts/unsloth/test_carry_vintage.py \ - scripts/unsloth/test_pin_contract.py \ - scripts/unsloth/test_feature_matrix.py; do - echo "::group::$t" - python3 "$t" || fail=1 - echo "::endgroup::" - done - exit "$fail" - - # A pin nobody decided about is the failure this whole file exists to stop. - # Being in `unchecked` with a reason is a fine answer; being in neither map - # is how DiffusionGemma went five weeks with no coverage and no record of it. - - name: Every pin is either checked or knowingly unchecked - run: | - set -euo pipefail - python3 - <<'PY' - import json, re, sys - pins = json.load(open("scripts/unsloth/pr-set.json"))["prs"] - doc = json.load(open("scripts/unsloth/feature-checks.json")) - owned = {f["owner"] for f in doc["features"].values() if f.get("owner")} - known = owned | set(doc.get("unchecked", {})) - fail = 0 - for entry in pins: - url = entry if isinstance(entry, str) else entry["url"] - m = re.match(r"https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/", url) - pin = f"{m.group(1)}#{m.group(2)}" - if pin not in known: - print(f"::error file=scripts/unsloth/feature-checks.json::{pin} is pinned " - "and appears in neither `features` nor `unchecked`; say which it is") - fail = 1 - for pin in sorted(owned & set(doc.get("unchecked", {}))): - print(f"::error file=scripts/unsloth/feature-checks.json::{pin} is in both " - "`features` and `unchecked`") - fail = 1 - print(f"{len(pins)} pin(s), {len(owned)} with a feature check, " - f"{len(doc.get('unchecked', {}))} knowingly unchecked") - sys.exit(fail) - PY - - # An over-limit run: script makes the whole file uncompilable, and nothing else sees it: yaml, actionlint and GitHub's own parser all pass it. See check_workflow_scalars.py. - - name: Check no workflow string is near GitHub's size limit - run: python3 scripts/unsloth/check_workflow_scalars.py --root . - diff --git a/.github/workflows/unsloth-prebuilt-cpu.yml b/.github/workflows/unsloth-prebuilt-cpu.yml deleted file mode 100644 index 45e609ae2d0f..000000000000 --- a/.github/workflows/unsloth-prebuilt-cpu.yml +++ /dev/null @@ -1,376 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: "Unsloth prebuilt: CPU" - -# Reusable child of unsloth-prebuilt.yml. Builds the CPU-only bundles for -# Linux + Windows, each on x64 and arm64 (one matrix entry per arch). Every -# matrix entry uploads a single app-*.{tar.gz|zip} artifact for the parent's -# assemble step to pick up. -# -# The Linux build mirrors oobabooga/llama-cpp-binaries' build-wheels-cpu.yml -# (GGML_BACKEND_DL + GGML_CPU_ALL_VARIANTS + GGML_RPC, no GPU backend). The -# Windows build instead matches ggml-org/llama.cpp release.yml's windows-cpu -# job (clang/LLVM toolchain file + "Ninja Multi-Config" + OpenMP + BoringSSL, -# arm64 cross-compiled from x64) -- clang is upstream's proven CPU path, so we -# follow it rather than llama-cpp-binaries' MSVC recipe; MSVC stays only where -# it is mandatory (CUDA/nvcc). Adapted to this repo's conventions: -# app-<tag>-<platform>-<arch>-cpu archives packaged straight from build/bin like -# the ROCm/macOS children (no embedded UNSLOTH_PREBUILT_INFO.json -- -# assemble_metadata.py derives the manifest entry from the filename), $ORIGIN -# RPATH on Linux. arm64 extends llama-cpp-binaries (x64-only) so the release -# covers the arm64 CPU hosts that previously fell back to ggml-org upstream. -# Both Linux legs build on ubuntu-22.04 so the bundles keep a glibc 2.35 / -# GLIBCXX <= 3.4.30 floor and load on Ubuntu 22.04 and Debian 12 hosts. - -on: - workflow_call: - inputs: - tag: - description: 'Upstream llama.cpp release tag (b####), resolved by parent' - required: true - type: string - repo: - description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' - required: false - default: 'ggml-org/llama.cpp' - type: string - source_artifact: - description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' - required: false - default: '' - type: string - -permissions: - contents: read - -jobs: - build-linux: - name: linux/${{ matrix.arch }} - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - { arch: x64, runner: ubuntu-22.04 } - - { arch: arm64, runner: ubuntu-22.04-arm } - steps: - # The parent's resolve job built the source tree (upstream base + any mix - # PRs, with the build number/commit and Unsloth fingerprint baked - # into cmake/build-info.cmake) and uploaded it as an artifact; extract it - # instead of cloning -- no .git needed, the build number is already baked. - - name: Download source @ ${{ inputs.tag }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: ${{ inputs.source_artifact }} - path: srcpkg - - name: Extract source - shell: bash - run: | - set -eux - mkdir -p src - tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 - - - name: Install build dependencies - run: | - set -eux - sudo apt-get update - sudo apt-get install -y build-essential libssl-dev ninja-build - - # arm64 builds on ubuntu-22.04-arm so the bundle keeps the x64 leg's - # loader floor (a 24.04 build needs GLIBC_2.38 and fails to load on - # Ubuntu 22.04 / Debian 12). Jammy's gcc can't target armv9.2-a+sme and - # a PPA gcc would raise the libstdc++ floor back, so use clang, which - # links against the system libstdc++. - - name: Toolchain (clang 19 on arm64) - if: matrix.arch == 'arm64' - run: | - set -eux - wget -q https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 19 - sudo apt-get install -y libomp-19-dev - { - echo "CC=clang-19" - echo "CXX=clang++-19" - } >> "$GITHUB_ENV" - - - name: ccache - uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 - with: - key: cpu-linux-${{ matrix.arch }}-${{ inputs.tag }} - restore-keys: | - cpu-linux-${{ matrix.arch }} - append-timestamp: false - variant: ccache - max-size: 2G - save: false - - - name: Configure - working-directory: src - run: | - set -eux - # Build recipe mirrors llama-cpp-binaries' CPU wheel (backend-DL + - # all CPU variants + RPC, no GPU backend). RPATH=$ORIGIN so the - # bundle's sibling .so files resolve from the binary's own directory. - # LLAMA_FATAL_WARNINGS below is -Werror. The arm64 image compiles with - # clang-19 against GCC 12's libstdc++, where std::stable_sort still - # reaches the deprecated get_temporary_buffer; GCC buries that in a - # system header, clang reports it at our instantiation. That failed - # this leg on 08-27 over a deprecation in code we do not own, so that - # one diagnostic is off. Every other warning stays fatal. - cmake -S . -B build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DGGML_NATIVE=OFF \ - -DGGML_BACKEND_DL=ON \ - -DGGML_CPU_ALL_VARIANTS=ON \ - -DGGML_RPC=ON \ - -DLLAMA_FATAL_WARNINGS=ON \ - -DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations \ - -DLLAMA_BUILD_TESTS=OFF \ - -DLLAMA_BUILD_EXAMPLES=OFF \ - -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_SERVER=ON \ - -DCMAKE_INSTALL_RPATH='$ORIGIN' \ - -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - - name: Build - working-directory: src - run: | - set -eux - # Build the full tool + server set (no --target), matching the macOS - # bins. Backend modules (CPU variants, RPC) build as ggml deps. - cmake --build build --config Release -j "$(nproc)" - strip build/bin/llama-* || true - - - name: Bundle OpenMP runtime (arm64) - if: matrix.arch == 'arm64' - run: cp /usr/lib/llvm-19/lib/libomp.so.5 src/build/bin/ - - # DiffusionGemma binaries (example targets present only in #24423 mix - # builds): best-effort, never fail the job. See the CUDA child for the - # rationale. The bundle tars all of build/bin, so anything produced here - # is shipped automatically. - - name: Build DiffusionGemma binaries (best-effort; mix builds only) - working-directory: src - run: | - set -u - if [ ! -d examples/diffusion-gemma-server ]; then - echo "no DiffusionGemma sources in this tree; skipping" - exit 0 - fi - cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON \ - || { echo "reconfigure for examples failed; skipping DiffusionGemma binaries"; exit 0; } - if cmake --build build --config Release -j "$(nproc)" \ - --target llama-diffusion-gemma-visual-server llama-diffusion-cli; then - strip build/bin/llama-diffusion-gemma-visual-server build/bin/llama-diffusion-cli || true - echo "built DiffusionGemma binaries" - else - echo "warning: DiffusionGemma binaries failed to build; bundle will omit them" - fi - exit 0 - - - name: Package bundle (tar.gz) - run: | - set -eux - ASSET="app-${{ inputs.tag }}-linux-${{ matrix.arch }}-cpu.tar.gz" - cp src/LICENSE src/build/bin/ - mkdir -p dist - (cd src/build/bin && tar -czf "${GITHUB_WORKSPACE}/dist/${ASSET}" .) - ls -la dist - - - name: Upload bundle artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: app-${{ inputs.tag }}-linux-${{ matrix.arch }}-cpu - path: dist/app-${{ inputs.tag }}-linux-${{ matrix.arch }}-cpu.tar.gz - if-no-files-found: error - - - name: Evict stale ccache files - # !cancelled(), unlike the save below: on a timeout the job gets a - # single ~5 minute teardown window (measured ~4m50s after process - # kill), shared by every remaining step and not replenished. Evicting - # spends that window on housekeeping; the save is what actually needs - # it, and a 2 GB cache is not quick to write. - if: ${{ !cancelled() }} - continue-on-error: true - run: ccache --evict-older-than 14d - - - name: Save ccache - # Save even when the build failed: the objects compiled before the - # failure are still worth keeping, and a job that saves nothing leaves - # a hole in the cache lineage that widens the next run's tag gap. - # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. - # - # always(), not !cancelled(): a timeout-minutes expiry puts the job on - # the CANCELLATION path, not the failure path, so !cancelled() would - # skip the save on the single most expensive case -- a leg that - # compiled for hours and then hit the cap. - if: ${{ always() }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}/.ccache - key: ccache-cpu-linux-${{ matrix.arch }}-${{ inputs.tag }}- - - build-windows: - name: windows/${{ matrix.arch }} - # Single x64 runner for both arches: arm64 is cross-compiled with the clang - # toolchain (vcvars amd64_arm64), exactly like ggml-org's release.yml - # windows-cpu job. CUDA stays on MSVC (mandatory for nvcc on Windows), but - # the CPU build uses clang/LLVM to match upstream's proven CPU recipe. - runs-on: windows-2025-vs2026 - strategy: - fail-fast: false - matrix: - include: - - { arch: x64, vcvars: x64, omp_arch: x86_64, cpu_variants: 'ON' } - - { arch: arm64, vcvars: amd64_arm64, omp_arch: aarch64, cpu_variants: 'OFF' } - defaults: - run: - shell: pwsh - steps: - # The parent's resolve job built the source tree (upstream base + any mix - # PRs, with the build number/commit and Unsloth fingerprint baked - # into cmake/build-info.cmake) and uploaded it as an artifact; extract it - # instead of cloning -- no .git needed, the build number is already baked. - - name: Download source @ ${{ inputs.tag }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: ${{ inputs.source_artifact }} - path: srcpkg - - name: Extract source - shell: bash - run: | - set -eux - mkdir -p src - tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 - - - name: ccache - uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 - with: - key: cpu-windows-${{ matrix.arch }}-${{ inputs.tag }} - restore-keys: | - cpu-windows-${{ matrix.arch }} - append-timestamp: false - variant: ccache - max-size: 2G - save: false - - - name: Install Ninja - run: choco install ninja --no-progress - - # Build recipe copied from ggml-org/llama.cpp release.yml (windows-cpu): - # clang via the per-arch LLVM toolchain file, "Ninja Multi-Config", OpenMP, - # BoringSSL, and GGML_CPU_ALL_VARIANTS only on x64 (it is an x86 microarch - # fan-out). vcvarsall sets the env (incl. the amd64_arm64 cross toolchain), - # so this runs in cmd. CMAKE_ARGS mirrors upstream's env.CMAKE_ARGS and - # builds the full tool + server set (no --target), matching the macOS bins. - - name: Build - working-directory: src - shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.vcvars }} - cmake -S . -B build -G "Ninja Multi-Config" ^ - -D CMAKE_TOOLCHAIN_FILE=cmake/${{ matrix.arch }}-windows-llvm.cmake ^ - -DLLAMA_BUILD_BORINGSSL=ON ^ - -DGGML_NATIVE=OFF ^ - -DGGML_BACKEND_DL=ON ^ - -DGGML_CPU_ALL_VARIANTS=${{ matrix.cpu_variants }} ^ - -DGGML_OPENMP=ON ^ - -DGGML_RPC=ON ^ - -DLLAMA_BUILD_TESTS=OFF ^ - -DLLAMA_BUILD_EXAMPLES=OFF ^ - -DLLAMA_BUILD_TOOLS=ON ^ - -DLLAMA_BUILD_SERVER=ON ^ - -DCMAKE_C_COMPILER_LAUNCHER=ccache ^ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - if errorlevel 1 exit /b 1 - cmake --build build --config Release - if errorlevel 1 exit /b 1 - - # DiffusionGemma binaries (#24423 mix builds only): best-effort, never fail - # the job (trailing `exit /b 0`). vcvarsall is re-called -- step env does - # not persist -- and the reconfigure reuses the cached clang toolchain. - - name: Build DiffusionGemma binaries (best-effort; mix builds only) - working-directory: src - shell: cmd - run: | - if not exist examples\diffusion-gemma-server ( - echo no DiffusionGemma sources in this tree; skipping - exit /b 0 - ) - call "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.vcvars }} - cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON - cmake --build build --config Release --target llama-diffusion-gemma-visual-server llama-diffusion-cli - exit /b 0 - - # Ninja Multi-Config emits to build/bin/Release. Ship the OpenMP runtime - # (GGML_OPENMP=ON) from the VS LLVM redist -- exactly as upstream does -- - # globbing the MSVC version dir so an image bump does not break the path. - # Sort newest-first: older toolsets' libomp lacks entry points current - # clang imports (__kmpc_dispatch_deinit -> STATUS_ENTRYPOINT_NOT_FOUND). - - name: Package bundle (zip) - run: | - $rel = "src/build/bin/Release" - Copy-Item src/LICENSE $rel/ - $redist = "C:\Program Files\Microsoft Visual Studio\18\Enterprise\VC\Redist\MSVC" - $omp = Get-ChildItem "$redist\*\debug_nonredist\${{ matrix.arch }}\Microsoft.VC*.OpenMP.LLVM\libomp140.${{ matrix.omp_arch }}.dll" -ErrorAction SilentlyContinue | - Sort-Object { [version]$_.Directory.Parent.Parent.Parent.Name } -Descending | - Select-Object -First 1 - if (-not $omp) { Write-Error "libomp140.${{ matrix.omp_arch }}.dll not found in the VS LLVM redist"; exit 1 } - Write-Host "shipping OpenMP runtime: $($omp.FullName)" - Copy-Item $omp.FullName $rel/ - New-Item -ItemType Directory -Force -Path dist | Out-Null - $asset = "app-${{ inputs.tag }}-windows-${{ matrix.arch }}-cpu.zip" - Push-Location $rel - 7z a -tzip "$env:GITHUB_WORKSPACE/dist/$asset" . - Pop-Location - Get-ChildItem dist - - # Smoke the packaged x64 bundle: launching an exe loads ggml-base and the - # picked libomp, so --version fails on a bad pick. timeout-minutes bounds - # a loader hard-error that can block instead of exiting. arm64 is - # cross-compiled and cannot run here. - - name: Smoke test bundle (x64 only) - if: matrix.arch == 'x64' - timeout-minutes: 5 - run: | - $rel = "src/build/bin/Release" - & "$rel\llama-server.exe" --version - if ($LASTEXITCODE -ne 0) { Write-Error "llama-server --version failed: $LASTEXITCODE"; exit 1 } - - - name: Upload bundle artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: app-${{ inputs.tag }}-windows-${{ matrix.arch }}-cpu - path: dist/app-${{ inputs.tag }}-windows-${{ matrix.arch }}-cpu.zip - if-no-files-found: error - - - name: Evict stale ccache files - # !cancelled(), unlike the save below: on a timeout the job gets a - # single ~5 minute teardown window (measured ~4m50s after process - # kill), shared by every remaining step and not replenished. Evicting - # spends that window on housekeeping; the save is what actually needs - # it, and a 2 GB cache is not quick to write. - if: ${{ !cancelled() }} - continue-on-error: true - run: ccache --evict-older-than 14d - - - name: Save ccache - # Save even when the build failed: the objects compiled before the - # failure are still worth keeping, and a job that saves nothing leaves - # a hole in the cache lineage that widens the next run's tag gap. - # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. - # - # always(), not !cancelled(): a timeout-minutes expiry puts the job on - # the CANCELLATION path, not the failure path, so !cancelled() would - # skip the save on the single most expensive case -- a leg that - # compiled for hours and then hit the cap. - if: ${{ always() }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}\.ccache - key: ccache-cpu-windows-${{ matrix.arch }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-cuda-windows.yml b/.github/workflows/unsloth-prebuilt-cuda-windows.yml deleted file mode 100644 index 90e28e1749b5..000000000000 --- a/.github/workflows/unsloth-prebuilt-cuda-windows.yml +++ /dev/null @@ -1,310 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: "Unsloth prebuilt: CUDA Windows" - -# Reusable child of unsloth-prebuilt.yml. Builds the per-profile CUDA Windows -# bundles the parent's `resolve` job filtered into `matrix`. Each entry uploads -# a single app-*.zip artifact for the parent's assemble step to pick up. -# -# Mirrors unsloth-prebuilt-cuda.yml (Linux), adapted for Windows: MSVC + Ninja -# toolchain, BoringSSL instead of system OpenSSL, no RPATH (Windows resolves -# DLLs from the executable's directory), and a .zip archive. CUDA compiles -# entirely in software here -- no GPU is present on the runner. - -on: - workflow_call: - inputs: - tag: - description: 'Upstream llama.cpp release tag (b####), resolved by parent' - required: true - type: string - commit: - description: 'Upstream commit SHA for that tag, resolved by parent' - required: true - type: string - repo: - description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' - required: false - default: 'ggml-org/llama.cpp' - type: string - source_artifact: - description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' - required: false - default: '' - type: string - matrix: - description: 'Matrix JSON {include:[...]} produced by the parent resolve job' - required: true - type: string - -permissions: - contents: read - -jobs: - build: - name: x64/${{ matrix.profile }} - runs-on: ${{ matrix.runner }} - # Hang guard only. Without it the job inherits GitHub's 360-minute default, - # which is above both assemble's `timeout-minutes: 350` and, more to the - # point, the "Wait for the build matrix" step's own 330-minute deadline in - # unsloth-prebuilt.yml -- so a wedged leg burns a runner for an hour after - # the publish it was feeding has already given up. - # - # 345, not something tighter: x64/cuda12-portable legitimately reaches the - # low 300s on a cold ccache. Run 27347317849 took 305.1 minutes and - # SUCCEEDED (286.9 of it genuine compile). Across 62 historical runs of this - # leg the max is 305.1 and nothing falls between 215 and 305, so a tighter - # cap buys no hang detection and would have destroyed that publish -- one - # killed leg fails the bundle-coverage gate and the whole night ships - # nothing. - # - # Note this cannot by itself rescue a slow run: the waiter's 330-minute - # deadline is wall-clock from assemble start and INCLUDES the child's queue - # wait, while timeout-minutes starts at job start and excludes it. Raising - # the waiter deadline is the separate change that would actually save runs. - timeout-minutes: 345 - strategy: - fail-fast: false - matrix: ${{ fromJSON(inputs.matrix) }} - defaults: - run: - shell: pwsh - steps: - # Report both fixed volumes at job start. The work is split across two: - # the CUDA toolkit, the tool cache and Program Files land on C:, while - # GITHUB_WORKSPACE -- the source tree, the CMake build tree and the ccache - # -- is on D:. A leg that dies mid-build can only be read against the - # volume it was writing to, so both are printed here and again after the - # toolkit install. - # - # No cleanup step, deliberately. One was written and then measured on a - # live windows-2022 runner: D: starts at 147.0 GB free of 150.0 and C: at - # 84.3 of 255.4, and the CUDA 12.8 toolkit is 4.79 GB, so C: never drops - # below about 106 GB. Reclaiming ~33 GB of preinstalled SDKs on every leg - # would have freed the volume that was not under pressure, on every - # release, forever. The 87-minute "runner lost communication" failure was - # not disk exhaustion, and unsloth-prebuilt-retry.yml already covers that - # class of infrastructure loss. - - name: Report disk space - run: | - "workspace: $env:GITHUB_WORKSPACE" - Get-CimInstance Win32_LogicalDisk -Filter 'DriveType = 3' | Sort-Object DeviceID | ForEach-Object { - "{0} {1:N1} GB free of {2:N1} GB" -f $_.DeviceID, ($_.FreeSpace / 1GB), ($_.Size / 1GB) - } - - - name: Checkout build tooling (this repo) - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: - path: tooling - - # The parent's resolve job built the source tree (upstream base + any mix - # PRs, with the build number/commit and Unsloth fingerprint baked - # into cmake/build-info.cmake) and uploaded it as an artifact; extract it - # instead of cloning -- no .git needed, the build number is already baked. - - name: Download source @ ${{ inputs.tag }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: ${{ inputs.source_artifact }} - path: srcpkg - - name: Extract source - shell: bash - run: | - set -eux - mkdir -p src - tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 - - - name: ccache - uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 - with: - key: cuda-${{ matrix.cuda }}-windows-${{ matrix.profile }}-${{ inputs.tag }} - restore-keys: | - cuda-${{ matrix.cuda }}-windows-${{ matrix.profile }} - append-timestamp: false - variant: ccache - max-size: 2G - save: false - - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: "3.11" - - - name: Install Ninja - run: choco install ninja --no-progress - - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 - - # Jimver's action has no mapping for CUDA 13.3 yet, so for that version we - # fall back to llama.cpp's own install method: curl the individual NVIDIA - # redist component archives and assemble the toolkit by hand. Block copied - # verbatim from ggml-org/llama.cpp .github/actions/windows-setup-cuda - # (the cuda_version == '13.3' case). Any other version uses Jimver. - - name: Install CUDA toolkit 13.3 (NVIDIA redist) - if: matrix.cuda == '13.3' - run: | - mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" - choco install unzip -y - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.3.29-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.5.1.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.3.33-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.3.29-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.3.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.3.27-archive.zip" - curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.3.3.1-archive.zip" - unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_crt-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_cudart-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvcc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvrtc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libcublas-windows-x86_64-13.5.1.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libnvvm-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvtx-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_profiler_api-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\visual_studio_integration-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cccl-windows-x86_64-13.3.3.3.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y - echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - - - name: Install CUDA toolkit ${{ matrix.cuda }} (Jimver) - if: matrix.cuda != '13.3' - uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 - id: cuda-toolkit - with: - cuda: ${{ matrix.cuda }} - method: 'network' - - - name: Set up CUDA environment (Jimver) - if: matrix.cuda != '13.3' - run: | - echo "CUDA_PATH=$env:CUDA_PATH" >> $env:GITHUB_ENV - echo "CUDA_HOME=$env:CUDA_PATH" >> $env:GITHUB_ENV - - - name: Verify CUDA - run: nvcc --version - - # Headroom going into the build, with the toolkit on C: and the restored - # ccache in the workspace on D:. First numbers to read if a leg dies - # mid-build; the build itself writes to the workspace volume. - - name: Report disk space before build - run: | - "workspace: $env:GITHUB_WORKSPACE" - Get-CimInstance Win32_LogicalDisk -Filter 'DriveType = 3' | Sort-Object DeviceID | ForEach-Object { - "{0} {1:N1} GB free of {2:N1} GB" -f $_.DeviceID, ($_.FreeSpace / 1GB), ($_.Size / 1GB) - } - - - name: Configure - working-directory: src - run: | - # CMAKE_CUDA_ARCHITECTURES is the explicit per-profile arch list (the - # whole point of the matrix). ccache launchers cache nvcc + cl.exe; - # no RPATH knobs -- Windows loads sibling DLLs from the binary's dir. - $archs = "${{ matrix.archs }}".Replace(' ', ';') - cmake -S . -B build -G Ninja ` - -DCMAKE_BUILD_TYPE=Release ` - -DGGML_NATIVE=OFF ` - -DGGML_BACKEND_DL=ON ` - -DGGML_CPU_ALL_VARIANTS=ON ` - -DGGML_RPC=ON ` - -DGGML_CUDA=ON ` - -DGGML_CUDA_CUB_3DOT2=ON ` - -DLLAMA_BUILD_TESTS=OFF ` - -DLLAMA_BUILD_EXAMPLES=OFF ` - -DLLAMA_BUILD_TOOLS=ON ` - -DLLAMA_BUILD_SERVER=ON ` - -DLLAMA_BUILD_BORINGSSL=ON ` - -DCMAKE_CUDA_ARCHITECTURES="$archs" ` - -DCMAKE_C_COMPILER_LAUNCHER=ccache ` - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ` - -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache - - - name: Build - working-directory: src - run: | - # -j 3: this multi-arch nvcc build peaks at ~3 GB host RSS on the - # 4 vCPU / 16 GB GitHub-hosted runners (measured over a 1s sampler), - # so 3 parallel TUs leave ~13 GB free. -j 4 adds no speed (the runner - # is ~2 physical cores + HT), so 3 is the sweet spot. - cmake --build build --config Release -j 3 - - # DiffusionGemma binaries (example targets present only in #24423 mix - # builds): best-effort, never fail the job. See the Linux CUDA child for - # the rationale. package_bundle.py ships the .exe only if it was produced. - - name: Build DiffusionGemma binaries (best-effort; mix builds only) - working-directory: src - run: | - if (-not (Test-Path "examples/diffusion-gemma-server")) { - Write-Host "no DiffusionGemma sources in this tree; skipping" - exit 0 - } - cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON - if ($LASTEXITCODE -ne 0) { - Write-Host "reconfigure for examples failed; skipping DiffusionGemma binaries" - exit 0 - } - cmake --build build --config Release -j 3 ` - --target llama-diffusion-gemma-visual-server llama-diffusion-cli - if ($LASTEXITCODE -ne 0) { - Write-Host "warning: DiffusionGemma binaries failed to build; bundle will omit them" - } else { - Write-Host "built DiffusionGemma binaries" - } - exit 0 - - - name: Package bundle - env: - PLATFORM: windows - ARCH: x64 - BIN_DIR: ${{ github.workspace }}/src/build/bin - SRC_DIR: ${{ github.workspace }}/src - OUT_DIR: ${{ github.workspace }}/dist - TAG: ${{ inputs.tag }} - SOURCE_COMMIT: ${{ inputs.commit }} - SOURCE_REPO: ${{ inputs.repo }} - SOURCE_REF_KIND: ${{ inputs.repo == 'ggml-org/llama.cpp' && 'tag' || 'mix' }} - PROFILE: ${{ matrix.profile }} - LINE: ${{ matrix.line }} - KLASS: ${{ matrix.klass }} - RANK: ${{ matrix.rank }} - TOOLKIT_LINE: ${{ matrix.toolkit_line }} - DOCKER_IMAGE: github-hosted ${{ matrix.runner }}, CUDA ${{ matrix.cuda }} ${{ matrix.cuda == '13.3' && '(NVIDIA redist)' || '(Jimver)' }} - ARCHS: ${{ matrix.archs }} - SMS: ${{ matrix.sms }} - run: python tooling/scripts/unsloth/package_bundle.py - - - name: Upload bundle artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: app-${{ inputs.tag }}-windows-x64-${{ matrix.profile }} - path: dist/app-${{ inputs.tag }}-windows-x64-${{ matrix.profile }}.zip - if-no-files-found: error - - - name: Evict stale ccache files - # !cancelled(), unlike the save below: on a timeout the job gets a - # single ~5 minute teardown window (measured ~4m50s after process - # kill), shared by every remaining step and not replenished. Evicting - # spends that window on housekeeping; the save is what actually needs - # it, and a 2 GB cache is not quick to write. - if: ${{ !cancelled() }} - continue-on-error: true - run: ccache --evict-older-than 14d - - - name: Save ccache - # Save even when the build failed: the objects compiled before the - # failure are still worth keeping, and a job that saves nothing leaves - # a hole in the cache lineage that widens the next run's tag gap. - # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. - # - # always(), not !cancelled(): a timeout-minutes expiry puts the job on - # the CANCELLATION path, not the failure path, so !cancelled() would - # skip the save on the single most expensive case -- a leg that - # compiled for hours and then hit the cap. - if: ${{ always() }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}\.ccache - key: ccache-cuda-${{ matrix.cuda }}-windows-${{ matrix.profile }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-cuda.yml b/.github/workflows/unsloth-prebuilt-cuda.yml deleted file mode 100644 index b64ebe41f4a1..000000000000 --- a/.github/workflows/unsloth-prebuilt-cuda.yml +++ /dev/null @@ -1,290 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: "Unsloth prebuilt: CUDA" - -# Reusable child of unsloth-prebuilt.yml. Builds the per-profile CUDA bundles -# the parent's `resolve` job filtered into `matrix`. Each entry uploads a -# single app-*.tar.gz artifact for the parent's assemble step to pick up. - -on: - workflow_call: - inputs: - tag: - description: 'Upstream llama.cpp release tag (b####), resolved by parent' - required: true - type: string - commit: - description: 'Upstream commit SHA for that tag, resolved by parent' - required: true - type: string - repo: - description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' - required: false - default: 'ggml-org/llama.cpp' - type: string - source_artifact: - description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' - required: false - default: '' - type: string - matrix: - description: 'Matrix JSON {include:[...]} produced by the parent resolve job' - required: true - type: string - -permissions: - contents: read - -jobs: - build: - name: ${{ matrix.arch }}/${{ matrix.profile }} - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: ${{ fromJSON(inputs.matrix) }} - steps: - - name: Free disk space - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - tool-cache: true - android: true - dotnet: true - haskell: true - large-packages: false - swap-storage: true - - - name: Checkout build tooling (this repo) - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: - path: tooling - - # The parent's resolve job built the source tree (upstream base + any mix - # PRs, with the build number/commit and Unsloth fingerprint baked - # into cmake/build-info.cmake) and uploaded it as an artifact; extract it - # instead of cloning -- no .git needed, the build number is already baked. - - name: Download source @ ${{ inputs.tag }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: ${{ inputs.source_artifact }} - path: srcpkg - - name: Extract source - shell: bash - run: | - set -eux - mkdir -p src - tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 - - - name: Install build dependencies - run: | - set -eux - sudo apt-get update - sudo apt-get install -y build-essential libssl-dev - - - name: Install ARM64 host compiler (gcc-14) - if: matrix.arch == 'arm64' - run: | - set -eux - sudo apt-get install -y gcc-14 g++-14 - { - echo "CC=gcc-14" - echo "CXX=g++-14" - echo "CUDAHOSTCXX=g++-14" - } >> "$GITHUB_ENV" - - - name: ccache - uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 - with: - key: cuda-${{ matrix.cuda }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ inputs.tag }} - restore-keys: | - cuda-${{ matrix.cuda }}-${{ matrix.arch }}-${{ matrix.profile }} - append-timestamp: false - variant: ccache - max-size: 2G - save: false - - # Jimver has no mapping for CUDA 13.3 yet, so for that version we pull the - # toolkit components straight from NVIDIA's redist CDN (same source Jimver - # and ggml-org use) and assemble a prefix by hand. Component versions are - # from NVIDIA's redistrib_13.3.0.json. Runs on the same ubuntu runner, so - # the glibc floor is unchanged. Any other version uses Jimver. - - name: Install CUDA toolkit 13.3 (NVIDIA redist) - if: matrix.cuda == '13.3' - run: | - set -eux - case "${{ matrix.arch }}" in - x64) PLAT=linux-x86_64 ;; - arm64) PLAT=linux-sbsa ;; - *) echo "unsupported arch ${{ matrix.arch }}" >&2; exit 1 ;; - esac - PREFIX="$HOME/cuda-13.3" - mkdir -p "$PREFIX" - BASE="https://developer.download.nvidia.com/compute/cuda/redist" - for cv in \ - cuda_crt:13.3.33 cuda_cudart:13.3.29 cuda_nvcc:13.3.33 \ - cuda_nvrtc:13.3.33 libcublas:13.5.1.27 libnvvm:13.3.33 \ - cuda_nvtx:13.3.29 cuda_profiler_api:13.3.27 cccl:13.3.3.3.1; do - comp="${cv%:*}"; ver="${cv#*:}" - name="${comp}-${PLAT}-${ver}-archive" - curl -fsSL -o comp.tar.xz "$BASE/${comp}/${PLAT}/${name}.tar.xz" - tar -xf comp.tar.xz - cp -a "${name}/." "$PREFIX"/ - rm -rf comp.tar.xz "${name}" - done - # Redist ships libs under lib/; some CMake CUDA discovery expects lib64. - ln -sfn lib "$PREFIX/lib64" - { - echo "CUDA_PATH=$PREFIX" - echo "CUDA_HOME=$PREFIX" - echo "LD_LIBRARY_PATH=$PREFIX/lib:${LD_LIBRARY_PATH:-}" - } >> "$GITHUB_ENV" - echo "$PREFIX/bin" >> "$GITHUB_PATH" - - - name: Install CUDA toolkit ${{ matrix.cuda }} (Jimver) - if: matrix.cuda != '13.3' - uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 - id: cuda-toolkit - with: - cuda: ${{ matrix.cuda }} - method: 'network' - - - name: Set up CUDA environment (Jimver) - if: matrix.cuda != '13.3' - run: | - echo "CUDA_PATH=${{ steps.cuda-toolkit.outputs.CUDA_PATH }}" >> "$GITHUB_ENV" - echo "CUDA_HOME=${{ steps.cuda-toolkit.outputs.CUDA_PATH }}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${{ steps.cuda-toolkit.outputs.CUDA_PATH }}/lib64:${LD_LIBRARY_PATH:-}" >> "$GITHUB_ENV" - - - name: Verify CUDA - run: nvcc --version - - - name: Configure - working-directory: src - run: | - set -eux - # Three non-obvious flags: - # - CMAKE_INSTALL_RPATH=$ORIGIN + RPATH knobs: the tar.gz layout - # ships sibling .so files in the same directory as the binaries. - # - CMAKE_CUDA_ARCHITECTURES: explicit per-profile arch list, the - # whole point of the matrix. - # - CMAKE_*_COMPILER_LAUNCHER=ccache: Jimver installs nvcc outside - # /usr/local/bin, so PATH-symlinked ccache misses CUDA TUs -- - # setting the launcher explicitly caches nvcc too. - # No LLAMA_FATAL_WARNINGS on this leg, unlike cpu/vulkan/macos. Here it also - # sets nvcc -Werror all-warnings, and the host warnings arrive through an - # -Xcompiler list llama.cpp composes itself, which CMAKE_CXX_FLAGS does not - # reach -- verified on 08-27, where the suppression added that morning was - # absent from the actual nvcc command line. So the warning set here is - # all-or-nothing, and 'all' means every libstdc++ false positive fails a - # release: cuda13-newer and cuda13-portable both died on a GCC 11 - # -Wstringop-overflow inside std::copy, from ggml_cuda_try_fuse. - cmake -S . -B build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DGGML_NATIVE=OFF \ - -DGGML_BACKEND_DL=ON \ - -DGGML_CPU_ALL_VARIANTS=ON \ - -DGGML_RPC=ON \ - -DGGML_CUDA=ON \ - -DGGML_CUDA_CUB_3DOT2=ON \ - -DLLAMA_BUILD_TESTS=OFF \ - -DLLAMA_BUILD_EXAMPLES=OFF \ - -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_SERVER=ON \ - -DCMAKE_CUDA_ARCHITECTURES="$(echo '${{ matrix.archs }}' | tr ' ' ';')" \ - -DCMAKE_INSTALL_RPATH='$ORIGIN' \ - -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache - - - name: Build - working-directory: src - run: | - set -eux - # Backend modules (CPU variants, CUDA, RPC) build as ggml dependencies, - # so every tool pulls them in. - # -j 3: this multi-arch nvcc build peaks at ~3 GB host RSS on the - # 4 vCPU / 16 GB GitHub-hosted runners (x64 and arm64 alike, measured - # over a 1s sampler), so 3 parallel TUs leave ~13 GB free. -j 4 adds - # no speed (the runner is ~2 physical cores + HT), so 3 is the sweet spot. - cmake --build build --config Release -j 3 - strip build/bin/llama-* || true - - # The DiffusionGemma visual server + cli are example targets that exist - # only when the source tree carries ggml-org/llama.cpp#24423 (a mix build). - # Build them best-effort and NEVER fail the job: the full tool bundle - # above must publish even if these do not compile on a given toolchain. - # package_bundle.py ships them only if they were produced. - - name: Build DiffusionGemma binaries (best-effort; mix builds only) - working-directory: src - run: | - set -u - if [ ! -d examples/diffusion-gemma-server ]; then - echo "no DiffusionGemma sources in this tree; skipping" - exit 0 - fi - cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON \ - || { echo "reconfigure for examples failed; skipping DiffusionGemma binaries"; exit 0; } - if cmake --build build --config Release -j 3 \ - --target llama-diffusion-gemma-visual-server llama-diffusion-cli; then - strip build/bin/llama-diffusion-gemma-visual-server build/bin/llama-diffusion-cli || true - echo "built DiffusionGemma binaries" - else - echo "warning: DiffusionGemma binaries failed to build; bundle will omit them" - fi - exit 0 - - - name: Package bundle - env: - PLATFORM: linux - ARCH: ${{ matrix.arch }} - BIN_DIR: ${{ github.workspace }}/src/build/bin - SRC_DIR: ${{ github.workspace }}/src - OUT_DIR: ${{ github.workspace }}/dist - TAG: ${{ inputs.tag }} - SOURCE_COMMIT: ${{ inputs.commit }} - SOURCE_REPO: ${{ inputs.repo }} - SOURCE_REF_KIND: ${{ inputs.repo == 'ggml-org/llama.cpp' && 'tag' || 'mix' }} - PROFILE: ${{ matrix.profile }} - LINE: ${{ matrix.line }} - KLASS: ${{ matrix.klass }} - RANK: ${{ matrix.rank }} - TOOLKIT_LINE: ${{ matrix.toolkit_line }} - DOCKER_IMAGE: github-hosted ${{ matrix.runner }}, CUDA ${{ matrix.cuda }} ${{ matrix.cuda == '13.3' && '(NVIDIA redist)' || '(Jimver)' }} - ARCHS: ${{ matrix.archs }} - SMS: ${{ matrix.sms }} - run: python3 tooling/scripts/unsloth/package_bundle.py - - - name: Upload bundle artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: app-${{ inputs.tag }}-linux-${{ matrix.arch }}-${{ matrix.profile }} - path: dist/app-${{ inputs.tag }}-linux-${{ matrix.arch }}-${{ matrix.profile }}.tar.gz - if-no-files-found: error - - - name: Evict stale ccache files - # !cancelled(), unlike the save below: on a timeout the job gets a - # single ~5 minute teardown window (measured ~4m50s after process - # kill), shared by every remaining step and not replenished. Evicting - # spends that window on housekeeping; the save is what actually needs - # it, and a 2 GB cache is not quick to write. - if: ${{ !cancelled() }} - continue-on-error: true - run: ccache --evict-older-than 14d - - - name: Save ccache - # Save even when the build failed: the objects compiled before the - # failure are still worth keeping, and a job that saves nothing leaves - # a hole in the cache lineage that widens the next run's tag gap. - # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. - # - # always(), not !cancelled(): a timeout-minutes expiry puts the job on - # the CANCELLATION path, not the failure path, so !cancelled() would - # skip the save on the single most expensive case -- a leg that - # compiled for hours and then hit the cap. - if: ${{ always() }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}/.ccache - key: ccache-cuda-${{ matrix.cuda }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-deadman.yml b/.github/workflows/unsloth-prebuilt-deadman.yml deleted file mode 100644 index 9aa74bd69904..000000000000 --- a/.github/workflows/unsloth-prebuilt-deadman.yml +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: Unsloth prebuilt dead-man check - -# Watches the output, not the process: a run can go green while publishing -# nothing, which no failure-triggered alert can see. Upstream cuts releases -# several times a day, so two days of silence here is a fault, not a quiet -# upstream. - -on: - schedule: - - cron: '23 9 * * *' - workflow_dispatch: - -permissions: - contents: read - issues: write - -env: - STALE_DAYS: '2' - -jobs: - deadman: - name: Check publish freshness - runs-on: ubuntu-24.04 - steps: - - name: Checkout (for the composite action) - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: { persist-credentials: false } - - - name: Check newest published release - id: c - env: - GH_TOKEN: ${{ github.token }} - run: | - set -uo pipefail - LATEST="$(gh api "repos/${GITHUB_REPOSITORY}/releases?per_page=30" \ - --jq '[.[] | select(.draft==false and .prerelease==false)] - | max_by(.published_at|fromdateiso8601) - | "\(.tag_name)\t\(.published_at)"' 2>/dev/null || true)" - - if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then - # Do not manufacture an alert from an API hiccup. - echo "::warning::could not read the release list; skipping this check" - echo "status=skip" >> "$GITHUB_OUTPUT" - exit 0 - fi - - TAG="${LATEST%%$'\t'*}" - PUB="${LATEST##*$'\t'}" - AGE_D=$(( ( $(date -u +%s) - $(date -u -d "$PUB" +%s) ) / 86400 )) - echo "newest release ${TAG} published ${PUB} (${AGE_D}d ago); threshold ${STALE_DAYS}d" - - if [ "$AGE_D" -le "${STALE_DAYS}" ]; then - echo "status=success" >> "$GITHUB_OUTPUT" - echo "details=" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "status=failure" >> "$GITHUB_OUTPUT" - { - echo 'details<<ALERT_EOF' - echo "No prebuilt release has been published in **${AGE_D} days** (threshold ${STALE_DAYS})." - echo - echo "Newest release: \`${TAG}\`, published ${PUB}." - echo - echo "The nightly may be failing, or it may be completing green while" - echo "publishing nothing. Check the most recent runs of \`Unsloth prebuilt (full release)\`." - echo 'ALERT_EOF' - } >> "$GITHUB_OUTPUT" - - - name: Alert - if: ${{ steps.c.outputs.status != 'skip' }} - uses: ./.github/actions/prebuilt-alert - with: - status: ${{ steps.c.outputs.status }} - key: llama-prebuilt-stale - title: 'No llama.cpp prebuilt release published recently' - details: ${{ steps.c.outputs.details }} - token: ${{ github.token }} diff --git a/.github/workflows/unsloth-prebuilt-macos.yml b/.github/workflows/unsloth-prebuilt-macos.yml deleted file mode 100644 index 2bfe3e832d80..000000000000 --- a/.github/workflows/unsloth-prebuilt-macos.yml +++ /dev/null @@ -1,211 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: "Unsloth prebuilt: macOS" - -# Reusable child of unsloth-prebuilt.yml. Builds the macOS slices (arm64 Metal + -# x64 CPU) the parent's `resolve` job placed in `matrix`. Each entry uploads a -# single app-*.tar.gz artifact for the parent's assemble step to pick up. -# -# The only change over ggml-org's own macos build is an explicit -# -DCMAKE_OSX_DEPLOYMENT_TARGET per slice, so the binaries declare their load -# floor instead of inheriting the runner OS. Upstream stopped pinning this on -# arm64 (their macos-26 runner stamps minos=26, which fails to dyld-load on -# macOS < 26); owning the build is how we keep a loadable floor. - -on: - workflow_call: - inputs: - tag: - description: 'Upstream llama.cpp release tag (b####), resolved by parent' - required: true - type: string - repo: - description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' - required: false - default: 'ggml-org/llama.cpp' - type: string - source_artifact: - description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' - required: false - default: '' - type: string - matrix: - description: 'Matrix JSON {include:[...]} produced by the parent resolve job' - required: true - type: string - -permissions: - contents: read - -jobs: - build: - name: ${{ matrix.build }} - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: ${{ fromJSON(inputs.matrix) }} - steps: - - name: Checkout build tooling (this repo) - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: - path: tooling - persist-credentials: false - - # The parent's resolve job built the source tree (upstream base + any mix - # PRs, with the build number/commit and Unsloth fingerprint baked - # into cmake/build-info.cmake) and uploaded it as an artifact; extract it - # instead of cloning -- no .git needed, the build number is already baked. - - name: Download source @ ${{ inputs.tag }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: ${{ inputs.source_artifact }} - path: srcpkg - - name: Extract source - shell: bash - run: | - set -eux - mkdir -p src - tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 - - # The macOS legs were the last build jobs without a compiler cache. Metal's - # backend is Objective-C, and the arm64 leg is the only one that builds it - # (the x64 leg passes -DGGML_METAL=OFF), so the OBJC launchers matter here - # in a way they do not elsewhere. Setting a launcher for a language this - # build never enables is inert, so both are set unconditionally. - - name: ccache - uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 - with: - key: macos-${{ matrix.build }}-${{ matrix.deploy_target }}-${{ inputs.tag }} - restore-keys: | - macos-${{ matrix.build }}-${{ matrix.deploy_target }} - append-timestamp: false - variant: ccache - max-size: 2G - save: false - - - name: Build (deployment target ${{ matrix.deploy_target }}) - working-directory: src - run: | - set -euo pipefail - cmake -B build \ - ${{ matrix.defines }} \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_OBJC_COMPILER_LAUNCHER=ccache \ - -DCMAKE_OBJCXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ matrix.deploy_target }} \ - -DCMAKE_INSTALL_RPATH='@loader_path' \ - -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ - -DLLAMA_FATAL_WARNINGS=ON \ - -DLLAMA_BUILD_BORINGSSL=ON \ - -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_RPC_RDMA=OFF - cmake --build build --config Release -j "$(sysctl -n hw.logicalcpu)" - - # ggml-rpc AUTO-DETECTS RDMA: on Apple it find_library(rdma)s and, when the - # runner has one, defaults GGML_RPC_RDMA to ON and links /usr/lib/librdma.dylib - # into libggml-rpc. That library ships with the runner image, not with macOS, - # so the bundle loaded on the builder and died everywhere else: - # - # dyld: Library not loaded: /usr/lib/librdma.dylib - # Referenced from: .../libggml-rpc.0.dylib - # - # (b10639-mix-f6f92fe, the first release carrying upstream b114b4739.) A - # redistributable must not have its configuration decided by whatever happened - # to be installed where it was built, so the value is pinned rather than - # detected. Pinned OFF specifically because RDMA-over-Thunderbolt is not - # something the prebuilt's consumers can use. - - name: Assert the RDMA transport stayed off - working-directory: src - run: | - set -euo pipefail - grep -q '^GGML_RPC_RDMA:BOOL=OFF$' build/CMakeCache.txt || { - echo "::error::GGML_RPC_RDMA is not OFF in the CMake cache; a macOS"\ - "prebuilt that links librdma cannot load on a consumer Mac" - grep -i 'rdma' build/CMakeCache.txt || true - exit 1 - } - # The cache says what was asked for; otool says what was linked. The - # launch gate below cannot see this, because it runs on the one host - # where the library does exist. - # No xargs -r: that is a GNU extension and these are BSD userland runners. - # An empty find just makes otool complain to a discarded stderr and grep - # match nothing, which is the answer we want anyway. - if find build/bin -type f \( -name '*.dylib' -o -perm -u+x \) -print0 \ - | xargs -0 otool -L 2>/dev/null | grep -i 'librdma'; then - echo "::error::a shipped Mach-O still links librdma" - exit 1 - fi - echo "rdma gate passed: GGML_RPC_RDMA=OFF and nothing links librdma" - - - name: Load gate (minos <= ${{ matrix.deploy_target }}, arch, launch) - run: bash tooling/scripts/unsloth/assert_macho_minos.sh src/build/bin "${{ matrix.expect_arch }}" "${{ matrix.deploy_target }}" - - # DiffusionGemma binaries (example targets present only in #24423 mix - # builds): best-effort, never fail the job. Built after the load gate so - # they do not affect the required-binary minos check; the same cached - # deployment target applies. The bundle tars all of build/bin, so anything - # produced here is shipped automatically. - - name: Build DiffusionGemma binaries (best-effort; mix builds only) - working-directory: src - run: | - set -u - if [ ! -d examples/diffusion-gemma-server ]; then - echo "no DiffusionGemma sources in this tree; skipping" - exit 0 - fi - cmake -B build -DLLAMA_BUILD_EXAMPLES=ON \ - || { echo "reconfigure for examples failed; skipping DiffusionGemma binaries"; exit 0; } - if cmake --build build --config Release -j "$(sysctl -n hw.logicalcpu)" \ - --target llama-diffusion-gemma-visual-server llama-diffusion-cli; then - echo "built DiffusionGemma binaries" - else - echo "warning: DiffusionGemma binaries failed to build; bundle will omit them" - fi - exit 0 - - - name: Package bundle - working-directory: src - run: | - set -euo pipefail - TAG="${{ inputs.tag }}" - cp LICENSE build/bin/ - mkdir -p "$GITHUB_WORKSPACE/dist" - # BSD tar (-s) rewrites the leading ./ to llama-<tag>/ so the archive - # unpacks into a named dir, matching upstream's own macos tarball layout. - tar -czf "$GITHUB_WORKSPACE/dist/llama-${TAG}-bin-macos-${{ matrix.build }}.tar.gz" \ - -s ",^\.,llama-${TAG}," -C build/bin . - - - name: Upload bundle artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: app-${{ inputs.tag }}-macos-${{ matrix.build }} - path: dist/llama-${{ inputs.tag }}-bin-macos-${{ matrix.build }}.tar.gz - if-no-files-found: error - - - name: Evict stale ccache files - # !cancelled(), unlike the save below: on a timeout the job gets a - # single ~5 minute teardown window (measured ~4m50s after process - # kill), shared by every remaining step and not replenished. Evicting - # spends that window on housekeeping; the save is what actually needs - # it, and a 2 GB cache is not quick to write. - if: ${{ !cancelled() }} - continue-on-error: true - run: ccache --evict-older-than 14d - - - name: Save ccache - # Save even when the build failed: the objects compiled before the - # failure are still worth keeping, and a job that saves nothing leaves - # a hole in the cache lineage that widens the next run's tag gap. - # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. - # - # always(), not !cancelled(): a timeout-minutes expiry puts the job on - # the CANCELLATION path, not the failure path, so !cancelled() would - # skip the save on the single most expensive case -- a leg that - # compiled for hours and then hit the cap. - if: ${{ always() }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}/.ccache - key: ccache-macos-${{ matrix.build }}-${{ matrix.deploy_target }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-retry.yml b/.github/workflows/unsloth-prebuilt-retry.yml deleted file mode 100644 index 6f90e788baaa..000000000000 --- a/.github/workflows/unsloth-prebuilt-retry.yml +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: Unsloth prebuilt retry - -# Re-runs a nightly that stopped for a reason outside the build itself. -# -# Two causes so far, both of which leave a finished build set unpublished: -# -# Cancelled with nothing failing. On 08-05 all 40 builds passed and the -# publish job was cancelled 15s in, with no failing job, no superseding run -# and nothing in the workflow that cancels. Every artifact was still there; -# one rerun published the release. -# -# A runner that dies mid-build. On 08-07 39 of 40 builds passed and -# `CUDA Windows / x64/cuda12-portable` failed after 78 minutes of nvcc with -# no compile error in its log, which simply stopped 46 minutes before the -# job ended. GitHub's own annotation was "The hosted runner lost -# communication with the server". assemble is `needs:` every build, so it -# skipped and no release was cut. -# -# Without this the pipeline sits on a finished build set until a human -# notices, which is the silent no-publish the alerting exists to catch, -# reached one step later. - -on: - workflow_run: - workflows: ['Unsloth prebuilt (full release)'] - types: [completed] - -permissions: - contents: read - actions: write - # The runner-loss test reads each failed job's annotation, which lives on - # the check run rather than in the job log. - checks: read - -jobs: - retry: - name: Rerun an externally broken run once - # run_attempt caps this at one retry: the rerun fires this workflow again - # as attempt 2, which no longer matches. A hand-cancelled workflow_dispatch - # is someone stopping their own build, so leave it stopped. - if: >- - (github.event.workflow_run.conclusion == 'cancelled' - || github.event.workflow_run.conclusion == 'failure') - && github.event.workflow_run.run_attempt == 1 - && github.event.workflow_run.event != 'workflow_dispatch' - runs-on: ubuntu-24.04 - steps: - - name: Rerun the run when nothing about the build actually failed - env: - GH_TOKEN: ${{ github.token }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_URL: ${{ github.event.workflow_run.html_url }} - CONCLUSION: ${{ github.event.workflow_run.conclusion }} - run: | - set -uo pipefail - - JOBS="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs" \ - --jq '.jobs[] | [.id, .conclusion, .name] | @tsv')" - - # A rerun of a run where nothing succeeded buys nothing: there is no - # salvageable work, and whatever stopped it will stop it again. - if ! cut -f2 <<<"$JOBS" | grep -qx 'success'; then - echo "not retrying ${RUN_ID}: nothing succeeded, so there is no work to salvage" \ - | tee -a "$GITHUB_STEP_SUMMARY" - exit 0 - fi - - FAILED="$(awk -F'\t' '$2 == "failure"' <<<"$JOBS")" - - # assemble waits on the build matrix from inside the job, so a leg - # that dies fails assemble too. Alongside another failure that is a - # consequence, not a cause; read it only when it failed alone. - # The name tracks assemble's `name:` in unsloth-prebuilt.yml. - INDEPENDENT="$(awk -F'\t' 'NF && $3 != "Assemble metadata + publish"' <<<"$FAILED")" - if [ -n "$INDEPENDENT" ]; then - FAILED="$INDEPENDENT" - fi - - # A cancellation that follows a real failure is a build problem to - # read, not to repeat. - if [ "$CONCLUSION" = cancelled ] && [ -n "$FAILED" ]; then - echo "not retrying ${RUN_ID}: it has a failed job, so the cancel is a consequence" \ - | tee -a "$GITHUB_STEP_SUMMARY" - exit 0 - fi - - # For an outright failure, retry only when every failed job died - # because its runner went away. A compile error repeats identically - # and must be read, not rerun; a lost runner is infrastructure and a - # rerun is the whole fix. Anything we cannot positively identify as - # runner loss counts as a real failure, so an unreadable annotation - # fails closed and the run stays put. - if [ "$CONCLUSION" = failure ]; then - if [ -z "$FAILED" ]; then - echo "not retrying ${RUN_ID}: it concluded failure with no failed job, so there is nothing to rerun" \ - | tee -a "$GITHUB_STEP_SUMMARY" - exit 0 - fi - while IFS=$'\t' read -r JOB_ID _ JOB_NAME; do - [ -n "${JOB_ID:-}" ] || continue - NOTES="$(gh api "repos/${GITHUB_REPOSITORY}/check-runs/${JOB_ID}/annotations" \ - --jq '.[].message' 2>/dev/null)" - if ! grep -qF 'lost communication with the server' <<<"$NOTES"; then - echo "not retrying ${RUN_ID}: \`${JOB_NAME}\` failed for a reason other than runner loss, so read it rather than repeat it" \ - | tee -a "$GITHUB_STEP_SUMMARY" - exit 0 - fi - echo "runner loss: ${JOB_NAME}" | tee -a "$GITHUB_STEP_SUMMARY" - done <<<"$FAILED" - fi - - if gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/rerun-failed-jobs"; then - echo "rerunning ${CONCLUSION} jobs of ${RUN_URL}" | tee -a "$GITHUB_STEP_SUMMARY" - else - echo "::warning::could not rerun ${RUN_URL}; rerun it by hand" | tee -a "$GITHUB_STEP_SUMMARY" - fi diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml deleted file mode 100644 index b650874f8ecf..000000000000 --- a/.github/workflows/unsloth-prebuilt-rocm.yml +++ /dev/null @@ -1,878 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: "Unsloth prebuilt: ROCm" - -# Reusable child of unsloth-prebuilt.yml. Per-gfx-target ROCm bundles on -# Windows + Ubuntu. -# -# Huge thanks to the lemonade-sdk team -- this is adapted from their workflow: -# https://github.com/lemonade-sdk/llamacpp-rocm/blob/main/.github/workflows/build-llamacpp-rocm.yml -# -# Build mechanics taken from that file: TheRock multi-arch nightly download + -# version auto-detect, gfx target name mapping, HIP/clang cmake invocation, the -# full hardcoded ROCm runtime lib copy list, patchelf RPATH. -# -# Local adaptations: -# - tag is passed in (the parent resolves it against ggml-org upstream) -# instead of lemonade's auto-incrementing b1000+ scheme. -# - artifacts pre-packaged into app-<tag>-<platform>-x64-rocm-<gfx>.{tar.gz|zip} -# so the parent assemble step picks them up via the same merge-multiple -# download pattern it uses for CUDA bundles. -# - test jobs (stx-halo, stx) dropped -- those need self-hosted AMD runners. -# - build-summary + post-build cleanup steps dropped (clutter). - -on: - workflow_call: - inputs: - tag: - description: 'Upstream llama.cpp release tag (b####), resolved by parent' - required: true - type: string - repo: - description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' - required: false - default: 'ggml-org/llama.cpp' - type: string - source_artifact: - description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' - required: false - default: '' - type: string - matrix: - description: 'gfx_target matrix JSON ({"gfx_target":[...]}), built by parent' - required: true - type: string - operating_systems: - description: 'OSes to build for (comma-separated: windows,ubuntu)' - required: false - default: 'windows,ubuntu' - type: string - rocm_version: - description: 'TheRock ROCm version (e.g., 10.1.0a20260807), "weekly" or "latest"' - required: false - default: 'weekly' - type: string - rocm_cutoff: - description: 'For "weekly": YYYYMMDD cutoff resolved once by the parent. Blank means each leg computes its own.' - required: false - default: '' - type: string - -permissions: - contents: read - -jobs: - build-windows: - name: windows/${{ matrix.gfx_target }} - runs-on: windows-2022 - if: contains(inputs.operating_systems, 'windows') - strategy: - matrix: ${{ fromJson(inputs.matrix) }} - fail-fast: false - - steps: - - name: Clean up existing directories (safety precaution) - run: | - # Remove existing llama.cpp directory if it exists - if (Test-Path "llama.cpp") { - Write-Host "Removing existing llama.cpp directory..." - Remove-Item -Recurse -Force "llama.cpp" - } - - # Remove existing C:\opt\rocm directory if it exists - if (Test-Path "C:\opt\rocm") { - Write-Host "Removing existing C:\opt\rocm directory..." - Remove-Item -Recurse -Force "C:\opt\rocm" - } - - # Remove any existing ROCm tarball - if (Test-Path "rocm.tar.gz") { - Write-Host "Removing existing rocm.tar.gz..." - Remove-Item -Force "rocm.tar.gz" - } - - Write-Host "Cleanup completed successfully" - - - name: Install Visual Studio Build Tools - run: | - # Retry helper: Invoke-WebRequest's -MaximumRetryCount only retries on - # HTTP status failures (400-599/304), not on the TCP-level connection - # timeouts these download hosts intermittently throw, so wrap each - # download in an explicit catch-all retry with linear backoff. - function Invoke-DownloadWithRetry { - param([string]$Uri, [string]$OutFile, [int]$Retries = 5, [int]$DelaySec = 10) - for ($i = 1; $i -le $Retries; $i++) { - try { - Invoke-WebRequest -Uri $Uri -OutFile $OutFile -ErrorAction Stop - return - } catch { - Write-Host "Download attempt $i/$Retries for $Uri failed: $($_.Exception.Message)" - if ($i -eq $Retries) { throw } - Start-Sleep -Seconds ($DelaySec * $i) - } - } - } - - # Download and install Visual Studio Build Tools - $vsInstallerUrl = "https://aka.ms/vs/17/release/vs_buildtools.exe" - $vsInstallerPath = "$env:TEMP\vs_buildtools.exe" - - Write-Host "Downloading Visual Studio Build Tools..." - Invoke-DownloadWithRetry -Uri $vsInstallerUrl -OutFile $vsInstallerPath - - Write-Host "Installing Visual Studio Build Tools..." - Start-Process -FilePath $vsInstallerPath -ArgumentList "--quiet", "--wait", "--norestart", "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "--add", "Microsoft.VisualStudio.Component.VC.CMake.Project", "--add", "Microsoft.VisualStudio.Component.VC.ATL", "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621" -Wait - - # Clean up installer - Remove-Item $vsInstallerPath -Force - - - name: Install build dependencies - run: | - Write-Host "Installing build dependencies using manual methods..." - - # Retry helper: Invoke-WebRequest's -MaximumRetryCount only retries on - # HTTP status failures (400-599/304), not on the TCP-level connection - # timeouts these download hosts intermittently throw, so wrap each - # download in an explicit catch-all retry with linear backoff. - function Invoke-DownloadWithRetry { - param([string]$Uri, [string]$OutFile, [int]$Retries = 5, [int]$DelaySec = 10) - for ($i = 1; $i -le $Retries; $i++) { - try { - Invoke-WebRequest -Uri $Uri -OutFile $OutFile -ErrorAction Stop - return - } catch { - Write-Host "Download attempt $i/$Retries for $Uri failed: $($_.Exception.Message)" - if ($i -eq $Retries) { throw } - Start-Sleep -Seconds ($DelaySec * $i) - } - } - } - - # Install Ninja - Write-Host "Installing Ninja..." - $ninjaUrl = "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip" - $ninjaPath = "$env:TEMP\ninja-win.zip" - $ninjaDir = "C:\ninja" - New-Item -ItemType Directory -Force -Path $ninjaDir - Invoke-DownloadWithRetry -Uri $ninjaUrl -OutFile $ninjaPath - Expand-Archive -Path $ninjaPath -DestinationPath $ninjaDir -Force - - # Install Strawberry Perl via Chocolatey (already on the runner) instead - # of downloading the MSI from the frequently-unreachable strawberryperl.com - # host that was timing out and failing these builds. It ships preinstalled - # on the windows-2022 image, so this is normally a fast no-op. - Write-Host "Installing Strawberry Perl via Chocolatey..." - choco install strawberryperl -y --no-progress - - # Verify installations - $env:PATH = "C:\ninja;C:\Strawberry\perl\bin;C:\Strawberry\c\bin;$env:PATH" - Write-Host "Verifying installations..." - ninja --version - perl --version - - Write-Host "Manual installation of build dependencies completed" - - - name: Download ROCm nightly tarball - run: | - # Retry helper: Invoke-WebRequest's -MaximumRetryCount only retries on - # HTTP status failures (400-599/304), not on the TCP-level connection - # timeouts these download hosts intermittently throw, so wrap each - # download in an explicit catch-all retry with linear backoff. - function Invoke-DownloadWithRetry { - param([string]$Uri, [string]$OutFile, [int]$Retries = 5, [int]$DelaySec = 10) - for ($i = 1; $i -le $Retries; $i++) { - try { - Invoke-WebRequest -Uri $Uri -OutFile $OutFile -ErrorAction Stop - return - } catch { - Write-Host "Download attempt $i/$Retries for $Uri failed: $($_.Exception.Message)" - if ($i -eq $Retries) { throw } - Start-Sleep -Seconds ($DelaySec * $i) - } - } - } - - # Determine ROCm version to use - $rocmVersion = "${{ inputs.rocm_version }}" - $currentTarget = "${{ matrix.gfx_target }}" - - # Map the build target to the matching TheRock archive family - $archiveTarget = $currentTarget - if ($currentTarget -eq "gfx103X" -or $currentTarget -eq "gfx110X" -or $currentTarget -eq "gfx120X") { - $archiveTarget = "$currentTarget-all" - Write-Host "Using target with -all suffix: $archiveTarget" - } - - # TheRock publishes nightlies to the multi-arch tarball index. The - # static HTML page embeds a JSON `files` array with names and mtimes. - $baseUrl = "https://rocm.nightlies.amd.com/tarball-multi-arch" - if ($rocmVersion -eq "latest" -or $rocmVersion -eq "weekly") { - # weekly: see the Linux job. - $cutoff = "99999999" - if ($rocmVersion -eq "weekly") { - $cutoff = "${{ inputs.rocm_cutoff }}" - if (-not $cutoff) { - # Windows uses its own zone ids; the IANA name is the fallback. - try { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("Pacific Standard Time") } - catch { $tz = [System.TimeZoneInfo]::FindSystemTimeZoneById("America/Los_Angeles") } - $sf = [System.TimeZoneInfo]::ConvertTimeFromUtc([DateTime]::UtcNow, $tz) - $cutoff = $sf.Date.AddDays(-([int]$sf.DayOfWeek + 1)).ToString("yyyyMMdd") - } - Write-Host "Weekly pin: newest build dated on or before $cutoff (week of the last SF Sunday)" - } else { - Write-Host "Auto-detecting latest ROCm version for target: $currentTarget" - } - $indexHtml = (Invoke-WebRequest "$baseUrl/" -UseBasicParsing).Content - $filesMatch = [regex]::Match($indexHtml, 'const files = (\[.*?\]);', [System.Text.RegularExpressions.RegexOptions]::Singleline) - if (-not $filesMatch.Success) { - Write-Error "Failed to parse file index from $baseUrl/" - exit 1 - } - - # Pick the newest build date and exclude sibling test archives. - $allFiles = $filesMatch.Groups[1].Value | ConvertFrom-Json - $prefix = "therock-dist-windows-$archiveTarget-" - $versionPattern = "^$([regex]::Escape($prefix))\d+\.\d+\.\d+(a|rc)\d+\.tar\.gz$" - $latest = $allFiles | - Where-Object { $_.name -match $versionPattern } | - Where-Object { [regex]::Match($_.name, '(\d{8})\.tar\.gz$').Groups[1].Value -le $cutoff } | - Sort-Object { [regex]::Match($_.name, '(\d{8})\.tar\.gz$').Groups[1].Value } | - Select-Object -Last 1 - if (-not $latest) { - Write-Error "No tarball found for prefix '$prefix' at or before $cutoff at $baseUrl/" - exit 1 - } - $latestFile = $latest.name - Write-Host "Found latest file: $latestFile" - - # Extract version from the filename for environment variable - if ($latestFile -match "therock-dist-windows-$archiveTarget-(\d+\.\d+\.\d+(?:a|rc)\d+)\.tar\.gz") { - $rocmVersion = $matches[1] - Write-Host "Detected latest ROCm version: $rocmVersion" - } else { - Write-Error "Failed to extract ROCm version from latest file: $latestFile" - Write-Error "Expected pattern: therock-dist-windows-$archiveTarget-<version>.tar.gz" - exit 1 - } - - $rocmUrl = "$baseUrl/$latestFile" - } else { - $rocmUrl = "$baseUrl/therock-dist-windows-$archiveTarget-$rocmVersion.tar.gz" - } - - # Store the version for use in other steps - echo "DETECTED_ROCM_VERSION=$rocmVersion" >> $env:GITHUB_ENV - - Write-Host "Downloading ROCm from: $rocmUrl" - Invoke-DownloadWithRetry -Uri $rocmUrl -OutFile "rocm.tar.gz" - - - name: Extract ROCm to C:\opt\rocm - run: | - # Create directory if it doesn't exist - New-Item -ItemType Directory -Force -Path "C:\opt\rocm" - - # Extract the tarball - tar -xzf rocm.tar.gz -C C:\opt\rocm --strip-components=1 - - # Keyed per gfx target and OS: the seven targets compile different device - # code from the same host sources, so a shared cache would mostly miss. - # save: false -- the explicit actions/cache/save step at the end of the job - # runs after packaging, so a failed package step does not persist a cache - # for a bundle that never shipped (same pattern as the CPU/CUDA children). - # - # The ROCm version is in the key, and in the restore prefix, because ccache hashes the compiler into every entry: a cache from another nightly can never hit. - - name: ccache - uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 - with: - key: rocm-windows-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }} - restore-keys: | - rocm-windows-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }} - append-timestamp: false - variant: ccache - max-size: 2G - save: false - - # The parent's resolve job built the source tree (upstream base + any mix - # PRs, with the build number/commit and Unsloth fingerprint baked - # into cmake/build-info.cmake) and uploaded it as an artifact; extract it - # instead of cloning -- no .git needed, the build number is already baked. - - name: Download source @ ${{ inputs.tag }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: ${{ inputs.source_artifact }} - path: srcpkg - - name: Extract source - shell: bash - run: | - set -eux - mkdir -p llama.cpp - tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C llama.cpp --strip-components=1 - - - name: Build Llama.cpp + ROCm - shell: cmd - run: | - - REM Map GPU targets - set "current_target=${{ matrix.gfx_target }}" - echo Input target: %current_target% - - if "%current_target%"=="gfx110X" ( - set "mapped_target=gfx1100;gfx1101;gfx1102;gfx1103" - ) else if "%current_target%"=="gfx103X" ( - set "mapped_target=gfx1030;gfx1031;gfx1032;gfx1034" - ) else if "%current_target%"=="gfx1151" ( - set "mapped_target=gfx1151" - ) else if "%current_target%"=="gfx1150" ( - set "mapped_target=gfx1150" - ) else if "%current_target%"=="gfx120X" ( - set "mapped_target=gfx1200;gfx1201" - ) else ( - set "mapped_target=%current_target%" - ) - echo Mapped target: %mapped_target% - - REM Set up environment variables and PATH - set HIP_PATH=C:\opt\rocm - set HIP_PLATFORM=amd - set PATH=%HIP_PATH%\lib\llvm\bin;%HIP_PATH%\bin;%PATH% - - REM Set up x64 Native Tools Command Prompt environment. - REM Pin to the VS 2022 line (-version "[17.0,18.0)") so vswhere -latest does not - REM pick up the runner image's VS 2026 (MSVC 14.51), whose STL adds constexpr to - REM math builtins like isgreater; that constexpr (implicitly __host__ __device__) - REM collides with clang's HIP __device__ forward declares and breaks the ggml-cuda - REM .cu compile (ggml-org/llama.cpp#22570). VS 2022 (MSVC 14.44) is unaffected. - call "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -latest -version "[17.0,18.0)" -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath > vs_path.txt - set /p VS_PATH=<vs_path.txt - call "%VS_PATH%\VC\Auxiliary\Build\vcvars64.bat" - - REM Create build directory - cd llama.cpp - mkdir build - cd build - - REM Configure the project - REM Only the C/CXX launchers here: ggml-hip forces CXX_IS_HIPCC on - REM WIN32, so the device sources compile as C++, not the HIP language. - cmake .. -G Ninja ^ - -DCMAKE_C_COMPILER="C:\opt\rocm\lib\llvm\bin\clang.exe" ^ - -DCMAKE_CXX_COMPILER="C:\opt\rocm\lib\llvm\bin\clang++.exe" ^ - -DCMAKE_CXX_FLAGS="-IC:\opt\rocm\include" ^ - -DCMAKE_CROSSCOMPILING=ON ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DGPU_TARGETS="%mapped_target%" ^ - -DBUILD_SHARED_LIBS=ON ^ - -DLLAMA_BUILD_TESTS=OFF ^ - -DGGML_HIP=ON ^ - -DGGML_OPENMP=OFF ^ - -DGGML_CUDA_FORCE_CUBLAS=OFF ^ - -DGGML_RPC=ON ^ - -DGGML_HIP_ROCWMMA_FATTN=OFF ^ - -DLLAMA_BUILD_BORINGSSL=ON ^ - -DGGML_NATIVE=OFF ^ - -DGGML_STATIC=OFF ^ - -DCMAKE_C_COMPILER_LAUNCHER=ccache ^ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ^ - -DCMAKE_SYSTEM_NAME=Windows - - REM Build the project - cmake --build . -j %NUMBER_OF_PROCESSORS% - - - name: Copy ROCm core DLLs to build directory - run: | - $rocmVersion = if ($env:DETECTED_ROCM_VERSION) { $env:DETECTED_ROCM_VERSION } else { "${{ inputs.rocm_version }}" } - $buildBinPath = "llama.cpp\build\bin" - $rocmBinPath = "C:\opt\rocm\bin" - - Write-Host "Copying ROCm core DLL files to build directory..." - Write-Host "Source: $rocmBinPath" - Write-Host "Destination: $buildBinPath" - - if (Test-Path $rocmBinPath) { - # Copy files matching patterns and specific names - $filesToCopy = @( - "amdhip64_*.dll", - "rocm_kpack.dll", - "amd_comgr*.dll", - "libhipblas.dll", - "rocblas.dll", - "rocsolver.dll", - "hipblaslt.dll", - "libhipblaslt.dll", - "hipblas.dll", - "origami.dll" - ) - - foreach ($pattern in $filesToCopy) { - $matchingFiles = Get-ChildItem -Path $rocmBinPath -Name $pattern -ErrorAction SilentlyContinue - if ($matchingFiles) { - foreach ($file in $matchingFiles) { - $sourcePath = Join-Path $rocmBinPath $file - $destPath = Join-Path $buildBinPath $file - Copy-Item $sourcePath $destPath - Write-Host "Copied: $file" - } - } else { - Write-Host "Warning: No files found matching pattern: $pattern" - } - } - - # Copy the rocblas\library folder and all its contents - $rocblasLibPath = Join-Path $rocmBinPath "rocblas\library" - if (Test-Path $rocblasLibPath) { - Write-Host "Copying rocblas\library folder and all contents..." - $destRocblasPath = Join-Path $buildBinPath "rocblas\library" - Copy-Item -Path $rocblasLibPath -Destination $destRocblasPath -Recurse -Force - Write-Host "Copied: rocblas\library folder with all contents" - - # List the contents of the copied rocblas\library folder - Write-Host "Contents of rocblas\library:" - Get-ChildItem $destRocblasPath -Recurse | Select-Object FullName, Length | Format-Table -AutoSize - } else { - Write-Host "Warning: rocblas\library folder not found at: $rocblasLibPath" - } - - # Copy the hipblaslt\library folder and all its contents - $hipblasltLibPath = Join-Path $rocmBinPath "hipblaslt\library" - if (Test-Path $hipblasltLibPath) { - Write-Host "Copying hipblaslt\library folder and all contents..." - $destHipblasltPath = Join-Path $buildBinPath "hipblaslt\library" - Copy-Item -Path $hipblasltLibPath -Destination $destHipblasltPath -Recurse -Force - Write-Host "Copied: hipblaslt\library folder with all contents" - - # List the contents of the copied hipblaslt\library folder - Write-Host "Contents of hipblaslt\library:" - Get-ChildItem $destHipblasltPath -Recurse | Select-Object FullName, Length | Format-Table -AutoSize - } else { - Write-Host "Warning: rocblas\library folder not found at: $rocblasLibPath" - } - - Write-Host "ROCm core files successfully copied to build directory" - } else { - Write-Error "ROCm bin directory not found: $rocmBinPath" - exit 1 - } - - - name: List build artifacts (including ROCm files) - run: | - cd llama.cpp\build\bin - Write-Host "Final build artifacts (including ROCm core files):" - Get-ChildItem -Recurse | Format-Table Name, Length, LastWriteTime - - - name: Package bundle (zip) - shell: bash - run: | - set -eux - ASSET="app-${{ inputs.tag }}-windows-x64-rocm-${{ matrix.gfx_target }}.zip" - mkdir -p dist - (cd llama.cpp/build/bin && 7z a -tzip "${GITHUB_WORKSPACE}/dist/${ASSET}" .) - ls -la dist - - - name: Upload bundle artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: app-${{ inputs.tag }}-windows-x64-rocm-${{ matrix.gfx_target }} - path: dist/app-${{ inputs.tag }}-windows-x64-rocm-${{ matrix.gfx_target }}.zip - if-no-files-found: error - - # ROCm is the first backend here to run device code through ccache, so log - # the hit rate: a non-zero "unsupported source language" would mean the - # device TUs are falling through to the real compiler uncached. - - name: ccache stats - continue-on-error: true - run: ccache --show-stats -v - - - name: Evict stale ccache files - # !cancelled(), unlike the save below: on a timeout the job gets a - # single ~5 minute teardown window (measured ~4m50s after process - # kill), shared by every remaining step and not replenished. Evicting - # spends that window on housekeeping; the save is what actually needs - # it, and a 2 GB cache is not quick to write. - if: ${{ !cancelled() }} - continue-on-error: true - run: ccache --evict-older-than 14d - - - name: Save ccache - # Save even when the build failed: the objects compiled before the - # failure are still worth keeping, and a job that saves nothing leaves - # a hole in the cache lineage that widens the next run's tag gap. - # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. - # - # always(), not !cancelled(): a timeout-minutes expiry puts the job on - # the CANCELLATION path, not the failure path, so !cancelled() would - # skip the save on the single most expensive case -- a leg that - # compiled for hours and then hit the cap. - if: ${{ always() }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}\.ccache - key: ccache-rocm-windows-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }}- - - build-ubuntu: - name: linux/${{ matrix.gfx_target }} - runs-on: ubuntu-22.04 - if: contains(inputs.operating_systems, 'ubuntu') - strategy: - matrix: ${{ fromJson(inputs.matrix) }} - fail-fast: false - - steps: - - name: Free disk space - # Remove unused runner files to free up disk space - run: curl -fsSL https://raw.githubusercontent.com/kou/arrow/e49d8ae15583ceff03237571569099a6ad62be32/ci/scripts/util_free_space.sh | bash - - - name: Clean up existing directories (safety precaution) - run: | - # Remove existing llama.cpp directory if it exists - if [ -d "llama.cpp" ]; then - echo "Removing existing llama.cpp directory..." - rm -rf llama.cpp - fi - - # Remove existing /opt/rocm directory if it exists - if [ -d "/opt/rocm" ]; then - echo "Removing existing /opt/rocm directory..." - sudo rm -rf /opt/rocm - fi - - # Remove any existing ROCm tarball - if [ -f "rocm.tar.gz" ]; then - echo "Removing existing rocm.tar.gz..." - rm -f rocm.tar.gz - fi - - echo "Cleanup completed successfully" - - - name: Install build dependencies - run: | - echo "Installing build dependencies..." - sudo apt update - sudo apt install -y cmake ninja-build unzip curl - - # Verify installations - echo "Verifying installations..." - cmake --version - ninja --version - echo "Build dependencies installation completed" - - - name: Download and extract ROCm directly to /opt/rocm - run: | - # Determine ROCm version to use - rocm_version="${{ inputs.rocm_version }}" - current_target="${{ matrix.gfx_target }}" - - # Map the build target to the matching TheRock archive family - archive_target="$current_target" - if [[ "$current_target" = "gfx103X" || "$current_target" = "gfx110X" || "$current_target" = "gfx120X" ]]; then - archive_target="${current_target}-all" - echo "Using target with -all suffix: $archive_target" - fi - - # TheRock publishes nightlies to the multi-arch tarball index. The - # static HTML page embeds a JSON `files` array with names and mtimes. - base_url="https://rocm.nightlies.amd.com/tarball-multi-arch" - if [ "$rocm_version" = "latest" ] || [ "$rocm_version" = "weekly" ]; then - # weekly: take the newest alpha up to the Saturday before the last SF Sunday, so a whole week uses one toolchain and the ccache hits. - # The cutoff day must be settled before the first run reads it: TheRock usually publishes the evening before, but has landed as late as 18:33 PT on the named day, which splits the week. - # The parent sends one cutoff for the run, so every leg agrees; the fallback is for a standalone call. - cutoff=99999999 - if [ "$rocm_version" = "weekly" ]; then - cutoff="${{ inputs.rocm_cutoff }}" - [ -n "$cutoff" ] || cutoff="$(TZ=America/Los_Angeles date -d "-$(( $(TZ=America/Los_Angeles date +%w) + 1 )) days" +%Y%m%d)" - echo "Weekly pin: newest build dated on or before $cutoff (week of the last SF Sunday)" - else - echo "Auto-detecting latest ROCm version for target: $current_target" - fi - prefix="therock-dist-linux-${archive_target}-" - files_json=$(curl -s "$base_url/" | tr '\n' ' ' | grep -oP 'const files = \K\[.*?\](?=\s*;)') - if [ -z "$files_json" ]; then - echo "Failed to parse file index from $base_url/" - exit 1 - fi - - # Pick the newest build date and exclude sibling test archives. - latest_file=$(echo "$files_json" | jq -r --arg p "$prefix" --arg c "$cutoff" \ - '[.[] | select(.name | test("^" + $p + "[0-9]+\\.[0-9]+\\.[0-9]+(a|rc)[0-9]+\\.tar\\.gz$")) | select((.name | capture("(?<d>[0-9]{8})\\.tar\\.gz$").d) <= $c)] | sort_by(.name | capture("(?<d>[0-9]{8})\\.tar\\.gz$").d) | last | .name // empty') - if [ -z "$latest_file" ]; then - echo "No tarball found for prefix '$prefix' at or before $cutoff at $base_url/" - exit 1 - fi - echo "Found latest file: $latest_file" - - # Extract version from the filename for environment variable - if [[ "$latest_file" =~ therock-dist-linux-${archive_target}-([0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+)\.tar\.gz ]]; then - rocm_version="${BASH_REMATCH[1]}" - echo "Detected latest ROCm version: $rocm_version" - else - echo "Failed to extract ROCm version from latest file: $latest_file" - echo "Expected pattern: therock-dist-linux-${archive_target}-<version>.tar.gz" - exit 1 - fi - - rocm_url="$base_url/$latest_file" - else - rocm_url="$base_url/therock-dist-linux-${archive_target}-${rocm_version}.tar.gz" - fi - - # Store the version for use in other steps - echo "DETECTED_ROCM_VERSION=$rocm_version" >> $GITHUB_ENV - - echo "Streaming ROCm from: $rocm_url directly to extraction" - - # Create directory if it doesn't exist - sudo mkdir -p /opt/rocm - - # Stream download directly into tar extraction (no intermediate file) - curl -sL "$rocm_url" | sudo tar --use-compress-program=gzip -xf - -C /opt/rocm --strip-components=1 - - - name: Set ROCm environment variables - run: | - echo "Setting ROCm environment variables..." - - # Set environment variables for this step and subsequent steps - echo "HIP_PATH=/opt/rocm" >> $GITHUB_ENV - echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV - echo "HIP_PLATFORM=amd" >> $GITHUB_ENV - echo "HIP_CLANG_PATH=/opt/rocm/llvm/bin" >> $GITHUB_ENV - echo "HIP_INCLUDE_PATH=/opt/rocm/include" >> $GITHUB_ENV - echo "HIP_LIB_PATH=/opt/rocm/lib" >> $GITHUB_ENV - echo "HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode" >> $GITHUB_ENV - - # Update PATH - echo "/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH" >> $GITHUB_PATH - - # Set library paths - echo "LD_LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV - echo "LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:${LIBRARY_PATH:-}" >> $GITHUB_ENV - echo "CPATH=/opt/rocm/include:${CPATH:-}" >> $GITHUB_ENV - echo "PKG_CONFIG_PATH=/opt/rocm/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> $GITHUB_ENV - - echo "ROCm environment variables set successfully" - - # See the Windows job for why the key is per gfx target and ROCm version, and why save: false. - - name: ccache - uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 - with: - key: rocm-linux-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }} - restore-keys: | - rocm-linux-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }} - append-timestamp: false - variant: ccache - max-size: 2G - save: false - - # The action sets this on Windows and macOS but leaves Linux on mtime, and ROCm is re-extracted every run, so mtime is not a reliable compiler identity. - - name: Hash the compiler by content, not mtime - run: ccache --set-config=compiler_check=content - - # The parent's resolve job built the source tree (upstream base + any mix - # PRs, with the build number/commit and Unsloth fingerprint baked - # into cmake/build-info.cmake) and uploaded it as an artifact; extract it - # instead of cloning -- no .git needed, the build number is already baked. - - name: Download source @ ${{ inputs.tag }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: ${{ inputs.source_artifact }} - path: srcpkg - - name: Extract source - shell: bash - run: | - set -eux - mkdir -p llama.cpp - tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C llama.cpp --strip-components=1 - - - name: Build Llama.cpp + ROCm - run: | - # Map GPU targets - current_target="${{ matrix.gfx_target }}" - echo "Input target: $current_target" - - if [ "$current_target" = "gfx110X" ]; then - mapped_target="gfx1100;gfx1101;gfx1102;gfx1103" - elif [ "$current_target" = "gfx103X" ]; then - mapped_target="gfx1030;gfx1031;gfx1032;gfx1034" - elif [ "$current_target" = "gfx1151" ]; then - mapped_target="gfx1151" - elif [ "$current_target" = "gfx1150" ]; then - mapped_target="gfx1150" - elif [ "$current_target" = "gfx120X" ]; then - mapped_target="gfx1200;gfx1201" - else - mapped_target="$current_target" - fi - echo "Mapped target: $mapped_target" - - # Create build directory - cd llama.cpp - mkdir build - cd build - - # Configure the project - # CMAKE_HIP_COMPILER_LAUNCHER is needed here but not on Windows: the - # CXX compiler is ROCm's clang++ (not hipcc), so ggml-hip leaves - # CXX_IS_HIPCC false and calls enable_language(HIP), putting the device - # sources on the HIP language rather than CXX. - cmake .. -G Ninja \ - -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang \ - -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ - -DCMAKE_CXX_FLAGS="-I/opt/rocm/include" \ - -DCMAKE_CROSSCOMPILING=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DGPU_TARGETS="$mapped_target" \ - -DBUILD_SHARED_LIBS=ON \ - -DLLAMA_BUILD_TESTS=OFF \ - -DGGML_HIP=ON \ - -DGGML_OPENMP=OFF \ - -DGGML_CUDA_FORCE_CUBLAS=OFF \ - -DGGML_RPC=ON \ - -DGGML_HIP_ROCWMMA_FATTN=OFF \ - -DLLAMA_BUILD_BORINGSSL=ON \ - -DGGML_NATIVE=OFF \ - -DGGML_STATIC=OFF \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_HIP_COMPILER_LAUNCHER=ccache \ - -DCMAKE_SYSTEM_NAME=Linux - - # Build the project - cmake --build . -j $(nproc) - - - name: Copy ROCm core libs to build directory - run: | - build_bin_path="llama.cpp/build/bin" - rocm_bin_path="/opt/rocm/bin" - - # Copy the rocblas/library folder and all its contents - rocblas_lib_path="/opt/rocm/lib/rocblas/library" - if [ -d "$rocblas_lib_path" ]; then - echo "Copying rocblas/library folder and all contents..." - dest_rocblas_path="$build_bin_path/rocblas/library" - mkdir -p "$(dirname "$dest_rocblas_path")" - cp -r "$rocblas_lib_path" "$(dirname "$dest_rocblas_path")/" - echo "Copied: rocblas/library folder with all contents" - - # List the contents of the copied rocblas/library folder - echo "Contents of rocblas/library:" - find "$dest_rocblas_path" -type f -exec ls -la {} \; | head -20 - else - echo "Warning: rocblas/library folder not found at: $rocblas_lib_path" - fi - - # Copy the hipblaslt/library folder and all its contents - hipblaslt_lib_path="/opt/rocm/lib/hipblaslt/library" - if [ -d "$hipblaslt_lib_path" ]; then - echo "Copying hipblaslt/library folder and all contents..." - dest_hipblaslt_path="$build_bin_path/hipblaslt/library" - mkdir -p "$(dirname "$dest_hipblaslt_path")" - cp -r "$hipblaslt_lib_path" "$(dirname "$dest_hipblaslt_path")/" - echo "Copied: hipblaslt/library folder with all contents" - - # List the contents of the copied hipblaslt/library folder - echo "Contents of hipblaslt/library:" - find "$dest_hipblaslt_path" -type f -exec ls -la {} \; | head -20 - else - echo "Warning: hipblaslt/library folder not found at: $hipblaslt_lib_path" - fi - - # Copy required ROCm libraries to build directory - # If artifacts from ROCm or Llama.cpp change, you may need to update this list - # To get a new list of all libraries, run: - # gather_required_libs.py --rocm-dir /opt/rocm --dest-dir llama.cpp/build/bin - echo "Copying required ROCm libraries to build directory..." - cp -v /opt/rocm/lib/libhipblas.so* "$build_bin_path/" 2>/dev/null || echo "libhipblas.so* not found" - cp -v /opt/rocm/lib/librocblas.so* "$build_bin_path/" 2>/dev/null || echo "librocblas.so* not found" - cp -v /opt/rocm/lib/libamdhip64.so* "$build_bin_path/" 2>/dev/null || echo "libamdhip64.so* not found" - cp -v /opt/rocm/lib/librocsolver.so* "$build_bin_path/" 2>/dev/null || echo "librocsolver.so* not found" - cp -v /opt/rocm/lib/libroctx64.so* "$build_bin_path/" 2>/dev/null || echo "libroctx64.so* not found" - cp -v /opt/rocm/lib/libhipblaslt.so* "$build_bin_path/" 2>/dev/null || echo "libhipblaslt.so* not found" - cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_liblzma.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_liblzma.so* not found" - cp -v /opt/rocm/lib/librocprofiler-register.so* "$build_bin_path/" 2>/dev/null || echo "librocprofiler-register.so* not found" - cp -v /opt/rocm/lib/libamd_comgr.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr.so* not found" - cp -v /opt/rocm/lib/libamd_comgr_loader.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr_loader.so* not found" - cp -v /opt/rocm/lib/libhsa-runtime64.so* "$build_bin_path/" 2>/dev/null || echo "libhsa-runtime64.so* not found" - cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_numa.so* not found" - cp -v /opt/rocm/lib/librocroller.so* "$build_bin_path/" 2>/dev/null || echo "librocroller.so* not found" - cp -v /opt/rocm/lib/liborigami.so* "$build_bin_path/" 2>/dev/null || echo "liborigami.so* not found" - cp -v /opt/rocm/lib/librocm_kpack.so* "$build_bin_path/" 2>/dev/null || echo "librocm_kpack.so* not found" - cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_z.so* not found" - cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_zstd.so* not found" - cp -v /opt/rocm/lib/llvm/lib/libLLVM.so* "$build_bin_path/" 2>/dev/null || echo "libLLVM.so* not found" - cp -v /opt/rocm/lib/llvm/lib/libclang-cpp.so* "$build_bin_path/" 2>/dev/null || echo "libclang-cpp.so* not found" - - cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_elf.so* not found" - cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm.so* not found" - cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm_amdgpu.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm_amdgpu.so* not found" - cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_bz2.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_bz2.so* not found" - - # libatomic.so.1 is a transitive dependency of librocm_sysdeps_numa - # (libhsa-runtime64 -> numa -> libatomic) but it is a system GCC runtime - # library, not shipped under /opt/rocm. Bundle it so the runtime stays - # self-contained on hosts that lack it. lemonade-sdk/lemonade#1349 hit - # exactly this ("libatomic.so.1: cannot open shared object file"). - sudo apt-get install -y libatomic1 >/dev/null 2>&1 || true - libatomic_path="$(ldconfig -p | awk -F'=> ' '/libatomic\.so\.1/{print $2; exit}')" - cp -v "$libatomic_path" "$build_bin_path/" 2>/dev/null || echo "libatomic.so.1 not found" - - echo "Finished copying required ROCm libraries" - - - name: Set RPATH for portable distribution - run: | - sudo apt-get install -y patchelf - cd llama.cpp/build/bin - # Set RPATH to $ORIGIN so all libraries (including the comgr stub loader) find deps locally - for file in *.so* llama-*; do - [ -f "$file" ] && [ ! -L "$file" ] && patchelf --set-rpath '$ORIGIN' "$file" 2>/dev/null || true - done - - - name: List build artifacts (including ROCm files) - run: | - cd llama.cpp/build/bin - echo "Final build artifacts (including ROCm library files):" - ls -la - - - name: Package bundle (tar.gz) - run: | - set -eux - ASSET="app-${{ inputs.tag }}-linux-x64-rocm-${{ matrix.gfx_target }}.tar.gz" - mkdir -p dist - (cd llama.cpp/build/bin && tar -czf "${GITHUB_WORKSPACE}/dist/${ASSET}" .) - ls -la dist - - - name: Upload bundle artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: app-${{ inputs.tag }}-linux-x64-rocm-${{ matrix.gfx_target }} - path: dist/app-${{ inputs.tag }}-linux-x64-rocm-${{ matrix.gfx_target }}.tar.gz - if-no-files-found: error - - # See the Windows job: logs whether the HIP-language TUs actually cache. - - name: ccache stats - continue-on-error: true - run: ccache --show-stats -v - - - name: Evict stale ccache files - # !cancelled(), unlike the save below: on a timeout the job gets a - # single ~5 minute teardown window (measured ~4m50s after process - # kill), shared by every remaining step and not replenished. Evicting - # spends that window on housekeeping; the save is what actually needs - # it, and a 2 GB cache is not quick to write. - if: ${{ !cancelled() }} - continue-on-error: true - run: ccache --evict-older-than 14d - - - name: Save ccache - # Save even when the build failed: the objects compiled before the - # failure are still worth keeping, and a job that saves nothing leaves - # a hole in the cache lineage that widens the next run's tag gap. - # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. - # - # always(), not !cancelled(): a timeout-minutes expiry puts the job on - # the CANCELLATION path, not the failure path, so !cancelled() would - # skip the save on the single most expensive case -- a leg that - # compiled for hours and then hit the cap. - if: ${{ always() }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}/.ccache - key: ccache-rocm-linux-${{ matrix.gfx_target }}-${{ env.DETECTED_ROCM_VERSION }}-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt-vulkan.yml b/.github/workflows/unsloth-prebuilt-vulkan.yml deleted file mode 100644 index 2d1d1631b443..000000000000 --- a/.github/workflows/unsloth-prebuilt-vulkan.yml +++ /dev/null @@ -1,409 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: "Unsloth prebuilt: Vulkan" - -# Reusable child of unsloth-prebuilt.yml. Builds the Vulkan bundles for -# Linux x64/arm64 + Windows x64. Each matrix entry uploads a single -# app-*.{tar.gz|zip} artifact for the parent's assemble step to pick up. -# -# Mirrors oobabooga/llama-cpp-binaries' build-wheels-vulkan.yml build recipe -# (the CPU recipe plus GGML_VULKAN=ON and a Vulkan SDK install), adapted to -# this repo's conventions: app-<tag>-<platform>-<arch>-vulkan archives packaged -# straight from build/bin like the ROCm/macOS children (no embedded -# UNSLOTH_PREBUILT_INFO.json -- assemble_metadata.py derives the manifest entry -# from the filename), MSVC + BoringSSL on Windows, $ORIGIN RPATH on Linux. -# -# The Vulkan loader (libvulkan.so.1 / vulkan-1.dll) is a system/driver library -# resolved by the OS at runtime, so -- like llama-cpp-binaries -- it is not -# bundled; only ggml's own libggml-vulkan backend module ships in the archive. -# Both Linux legs use ubuntu-22.04 to keep a glibc 2.35 / GLIBCXX <= 3.4.30 -# floor. The arm64 leg supplies newer header-only SDK pieces at build time. - -on: - workflow_call: - inputs: - tag: - description: 'Upstream llama.cpp release tag (b####), resolved by parent' - required: true - type: string - repo: - description: 'Source repo (owner/name): ggml-org/llama.cpp for plain builds, or this repo for mix tags' - required: false - default: 'ggml-org/llama.cpp' - type: string - source_artifact: - description: 'Workflow artifact (app-source-*) holding the stamped source tree; set by resolve for every build' - required: false - default: '' - type: string - -permissions: - contents: read - -env: - # LunarG SDK version used by ggml-org's Windows release build. - VULKAN_VERSION: 1.4.357.0 - -jobs: - build-linux: - name: linux/${{ matrix.arch }} - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - { arch: x64, runner: ubuntu-22.04 } - - { arch: arm64, runner: ubuntu-22.04-arm } - steps: - # The parent's resolve job built the source tree (upstream base + any mix - # PRs, with the build number/commit and Unsloth fingerprint baked - # into cmake/build-info.cmake) and uploaded it as an artifact; extract it - # instead of cloning -- no .git needed, the build number is already baked. - - name: Download source @ ${{ inputs.tag }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: ${{ inputs.source_artifact }} - path: srcpkg - - name: Extract source - shell: bash - run: | - set -eux - mkdir -p src - tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 - - - name: Install Vulkan SDK + build dependencies - run: | - set -eux - if [ "${{ matrix.arch }}" = x64 ]; then - wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add - - sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list - sudo apt-get update -y - sudo apt-get install -y build-essential mesa-vulkan-drivers vulkan-sdk libssl-dev ninja-build - else - sudo apt-get update -y - sudo apt-get install -y build-essential libvulkan-dev spirv-headers libssl-dev ninja-build - fi - - # Match the CPU arm64 leg so the bundle retains Jammy's loader and - # libstdc++ floors. Jammy's gcc cannot target armv9.2-a+sme. - - name: Toolchain (clang 19 on arm64) - if: matrix.arch == 'arm64' - run: | - set -eux - wget -q https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 19 - sudo apt-get install -y libomp-19-dev - { - echo "CC=clang-19" - echo "CXX=clang++-19" - } >> "$GITHUB_ENV" - - - name: ccache - uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 - with: - key: vulkan-linux-${{ matrix.arch }}-${{ inputs.tag }} - restore-keys: | - vulkan-linux-${{ matrix.arch }} - append-timestamp: false - variant: ccache - max-size: 2G - save: false - - # Ubuntu 22.04 has an arm64 Vulkan loader, but its headers are too old for - # the current backend and it has no glslc package. Install matching pinned - # headers and build the shader compiler shipped with the LunarG SDK. - - name: Install Vulkan headers (arm64) - if: matrix.arch == 'arm64' - run: | - set -eux - package="$RUNNER_TEMP/vulkan-headers.deb" - staging="$RUNNER_TEMP/vulkan-headers" - curl -fsSL \ - "https://packages.lunarg.com/vulkan/pool/main/v/vulkan-headers/vulkan-headers_1.4.313.0~rc1-1lunarg22.04-1_all.deb" \ - -o "$package" - echo "587b2d8e79416b394170ab61557c98765570cd153730f819a917866d78f45e1a $package" | sha256sum -c - - dpkg-deb -x "$package" "$staging" - sudo cp -a "$staging/usr/." /usr/local/ - - - name: Build glslc (arm64) - if: matrix.arch == 'arm64' - run: | - set -eux - archive="$RUNNER_TEMP/shaderc.tar.gz" - source="$RUNNER_TEMP/shaderc" - curl -fsSL \ - "https://packages.lunarg.com/vulkan/pool/main/s/shaderc/shaderc_2025.2~rc1-1lunarg22.04.orig.tar.gz" \ - -o "$archive" - echo "59c0c478f2f40a076e610587d099e39ed059cb7319fe464f8ba1bd07c6bf02c5 $archive" | sha256sum -c - - mkdir -p "$source" - tar -xzf "$archive" -C "$source" - cmake -S "$source" -B "$source/build" -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DSHADERC_SKIP_TESTS=ON \ - -DSHADERC_SKIP_EXAMPLES=ON \ - -DSHADERC_SKIP_COPYRIGHT_CHECK=ON \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - cmake --build "$source/build" --target glslc_exe -j "$(nproc)" - sudo install -m 0755 "$source/build/glslc/glslc" /usr/local/bin/glslc - glslc --version - - - name: Configure - working-directory: src - run: | - set -eux - # Build recipe mirrors llama-cpp-binaries' Vulkan wheel: the CPU recipe - # (backend-DL + all CPU variants + RPC) plus GGML_VULKAN. RPATH=$ORIGIN - # so the bundle's sibling .so files resolve from the binary's own dir. - # LLAMA_FATAL_WARNINGS below is -Werror. The arm64 image compiles with - # clang-19 against GCC 12's libstdc++, where std::stable_sort still - # reaches the deprecated get_temporary_buffer; GCC buries that in a - # system header, clang reports it at our instantiation. That failed - # this leg on 08-27 over a deprecation in code we do not own, so that - # one diagnostic is off. Every other warning stays fatal. - cmake -S . -B build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DGGML_NATIVE=OFF \ - -DGGML_BACKEND_DL=ON \ - -DGGML_CPU_ALL_VARIANTS=ON \ - -DGGML_RPC=ON \ - -DGGML_VULKAN=ON \ - -DLLAMA_FATAL_WARNINGS=ON \ - -DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations \ - -DLLAMA_BUILD_TESTS=OFF \ - -DLLAMA_BUILD_EXAMPLES=OFF \ - -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_SERVER=ON \ - -DCMAKE_INSTALL_RPATH='$ORIGIN' \ - -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - - name: Build - working-directory: src - run: | - set -eux - # Backend modules (CPU variants, Vulkan, RPC) build as ggml - # dependencies, so every tool pulls them in. - cmake --build build --config Release -j "$(nproc)" - strip build/bin/llama-* || true - - - name: Bundle OpenMP runtime (arm64) - if: matrix.arch == 'arm64' - run: cp /usr/lib/llvm-19/lib/libomp.so.5 src/build/bin/ - - # DiffusionGemma binaries (example targets present only in #24423 mix - # builds): best-effort, never fail the job. See the CUDA child for the - # rationale. The bundle tars all of build/bin, so anything produced here - # is shipped automatically. - - name: Build DiffusionGemma binaries (best-effort; mix builds only) - working-directory: src - run: | - set -u - if [ ! -d examples/diffusion-gemma-server ]; then - echo "no DiffusionGemma sources in this tree; skipping" - exit 0 - fi - cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON \ - || { echo "reconfigure for examples failed; skipping DiffusionGemma binaries"; exit 0; } - if cmake --build build --config Release -j "$(nproc)" \ - --target llama-diffusion-gemma-visual-server llama-diffusion-cli; then - strip build/bin/llama-diffusion-gemma-visual-server build/bin/llama-diffusion-cli || true - echo "built DiffusionGemma binaries" - else - echo "warning: DiffusionGemma binaries failed to build; bundle will omit them" - fi - exit 0 - - - name: Package bundle (tar.gz) - run: | - set -eux - ASSET="app-${{ inputs.tag }}-linux-${{ matrix.arch }}-vulkan.tar.gz" - cp src/LICENSE src/build/bin/ - mkdir -p dist - (cd src/build/bin && tar -czf "${GITHUB_WORKSPACE}/dist/${ASSET}" .) - ls -la dist - - - name: Upload bundle artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: app-${{ inputs.tag }}-linux-${{ matrix.arch }}-vulkan - path: dist/app-${{ inputs.tag }}-linux-${{ matrix.arch }}-vulkan.tar.gz - if-no-files-found: error - - - name: Evict stale ccache files - # !cancelled(), unlike the save below: on a timeout the job gets a - # single ~5 minute teardown window (measured ~4m50s after process - # kill), shared by every remaining step and not replenished. Evicting - # spends that window on housekeeping; the save is what actually needs - # it, and a 2 GB cache is not quick to write. - if: ${{ !cancelled() }} - continue-on-error: true - run: ccache --evict-older-than 14d - - - name: Save ccache - # Save even when the build failed: the objects compiled before the - # failure are still worth keeping, and a job that saves nothing leaves - # a hole in the cache lineage that widens the next run's tag gap. - # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. - # - # always(), not !cancelled(): a timeout-minutes expiry puts the job on - # the CANCELLATION path, not the failure path, so !cancelled() would - # skip the save on the single most expensive case -- a leg that - # compiled for hours and then hit the cap. - if: ${{ always() }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}/.ccache - key: ccache-vulkan-linux-${{ matrix.arch }}-${{ inputs.tag }}- - - build-windows: - name: windows/x64 - runs-on: windows-2022 - defaults: - run: - shell: pwsh - steps: - # The parent's resolve job built the source tree (upstream base + any mix - # PRs, with the build number/commit and Unsloth fingerprint baked - # into cmake/build-info.cmake) and uploaded it as an artifact; extract it - # instead of cloning -- no .git needed, the build number is already baked. - - name: Download source @ ${{ inputs.tag }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - name: ${{ inputs.source_artifact }} - path: srcpkg - - name: Extract source - shell: bash - run: | - set -eux - mkdir -p src - tar -xzf "srcpkg/llama.cpp-source-${{ inputs.tag }}.tar.gz" -C src --strip-components=1 - - - name: ccache - uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03 # v1.2.23 - with: - key: vulkan-windows-x64-${{ inputs.tag }} - restore-keys: | - vulkan-windows-x64 - append-timestamp: false - variant: ccache - max-size: 2G - save: false - - - name: Install Ninja - run: choco install ninja --no-progress - - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 - - - name: Install Vulkan SDK - run: | - curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe" - & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install - Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}" - Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin" - - - name: Configure - working-directory: src - run: | - # Same recipe as Linux plus BoringSSL (statically linked, no system - # OpenSSL on the runner). No RPATH -- Windows loads sibling DLLs from - # the binary's directory. CMAKE_PREFIX_PATH points at the Vulkan SDK - # so CMake finds its SPIRV-Headers config (set via env, not -D, so the - # backslashes survive); mirrors llama-cpp-binaries' Vulkan wheel. - # GGML_OPENMP=OFF keeps this MSVC bundle self-contained: a default-ON - # MSVC build would link vcomp140.dll (not shipped). Upstream sidesteps - # this by building its Windows Vulkan artifact with GGML_CPU=OFF (no - # OpenMP at all); we ship a full bundle, so we disable OpenMP instead - # (the CPU backend is a GPU-offload fallback and uses ggml's threadpool). - $env:CMAKE_PREFIX_PATH = $env:VULKAN_SDK - cmake -S . -B build -G Ninja ` - -DCMAKE_BUILD_TYPE=Release ` - -DGGML_NATIVE=OFF ` - -DGGML_BACKEND_DL=ON ` - -DGGML_CPU_ALL_VARIANTS=ON ` - -DGGML_OPENMP=OFF ` - -DGGML_RPC=ON ` - -DGGML_VULKAN=ON ` - -DLLAMA_BUILD_TESTS=OFF ` - -DLLAMA_BUILD_EXAMPLES=OFF ` - -DLLAMA_BUILD_TOOLS=ON ` - -DLLAMA_BUILD_SERVER=ON ` - -DLLAMA_BUILD_BORINGSSL=ON ` - -DCMAKE_C_COMPILER_LAUNCHER=ccache ` - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - - name: Build - working-directory: src - run: | - cmake --build build --config Release -j 3 - - # DiffusionGemma binaries (#24423 mix builds only): best-effort, never - # fail the job. See the CUDA child for the rationale. - - name: Build DiffusionGemma binaries (best-effort; mix builds only) - working-directory: src - run: | - if (-not (Test-Path "examples/diffusion-gemma-server")) { - Write-Host "no DiffusionGemma sources in this tree; skipping" - exit 0 - } - cmake -S . -B build -DLLAMA_BUILD_EXAMPLES=ON - if ($LASTEXITCODE -ne 0) { - Write-Host "reconfigure for examples failed; skipping DiffusionGemma binaries" - exit 0 - } - cmake --build build --config Release -j 3 ` - --target llama-diffusion-gemma-visual-server llama-diffusion-cli - if ($LASTEXITCODE -ne 0) { - Write-Host "warning: DiffusionGemma binaries failed to build; bundle will omit them" - } else { - Write-Host "built DiffusionGemma binaries" - } - exit 0 - - - name: Package bundle (zip) - shell: bash - run: | - set -eux - ASSET="app-${{ inputs.tag }}-windows-x64-vulkan.zip" - cp src/LICENSE src/build/bin/ - mkdir -p dist - (cd src/build/bin && 7z a -tzip "${GITHUB_WORKSPACE}/dist/${ASSET}" .) - ls -la dist - - - name: Upload bundle artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: app-${{ inputs.tag }}-windows-x64-vulkan - path: dist/app-${{ inputs.tag }}-windows-x64-vulkan.zip - if-no-files-found: error - - - name: Evict stale ccache files - # !cancelled(), unlike the save below: on a timeout the job gets a - # single ~5 minute teardown window (measured ~4m50s after process - # kill), shared by every remaining step and not replenished. Evicting - # spends that window on housekeeping; the save is what actually needs - # it, and a 2 GB cache is not quick to write. - if: ${{ !cancelled() }} - continue-on-error: true - run: ccache --evict-older-than 14d - - - name: Save ccache - # Save even when the build failed: the objects compiled before the - # failure are still worth keeping, and a job that saves nothing leaves - # a hole in the cache lineage that widens the next run's tag gap. - # Measured: gap 7 -> 68% hit rate, gap 13 -> 6%. - # - # always(), not !cancelled(): a timeout-minutes expiry puts the job on - # the CANCELLATION path, not the failure path, so !cancelled() would - # skip the save on the single most expensive case -- a leg that - # compiled for hours and then hit the cap. - if: ${{ always() }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}\.ccache - key: ccache-vulkan-windows-x64-${{ inputs.tag }}- diff --git a/.github/workflows/unsloth-prebuilt.yml b/.github/workflows/unsloth-prebuilt.yml deleted file mode 100644 index 834ff5449093..000000000000 --- a/.github/workflows/unsloth-prebuilt.yml +++ /dev/null @@ -1,1317 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: Unsloth prebuilt (full release) - -# Atomic daily build for unslothai/llama.cpp. One cron, one workflow run, one -# release per upstream b#### tag. Splits the heavy build work into six reusable -# children: -# unsloth-prebuilt-cuda.yml -- Linux CUDA bundles (x64 + arm64, matrix profiles) -# unsloth-prebuilt-cuda-windows.yml -- CUDA Windows bundles (x64, matrix profiles) -# unsloth-prebuilt-rocm.yml -- ROCm bundles (Windows + Ubuntu, per gfx target) -# unsloth-prebuilt-macos.yml -- macOS bundles (arm64 Metal + x64 CPU) -# unsloth-prebuilt-cpu.yml -- CPU-only bundles (Linux + Windows, x64 + arm64) -# unsloth-prebuilt-vulkan.yml -- Vulkan bundles (Linux x64/arm64 + Windows x64) -# -# Atomicity: the `assemble` job depends on all children. GitHub's default -# `needs` semantics require all needs to succeed -- if any matrix entry in -# any child fails, `assemble` skips and no release is published. The -# installer needs the full bundle set at the same tag or it'll dispatch to -# something that isn't there. -# -# Mix builds: scripts/unsloth/pr-set.json can list ggml-org/llama.cpp or -# unslothai/llama.cpp PRs (each pinned to an exact commit) to merge into the -# build. `resolve` merges the open ones onto the base tag and uploads the -# merged tree as a workflow artifact that the children extract instead of -# cloning; the release is tagged b####-mix-<hash>. Empty list = vanilla -# upstream build. - -on: - schedule: - - cron: '13 20 * * *' # ~1PM San Francisco PDT / ~12PM PST (UTC; not DST adjusted) - workflow_dispatch: - inputs: - tag: - description: 'ggml-org tag (b#### or "latest")' - default: 'latest' - required: true - type: string - min_age_hours: - description: 'For "latest": only build a release public for at least this many hours (blank = default 6)' - default: '' - required: false - type: string - only_profile: - description: 'CUDA profile to build' - default: 'all' - required: false - type: choice - options: [all, cuda12-legacy, cuda12-older, cuda12-newer, cuda12-portable, cuda13-older, cuda13-newer, cuda13-portable] - operating_systems: - description: 'OSes for ROCm builds' - default: 'windows,ubuntu' - required: false - type: string - gfx_target: - description: 'GPU targets for ROCm builds' - default: 'gfx1151,gfx1150,gfx120X,gfx110X,gfx103X,gfx90a,gfx908' - required: false - type: string - rocm_version: - description: 'ROCm version, "weekly" (newest alpha as of the last SF Sunday) or "latest"' - default: 'weekly' - required: false - type: string - publish: - description: 'Publish to GitHub Releases' - default: false - required: false - type: boolean - keep_artifacts: - description: 'Keep this run''s artifacts even if it publishes nothing (artifact-only test runs)' - default: false - required: false - type: boolean - -permissions: - contents: write - -env: - # Supply-chain aging: when resolving "latest", only build an upstream release - # that has been public for at least this many hours, so a malicious or broken - # release has time to be caught and yanked before we compile and ship it. An - # explicit b#### tag (manual run) skips it. Per-run override via min_age_hours. - UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS: "6" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.tag || 'scheduled' }} - # A nightly still in flight when the next one starts is stuck, not busy: a - # healthy run finishes in a couple of hours. Leaving this false lets one - # wedged run block every following nightly with nothing to show for it. - # Only the schedule supersedes; a manual dispatch never kills a live nightly. - cancel-in-progress: ${{ github.event_name == 'schedule' }} - -jobs: - resolve: - name: Resolve tag - runs-on: ubuntu-24.04 - # Read-only: resolve merges third-party PR content but pushes nothing; - # only assemble needs the workflow-level contents:write (publish). - permissions: - contents: read - outputs: - tag: ${{ steps.r.outputs.tag }} - repo: ${{ steps.r.outputs.repo }} - base: ${{ steps.r.outputs.base }} - prs: ${{ steps.r.outputs.prs }} - source_artifact: ${{ steps.r.outputs.source_artifact }} - commit: ${{ steps.r.outputs.commit }} - ggml_tree: ${{ steps.r.outputs.ggml_tree }} - ggml_version: ${{ steps.r.outputs.ggml_version }} - exists: ${{ steps.r.outputs.exists }} - cuda_matrix: ${{ steps.r.outputs.cuda_matrix }} - win_cuda_matrix: ${{ steps.r.outputs.win_cuda_matrix }} - rocm_matrix: ${{ steps.r.outputs.rocm_matrix }} - rocm_cutoff: ${{ steps.r.outputs.rocm_cutoff }} - macos_matrix: ${{ steps.r.outputs.macos_matrix }} - env: - GH_TOKEN: ${{ github.token }} - steps: - # Shallow by default (only pr-set.json is needed); a mix build unshallows - # in-place to merge the PRs. - - name: Checkout (pr-set.json + mix merge workspace) - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - id: r - run: | - set -euo pipefail - REQ='${{ github.event.inputs.tag || 'latest' }}' - ONLY='${{ github.event.inputs.only_profile || 'all' }}' - GFX_DEFAULT='gfx1151,gfx1150,gfx120X,gfx110X,gfx103X,gfx90a,gfx908' - GFX='${{ github.event.inputs.gfx_target }}'; GFX="${GFX:-$GFX_DEFAULT}" - OS_LIST='${{ github.event.inputs.operating_systems || 'windows,ubuntu' }}' - AGE_H='${{ github.event.inputs.min_age_hours }}' - - # publish does delete-then-create on the whole release, and the installer - # treats the published manifest as the authoritative bundle set: a partial - # build drops bundles and silently downgrades uncovered hosts to slow - # source builds. Refuse to publish unless the full default set is built; - # publish=false test runs may use subsets. - if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ inputs.publish }}" = "true" ]; then - [ "$ONLY" = "all" ] || { echo "refusing to publish only_profile=$ONLY: a partial CUDA set clobbers manifest coverage. Use only_profile=all to publish, or publish=false for an artifact-only test." >&2; exit 1; } - [ "$GFX" = "$GFX_DEFAULT" ] || { echo "refusing to publish gfx_target='$GFX': publish requires the full default set ($GFX_DEFAULT); use publish=false for a subset test." >&2; exit 1; } - case "$OS_LIST" in - *windows*ubuntu*|*ubuntu*windows*) : ;; - *) echo "refusing to publish operating_systems='$OS_LIST': both windows and ubuntu ROCm bundles are required. Use publish=false for a subset test." >&2; exit 1 ;; - esac - fi - [ -n "$AGE_H" ] || AGE_H="${UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS:-6}" - if [ "$REQ" = "latest" ]; then - # Newest published b#### build release that has been public for - # >= AGE_H hours -- the supply-chain aging window. GitHub does not - # guarantee the list order, so pick the max by published_at - # explicitly rather than trusting `first`. - # Select on the tag shape, not on prerelease: since 08-21 upstream - # marks the b#### builds prerelease and keeps that flag clear only - # for the semver v#.#.# releases, which are not what we build. - CUTOFF="$(date -u -d "-${AGE_H} hours" +%s)" - BASE="$(gh api 'repos/ggml-org/llama.cpp/releases?per_page=100' \ - | jq -r --argjson cutoff "$CUTOFF" '[.[] | select(.draft==false) | select(.tag_name|test("^b[0-9]+$")) | select((.published_at|fromdateiso8601) <= $cutoff)] | max_by(.published_at|fromdateiso8601) | .tag_name')" - if [ -z "$BASE" ] || [ "$BASE" = "null" ]; then - echo "refusing: no ggml-org b#### release older than ${AGE_H}h in the last 100 releases" >&2 - exit 1 - fi - echo "selected $BASE (aged >= ${AGE_H}h)" - else - BASE="$REQ" # explicit b#### override skips the aging filter - fi - printf '%s' "$BASE" | grep -qE '^b[0-9]+$' || { echo "refusing non-release tag '$BASE'" >&2; exit 1; } - - # Resolve the PR mix set (scripts/unsloth/pr-set.json): PR commit - # urls from ggml-org/llama.cpp or unslothai/llama.cpp (no other - # repos). Pins are mandatory -- only an exact, reviewed commit - # is ever built, so a PR author pushing more commits cannot change - # what the nightly ships. Non-open PRs are still merged in -- upstream - # tags lag merges, so dropping a pin on merge leaves the arch in - # neither the base nor the mix; see the state gate below and the - # pr-set.json _doc. unsloth-pr-set-lint.yml runs the same checks on every - # push that edits the file, but that is only a tripwire -- a red - # lint does not stop the schedule, so the gate must live here. - # An entry is a bare url string (required) or {"url", "required": false}. - # Bare strings stay valid, so the file needs no migration. - jq -e '.prs | type == "array" and all(.[]; - type == "string" - or (type == "object" and (.url | type == "string") - and ((if .required == null then true else .required end) | type == "boolean")))' \ - scripts/unsloth/pr-set.json >/dev/null \ - || { echo "scripts/unsloth/pr-set.json: .prs must be an array of PR url strings, or {url, required} objects" >&2; exit 1; } - PRS='[]' - URL_RE='^https://github\.com/(ggml-org|unslothai)/llama\.cpp/pull/([0-9]+)/commits/([0-9a-f]{40})/?$' - while read -r url REQUIRED; do - [[ "$url" =~ $URL_RE ]] || { echo "refusing malformed PR url '$url' (expected https://github.com/{ggml-org,unslothai}/llama.cpp/pull/<n>/commits/<40-hex-sha>)" >&2; exit 1; } - SRC="${BASH_REMATCH[1]}/llama.cpp"; NUM="${BASH_REMATCH[2]}"; SHA="${BASH_REMATCH[3]}" - # Abort naming the entry on a gh failure (typo'd numbers 404). - # Title can contain spaces, so it can't ride a space-delimited - # read -- pull each field out on its own. - PR_JSON="$(gh api "repos/${SRC}/pulls/${NUM}")" \ - || { echo "refusing ${SRC}#${NUM}: could not fetch PR metadata (nonexistent PR number in '$url', or a transient API failure); fix the pin in scripts/unsloth/pr-set.json or retry" >&2; exit 1; } - STATE="$(jq -r '.state' <<<"$PR_JSON")" - HEAD="$(jq -r '.head.sha' <<<"$PR_JSON")" - N_COMMITS="$(jq -r '.commits' <<<"$PR_JSON")" - TITLE="$(jq -r '.title' <<<"$PR_JSON")" - MERGED_AT="$(jq -r '.merged_at // ""' <<<"$PR_JSON")" - if [ "$STATE" != "open" ]; then - # Merging upstream does NOT put an arch in the build: upstream tags - # lag their merges, and BASE is then aged a further - # UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS on top of that. 26841 merged - # at 11:07 with the newest tag cut at 07:53, so dropping the pin on - # merge left the arch in neither the base nor the mix, and repinning - # was impossible because the old gate refused a non-open pin. So a - # non-open pin keeps being merged: once BASE contains the commit the - # merge is an empty no-op, and the pin can be deleted at leisure. - # "required": false keeps the old rot-away behaviour for an entry - # that should disappear the moment it stops being open. - if [ "$REQUIRED" = "false" ]; then - echo "::warning::skipping optional pin ${SRC}#${NUM} (${STATE}): $url" - continue - fi - if [ -n "$MERGED_AT" ]; then - echo "::warning::${SRC}#${NUM} merged upstream at ${MERGED_AT}; still mixing its pinned commit until a base tag contains it" - else - # Closed unmerged means upstream declined it. Nothing stops it - # shipping now, so the warning is the only signal -- drop the pin - # once you have decided you do not want that code. - echo "::warning::${SRC}#${NUM} is closed without being merged (upstream declined it); still mixing its pinned commit because the entry is required" - fi - fi - # A pin pasted from the wrong PR would build arbitrary code while - # the manifest blames PR #<num>; require the pinned commit to be - # a commit of that PR. The commits listing is capped at 250 by - # the API; past that, skip rather than false-fail. - if [ "$N_COMMITS" -gt 250 ]; then - echo "note: ${SRC}#${NUM} has ${N_COMMITS} commits (over the API listing cap); skipping pin membership check" - elif ! gh api "repos/${SRC}/pulls/${NUM}/commits" --paginate --jq '.[].sha' | grep -qx "$SHA"; then - echo "refusing ${SRC}#${NUM}: pinned commit ${SHA} is not a commit of that PR (wrong paste, or force-pushed away); fix the pin in scripts/unsloth/pr-set.json" >&2 - exit 1 - fi - [ "$SHA" = "$HEAD" ] || echo "note: ${SRC}#${NUM} is pinned to ${SHA} but its head has moved to ${HEAD}" - echo "including ${SRC}#${NUM} @ ${SHA}" - PRS="$(jq -c --arg r "$SRC" --arg n "$NUM" --arg s "$SHA" --arg u "$url" --arg t "$TITLE" '. + [{repo: $r, number: ($n|tonumber), sha: $s, url: $u, title: $t}]' <<<"$PRS")" - done < <(jq -r '.prs[] | if type == "string" then {url: ., required: true} else . end - | "\(.url)\t\(if .required == null then true else .required end)"' scripts/unsloth/pr-set.json | tr '\t' ' ') - - # Decide the tag and source repo first; the source tree is built - # further down, only if this release isn't already published. - if [ "$(jq length <<<"$PRS")" = 0 ]; then - # Plain build: pristine upstream base, no merge. REPO stays the real - # upstream repo even though we ship our own stamped tarball. - TAG="$BASE" - REPO='ggml-org/llama.cpp' - else - # The synthetic tag embeds a hash of the pinned repo#number:sha - # triples in listed order (merge order can matter for conflicts), so - # a pin update or a reorder yields a new tag and build, while the - # "exists" check below still skips rebuilding an already-published - # set. The repo is part of the key so PR #n in the two repos can't - # hash to the same set. The merged tree exists only as this repo's - # release assets, never upstream. - SETHASH="$(jq -r 'map("\(.repo)#\(.number):\(.sha)") | join("\n")' <<<"$PRS" | sha256sum | cut -c1-7)" - TAG="${BASE}-mix-${SETHASH}" - REPO="$GITHUB_REPOSITORY" - fi - SRC_ARTIFACT="app-source-${TAG}" - COMMIT="" - GGML_TREE="" - GGML_VERSION="" - - # Only PUBLISHED releases count as "exists"; a leftover draft (from - # a failed prior publish) should not block today's rebuild. - EXISTS=false - if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft 2>/dev/null || true)" = "false" ]; then - EXISTS=true - fi - - # Build the source tree every child compiles, ONCE, here -- but only - # when something will actually be built. On a scheduled no-op (the aged - # "latest" was already published) skip the whole checkout/merge/tar + - # upload; a manual dispatch always rebuilds. Check out the upstream base, - # merge any pinned PRs (mix builds), then bake the build number/commit - # and the Unsloth fingerprint into cmake/build-info.cmake. The - # result is uploaded as the app-source-* artifact every child extracts -- - # one identical tree everywhere, no child clones, fingerprint patch in one - # place. Nothing is pushed (GITHUB_TOKEN may never push commits touching - # .github/workflows, which upstream history routinely does). - if [ "$EXISTS" != "true" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - git remote add upstream https://github.com/ggml-org/llama.cpp.git - # The upstream checkout below takes scripts/unsloth/ away. Copy the - # whole dir out, not file by file: see the note above the step. - cp -r scripts/unsloth "${RUNNER_TEMP}/us" - ADDITIVE_MERGE="${RUNNER_TEMP}/us/additive_merge.py" - if [ "$(jq length <<<"$PRS")" != 0 ]; then - # Merges need a merge-base, so unshallow first. - if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then - git fetch -q --unshallow --no-tags origin - fi - git fetch -q --no-tags upstream "refs/tags/${BASE}:refs/tags/${BASE}" - git checkout -q --detach "refs/tags/${BASE}" - # Each pin is fetched from the repo its PR lives in (repo/number/ - # sha are machine fields, so the space-delimited read is safe; - # titles stay out of this loop). - while read -r SRC NUM SHA; do - # The PR's own repo first, then our refs/pins mirror, which is - # the only copy left once an author force-pushes the reviewed - # commit out of the PR. That is what took the nightly down on - # 07-31, and it is unrecoverable without a ref of our own. - git fetch -q --no-tags "https://github.com/${SRC}.git" "$SHA" 2>/dev/null \ - || git fetch -q --no-tags origin "refs/pins/${SHA}" 2>/dev/null \ - || { echo "could not fetch commit ${SHA} for ${SRC}#${NUM}; it is gone from ${SRC} and was never mirrored to refs/pins -- update or remove the pin in scripts/unsloth/pr-set.json" >&2; exit 1; } - if ! git rev-parse --verify -q "${SHA}^{commit}" >/dev/null; then - echo "fetched something for ${SRC}#${NUM} but ${SHA} is still missing" >&2; exit 1 - fi - # diff3 so additive_merge.py can see the merge base and refuse - # anything that is not a pure add/add. - if ! git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ - -c merge.conflictStyle=diff3 \ - merge --no-ff --no-edit -m "Merge ${SRC}#${NUM} @ ${SHA}" "$SHA"; then - # The recurring conflict is two PRs adding a line to the same - # architecture table, where the answer is always "keep both". - # additive_merge.py resolves only that, and refuses when - # either side edited existing text, so a real disagreement - # still hard-fails here rather than being papered over. - if python3 "$ADDITIVE_MERGE" \ - && [ -z "$(git diff --name-only --diff-filter=U)" ]; then - git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ - commit -q --no-edit - echo "::warning::${SRC}#${NUM} needed an additive merge; every conflict was a pure add/add and both sides were kept" - else - git merge --abort 2>/dev/null - echo "${SRC}#${NUM} (${SHA}) does not merge cleanly onto ${BASE} + the PRs listed before it; reorder or drop it in scripts/unsloth/pr-set.json" >&2; exit 1 - fi - fi - done < <(jq -r '.[] | "\(.repo) \(.number) \(.sha)"' <<<"$PRS") - echo "MERGED_PINS=1" >> "$GITHUB_ENV" - else - # Plain build: only the base tree is needed. Shallow is fine -- the - # build number is baked below, so no git history is needed at build time. - git fetch -q --depth 1 --no-tags upstream "refs/tags/${BASE}:refs/tags/${BASE}" - git checkout -q --detach "refs/tags/${BASE}" - fi - COMMIT="$(git rev-parse HEAD)" - # ABI key for anything compiled against our ggml (whisper.cpp slim - # bundles). The tree id changes only when ggml/ contents change, so - # a release that touches nothing under ggml/ does not force a - # rebuild downstream. The -mix- tag suffix is a hash of the PR set, - # not a ggml identity: it stays constant while the base tag moves. - GGML_TREE="$(git rev-parse HEAD:ggml)" - GGML_VERSION="$(sed -nE 's/^set\(GGML_VERSION_(MAJOR|MINOR|PATCH) ([0-9]+)\)$/\2/p' ggml/CMakeLists.txt | paste -sd. -)" - # Warn, do not fail: these only tighten downstream pairing, and - # whisper falls back to the old comparison when they are absent. - # Killing a 39-job release over a metadata field would be worse - # than publishing without it. - printf '%s' "$GGML_TREE" | grep -qE '^[0-9a-f]{40}$' \ - || { echo "::warning::could not resolve the ggml tree id"; GGML_TREE=""; } - printf '%s' "$GGML_VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' \ - || { echo "::warning::could not parse GGML_VERSION from ggml/CMakeLists.txt"; GGML_VERSION=""; } - echo "ggml tree ${GGML_TREE:-unknown} (version ${GGML_VERSION:-unknown})" - - # The tarball ships without .git, where cmake/build-info.cmake falls - # back to its defaults; bake real values into those defaults so - # llama-server --version doesn't report b0 (unknown). The build number - # stays the BASE's b#### number (the upstream tag's commit count), so - # anything comparing versions against upstream releases keeps working; - # BUILD_COMMIT identifies the (possibly merged) head. - COUNT="${BASE#b}" - SHORT="$(git rev-parse --short HEAD)" - sed -i "s/^set(BUILD_NUMBER 0)$/set(BUILD_NUMBER ${COUNT})/" cmake/build-info.cmake - sed -i "s/^set(BUILD_COMMIT \"unknown\")$/set(BUILD_COMMIT \"${SHORT}\")/" cmake/build-info.cmake - grep -q "set(BUILD_NUMBER ${COUNT})" cmake/build-info.cmake \ - && grep -q "set(BUILD_COMMIT \"${SHORT}\")" cmake/build-info.cmake \ - || { echo "cmake/build-info.cmake no longer has the expected fallback lines; cannot bake the build number into the source artifact" >&2; exit 1; } - - # Unsloth fingerprint: fold "Compiled by the Unsloth team" - # into BUILD_TARGET, which cmake bakes into LLAMA_BUILD_TARGET, so it - # prints on llama-server --version ("built with ... for ...") and shows - # up in `strings` on every binary that links common. Append so it wraps - # whatever BUILD_TARGET the build computes. Keep this string byte-identical - # to MARK in the assemble job's verify gate, which re-checks it landed. - grep -q 'set(BUILD_TARGET' cmake/build-info.cmake \ - || { echo "cmake/build-info.cmake has no BUILD_TARGET line to stamp" >&2; exit 1; } - printf '\n# Unsloth fingerprint: shows in --version and strings.\nset(BUILD_TARGET "${BUILD_TARGET} (Compiled by the Unsloth team)")\n' >> cmake/build-info.cmake - grep -q 'Compiled by the Unsloth team' cmake/build-info.cmake \ - || { echo "failed to stamp the Unsloth fingerprint into cmake/build-info.cmake" >&2; exit 1; } - - tar -czf "${RUNNER_TEMP}/llama.cpp-source-${TAG}.tar.gz" --exclude-vcs --transform "s,^\.,llama.cpp-${TAG}," . - echo "prepared ${TAG} (${COMMIT}, build ${COUNT})" - else - echo "release ${TAG} already published; skipping source prep (nothing to build)" - fi - - # ubuntu-22.04 is x64 (glibc 2.35). arm64 only has ubuntu-24.04-arm - # available on GitHub-hosted runners (glibc 2.39). arm64 is cuda13-only - # (no cuda12 SBSA) and ships the single "portable" coverage class. - # cuda12 installs via Jimver; cuda13 is pinned to 13.3 (matching - # upstream) which Jimver lacks, so those install via NVIDIA redist in - # the build children -- on the same runners, so the glibc floor holds. - ALL='[ - {"profile":"cuda12-legacy", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"legacy", "rank":5, "cuda":"12.8.0","toolkit_line":"12.8","archs":"50-virtual 61-virtual","sms":"50 52 60 61"}, - {"profile":"cuda12-older", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"older", "rank":10,"cuda":"12.8.0","toolkit_line":"12.8","archs":"70 75 80 86 89"}, - {"profile":"cuda12-newer", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"newer", "rank":20,"cuda":"12.8.0","toolkit_line":"12.8","archs":"86 89 90 100 120"}, - {"profile":"cuda12-portable","arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"portable","rank":30,"cuda":"12.8.0","toolkit_line":"12.8","archs":"70 75 80 86 89 90 100 120"}, - {"profile":"cuda13-older", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda13","klass":"older", "rank":40,"cuda":"13.3","toolkit_line":"13.3","archs":"75 80 86 89"}, - {"profile":"cuda13-newer", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda13","klass":"newer", "rank":50,"cuda":"13.3","toolkit_line":"13.3","archs":"86 89 90 100 120"}, - {"profile":"cuda13-portable","arch":"x64", "runner":"ubuntu-22.04", "line":"cuda13","klass":"portable","rank":60,"cuda":"13.3","toolkit_line":"13.3","archs":"75 80 86 89 90 100 120"}, - {"profile":"cuda13-portable","arch":"arm64","runner":"ubuntu-24.04-arm","line":"cuda13","klass":"portable","rank":60,"cuda":"13.3","toolkit_line":"13.3","archs":"90 100 120 121"} - ]' - if [ "$ONLY" = "all" ]; then - CUDA_INCLUDE="$(echo "$ALL" | jq -c .)" - else - CUDA_INCLUDE="$(echo "$ALL" | jq -c --arg p "$ONLY" '[.[] | select(.profile==$p)]')" - fi - # CUDA Windows reuses the x64 profiles (same arch lists / CUDA - # versions), just on a Windows runner. arm64 has no CUDA Windows target. - WIN_CUDA_INCLUDE="$(echo "$CUDA_INCLUDE" | jq -c '[.[] | select(.arch=="x64") | .runner="windows-2022"]')" - [ -n "$GFX" ] || { echo "refusing empty gfx_target (would publish a CUDA-only release labeled CUDA + ROCm)" >&2; exit 1; } - ROCM_MATRIX="$(jq -cn --arg g "$GFX" '{gfx_target: ($g | split(",") | map(gsub("^\\s+|\\s+$"; "")))}')" - - # One weekly cutoff for the whole run, before fan-out. See the ROCm child for why it is the Saturday before. - # Each leg reads its own clock otherwise, so a run crossing the SF Sat-to-Sun boundary would mix two toolchains in one release. - ROCM_CUTOFF="$(TZ=America/Los_Angeles date -d "-$(( $(TZ=America/Los_Angeles date +%w) + 1 )) days" +%Y%m%d)" - - # macOS slices are static: two fixed runners with per-slice deployment - # targets. arm64 builds on macos-26 (newest Metal SDK; avoids the - # M5/A19 "error compiling source" the macos-14 SDK emits) while both - # slices pin 13.3 to match upstream's Ventura compatibility floor. - MACOS_INCLUDE='[ - {"build":"arm64","runner":"macos-26", "expect_arch":"arm64", "deploy_target":"13.3","defines":"-DGGML_METAL_EMBED_LIBRARY=ON"}, - {"build":"x64", "runner":"macos-15-intel","expect_arch":"x86_64","deploy_target":"13.3","defines":"-DGGML_METAL=OFF"} - ]' - MACOS_INCLUDE="$(echo "$MACOS_INCLUDE" | jq -c .)" - - { - echo "tag=$TAG" - echo "repo=$REPO" - echo "base=$BASE" - echo "prs=$PRS" - echo "source_artifact=$SRC_ARTIFACT" - echo "commit=$COMMIT" - echo "ggml_tree=$GGML_TREE" - echo "ggml_version=$GGML_VERSION" - echo "exists=$EXISTS" - echo "cuda_matrix={\"include\":$CUDA_INCLUDE}" - echo "win_cuda_matrix={\"include\":$WIN_CUDA_INCLUDE}" - echo "rocm_matrix=$ROCM_MATRIX" - echo "rocm_cutoff=$ROCM_CUTOFF" - echo "macos_matrix={\"include\":$MACOS_INCLUDE}" - } >> "$GITHUB_OUTPUT" - echo "Resolved $REQ -> $TAG ($COMMIT); prs=$PRS; source_artifact=${SRC_ARTIFACT:-none}; release exists=$EXISTS; only=$ONLY; gfx=$GFX" - - # A bad pin resolution can still build fine, so it must be caught before the source artifact ships. See merge_checks.py. - # Its own step, not more script in `resolve`: GitHub caps one workflow string at 21000 chars and that step is near it. See check_workflow_scalars.py. - # That is also why `resolve` copies all of scripts/unsloth/ to ${RUNNER_TEMP}/us in one line rather than one cp per script: every check added - # here would otherwise cost another line inside the capped block, and going over silently disables the whole workflow. - - name: Check the merged tree for silently wrong resolutions - if: ${{ env.MERGED_PINS == '1' }} - run: | - set -euo pipefail - if ! python3 "${RUNNER_TEMP}/us/merge_checks.py" --root . ; then - echo "::error::the pinned PRs merged, but merge_checks.py found a resolution that is silently wrong; see the log for file and line" >&2 - exit 1 - fi - - # merge_checks.py asserts the ABSENCE of two known-bad shapes. This asserts the PRESENCE of what each pin carries, which is a different question and - # the one that goes unanswered when a pin rots into a no-op or a resolution quietly drops an arch registration. Free, so it runs before the compile gate. - - name: Check every pin still contributes what it carries - if: ${{ env.MERGED_PINS == '1' }} - # Through env, never interpolated into the script: `prs` carries PR - # titles, which are third-party text, and `${{ }}` pastes them into the - # shell source before bash ever sees it. - env: - PRS: ${{ steps.r.outputs.prs }} - BASE: ${{ steps.r.outputs.base }} - run: | - set -euo pipefail - if ! python3 "${RUNNER_TEMP}/us/pin_contract.py" --root . --base "$BASE" \ - --prs-json "$PRS" --report "${RUNNER_TEMP}/pin_contract.json" ; then - echo "::error::the pinned PRs merged, but the merged tree is missing code a pin carries; see the log for the pin and file" >&2 - exit 1 - fi - - # The gap this closes, observed 09-03: ggml-org#27754 merged with zero conflicts and did not compile, because upstream had added a parameter to - # build_attn_mha and the pin's new build_attn_sparse still called the old signature. Nothing before this point can see that, and without it the release - # dies in the CUDA leg after the 38-job fan-out. CPU only: a cold `llama` build took 59s at -j4 with no ccache, against 20-60 minutes for a CUDA build. - # mtmd is in the gate because `llama` alone is not enough: observed 09-04, ggml-org#25731 built `llama` clean while tools/mtmd did not compile at all, - # upstream having made mtmd_image_preprocessor::preprocess const while the pin's Inkling subclass stayed non-const, so it overrode nothing and the - # vision and audio towers were abstract. Every vision pin lands in mtmd, so a gate that skips it cannot see the whole class. - - name: Compile gate (CPU, llama and mtmd targets) - if: ${{ env.MERGED_PINS == '1' }} - run: | - set -euo pipefail - cmake -B "${RUNNER_TEMP}/gate" -DCMAKE_BUILD_TYPE=Release \ - -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_SERVER=OFF \ - -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_CURL=OFF > /dev/null - if ! cmake --build "${RUNNER_TEMP}/gate" --target llama mtmd -j "$(nproc)" ; then - echo "::error::the pinned PRs merged cleanly and the merged tree does not compile; fix or drop the pin rather than letting the build matrix find this" >&2 - exit 1 - fi - - # The stamped source tree (every build): every build child extracts this - # instead of cloning, and assemble ships it as the release's source-tarball - # asset, so a source build reproduces the same fingerprinted binary. Only - # produced when we build, so guard on the same condition as the build jobs - # (resolve skips source prep on a scheduled no-op, leaving no tarball). - - name: Upload source artifact - if: ${{ steps.r.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: ${{ steps.r.outputs.source_artifact }} - path: ${{ runner.temp }}/llama.cpp-source-${{ steps.r.outputs.tag }}.tar.gz - if-no-files-found: error - retention-days: 7 - - build-cuda: - name: CUDA - needs: resolve - if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} - uses: ./.github/workflows/unsloth-prebuilt-cuda.yml - with: - tag: ${{ needs.resolve.outputs.tag }} - repo: ${{ needs.resolve.outputs.repo }} - source_artifact: ${{ needs.resolve.outputs.source_artifact }} - commit: ${{ needs.resolve.outputs.commit }} - matrix: ${{ needs.resolve.outputs.cuda_matrix }} - - build-windows-cuda: - name: CUDA Windows - needs: resolve - if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} - uses: ./.github/workflows/unsloth-prebuilt-cuda-windows.yml - with: - tag: ${{ needs.resolve.outputs.tag }} - repo: ${{ needs.resolve.outputs.repo }} - source_artifact: ${{ needs.resolve.outputs.source_artifact }} - commit: ${{ needs.resolve.outputs.commit }} - matrix: ${{ needs.resolve.outputs.win_cuda_matrix }} - - build-rocm: - name: ROCm - needs: resolve - if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} - uses: ./.github/workflows/unsloth-prebuilt-rocm.yml - with: - tag: ${{ needs.resolve.outputs.tag }} - repo: ${{ needs.resolve.outputs.repo }} - source_artifact: ${{ needs.resolve.outputs.source_artifact }} - matrix: ${{ needs.resolve.outputs.rocm_matrix }} - operating_systems: ${{ github.event.inputs.operating_systems || 'windows,ubuntu' }} - rocm_version: ${{ github.event.inputs.rocm_version || 'weekly' }} - rocm_cutoff: ${{ needs.resolve.outputs.rocm_cutoff }} - - build-macos: - name: macOS - needs: resolve - if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} - uses: ./.github/workflows/unsloth-prebuilt-macos.yml - with: - tag: ${{ needs.resolve.outputs.tag }} - repo: ${{ needs.resolve.outputs.repo }} - source_artifact: ${{ needs.resolve.outputs.source_artifact }} - matrix: ${{ needs.resolve.outputs.macos_matrix }} - - build-cpu: - name: CPU - needs: resolve - if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} - uses: ./.github/workflows/unsloth-prebuilt-cpu.yml - with: - tag: ${{ needs.resolve.outputs.tag }} - repo: ${{ needs.resolve.outputs.repo }} - source_artifact: ${{ needs.resolve.outputs.source_artifact }} - - build-vulkan: - name: Vulkan - needs: resolve - if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} - uses: ./.github/workflows/unsloth-prebuilt-vulkan.yml - with: - tag: ${{ needs.resolve.outputs.tag }} - repo: ${{ needs.resolve.outputs.repo }} - source_artifact: ${{ needs.resolve.outputs.source_artifact }} - - assemble: - name: Assemble metadata + publish - # Only `resolve`, so this job starts alongside the build matrix instead of - # after it. It used to `needs:` every build child, which meant it began - # queueing for a runner only once the last leg went green: on the reference - # run that queue wait was 109 minutes to then run a 10-second job. Starting - # early overlaps the wait with the build. The cost is one runner slot held - # for the length of the run. - needs: [resolve] - # No `if: always()`. Atomicity is unchanged in effect, but it is now - # enforced by the "Wait for the build matrix" step below rather than by - # `needs:`: that step blocks until every sibling build job is finished and - # exits non-zero unless all of them succeeded, so a failed leg still - # publishes nothing. The installer needs the full bundle set or it'll fail. - if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-24.04 - # Consumed by the reclaim job below: 'true' only when THIS run's publish - # step actually ran and succeeded. - outputs: - published: ${{ steps.publish.outcome == 'success' }} - # The waiter holds this job open for the whole build. GitHub's 6h job cap - # would kill it with no useful message; stop short of that deliberately, - # leaving room for the download/verify/publish steps that follow. - timeout-minutes: 350 - # A job-level block replaces the workflow-level one, so contents:write has - # to be repeated here; actions:read is what lets the waiter read the run's - # job list (the `alert` job already needs it for the same reason). - permissions: - contents: write - actions: read - env: - GH_TOKEN: ${{ github.token }} - steps: - # Reimplements the `needs:` success check that the trimmed-down `needs:` - # above gave away. It has to be at least as strict as `needs:` was -- - # publishing a partial release is far worse than publishing late. - - name: Wait for the build matrix - env: - # Same expression that gates the verify/publish steps below. - PUBLISH_INTENT: ${{ github.event_name == 'schedule' || inputs.publish }} - run: | - set -euo pipefail - - # Build jobs live in called workflows, so they appear in this run's - # job list as "<caller job name> / <called job name>", e.g. - # "CUDA / x64/cuda12-legacy". One prefix per build child; every one of - # them must be represented or we are not looking at a complete run. - PREFIXES=('CUDA / ' 'CUDA Windows / ' 'ROCm / ' 'macOS / ' 'CPU / ' 'Vulkan / ') - SELF='Assemble metadata + publish' - ALERT='Report pipeline health' - - POLL=60 - DEADLINE=$(( $(date +%s) + 330 * 60 )) - # Called-workflow job records do not all exist the moment the run - # starts, so a missing prefix is only fatal once this has passed. - STARTUP_DEADLINE=$(( $(date +%s) + 45 * 60 )) - API_FAILS=0 - - while :; do - if ! JOBS="$(gh api --paginate \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" \ - --jq '.jobs[] | [.name, .status, (.conclusion // "none")] | @tsv' 2>/dev/null)"; then - # A blip in the jobs API must not throw away a finished build. - API_FAILS=$(( API_FAILS + 1 )) - if [ "$API_FAILS" -ge 10 ]; then - echo "ERROR: the jobs API failed 10 times running; cannot confirm the build matrix finished" >&2 - exit 1 - fi - echo "jobs API call failed (${API_FAILS}/10); retrying in ${POLL}s" - sleep "$POLL" - continue - fi - API_FAILS=0 - - # Everything in the run except this job and the alert job that - # reports on it. `resolve` stays in the set; it is already a - # `needs:`, so it is a free consistency check. - SIBS="$(printf '%s\n' "$JOBS" | awk -F'\t' -v self="$SELF" -v alert="$ALERT" 'NF && $1 != self && $1 != alert')" - - MISSING="" - for p in "${PREFIXES[@]}"; do - printf '%s\n' "$SIBS" | awk -F'\t' -v p="$p" 'index($1, p) == 1 { found = 1 } END { exit !found }' \ - || MISSING="${MISSING} '${p}'" - done - if [ -n "$MISSING" ]; then - if [ "$(date +%s)" -gt "$STARTUP_DEADLINE" ]; then - echo "ERROR: no job records for build child(ren):${MISSING}; refusing to publish without confirming they ran" >&2 - exit 1 - fi - echo "waiting for job records to appear for:${MISSING}" - sleep "$POLL" - continue - fi - - # Fail on the first finished leg that did not succeed rather than - # sitting on a runner for another hour to reach the same answer. - # `skipped` is the one conclusion that needs a judgement call: a - # publish run is pinned by `resolve` to the full default matrix, so - # a skipped leg there means something is wrong and must block, but a - # publish=false subset dispatch (say operating_systems=ubuntu) skips - # legs on purpose and `needs:` tolerated that before. - BAD="$(printf '%s\n' "$SIBS" | awk -F'\t' -v strict="$PUBLISH_INTENT" ' - !NF || $2 != "completed" { next } - $3 == "success" { next } - $3 == "skipped" && strict != "true" { next } - { printf " %s (%s)\n", $1, $3 } - ')" - if [ -n "$BAD" ]; then - echo "ERROR: refusing to publish, these build jobs did not succeed:" >&2 - printf '%s\n' "$BAD" >&2 - exit 1 - fi - - TOTAL="$(printf '%s\n' "$SIBS" | grep -c . || true)" - PENDING="$(printf '%s\n' "$SIBS" | awk -F'\t' 'NF && $2 != "completed"' | grep -c . || true)" - if [ "$PENDING" -eq 0 ]; then - echo "all ${TOTAL} build jobs finished and succeeded" - break - fi - if [ "$(date +%s)" -gt "$DEADLINE" ]; then - echo "ERROR: timed out waiting for the build matrix; ${PENDING} of ${TOTAL} jobs still running" >&2 - exit 1 - fi - echo "${PENDING} of ${TOTAL} build jobs still running; polling again in ${POLL}s" - sleep "$POLL" - done - - - name: Checkout build tooling (this repo) - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: - path: tooling - - # The first download attempt occasionally hits a transient ECONNRESET - # on GitHub's ListArtifacts API. Retrying once is cheaper than - # re-running the whole hour-long build matrix. - - name: Download built bundles - id: download - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - continue-on-error: true - with: - path: dist - pattern: app-* - merge-multiple: true - - - name: Download built bundles (retry) - if: steps.download.outcome == 'failure' - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 - with: - path: dist - pattern: app-* - merge-multiple: true - - # Fingerprint gate: every bundle must carry the fingerprint the build children - # fold into BUILD_TARGET (-> LLAMA_BUILD_TARGET, printed by `--version`). That - # string is compiled into common (build-info.cpp); on Linux/macOS it lands in - # llama-server's own .rodata, but on a Windows shared build it can live in the - # sibling llama-common DLL instead -- so scan every file in the bundle, not - # just the exe. Nothing but a compiled binary carries this string (no metadata - # file does), so a hit means the brand shipped. Scan bytes rather than running - # `--version`: cross-compiled (ROCm) and Windows binaries can't run on this - # Linux runner. MARK must stay byte-identical to the resolve job's stamp string. - # Refuse to publish if any bundle is unbranded. - - name: Verify Unsloth fingerprint in every bundle - run: | - set -euo pipefail - MARK='Compiled by the Unsloth team' - shopt -s nullglob - tmp="$(mktemp -d)" - fail=0 - checked=0 - for arc in dist/app-*.tar.gz dist/app-*.zip dist/llama-*-bin-macos-*.tar.gz; do - d="$tmp/extract" - rm -rf "$d"; mkdir -p "$d" - case "$arc" in - *.zip) unzip -qo "$arc" -d "$d" ;; - *.tar.gz) tar -xzf "$arc" -C "$d" ;; - esac - if grep -arq "$MARK" "$d"; then - checked=$((checked + 1)) - else - echo "ERROR: $(basename "$arc"): no file carries the Unsloth fingerprint" >&2; fail=1 - fi - done - rm -rf "$tmp" - [ "$checked" -gt 0 ] || { echo "ERROR: no bundles found to verify (expected app-*/llama-*-bin-macos-* in dist/)" >&2; exit 1; } - [ "$fail" = 0 ] || { echo "ERROR: refusing to publish unbranded binaries" >&2; exit 1; } - echo "fingerprint verified in $checked bundles" - - # Ship the stamped source tree as release assets so the installer's - # source-build fallback reproduces the same fingerprinted binary. The resolve - # job's app-source-* artifact lands the tag-named tarball in dist/ (via the - # app-* download above); copy it to the commit name too. Both are the exact - # local bytes assemble_metadata hashes into the sha256 index. - - name: Fetch source archives - run: | - set -eux - TAG='${{ needs.resolve.outputs.tag }}' - SHA='${{ needs.resolve.outputs.commit }}' - cp "dist/llama.cpp-source-${TAG}.tar.gz" "dist/llama.cpp-source-commit-${SHA}.tar.gz" - - - name: Generate manifest + sha256 index - # prs carries PR titles (arbitrary text), so pass it through env - # rather than inlining it: a title with a quote would otherwise break out - # of the shell command. Same handling in the publish step below. - env: - PRS_JSON: ${{ needs.resolve.outputs.prs }} - run: | - set -eux - python3 tooling/scripts/unsloth/assemble_metadata.py \ - --tag '${{ needs.resolve.outputs.tag }}' \ - --source-repo '${{ needs.resolve.outputs.repo }}' \ - --base-tag '${{ needs.resolve.outputs.base }}' \ - --pr-set "$PRS_JSON" \ - --ggml-tree '${{ needs.resolve.outputs.ggml_tree }}' \ - --ggml-version '${{ needs.resolve.outputs.ggml_version }}' \ - --commit '${{ needs.resolve.outputs.commit }}' \ - --dist dist --out dist \ - --publish-repo "$GITHUB_REPOSITORY" - ls -la dist - - - name: Verify full bundle coverage before publish - if: ${{ (github.event_name == 'schedule' || inputs.publish) && needs.resolve.outputs.exists != 'true' }} - run: | - set -eu - TAG='${{ needs.resolve.outputs.tag }}' - fail=0 - # A partial CUDA set (cuda13 without cuda12) silently strands the - # cuda12-runtime majority; require matching x64 coverage on both lines. - # cuda12-legacy has no cuda13 sibling, so `-nE ... p` drops it from - # this cross-line parity check; its presence is required separately below. - for os in linux windows; do - ext=$([ "$os" = windows ] && echo zip || echo tar.gz) - c12=$(ls dist/app-*-"$os"-x64-cuda12-*."$ext" 2>/dev/null | sed -nE 's/.*cuda12-(older|newer|portable)\..*/\1/p' | sort -u | paste -sd, -) - c13=$(ls dist/app-*-"$os"-x64-cuda13-*."$ext" 2>/dev/null | sed -nE 's/.*cuda13-(older|newer|portable)\..*/\1/p' | sort -u | paste -sd, -) - echo "$os x64 coverage: cuda12=[$c12] cuda13=[$c13]" - [ -n "$c12" ] && [ "$c12" = "$c13" ] || { echo "ERROR: $os x64 cuda12/cuda13 coverage mismatch" >&2; fail=1; } - done - # Children passing is not the same as files landing in dist/ (a - # download-artifact anomaly leaves green jobs and missing bundles), - # so assert presence of every line not covered by the parity check - # above (the non-CUDA-x64 lines, plus cuda12-legacy). The resolve-time - # input guard already pins publish runs to the full default matrix, so - # these names are exact. - for f in \ - "app-${TAG}-linux-x64-cuda12-legacy.tar.gz" \ - "app-${TAG}-windows-x64-cuda12-legacy.zip" \ - "llama-${TAG}-bin-macos-arm64.tar.gz" \ - "llama-${TAG}-bin-macos-x64.tar.gz" \ - "app-${TAG}-linux-arm64-cuda13-portable.tar.gz" \ - "app-${TAG}-linux-x64-cpu.tar.gz" \ - "app-${TAG}-windows-x64-cpu.zip" \ - "app-${TAG}-linux-arm64-cpu.tar.gz" \ - "app-${TAG}-windows-arm64-cpu.zip" \ - "app-${TAG}-linux-x64-vulkan.tar.gz" \ - "app-${TAG}-linux-arm64-vulkan.tar.gz" \ - "app-${TAG}-windows-x64-vulkan.zip"; do - [ -s "dist/$f" ] || { echo "ERROR: missing $f in dist/" >&2; fail=1; } - done - # Default ROCm set (mirrors the gfx_target input default): both OSes - # per family, or AMD hosts of that family silently lose their bundle. - for gfx in gfx1151 gfx1150 gfx120X gfx110X gfx103X gfx90a gfx908; do - [ -s "dist/app-${TAG}-linux-x64-rocm-${gfx}.tar.gz" ] || { echo "ERROR: missing linux rocm ${gfx} bundle in dist/" >&2; fail=1; } - [ -s "dist/app-${TAG}-windows-x64-rocm-${gfx}.zip" ] || { echo "ERROR: missing windows rocm ${gfx} bundle in dist/" >&2; fail=1; } - done - [ "$fail" = 0 ] || { echo "ERROR: refusing to publish a partial release" >&2; exit 1; } - - - name: Publish GitHub release - id: publish - # Last resort cap. The uploader has its own 90m phase deadline; if that - # ever fails to trip, this still fails the step with hours of job budget - # left instead of letting the 350m job cap kill it with no message, and - # it lets the rescue artifact step below run. - timeout-minutes: 120 - if: ${{ (github.event_name == 'schedule' || inputs.publish) && needs.resolve.outputs.exists != 'true' }} - # prs carries PR titles (arbitrary text), so pass it through env - # rather than inlining it: a title with a quote would otherwise break out - # of the shell command. - env: - PRS_JSON: ${{ needs.resolve.outputs.prs }} - run: | - set -eux - TAG='${{ needs.resolve.outputs.tag }}' - REPO="$GITHUB_REPOSITORY" - PRS="$PRS_JSON" - BASE='${{ needs.resolve.outputs.base }}' - # Link the base to the upstream release tag (always resolves). Each PR - # line is "<title> (#<n>, commit <sha>)": GitHub does not expand a bare - # reference into the PR title inline (only on hover), so we bake the - # title in ourselves. #<n> links to the PR in its home repo (full URL, - # so it resolves no matter which repo hosts the release, and GitHub - # still attaches its hovercard); non-upstream PRs spell the repo in - # the link text so an unslothai pin can't read as an upstream one. - # <sha> links to that pin's commit-in-PR URL. We deliberately do not - # link the merged commit: in mix mode it is a throwaway merge made on - # the runner and never pushed anywhere, so any repo@sha URL for it 404s. - NOTES="Automated Unsloth llama.cpp CUDA + ROCm + Vulkan + macOS + CPU prebuild for upstream [${BASE}](https://github.com/ggml-org/llama.cpp/releases/tag/${BASE})" - if [ "$(jq length <<<"$PRS")" = 0 ]; then - NOTES="${NOTES}." - else - PR_LIST="$(jq -r 'map("- \(.title) ([\(if .repo == "ggml-org/llama.cpp" then "" else .repo end)#\(.number)](https://github.com/\(.repo)/pull/\(.number)), commit [\(.sha[0:7])](\(.url)))") | join("\n")' <<<"$PRS")" - NOTES="$(printf '%s, merged with:\n\n%s' "$NOTES" "$PR_LIST")" - fi - - # Atomic publish: upload as draft (hidden from the anon GitHub API - # the installer uses), then flip draft=false only once every asset - # landed. Leftover drafts from failed runs are detected as "not - # exists" by resolve and rebuilt on the next run. - if [ "$(gh release view "$TAG" --repo "$REPO" --json isDraft --jq .isDraft 2>/dev/null || true)" = "true" ]; then - gh release delete "$TAG" --repo "$REPO" --yes - fi - # Create the draft empty, then upload through the uploader instead of - # passing dist/* here. `gh release create` uses a fixed 5-worker pool - # with no per-connection timeout, so a few wedged PUTs block the whole - # set: in run 31335302864 six large bundles stalled at ~0.03 MB/s and - # held the pool for 3h45m, while the other 25 assets took 37s total. - for attempt in 1 2 3; do - if gh release create "$TAG" --repo "$REPO" --draft \ - --title "llama.cpp prebuilt $TAG" \ - --notes "$NOTES"; then - break - fi - if [ "$attempt" = 3 ]; then - echo "ERROR: could not create draft release $TAG" >&2 - exit 1 - fi - sleep $(( attempt * 10 )) - done - - bash tooling/scripts/unsloth/upload_release_assets.sh \ - --tag "$TAG" --repo "$REPO" --dist dist - - # Reached only after the uploader verified every asset is present, - # byte-identical and in state "uploaded". - gh release edit "$TAG" --repo "$REPO" --draft=false - - # Debug fallback for a failed publish. This used to upload on EVERY run, - # ahead of the publish gate, which cost ~688 MiB of Actions artifact - # storage per run for a bundle nothing reads: no download-artifact in this - # repo references it, an org-wide code search for `actions/artifacts` - # returns nothing, and Studio's installer resolves release ASSETS - # (install_llama_prebuilt.py -> release_asset_download_url). Uploading - # ahead of the gate also published unverified builds -- on a public repo - # any signed-in user can download an artifact -- for runs that - # deliberately never released. - # - # Kept but moved after publish and gated on failure(), so it still rescues - # a failed-publish run without re-running the 40-job matrix, and costs - # nothing when the run is green. - - name: Upload full release set (artifacts) - # always() && !success(), not failure(): a cancelled assemble -- its own - # timeout, or a superseding nightly -- is not on the failure path, so - # failure() would skip the rescue bundle on exactly the runs a human - # most wants it for. Nothing else in this job competes for the teardown - # window, so attempting the upload there is free; if dist/ was never - # populated, if-no-files-found: warn keeps that quiet. - if: ${{ always() && !success() }} - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: unsloth-prebuilt-${{ needs.resolve.outputs.tag }} - path: dist/* - # warn, not error: an early failure can leave dist/ absent, and a - # missing debug bundle must not turn a diagnosable failure into a - # confusing second one. - if-no-files-found: warn - # rerun-failed-jobs reuses the run id, and artifacts persist across - # attempts, so a second failed attempt would 409 on the duplicate name. - overwrite: true - retention-days: 7 - - # whisper.cpp slim bundles are compiled against this release's ggml and - # ship no libggml*, so a new ggml means whisper has to republish. Without - # this it only finds out on its own cron, and Studio reports "no - # compatible prebuilt" until then. - - name: Notify whisper.cpp - if: ${{ (github.event_name == 'schedule' || inputs.publish) && needs.resolve.outputs.exists != 'true' }} - # Never fail a published release because the notification did not land. - continue-on-error: true - env: - DISPATCH_TOKEN: ${{ secrets.WHISPER_DISPATCH_TOKEN }} - TAG: ${{ needs.resolve.outputs.tag }} - GGML_TREE: ${{ needs.resolve.outputs.ggml_tree }} - run: | - # No -x here: it would trace the token into the log. - set -eu - if [ -z "${DISPATCH_TOKEN:-}" ]; then - echo "::warning::WHISPER_DISPATCH_TOKEN is not set; whisper.cpp will only pick this up on its own schedule" - exit 0 - fi - PAYLOAD="$(jq -cn --arg t "$TAG" --arg g "$GGML_TREE" --arg r "$GITHUB_RUN_ID" \ - '{event_type: "llama-published", client_payload: {llama_tag: $t, ggml_tree: $g, run_id: $r}}')" - CODE="$(curl -sS -o /dev/null -w '%{http_code}' -X POST \ - -H "Authorization: Bearer ${DISPATCH_TOKEN}" \ - -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/unslothai/whisper.cpp/dispatches \ - -d "$PAYLOAD")" - # 204 is "event accepted", not "whisper republished". The dead-man - # check is what catches a dropped or ignored event. - if [ "$CODE" = "204" ]; then - echo "notified whisper.cpp of ${TAG}" - else - echo "::warning::whisper.cpp dispatch returned HTTP ${CODE}; it will fall back to its own schedule" - fi - - # Reclaim this run's Actions artifact storage once the release is canonical. - # - # Every prebuilt is stored TWICE: as an Actions artifact (billed against the - # Actions storage quota) and as a release asset (not billed against it). - # Measured on run 31218133438: 36 of its 40 artifacts, 7.67 GiB, are byte-for - # -byte already on the release. On 2026-08-07 that duplication reached ~492 - # GiB org-wide, 449 GiB of it here, and GitHub stopped scheduling Actions runs - # across unslothai/* while personal-account repos ran the same workflows - # normally. Deleting artifacts restored scheduling within ~45 seconds. - # - # Its OWN job rather than a step of assemble, for two reasons: - # - # * Cancellation. As a trailing step, a cancel landing inside its ~13 second - # window left assemble `cancelled` with a partial app-* set, and - # unsloth-prebuilt-retry.yml reruns a cancelled run that has no failed job - # -- rerunning assemble itself, which then restarts at "Download built - # bundles", dies at the coverage gate, and reports "Nothing was published" - # for a release that HAD published. As a separate job, assemble has already - # succeeded, so a rerun re-runs only this job, and re-running it is a no-op: - # deleted artifacts are simply absent from the second listing. - # * Least privilege. `actions: write` also grants Actions CACHE deletion, so - # holding it across assemble's unzip/tar/python steps would put every - # ccache this pipeline depends on within reach of that job. Here it is - # confined to a job whose only action is deleting artifacts. - # - # Skipped, not failed, when there is nothing to reclaim: `needs.assemble - # .outputs.published` is 'true' only when THIS run's publish step ran and - # succeeded, so a dispatch that skips publishing (publish:false, or the tag - # already released) never reaches the delete -- its artifacts would otherwise - # be name-matched against a DIFFERENT build's release assets, since artifact - # names are keyed only on the tag. - # - # Residual, accepted: `published` proves THIS run's publish step succeeded, not - # that the release object is this run's. A scheduled run and a same-tag - # dispatch sit in different concurrency groups, so a dispatch can delete the - # scheduled run's draft and create its own; the scheduled run's - # `gh release edit --draft=false` then succeeds against the dispatch's release. - # The payloads are equivalent -- the resolve guard pins every publish run to - # the full default matrix, and the tag encodes base plus pr-set hash -- so the - # assets match whichever run wrote them. - reclaim: - name: Reclaim artifact storage - # Every build leg, not just assemble. - # assemble needs only resolve, so one failing leg fails it immediately while slower legs are still building, and with always() this job then deleted the app-source-* artifact out from under them. - # On 08-27 one arm64 CPU failure became ten: nine ROCm legs died on "Artifact not found" seconds later, which reads as a ROCm fault and is not. - needs: [resolve, build-cuda, build-windows-cuda, build-rocm, build-macos, build-cpu, build-vulkan, assemble] - # always(), so a run that publishes NOTHING still cleans up after itself. - # Gating this on `published` leaked every non-publishing run's bundles: a - # workflow_dispatch defaults to publish:false, and a cancelled run never - # reaches publish either. On 08-10 that was 5.9 GiB from one cancelled run - # plus 7.1 GiB from one publish:false run, both deleted by hand. The two - # cases are handled by different steps below -- a published run deletes only - # what is provably on the release, an unpublished one has nothing to match - # against and deletes its own bundles outright. - if: ${{ always() }} - runs-on: ubuntu-24.04 - timeout-minutes: 20 - permissions: - actions: write # delete this run's artifacts - contents: read # read the release asset list - steps: - - name: Delete artifacts already published as release assets - if: ${{ needs.assemble.outputs.published == 'true' }} - # Never fail a published release over cleanup. - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ needs.resolve.outputs.tag }} - run: | - set -euo pipefail - repo="$GITHUB_REPOSITORY" - - if [ -z "${TAG:-}" ]; then - echo "no tag resolved; leaving artifacts untouched" - exit 0 - fi - - # Gate 1: the release must exist and be published, not a draft. - draft="$(gh release view "$TAG" --repo "$repo" --json isDraft -q .isDraft 2>/dev/null || echo missing)" - if [ "$draft" != "false" ]; then - echo "release $TAG is '$draft', not a published release; leaving artifacts untouched" - exit 0 - fi - assets="$RUNNER_TEMP/reclaim-assets.txt" - arts="$RUNNER_TEMP/reclaim-arts.tsv" - gh release view "$TAG" --repo "$repo" --json assets -q '.assets[].name' | sort > "$assets" - echo "release $TAG has $(wc -l < "$assets") assets" - - # Gate 2: only THIS run's artifacts are even considered, so the step - # cannot reach another run's -- including a concurrent build's. - gh api "repos/$repo/actions/runs/$GITHUB_RUN_ID/artifacts" --paginate \ - -q '.artifacts[] | select(.expired==false) | "\(.id)\t\(.size_in_bytes)\t\(.name)"' > "$arts" || true - echo "this run has $(grep -c . "$arts" || true) live artifacts" - - freed=0; deleted=0; kept=0; failed=0 - while IFS="$(printf '\t')" read -r id size name; do - [ -z "${id:-}" ] && continue - # Gate 3: delete only what is provably already on the release. - # Build children upload `app-<tag>-<platform>`; assemble publishes - # it as `.tar.gz` (linux/macos) or `.zip` (windows). Anything that - # does not match is KEPT -- that is what protects a partial publish. - # By design this also keeps the macOS bundles (published under - # `llama-<tag>-bin-macos-*`) and the source tarball: 56 MiB of the - # 7.42 GiB, a cheap price for never deleting something early. - # -F: fixed string. Without it every `.` in the name is a regex - # wildcard, and gfx_target / only_profile are free-text - # workflow_dispatch inputs that flow into artifact names. - if grep -qxF -- "${name}.tar.gz" "$assets" || grep -qxF -- "${name}.zip" "$assets"; then - # < /dev/null so the command can never consume the loop's stdin - # and silently truncate the sweep to one artifact. - if err="$(gh api -X DELETE "repos/$repo/actions/artifacts/$id" --silent < /dev/null 2>&1)"; then - freed=$(( freed + size )); deleted=$(( deleted + 1 )) - else - # Keep the reason: a 403 from a permissions regression and a - # transient blip need different responses, and this step is - # continue-on-error so nothing else surfaces it. - printf ' could not delete %s: %s\n' "$name" "$err" - failed=$(( failed + 1 )) - fi - else - printf ' KEEP %s (no matching release asset)\n' "$name" - kept=$(( kept + 1 )) - fi - done < "$arts" - - echo "deleted $deleted artifacts, freed $(( freed / 1048576 )) MiB, kept $kept, failed $failed" - if [ "$failed" -gt 0 ]; then - echo "::warning::$failed artifact(s) could not be deleted; storage will be reclaimed by retention instead" - fi - { - echo "### Artifact storage reclaimed" - echo "" - echo "| metric | value |" - echo "| --- | --- |" - echo "| release | \`$TAG\` |" - echo "| artifacts deleted | $deleted |" - echo "| storage freed | $(( freed / 1048576 )) MiB |" - echo "| kept (no release asset) | $kept |" - echo "| delete failures | $failed |" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Delete artifacts of a run that published nothing - # The other step's name-match against release assets is meaningless - # here: either no release was written, or the tag belongs to a DIFFERENT - # run's release. So the rule is simply that nothing will ever consume - # these -- a publish:false dispatch is a test, and a cancelled or failed - # run is not resumable past the missing legs -- and they are deleted. - # - # Accepted: unsloth-prebuilt-retry.yml can rerun a cancelled run whose - # artifacts this step deleted. That rerun fails loudly at assemble's - # coverage gate rather than publishing a partial set, which is the - # failure mode this pipeline already prefers. Pass keep_artifacts:true - # on a dispatch whose bundles you intend to download by hand. - if: ${{ needs.assemble.outputs.published != 'true' && inputs.keep_artifacts != true }} - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - repo="$GITHUB_REPOSITORY" - arts="$RUNNER_TEMP/reclaim-unpublished.tsv" - - # Same containment as the published path: only THIS run's artifacts - # are listed, so the step cannot reach a concurrent build's. - gh api "repos/$repo/actions/runs/$GITHUB_RUN_ID/artifacts" --paginate \ - -q '.artifacts[] | select(.expired==false) | "\(.id)\t\(.size_in_bytes)\t\(.name)"' > "$arts" || true - n="$(grep -c . "$arts" || true)" - echo "run published nothing; deleting its $n live artifact(s)" - - freed=0; deleted=0; failed=0 - while IFS="$(printf '\t')" read -r id size name; do - [ -z "${id:-}" ] && continue - if err="$(gh api -X DELETE "repos/$repo/actions/artifacts/$id" --silent < /dev/null 2>&1)"; then - freed=$(( freed + size )); deleted=$(( deleted + 1 )) - else - printf ' could not delete %s: %s\n' "$name" "$err" - failed=$(( failed + 1 )) - fi - done < "$arts" - - echo "deleted $deleted artifacts, freed $(( freed / 1048576 )) MiB, failed $failed" - if [ "$failed" -gt 0 ]; then - echo "::warning::$failed artifact(s) could not be deleted; storage will be reclaimed by retention instead" - fi - { - echo "### Artifact storage reclaimed (unpublished run)" - echo "" - echo "| metric | value |" - echo "| --- | --- |" - echo "| artifacts deleted | $deleted |" - echo "| storage freed | $(( freed / 1048576 )) MiB |" - echo "| delete failures | $failed |" - } >> "$GITHUB_STEP_SUMMARY" - - # ── Keep the ccache budget inside the repo limit ── - # - # ccache entries are immutable, so every run writes a NEW cache per - # (cuda, os, profile) and finds the previous one by restore-keys prefix. - # restore-keys returns only the MOST RECENT match, so older generations can - # never be selected again -- they are pure landfill. Measured 2026-08-08: - # 122 caches / 31.72 GiB, of which only 40 / 9.13 GiB were reachable. - # - # That matters for build time, not tidiness. The repo cache limit is 50 GB - # and there are ~40 prefixes; once the total hits the limit GitHub evicts by - # its own LRU, which can take a LIVE cache. A partial cache is far worse - # than none: measured locally over 513 real translation units, capping a - # cache to 30% of what the build needs drops the hit rate to 14.2% and - # flips hits from direct to preprocessed -- the exact 3-direct / - # 52-preprocessed signature seen in CI when the cap was 500 MB, which cost - # a 204-minute build instead of 54. - # - # Keeping 2 rather than 1: the second generation is the fallback when a job - # dies before saving, which is precisely the hole that forces a cold build. - - name: Prune superseded ccache generations - # Unchanged trigger: this job now runs on every outcome for artifact - # cleanup, but cache pruning stays on published runs only. A cancelled - # run's children may not have written their caches yet, and pruning - # "superseded" generations against a half-written set could drop a live - # one -- a partial cache is worse than a stale one. - if: ${{ needs.assemble.outputs.published == 'true' }} - # Never fail a published release over cache housekeeping. - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - KEEP: 2 - run: | - set -euo pipefail - repo="$GITHUB_REPOSITORY" - all="$RUNNER_TEMP/caches.tsv" - gh api --paginate "repos/$repo/actions/caches?per_page=100" \ - -q '.actions_caches[] | "\(.id)\t\(.created_at)\t\(.size_in_bytes)\t\(.key)"' > "$all" || true - total=$(awk -F'\t' '{s+=$3} END {printf "%d", s+0}' "$all") - echo "$(grep -c . "$all" || true) caches, $(( total / 1073741824 )) GiB" - - # Group by the restore-keys prefix: the key minus its -<tag>- suffix. - # Newest first, so anything past $KEEP is unreachable by restore-keys. - # The ROCm version comes off too, else every weekly toolchain becomes its own group and keeps 2 caches that can never hit again. - freed=0; deleted=0 - while IFS=$'\t' read -r id created size key; do - [ -z "${id:-}" ] && continue - pre="$(printf '%s' "$key" | sed -E 's/-b[0-9]+(-mix-[0-9a-f]+)?-?$//; s/-[0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+$//')" - printf '%s\t%s\t%s\t%s\n' "$pre" "$created" "$id" "$size" - done < "$all" | sort -t"$(printf '\t')" -k1,1 -k2,2r > "$RUNNER_TEMP/grouped.tsv" - - prev=""; n=0 - while IFS=$'\t' read -r pre created id size; do - [ -z "${pre:-}" ] && continue - if [ "$pre" != "$prev" ]; then prev="$pre"; n=1; else n=$(( n + 1 )); fi - [ "$n" -le "$KEEP" ] && continue - if gh api -X DELETE "repos/$repo/actions/caches/$id" --silent < /dev/null 2>/dev/null; then - freed=$(( freed + size )); deleted=$(( deleted + 1 )) - fi - done < "$RUNNER_TEMP/grouped.tsv" - - after=$(( total - freed )) - echo "pruned $deleted superseded caches, freed $(( freed / 1073741824 )) GiB" - { - echo "### ccache budget" - echo "" - echo "| metric | value |" - echo "| --- | --- |" - echo "| kept per prefix | $KEEP |" - echo "| caches pruned | $deleted |" - echo "| freed | $(( freed / 1073741824 )) GiB |" - echo "| cache total after | $(( after / 1073741824 )) GiB of 50 GB |" - } >> "$GITHUB_STEP_SUMMARY" - if [ "$after" -gt 42949672960 ]; then - echo "::warning::ccache total is $(( after / 1073741824 )) GiB of the 50 GB repo limit; at the limit GitHub evicts by LRU and a partially evicted cache collapses the hit rate. Lower KEEP to 1, or raise the limit." - fi - - # Without this a failed scheduled run is silent: GitHub emails only whoever - # last touched the cron file, which is how three nightlies failed unnoticed. - # Runs on publish-intent runs only, so subset test dispatches stay quiet. - alert: - name: Report pipeline health - needs: [resolve, build-cuda, build-windows-cuda, build-rocm, build-macos, build-cpu, build-vulkan, assemble] - if: ${{ always() && (github.event_name == 'schedule' || inputs.publish) }} - runs-on: ubuntu-24.04 - permissions: - contents: read - issues: write - actions: read - steps: - - name: Checkout (for the composite action) - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: { persist-credentials: false } - - - name: Summarise run - id: s - env: - GH_TOKEN: ${{ github.token }} - NEEDS_JSON: ${{ toJSON(needs) }} - run: | - set -uo pipefail - # 'skipped' is healthy: a scheduled no-op skips every build job. - # 'cancelled' is not. On 08-05 every build passed and the publish job - # was cancelled 15s in by something outside the run, so nothing was - # released and this reported success: the silent no-publish we exist - # to catch. Anything that is not success or skipped counts. - FAILED="$(jq -r 'to_entries | map(select(.value.result != "success" and .value.result != "skipped") | "\(.key) (\(.value.result))") | join(", ")' <<<"$NEEDS_JSON")" - if [ -z "$FAILED" ]; then - echo "status=success" >> "$GITHUB_OUTPUT" - echo "details=" >> "$GITHUB_OUTPUT" - exit 0 - fi - echo "status=failure" >> "$GITHUB_OUTPUT" - - # Quote the actual refusal so the issue says why, not just that a job failed. - REASON="$(gh run view "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" --log-failed 2>/dev/null \ - | grep -aE 'refusing |does not merge cleanly onto |could not fetch commit |ERROR: ' \ - | sed -E 's/^[0-9-]+T[0-9:.]+Z //' | cut -c1-300 | head -5 || true)" - - { - echo 'details<<ALERT_EOF' - echo "**Failed jobs:** ${FAILED}" - if [ -n "$REASON" ]; then - echo - echo 'Reported reason:' - echo - echo '```' - echo "$REASON" - echo '```' - fi - echo - echo "Nothing was published; the previous release remains \`Latest\`." - echo 'ALERT_EOF' - } >> "$GITHUB_OUTPUT" - - - name: Alert - uses: ./.github/actions/prebuilt-alert - with: - status: ${{ steps.s.outputs.status }} - key: llama-prebuilt-nightly - title: 'Nightly llama.cpp prebuilt is failing' - details: ${{ steps.s.outputs.details }} - token: ${{ github.token }} diff --git a/.github/workflows/unsloth-repin-bot.yml b/.github/workflows/unsloth-repin-bot.yml deleted file mode 100644 index a074992511a0..000000000000 --- a/.github/workflows/unsloth-repin-bot.yml +++ /dev/null @@ -1,209 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: Unsloth repin bot - -# Preflight says the pins stopped merging. This does the mechanical half of the -# fix: merge the base tag into each pin branch we own, resolve the add/add -# collisions that cause almost all of these, and open a PR moving the pins. -# -# It opens a PR and stops. It never merges it, never touches a branch belonging -# to somebody else, and never repins past a commit a human reviewed -- the pin -# file exists to guarantee that only reviewed code ships, and a bot that can -# widen it on its own has removed the guarantee. - -on: - workflow_run: - workflows: ['Unsloth pin preflight'] - types: [completed] - workflow_dispatch: - -permissions: - contents: read - issues: write - -concurrency: - group: unsloth-repin-bot - cancel-in-progress: false - -env: - # Branch the bot parks its proposal on. Reused every run so a week of - # breakage is one PR to review, not seven. - REPIN_BRANCH: unsloth/auto-repin - -jobs: - repin: - name: Repin against the current base tag - # A preflight that passed has nothing to fix. - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'failure' }} - runs-on: ubuntu-24.04 - steps: - # Without this, checkout leaves a github.com extraheader carrying - # GITHUB_TOKEN, which would win over the REPIN_TOKEN in our push URLs. - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: { persist-credentials: false } - - - name: Resolve base tag - id: base - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - AGE_H="${UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS:-6}" - CUTOFF="$(date -u -d "-${AGE_H} hours" +%s)" - # Same base tag the nightly resolves: newest aged b#### build. - # Upstream marks those prerelease since 08-21, so match the tag shape. - BASE="$(gh api 'repos/ggml-org/llama.cpp/releases?per_page=100' \ - | jq -r --argjson cutoff "$CUTOFF" '[.[] | select(.draft==false) | select(.tag_name|test("^b[0-9]+$")) | select((.published_at|fromdateiso8601) <= $cutoff)] | max_by(.published_at|fromdateiso8601) | .tag_name')" - if [ -z "$BASE" ] || [ "$BASE" = "null" ]; then - echo "::warning::no aged upstream release found; nothing to repin onto" - echo "base=" >> "$GITHUB_OUTPUT"; exit 0 - fi - echo "base $BASE" - echo "base=$BASE" >> "$GITHUB_OUTPUT" - - - name: Merge and repin - id: repin - if: ${{ steps.base.outputs.base != '' }} - env: - GH_TOKEN: ${{ github.token }} - run: | - set -uo pipefail - python3 scripts/unsloth/repin.py \ - --pr-set scripts/unsloth/pr-set.json \ - --base "${{ steps.base.outputs.base }}" \ - --work "${RUNNER_TEMP}/repin" \ - --report "${RUNNER_TEMP}/repin.json" \ - --markdown "${RUNNER_TEMP}/repin.md" - CHANGED="$(jq -r '.changed' "${RUNNER_TEMP}/repin.json")" - BLOCKED="$(jq -r '[.results[] | select(.action == "conflict" or .action == "third-party")] | length' "${RUNNER_TEMP}/repin.json")" - echo "changed=$CHANGED" >> "$GITHUB_OUTPUT" - echo "blocked=$BLOCKED" >> "$GITHUB_OUTPUT" - - - name: Push the merged branches and open the PR - id: push - if: ${{ steps.repin.outputs.changed != '' && steps.repin.outputs.changed != '0' }} - env: - # Pushing to danielhanchen/llama.cpp is cross-repo, which GITHUB_TOKEN - # cannot do at all, and these merges carry upstream's own workflow - # changes, which needs workflow write. Without the secret the bot - # still reports; it just cannot act. - REPIN_TOKEN: ${{ secrets.REPIN_TOKEN }} - GH_TOKEN: ${{ github.token }} - BASE: ${{ steps.base.outputs.base }} - run: | - set -uo pipefail - if [ -z "${REPIN_TOKEN:-}" ]; then - echo "::warning::REPIN_TOKEN is not set; reporting the repin instead of pushing it" - echo "mode=report" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # Push every merged branch before touching the pin file: a pin whose - # commit is not on a remote is a pin the nightly cannot fetch. - FAILED="" - while read -r path head_repo head_ref new_sha; do - echo "pushing ${new_sha:0:10} to ${head_repo}:${head_ref}" - # No -x anywhere in this step; the URL carries the token. - if ! git -C "$path" push \ - "https://x-access-token:${REPIN_TOKEN}@github.com/${head_repo}.git" \ - "HEAD:refs/heads/${head_ref}" 2>&1 | sed "s/${REPIN_TOKEN}/***/g"; then - FAILED="${FAILED} ${head_repo}:${head_ref}" - fi - done < <(jq -r '.results[] | select(.action == "repin") - | "\(.repo_path) \(.head_repo) \(.head_ref) \(.new_sha)"' "${RUNNER_TEMP}/repin.json") - - if [ -n "$FAILED" ]; then - echo "::error::could not push:${FAILED}" - echo "mode=pushfail" >> "$GITHUB_OUTPUT" - echo "failed=${FAILED}" >> "$GITHUB_OUTPUT" - exit 0 - fi - - git config user.name 'unsloth-repin-bot' - git config user.email 'unsloth-repin-bot@users.noreply.github.com' - git checkout -q -B "${REPIN_BRANCH}" - git add scripts/unsloth/pr-set.json - git commit -qm "Repin PR set onto ${BASE}" - # Force: the branch is a rolling proposal against whatever base tag is - # current, so yesterday's version is not worth preserving. - git push -q --force \ - "https://x-access-token:${REPIN_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ - "HEAD:refs/heads/${REPIN_BRANCH}" 2>&1 | sed "s/${REPIN_TOKEN}/***/g" - - { - cat "${RUNNER_TEMP}/repin.md" - echo - echo "Opened automatically after [pin preflight](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/workflows/unsloth-pin-preflight.yml) failed. Review the resolutions above before merging; the bot does not merge its own PRs." - } > "${RUNNER_TEMP}/body.md" - - EXISTING="$(GH_TOKEN="${REPIN_TOKEN}" gh pr list --repo "${GITHUB_REPOSITORY}" \ - --head "${REPIN_BRANCH}" --state open --json number --jq '.[0].number' 2>/dev/null || true)" - if [ -n "$EXISTING" ] && [ "$EXISTING" != "null" ]; then - GH_TOKEN="${REPIN_TOKEN}" gh pr edit "$EXISTING" --repo "${GITHUB_REPOSITORY}" \ - --title "Repin PR set onto ${BASE}" --body-file "${RUNNER_TEMP}/body.md" >/dev/null - echo "updated PR #${EXISTING}" - echo "pr=${EXISTING}" >> "$GITHUB_OUTPUT" - else - URL="$(GH_TOKEN="${REPIN_TOKEN}" gh pr create --repo "${GITHUB_REPOSITORY}" \ - --base master --head "${REPIN_BRANCH}" \ - --title "Repin PR set onto ${BASE}" --body-file "${RUNNER_TEMP}/body.md" 2>&1 | tail -1)" - echo "opened ${URL}" - echo "pr=${URL}" >> "$GITHUB_OUTPUT" - fi - echo "mode=pushed" >> "$GITHUB_OUTPUT" - - - name: Report - id: report - if: ${{ always() && steps.base.outputs.base != '' }} - env: - CHANGED: ${{ steps.repin.outputs.changed }} - BLOCKED: ${{ steps.repin.outputs.blocked }} - OUTCOME: ${{ steps.repin.outcome }} - MODE: ${{ steps.push.outputs.mode }} - PR: ${{ steps.push.outputs.pr }} - FAILED: ${{ steps.push.outputs.failed }} - run: | - set -uo pipefail - # Only shout when a human has something to do. A run that repinned - # everything and opened a PR is already visible as a PR. - STATUS=success - # A crash in the repin step leaves no report at all, which must not - # read as "nothing to do" -- that is the green-run-does-nothing hole. - [ "${OUTCOME:-}" = "success" ] || STATUS=failure - { - echo 'details<<ALERT_EOF' - cat "${RUNNER_TEMP}/repin.md" 2>/dev/null \ - || echo "The repin step did not finish (outcome: ${OUTCOME:-unknown}); no report was produced." - echo - case "${MODE:-}" in - pushed) echo "Proposed in ${PR}." ;; - report) STATUS=failure - echo "\`REPIN_TOKEN\` is not configured, so nothing was pushed. Reproduce locally:" - echo - echo '```' - echo "git checkout <pin sha> && git merge <base tag>" - echo "python3 scripts/unsloth/additive_merge.py --repo ." - echo '```' ;; - pushfail) STATUS=failure - echo "Could not push:${FAILED}. \`REPIN_TOKEN\` likely lacks contents or workflow write on those repositories." ;; - *) [ "${CHANGED:-0}" = "0" ] && echo "Nothing could be repinned automatically." ;; - esac - if [ "${BLOCKED:-0}" != "0" ]; then - STATUS=failure - echo - echo "${BLOCKED} pin(s) need a human, see the table above." - fi - echo 'ALERT_EOF' - } >> "$GITHUB_OUTPUT" - echo "status=${STATUS}" >> "$GITHUB_OUTPUT" - - - name: Alert - if: ${{ always() && steps.report.outputs.status != '' }} - uses: ./.github/actions/prebuilt-alert - with: - status: ${{ steps.report.outputs.status }} - key: llama-repin-bot - title: 'Pins need a manual repin' - details: ${{ steps.report.outputs.details }} - token: ${{ github.token }} diff --git a/.github/workflows/unsloth-upstream-sync-guard.yml b/.github/workflows/unsloth-upstream-sync-guard.yml deleted file mode 100644 index 20dca54895d3..000000000000 --- a/.github/workflows/unsloth-upstream-sync-guard.yml +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. - -name: "Unsloth: upstream sync guard" - -# Holds the two properties that make a fork sync cheap, and that silently broke once. -# -# The 08-07 sync (PR #80) was squash-merged, so upstream 82bb48500 never became an ancestor of -# master. The files arrived; the ancestry did not. For three weeks every merge involving a -# master-derived branch three-way merged against a 2026-06-10 base and manufactured conflicts -# in files nobody had touched -- 539 of them, against 21 with the correct base. Nothing was red -# while that was true, which is the whole reason this exists. -# -# Uses the compare API rather than a checkout: the ancestry question is one request, and a -# full-history checkout of this repository is neither fast nor free. - -on: - push: - branches: [master] - workflow_dispatch: - -permissions: - contents: read - -jobs: - guard: - name: Upstream sync invariants - runs-on: ubuntu-24.04 - env: - GH_TOKEN: ${{ github.token }} - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: { fetch-depth: 1 } - - - name: Check the recorded sync point is still an ancestor of master - run: | - set -euo pipefail - FILE=scripts/unsloth/upstream-sync.json - SHA="$(jq -r .commit "$FILE")" - TAG="$(jq -r .tag "$FILE")" - case "$SHA" in - [0-9a-f]*) [ "${#SHA}" -eq 40 ] || { echo "::error file=$FILE::commit must be a 40-hex sha"; exit 1; } ;; - *) echo "::error file=$FILE::commit must be a 40-hex sha"; exit 1 ;; - esac - - # compare(base...head): 'ahead' or 'identical' means base is an ancestor of head. - # 'diverged' or 'behind' means it is not, which is what a squash-merged sync looks like. - STATUS="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${SHA}...master" --jq .status)" - echo "compare ${SHA:0:10} (${TAG}) ...master -> ${STATUS}" - case "$STATUS" in - ahead|identical) echo "ancestry OK" ;; - *) - echo "::error file=$FILE::upstream ${TAG} (${SHA:0:10}) is NOT an ancestor of master (compare says '${STATUS}')." - echo "::error::A sync PR was almost certainly squash- or rebase-merged. Squashing drops the upstream parent, so the merge base stays stale and every later merge invents hundreds of conflicts. Re-land the sync with a merge commit." - exit 1 ;; - esac - - - name: Check the fork still owns only CI - run: | - set -euo pipefail - FILE=scripts/unsloth/upstream-sync.json - SHA="$(jq -r .commit "$FILE")" - - # The compare API caps its file list. Say so rather than pass on a truncated answer. - RESP="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${SHA}...master")" - TOTAL="$(jq -r '.files | length' <<<"$RESP")" - if [ "$TOTAL" -ge 300 ]; then - echo "::error file=$FILE::compare returned ${TOTAL} files, at or over the API cap, so this check cannot be trusted. The fork delta should be well under 100 paths; if it is genuinely this large the invariant has already broken." - exit 1 - fi - - STRAY="$(jq -r '.files[].filename' <<<"$RESP" | grep -vE '^(\.github/|scripts/unsloth/)' || true)" - if [ -n "$STRAY" ]; then - echo "::error file=$FILE::the fork now diverges from upstream outside .github/ and scripts/unsloth/:" - echo "$STRAY" | sed 's/^/ /' - echo "::error::Syncs are provably additive only while this fork owns no llama.cpp source. Land source changes upstream, or pin them through scripts/unsloth/pr-set.json, rather than carrying them on master." - exit 1 - fi - echo "fork delta is ${TOTAL} path(s), all under .github/ or scripts/unsloth/" diff --git a/scripts/unsloth/additive_merge.py b/scripts/unsloth/additive_merge.py deleted file mode 100644 index 5364dc1b7c10..000000000000 --- a/scripts/unsloth/additive_merge.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Resolve merge conflicts that are provably pure add/add, and only those. - -The conflict that keeps breaking the nightly is always the same shape: upstream -registers a new architecture in a fallthrough group and one of our pinned PRs -registers another one at the same spot. Neither side changed a line the other -side touched -- both only added, at a place where the merge base had nothing. -The union of the two additions is the resolution, and it is mechanical. - -Anything else is left conflicted and reported. In particular a conflict where -the merge base is non-empty means at least one side *edited* shared text, and -picking a side or unioning them is a guess. This script never guesses. - -The two additions are compared on their CONTENT, not on the braces around it. -A case arm is `case X:`, a body, and `} break;`, and two arms for different -architectures share that last part whatever they do. Treating the scaffolding -as evidence that the same change was made twice refuses exactly the conflict -this script exists for; see STRUCTURAL below. - -Reads a conflicted work tree, writes resolutions in place, exits 0 if every -conflict in every file was resolved and 1 otherwise. `--report` emits JSON -describing what it did for the caller to quote in a PR body. -""" - -from __future__ import annotations - -import argparse -import json -import re -import subprocess -import sys -from pathlib import Path - -OURS = "<<<<<<< " -BASE = "||||||| " -SEP = "=======" -THEIRS = ">>>>>>> " - - -class Unresolvable(Exception): - """A conflict this script is not allowed to decide.""" - - -def parse_conflicts(lines: list[str]) -> list[tuple[int, int, list[str], list[str], list[str]]]: - """Split diff3-style content into (start, end, ours, base, theirs) regions. - - Raises Unresolvable if the markers do not nest as diff3 promises, which - means the file is not in the state we think it is. - """ - regions = [] - i = 0 - n = len(lines) - while i < n: - if not lines[i].startswith(OURS): - i += 1 - continue - start = i - ours: list[str] = [] - base: list[str] = [] - theirs: list[str] = [] - cur = ours - seen_base = False - i += 1 - while True: - if i >= n: - raise Unresolvable(f"unterminated conflict starting at line {start + 1}") - ln = lines[i] - if ln.startswith(OURS): - raise Unresolvable(f"nested conflict marker at line {i + 1}") - if ln.startswith(BASE): - cur = base - seen_base = True - elif ln.rstrip("\n") == SEP: - cur = theirs - elif ln.startswith(THEIRS): - i += 1 - break - else: - cur.append(ln) - i += 1 - if not seen_base: - # Without the base section we cannot tell add/add from edit/edit. - raise Unresolvable( - f"conflict at line {start + 1} has no base section; " - "re-checkout with --conflict=diff3" - ) - regions.append((start, i, ours, base, theirs)) - return regions - - -def nonblank(lines: list[str]) -> list[str]: - return [ln.strip() for ln in lines if ln.strip()] - - -# A line that closes or opens a block and nothing else. Two INDEPENDENT case -# arms in the same switch share these by construction -- `{`, `} break;`, `}` -# are what a case arm is made of, not what makes it that case arm -- so finding -# them on both sides says nothing about whether the two sides added the same -# construct. Matching them as "shared" is what refused the real add/add of -# PROJECTOR_TYPE_KIMIK3 next to PROJECTOR_TYPE_DEEPSEEK4V in tools/mtmd/clip.cpp -# with "one change made twice: {, } break;", when the two arms had no line of -# actual content in common. -# -# Deliberately narrow: braces, brackets, parens, semicolons and commas, around -# at most one bare block-terminating keyword. `break;` matches, `return true;` -# does not, and anything naming a type, a constant or a function does not. -STRUCTURAL = re.compile(r"^[\s{}()\[\];,]*(?:break|continue|return|pass)?[\s{}()\[\];,]*$") - - -def identifying(lines: list[str]) -> set[str]: - """The lines that say WHICH construct this is, ignoring block scaffolding.""" - return {ln for ln in nonblank(lines) if not STRUCTURAL.match(ln)} - - -# `case FOO:`, `case FOO :`, `default:`. A fallthrough label may carry no body -# at all, which is the shape the nightly hits most often. -CASE_LABEL = re.compile(r"^(?:case\s+[^:]+|default\s*):") - - -def case_arms(lines: list[str]) -> set[str] | None: - """The case labels this side adds, or None if it is not a run of case arms. - - None, not an empty set: "adds no case arm" and "adds case arms, none of - which the other side adds" have to be told apart, and only the second one - licenses the union below. - """ - ident = [ln for ln in nonblank(lines) if not STRUCTURAL.match(ln)] - if not ident or not CASE_LABEL.match(ident[0]): - return None - return {ln for ln in ident if CASE_LABEL.match(ln)} - - -def resolve_region(ours: list[str], base: list[str], theirs: list[str]) -> list[str]: - """Return the union, or raise if this region is not a pure add/add.""" - if nonblank(base): - raise Unresolvable( - "merge base is not empty, so at least one side edited existing text" - ) - if not nonblank(ours) or not nonblank(theirs): - # One side added and the other added nothing: git would not have - # conflicted, so seeing this means the region is not what we expect. - raise Unresolvable("one side of the conflict is empty") - if ours == theirs: - # Both sides added byte-identical text; one copy is the resolution. - return list(ours) - ours_arms, theirs_arms = case_arms(ours), case_arms(theirs) - if ours_arms and theirs_arms and ours_arms.isdisjoint(theirs_arms): - # Both sides added case arms, and not one label is on both sides. Two - # arms of the same switch labelled differently are two constructs, so - # any line they happen to share is body text, not a duplicate: the real - # tools/mtmd/clip.cpp collision has a KIMIK3 arm and a DEEPSEEK4V arm - # that both set `hparams.rope_theta = 10000.0f;`, and refusing on that - # coincidence is what the shared-line check is for, backwards. - # - # The same change made twice would keep its label, so it lands in the - # check below instead. This is the one place where a shared line is - # allowed, and it is allowed because the labels prove the arms are - # distinct -- a duplicated label would not even compile. - return list(theirs) + list(ours) - shared = identifying(ours) & identifying(theirs) - if shared: - # Overlapping content is the signature of one construct added twice, - # not two independent additions. Unioning it would duplicate code. - # Scaffolding lines are excluded above, so what is left is content both - # sides genuinely wrote, which is the thing that makes this a duplicate. - raise Unresolvable( - "both sides add the same line(s), so this is one change made twice: " - + ", ".join(sorted(shared)[:3]) - ) - if not identifying(ours) or not identifying(theirs): - # Everything one side added is scaffolding, so there is no content to - # tell the two additions apart and the exclusion above has nothing left - # to work with. Refuse rather than union braces onto braces. - raise Unresolvable( - "one side adds only block scaffolding, so the two additions cannot " - "be told apart" - ) - # Upstream first, then ours: the same order a human repin produces. - return list(theirs) + list(ours) - - -def decide_file(path: Path) -> tuple[str, list[dict]]: - """Return the resolved content and a per-hunk record, without writing.""" - lines = path.read_text(encoding="utf-8", errors="surrogateescape").splitlines(keepends=True) - regions = parse_conflicts(lines) - if not regions: - raise Unresolvable("no conflict markers found") - - out: list[str] = [] - prev = 0 - hunks = [] - for start, end, ours, base, theirs in regions: - resolution = resolve_region(ours, base, theirs) - out.extend(lines[prev:start]) - out.extend(resolution) - prev = end - hunks.append( - { - "ours": "".join(ours), - "theirs": "".join(theirs), - "resolution": "".join(resolution), - } - ) - out.extend(lines[prev:]) - return "".join(out), hunks - - -def conflicted_files(repo: Path) -> list[str]: - r = subprocess.run( - ["git", "diff", "--name-only", "--diff-filter=U"], - cwd=repo, capture_output=True, text=True, check=True, - ) - return [f for f in r.stdout.splitlines() if f] - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--repo", default=".", help="conflicted work tree") - ap.add_argument("--report", help="write a JSON report here") - ap.add_argument("--dry-run", action="store_true", help="decide, but do not write") - args = ap.parse_args() - - repo = Path(args.repo).resolve() - files = conflicted_files(repo) - report: dict = {"resolved": [], "refused": [], "ok": False} - - if not files: - report["refused"].append({"file": "-", "reason": "no conflicted files"}) - - # Decide every file before writing any of them. A refusal on the second - # file must not leave the first one already rewritten on disk: the caller - # would then be looking at a tree that is neither the conflict nor the - # resolution. - pending: list[tuple[Path, str]] = [] - for f in files: - try: - content, hunks = decide_file(repo / f) - pending.append((repo / f, content)) - report["resolved"].append({"file": f, "hunks": hunks}) - except Unresolvable as e: - report["refused"].append({"file": f, "reason": str(e)}) - except OSError as e: - report["refused"].append({"file": f, "reason": f"cannot read: {e}"}) - - report["ok"] = bool(files) and not report["refused"] - - if report["ok"] and not args.dry_run: - for p, content in pending: - p.write_text(content, encoding="utf-8", errors="surrogateescape") - subprocess.run(["git", "add", "--"] + files, cwd=repo, check=True) - - for r in report["resolved"]: - print(f"resolved {r['file']}") - for r in report["refused"]: - print(f"refused {r['file']}: {r['reason']}", file=sys.stderr) - - if args.report: - Path(args.report).write_text(json.dumps(report, indent=2)) - return 0 if report["ok"] else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/assemble_metadata.py b/scripts/unsloth/assemble_metadata.py deleted file mode 100644 index 87fa3cebe1e4..000000000000 --- a/scripts/unsloth/assemble_metadata.py +++ /dev/null @@ -1,533 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Assemble the release-level sidecars for an Unsloth llama.cpp prebuilt release. - -Produces, matching the schema consumed by unslothai/unsloth's installer: - - llama-prebuilt-manifest.json : describes every locally-built bundle in this - release (CUDA x64/arm64 profiles + ROCm Linux/Windows per gfx target + - macOS arm64/x64 + CPU Linux/Windows x64+arm64 + Vulkan Linux x64/arm64 and - Windows x64), with the dispatch metadata the installer needs to pick the - right one. - - llama-prebuilt-sha256.json : a cross-OS integrity index covering both the - locally-built bundles AND the upstream ggml-org assets the installer still - pulls (arm64 CPU + the Windows CUDA cudart/runtime) + the source tarballs. - -Run after the build matrix has dropped the app-*.{tar.gz,zip} bundles into --dist. -""" -from __future__ import annotations - -import argparse -import datetime as _dt -import hashlib -import json -import os -import re -import sys -import tarfile -import time -import urllib.error -import urllib.request -import zipfile -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -UPSTREAM_REPO = "ggml-org/llama.cpp" - -BUNDLE_RE = re.compile( - r"^app-(?P<tag>[^/]+)-(?P<platform>linux|windows)-(?P<arch>x64|arm64)-(?P<profile>cuda1[23]-(?:older|newer|portable|legacy))\.(?P<ext>tar\.gz|zip)$" -) - -ROCM_BUNDLE_RE = re.compile( - r"^app-(?P<tag>[^/]+)-(?P<platform>linux|windows)-x64-rocm-(?P<gfx>gfx[0-9a-zA-Z]+)\.(?P<ext>tar\.gz|zip)$" -) - -# CPU-only and Vulkan bundles, built locally by unsloth-prebuilt-cpu.yml / -# unsloth-prebuilt-vulkan.yml. Like ROCm/macOS they are raw build/bin archives -# with no embedded UNSLOTH_PREBUILT_INFO.json, so everything in the manifest -# entry is derived from the filename. CPU covers Linux/Windows x64 + arm64; -# Vulkan covers Linux x64 + arm64 and Windows x64. -CPU_BUNDLE_RE = re.compile( - r"^app-(?P<tag>[^/]+)-(?P<platform>linux|windows)-(?P<arch>x64|arm64)-cpu\.(?P<ext>tar\.gz|zip)$" -) - -VULKAN_BUNDLE_RE = re.compile( - r"^app-(?P<tag>[^/]+)-(?P<target>linux-(?:x64|arm64)|windows-x64)-vulkan\.(?P<ext>tar\.gz|zip)$" -) - -# macOS slices are built by unsloth-prebuilt-macos.yml and land in dist/ under -# upstream's own naming (the installer expects that name). They carry no -# embedded UNSLOTH_PREBUILT_INFO.json, so -- like ROCm -- everything is derived -# from the filename. -MACOS_BUNDLE_RE = re.compile( - r"^llama-(?P<tag>[^/]+)-bin-macos-(?P<arch>arm64|x64)\.tar\.gz$" -) - -# Per-(platform, arch) dispatch keys for the published manifest + sha256 index. -# Linux x64 keeps the historical "linux-cuda" so older unsloth installers stay -# compatible; the others get distinct kinds so installers cleanly ignore a -# bundle they can't run instead of trying to launch the wrong binary. -KIND_BY_CUDA = { - ("linux", "x64"): {"manifest": "linux-cuda", "sha": "linux-cuda-app"}, - ("linux", "arm64"): {"manifest": "linux-arm64-cuda", "sha": "linux-arm64-cuda-app"}, - ("windows", "x64"): {"manifest": "windows-cuda", "sha": "windows-cuda-app"}, -} - -KIND_BY_ROCM_PLATFORM = { - "linux": {"manifest": "linux-rocm", "sha": "linux-rocm-app"}, - "windows": {"manifest": "windows-rocm", "sha": "windows-rocm-app"}, -} - -# CPU + Vulkan slices. These supersede the upstream ggml-org CPU/Vulkan -# passthroughs (we now build them ourselves). The manifest kinds match what the -# installer selects per (platform, arch): x64 keeps the historical -# linux-cpu/windows-cpu, arm64 uses linux-arm64/windows-arm64 (the same kinds -# the installer's upstream-fallback path used). The "-app" sha kinds mark them -# as locally-built bundles. -KIND_BY_CPU = { - ("linux", "x64"): {"manifest": "linux-cpu", "sha": "linux-cpu-app"}, - ("linux", "arm64"): {"manifest": "linux-arm64", "sha": "linux-arm64-app"}, - ("windows", "x64"): {"manifest": "windows-cpu", "sha": "windows-cpu-app"}, - ("windows", "arm64"): {"manifest": "windows-arm64", "sha": "windows-arm64-app"}, -} - -KIND_BY_VULKAN_TARGET = { - "linux-x64": {"manifest": "linux-vulkan", "sha": "linux-vulkan-app"}, - "linux-arm64": {"manifest": "linux-vulkan", "sha": "linux-arm64-vulkan-app"}, - "windows-x64": {"manifest": "windows-vulkan", "sha": "windows-vulkan-app"}, -} - -# macOS slices: install_kind / sha-index kind / manifest bundle_profile per arch. -# We build these ourselves now (upstream's arm64 release stamps minos=26 and -# won't dyld-load on macOS < 26), so they are recorded as locally-built bundles -# rather than upstream passthroughs. -MACOS_SLICE = { - "arm64": {"manifest": "macos-arm64", "sha": "macos-arm64-app", "profile": "macos-metal-arm64"}, - "x64": {"manifest": "macos-x64", "sha": "macos-x64-app", "profile": "macos-cpu-x64"}, -} - -# Mapping from the umbrella gfx target name (as it appears in the asset -# filename) to the concrete gfx architectures it compiles for. Mirrors the -# `mapped_target` switch in unsloth-prebuilt-rocm.yml; kept duplicated so the -# manifest can stay self-describing without parsing the workflow. -ROCM_TARGET_MAP = { - "gfx1151": ["gfx1151"], - "gfx1150": ["gfx1150"], - "gfx120X": ["gfx1200", "gfx1201"], - "gfx110X": ["gfx1100", "gfx1101", "gfx1102", "gfx1103"], - "gfx103X": ["gfx1030", "gfx1031", "gfx1032", "gfx1034"], - "gfx90a": ["gfx90a"], - "gfx908": ["gfx908"], -} - - -def sha256_file(path: Path) -> str: - h = hashlib.sha256() - with open(path, "rb") as fh: - for chunk in iter(lambda: fh.read(1 << 20), b""): - h.update(chunk) - return h.hexdigest() - - -def read_bundle_info(bundle: Path) -> dict: - """Read the UNSLOTH_PREBUILT_INFO.json embedded in a built bundle. - - Linux/macOS bundles are .tar.gz; Windows bundles are .zip -- dispatch on the - extension so the Windows CUDA bundles can be read too. - """ - target = "UNSLOTH_PREBUILT_INFO.json" - if bundle.name.endswith(".zip"): - with zipfile.ZipFile(bundle) as zf: - for n in zf.namelist(): - if n.endswith(target): - return json.loads(zf.read(n)) - else: - with tarfile.open(bundle, "r:gz") as tar: - for m in tar.getmembers(): - if m.isfile() and m.name.endswith(target): - return json.loads(tar.extractfile(m).read()) - sys.exit(f"ERROR: {bundle.name} has no {target}") - - -def _request(url: str, token: str | None) -> urllib.request.Request: - req = urllib.request.Request(url, headers={"User-Agent": "unsloth-prebuilt-assembler"}) - if token and "api.github.com" in url: - req.add_header("Authorization", f"Bearer {token}") - req.add_header("Accept", "application/vnd.github+json") - return req - - -def _with_retry(fn, *, attempts: int = 4, base: float = 2.0): - for i in range(attempts): - try: - return fn() - except (urllib.error.URLError, TimeoutError, ConnectionError) as e: - code = getattr(e, "code", None) - # give up on the last try or on a non-transient 4xx (429 is transient) - if i == attempts - 1 or (code is not None and 400 <= code < 500 and code != 429): - raise - time.sleep(base * (2 ** i)) - - -def http_json(url: str, token: str | None) -> object: - def go(): - with urllib.request.urlopen(_request(url, token), timeout=120) as resp: - return json.loads(resp.read()) - return _with_retry(go) - - -def sha256_url(url: str, token: str | None) -> str: - def go(): - h = hashlib.sha256() - with urllib.request.urlopen(_request(url, token), timeout=300) as resp: - for chunk in iter(lambda: resp.read(1 << 20), b""): - h.update(chunk) - return h.hexdigest() - return _with_retry(go) - - -def upstream_assets(tag: str, token: str | None) -> dict[str, dict]: - """name -> {url, digest} for the upstream release at `tag`.""" - data = http_json(f"https://api.github.com/repos/{UPSTREAM_REPO}/releases/tags/{tag}", token) - out: dict[str, dict] = {} - for asset in data.get("assets", []): # type: ignore[union-attr] - out[asset["name"]] = { - "url": asset["browser_download_url"], - "digest": asset.get("digest"), # "sha256:<hex>" since 2024, else None - } - return out - - -def asset_digest_or_hash(asset: dict, token: str | None) -> str: - """Prefer GitHub's published asset digest; stream-hash as fallback.""" - raw = (asset.get("digest") or "").strip().lower() - if raw.startswith("sha256:"): - h = raw.split(":", 1)[1] - if len(h) == 64 and all(c in "0123456789abcdef" for c in h): - return h - return sha256_url(asset["url"], token) - - -def build_artifacts( - cuda_bundles: list[tuple[str, str, str, dict]], - rocm_bundles: list[tuple[str, str, str]], - macos_bundles: list[tuple[str, str]], - cpu_bundles: list[tuple[str, str, str]], - vulkan_bundles: list[tuple[str, str]], -) -> list[dict]: - """cuda_bundles: list of (asset_name, platform, arch, embedded UNSLOTH_PREBUILT_INFO). - rocm_bundles: list of (asset_name, platform, gfx_target). - macos_bundles: list of (asset_name, arch). - cpu_bundles: list of (asset_name, platform, arch). - vulkan_bundles: list of (asset_name, target). - - CUDA fields come from each bundle's own embedded metadata, so the manifest - can never disagree with what was actually compiled. ROCm, macOS, CPU and - Vulkan bundles are raw archives (no embedded info), so their manifest - entries are derived from the filename + the ROCM_TARGET_MAP / MACOS_SLICE - tables. - """ - artifacts = [] - for asset_name, platform, arch, info in cuda_bundles: - artifacts.append({ - "asset_name": asset_name, - "install_kind": KIND_BY_CUDA[(platform, arch)]["manifest"], - "bundle_profile": info["bundle_profile"], - "runtime_line": info["runtime_line"], - "coverage_class": info["coverage_class"], - "supported_sms": info["supported_sms"], - "min_sm": info["min_sm"], - "max_sm": info["max_sm"], - "rank": info["bundle_rank"], - "toolkit_version": info["toolkit_line"], - }) - for asset_name, platform, gfx in rocm_bundles: - artifacts.append({ - "asset_name": asset_name, - "install_kind": KIND_BY_ROCM_PLATFORM[platform]["manifest"], - "gfx_target": gfx, - "mapped_targets": ROCM_TARGET_MAP.get(gfx, [gfx]), - }) - for asset_name, arch in macos_bundles: - # No runtime_line/coverage_class for macOS (no CUDA/ROCm runtime to - # match); emitted as explicit null so the key set stays stable, and a - # fixed rank since there is a single slice per arch. - artifacts.append({ - "asset_name": asset_name, - "install_kind": MACOS_SLICE[arch]["manifest"], - "bundle_profile": MACOS_SLICE[arch]["profile"], - "runtime_line": None, - "coverage_class": None, - "rank": 50, - }) - # CPU + Vulkan: no CUDA/ROCm runtime to match, so runtime_line/coverage_class - # are explicit null (stable key set). A single slice per (backend, platform, - # arch), so a fixed rank; CPU ranks last (1000) as the universal fallback, - # matching the installer's own direct-scan rank for a CPU bundle. - for asset_name, platform, arch in cpu_bundles: - artifacts.append({ - "asset_name": asset_name, - "install_kind": KIND_BY_CPU[(platform, arch)]["manifest"], - "bundle_profile": f"{platform}-cpu-{arch}", - "runtime_line": None, - "coverage_class": None, - "rank": 1000, - }) - for asset_name, target in vulkan_bundles: - platform, arch = target.rsplit("-", 1) - artifacts.append({ - "asset_name": asset_name, - "install_kind": KIND_BY_VULKAN_TARGET[target]["manifest"], - "bundle_profile": f"{platform}-vulkan-{arch}", - "runtime_line": None, - "coverage_class": None, - "rank": 60, - }) - return artifacts - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--tag", required=True) - ap.add_argument("--ref", default=None, - help="git ref the source was built from; defaults to refs/tags/<tag>") - ap.add_argument("--source-repo", default=UPSTREAM_REPO, - help="repo holding the source ref: upstream, or the publish repo for merged mix tags") - ap.add_argument("--base-tag", default=None, - help="upstream release tag the build is based on; defaults to --tag (differs for mix builds)") - ap.add_argument("--pr-set", default="[]", - help='JSON array of merged PRs: [{"repo":..,"number":..,"sha":..,"url":..,"title":..},..]') - ap.add_argument("--commit", required=True) - ap.add_argument("--ggml-tree", default=None, - help="git tree id of ggml/ in the built source; ABI key for paired builds") - ap.add_argument("--ggml-version", default=None) - ap.add_argument("--dist", required=True, type=Path, help="dir holding the built app-*.tar.gz bundles") - ap.add_argument("--out", required=True, type=Path, help="dir to write the two JSON sidecars into") - ap.add_argument("--publish-repo", required=True, help="repo the bundles+manifest are published to") - ap.add_argument("--token", default=None, help="GitHub token (else $GH_TOKEN/$GITHUB_TOKEN)") - args = ap.parse_args() - - token = args.token or os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - tag, commit, short = args.tag, args.commit, args.commit[:7] - ref = args.ref or f"refs/tags/{tag}" - source_repo = args.source_repo - base_tag = args.base_tag or tag - pr_set = json.loads(args.pr_set) - # Upstream release assets exist only for a vanilla build of an upstream tag - # (a mix build's merged tree exists in no repo, only in its release assets). - is_upstream_release = source_repo == UPSTREAM_REPO and ref == f"refs/tags/{tag}" and not pr_set - ref_kind = "tag" if is_upstream_release else "mix" if pr_set else "ref" - source_ref = tag if ref == f"refs/tags/{tag}" else ref - args.out.mkdir(parents=True, exist_ok=True) - - def base_entry(kind: str, repo: str, digest: str) -> dict: - return { - "kind": kind, - "repo": repo, - "sha256": digest, - "source_commit": commit, - "source_commit_short": short, - "upstream_tag": base_tag, - } - - sha_artifacts: dict[str, dict] = {} - - # 1a) locally-built CUDA bundles (Linux x64/arm64 .tar.gz + Windows x64 - # .zip): hash in parallel. All carry embedded UNSLOTH_PREBUILT_INFO.json. - found: list[tuple[str, str, str, dict]] = [] - cuda_paths = sorted(args.dist.glob("app-*-linux-*.tar.gz")) + sorted(args.dist.glob("app-*-windows-*.zip")) - for p in cuda_paths: - m = BUNDLE_RE.match(p.name) - if not m: - continue - found.append((p.name, m.group("platform"), m.group("arch"), read_bundle_info(p))) - if not found: - print(f"ERROR: no app-* CUDA bundles in {args.dist}", file=sys.stderr) - return 1 - with ThreadPoolExecutor(max_workers=4) as pool: - local_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), found)) - for (name, platform, arch, _info), digest in zip(found, local_digests): - sha_artifacts[name] = base_entry(KIND_BY_CUDA[(platform, arch)]["sha"], args.publish_repo, digest) - - # 1b) locally-built ROCm bundles (linux .tar.gz + windows .zip): hash in - # parallel. No embedded metadata; we derive everything from the filename. - rocm_found: list[tuple[str, str, str]] = [] - for p in sorted(list(args.dist.glob("app-*-rocm-*.tar.gz")) + list(args.dist.glob("app-*-rocm-*.zip"))): - m = ROCM_BUNDLE_RE.match(p.name) - if not m: - continue - rocm_found.append((p.name, m.group("platform"), m.group("gfx"))) - if rocm_found: - with ThreadPoolExecutor(max_workers=4) as pool: - rocm_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), rocm_found)) - for (name, platform, _gfx), digest in zip(rocm_found, rocm_digests): - sha_artifacts[name] = base_entry(KIND_BY_ROCM_PLATFORM[platform]["sha"], args.publish_repo, digest) - else: - # Warning, not error: ROCm can legitimately be empty when a dispatch run - # narrows operating_systems to skip both Windows and Ubuntu. The daily - # schedule always builds the full set, so this fires only on manual runs. - print("WARNING: no app-*-rocm-*.{tar.gz,zip} bundles found", file=sys.stderr) - - # 1c) locally-built macOS slices (arm64 Metal + x64 CPU): hash in parallel. - # No embedded metadata; we derive everything from the filename. We build - # these ourselves now, so they are NOT recorded as upstream passthroughs in - # section 2. - macos_found: list[tuple[str, str]] = [] - for p in sorted(args.dist.glob("llama-*-bin-macos-*.tar.gz")): - m = MACOS_BUNDLE_RE.match(p.name) - if not m: - continue - macos_found.append((p.name, m.group("arch"))) - if macos_found: - with ThreadPoolExecutor(max_workers=4) as pool: - macos_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), macos_found)) - for (name, arch), digest in zip(macos_found, macos_digests): - sha_artifacts[name] = base_entry(MACOS_SLICE[arch]["sha"], args.publish_repo, digest) - else: - # Like ROCm: warn rather than error, so a partial dispatch run still - # assembles. The daily schedule always builds both slices. - print("WARNING: no llama-*-bin-macos-*.tar.gz bundles found", file=sys.stderr) - - # 1d) locally-built CPU + Vulkan bundles (Linux .tar.gz + Windows .zip). - # No embedded metadata; everything is derived from the filename. These - # replace the upstream ggml-org CPU/Vulkan passthroughs that section 2 used - # to record -- the release now ships our own builds for these slices. - def scan_bundles(regex) -> list[tuple[str, "re.Match[str]"]]: - out: list[tuple[str, "re.Match[str]"]] = [] - for p in sorted(list(args.dist.glob("app-*.tar.gz")) + list(args.dist.glob("app-*.zip"))): - m = regex.match(p.name) - if m: - out.append((p.name, m)) - return out - - cpu_found = [(name, m.group("platform"), m.group("arch")) for name, m in scan_bundles(CPU_BUNDLE_RE)] - if cpu_found: - with ThreadPoolExecutor(max_workers=4) as pool: - cpu_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), cpu_found)) - for (name, platform, arch), digest in zip(cpu_found, cpu_digests): - sha_artifacts[name] = base_entry(KIND_BY_CPU[(platform, arch)]["sha"], args.publish_repo, digest) - else: - print("WARNING: no app-*-cpu.{tar.gz,zip} bundles found", file=sys.stderr) - - vulkan_found = [(name, m.group("target")) for name, m in scan_bundles(VULKAN_BUNDLE_RE)] - if vulkan_found: - with ThreadPoolExecutor(max_workers=4) as pool: - vulkan_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), vulkan_found)) - for (name, target), digest in zip(vulkan_found, vulkan_digests): - sha_artifacts[name] = base_entry( - KIND_BY_VULKAN_TARGET[target]["sha"], args.publish_repo, digest - ) - else: - print("WARNING: no app-*-vulkan.{tar.gz,zip} bundles found", file=sys.stderr) - - # 2) upstream per-OS bundles: read GitHub's published asset.digest from the - # API response; fall back to a streaming hash if a digest is missing. - # macOS and the locally-built CPU/Vulkan slices are absent here on - # purpose -- we build those ourselves (1c/1d). - # A mix build has no upstream release for its tag, so the whole section - # is skipped; its uncovered hosts fall back to a source build of the - # merged tree instead of a vanilla upstream binary missing the PRs. - if not is_upstream_release: - print(f"WARNING: {source_repo}@{ref} is not an upstream release tag; " - "skipping upstream asset index entries", file=sys.stderr) - else: - assets = upstream_assets(tag, token) - wanted: list[tuple[str, str]] = [] # (name, kind) - for name in sorted(assets): - if re.fullmatch(r"cudart-llama-bin-win-cuda-\d+\.\d+-x64\.zip", name): - wanted.append((name, "windows-cuda-upstream")) - # The win-cuda BINARY zips must be recorded under their own names too: - # the installer resolves an attempt's hash by exact asset name first - # and only then falls back to the cudart alias, so without these - # entries every Windows CUDA binary gets paired with the cudart digest - # and fails download verification. - elif re.fullmatch( - rf"llama-{re.escape(tag)}-bin-win-cuda-\d+\.\d+-x64\.zip", name - ): - wanted.append((name, "windows-cuda-upstream")) - # x64 CPU and all current Vulkan targets are no longer passthroughs -- - # we build them ourselves (section 1d above). arm64 CPU is now built too - # (1d emits the locally-built linux-arm64/windows-arm64 bundles), but the - # installer still selects the upstream arm64 asset until it is switched to - # those bundles; keep these passthrough checksums until that installer - # flip lands, then drop them. - for name, kind in ( - (f"llama-{tag}-bin-ubuntu-arm64.tar.gz", "linux-arm64-upstream"), - (f"llama-{tag}-bin-win-cpu-arm64.zip", "windows-arm64-upstream"), - ): - if name not in assets: - print(f"WARNING: upstream asset {name} not found at {tag}; skipping", file=sys.stderr) - continue - wanted.append((name, kind)) - for name, kind in wanted: - sha_artifacts[name] = base_entry(kind, UPSTREAM_REPO, asset_digest_or_hash(assets[name], token)) - - # 3) source tarballs: prefer a local copy in dist -- the workflow downloads - # them from codeload so the published asset and its recorded checksum are - # the exact same bytes. Fall back to stream-hashing codeload if absent - # (e.g. a standalone/local run that didn't pre-fetch them). codeload - # doesn't expose pre-computed digests, so we always hash the content. - source_jobs = [ - (f"llama.cpp-source-{tag}.tar.gz", "upstream-source", - f"https://codeload.github.com/{source_repo}/tar.gz/{ref}"), - (f"llama.cpp-source-commit-{commit}.tar.gz", "exact-source", - f"https://codeload.github.com/{source_repo}/tar.gz/{commit}"), - ] - - def source_digest(name: str, url: str) -> str: - local = args.dist / name - return sha256_file(local) if local.is_file() else sha256_url(url, token) - - with ThreadPoolExecutor(max_workers=2) as pool: - source_digests = list(pool.map(lambda j: source_digest(j[0], j[2]), source_jobs)) - for (name, kind, _url), digest in zip(source_jobs, source_digests): - sha_artifacts[name] = base_entry(kind, source_repo, digest) - - # 4) manifest, then hash it into the index. Both sidecars share the same - # source-description header; merged_prs records the exact pinned PR SHAs - # a mix build compiled (empty for vanilla builds). - common = { - "schema_version": 1, - "component": "llama.cpp", - "source_repo": source_repo, - "source_repo_url": f"https://github.com/{source_repo}", - "source_ref_kind": ref_kind, - "requested_source_ref": source_ref, - "resolved_source_ref": source_ref, - "source_commit": commit, - "source_commit_short": short, - "upstream_repo": UPSTREAM_REPO, - "upstream_tag": base_tag, - "merged_prs": pr_set, - # ABI key for anything compiled against this release's ggml, e.g. - # whisper.cpp slim bundles. Changes only when ggml/ contents change. - # The -mix- tag suffix hashes the PR set, not ggml, so it stays - # constant while the base tag (and ggml with it) moves. - "ggml_tree": args.ggml_tree, - "ggml_version": args.ggml_version, - } - manifest = { - **common, - "generated_at_utc": _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "artifacts": build_artifacts(found, rocm_found, macos_found, cpu_found, vulkan_found), - } - manifest_path = args.out / "llama-prebuilt-manifest.json" - manifest_path.write_text(json.dumps(manifest, indent=2)) - sha_artifacts["llama-prebuilt-manifest.json"] = base_entry( - "published-manifest", args.publish_repo, sha256_file(manifest_path) - ) - - sha256_doc = { - **common, - "release_tag": tag, - "artifacts": sha_artifacts, - } - (args.out / "llama-prebuilt-sha256.json").write_text(json.dumps(sha256_doc, indent=2)) - - print(f"wrote manifest ({len(manifest['artifacts'])} artifacts) and sha256 index " - f"({len(sha_artifacts)} entries) to {args.out}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/unsloth/assert_macho_minos.sh b/scripts/unsloth/assert_macho_minos.sh deleted file mode 100755 index 01a1536c9a0d..000000000000 --- a/scripts/unsloth/assert_macho_minos.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Pre-publish gate for the Unsloth macOS llama.cpp prebuilt. Fails the build -# unless every shipped Mach-O declares a minimum macOS <= the pinned deployment -# target (so it dyld-loads on that floor or newer), carries the expected arch slice, and -# actually launches. This is what keeps a runner/SDK bump from silently shipping -# a minos=26 binary that fails on older Macs. -# -# Usage: assert_macho_minos.sh <bin_dir> <expect_arch> [max_minos] -# expect_arch: arm64 | x86_64 max_minos: default 14.0 -set -uo pipefail - -BIN_DIR="${1:?bin dir required}" -EXPECT_ARCH="${2:?expected arch required}" -MAX_MINOS="${3:-14.0}" - -fail() { echo "::error::$*"; exit 1; } -# Compare dotted major.minor as major*100+minor (14.0 -> 1400). -ver_key() { local v="${1%%-*}"; awk -F. '{printf "%d", $1*100 + ($2==""?0:$2)}' <<<"$v"; } -MAX_KEY="$(ver_key "$MAX_MINOS")" - -command -v vtool >/dev/null 2>&1 || fail "vtool not found (Xcode command line tools required)" - -# macOS ships bash 3.2, which has no `mapfile`; read into the array portably. -MACHOS=() -while IFS= read -r _macho; do MACHOS+=("$_macho"); done < <(find "$BIN_DIR" -type f \( -name '*.dylib' -o -name 'llama-server' -o -name 'llama-quantize' -o -name 'llama-cli' \) 2>/dev/null) -[ "${#MACHOS[@]}" -gt 0 ] || fail "no Mach-O binaries found under $BIN_DIR" - -for macho in "${MACHOS[@]}"; do - minos="$(vtool -show-build "$macho" 2>/dev/null | awk '/minos/{print $2; exit}')" - [ -n "$minos" ] || fail "$(basename "$macho") has no LC_BUILD_VERSION/minos" - if [ "$(ver_key "$minos")" -gt "$MAX_KEY" ]; then - fail "$(basename "$macho") minos=$minos exceeds deployment target $MAX_MINOS" - fi - if ! lipo -archs "$macho" 2>/dev/null | tr ' ' '\n' | grep -qx "$EXPECT_ARCH"; then - fail "$(basename "$macho") is missing the $EXPECT_ARCH slice (got: $(lipo -archs "$macho" 2>/dev/null))" - fi -done -echo "static check passed: ${#MACHOS[@]} Mach-O files, all minos<=$MAX_MINOS, arch=$EXPECT_ARCH" - -# Runtime launch forces dyld to resolve every linked dylib (incl. Metal). -for tool in llama-cli llama-quantize; do - bin="$(find "$BIN_DIR" -type f -name "$tool" 2>/dev/null | head -1)" - [ -n "$bin" ] || fail "$tool not found under $BIN_DIR" -done -CLI="$(find "$BIN_DIR" -type f -name llama-cli | head -1)" -QUANT="$(find "$BIN_DIR" -type f -name llama-quantize | head -1)" -"$CLI" --version >/dev/null 2>&1 || fail "llama-cli failed to launch (dyld load / symbol error)" -# llama-quantize's usage() ends in exit(1), so --help is non-zero by design. -# A dyld/symbol failure dies before main and prints nothing, so verify the -# binary actually reached main (printed usage) rather than trusting the code. -q_out="$("$QUANT" --help 2>&1 || true)" -printf '%s\n' "$q_out" | grep -q "usage:" || fail "llama-quantize failed to launch (dyld load / symbol error)" -echo "runtime launch passed: llama-cli --version and llama-quantize --help both ran" diff --git a/scripts/unsloth/carry_vintage.py b/scripts/unsloth/carry_vintage.py deleted file mode 100755 index 0f469db9cca9..000000000000 --- a/scripts/unsloth/carry_vintage.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Report which carry files are an unmodified older copy of the upstream PR. - -A carry branch replays an upstream PR onto an aged base tag. -When that PR moves, the question before every refresh is the same one: have we actually changed this file, or are we just holding a stale copy of theirs? -Answering it by hand means diffing every file the PR touches and reading each hunk, which is what made the 08-27 GLM-5-Next refresh expensive, and two of those hand answers were wrong. - -The mechanical answer: if our version of a file is byte-identical to the version at SOME commit of the upstream PR, then we never edited it, and their newer copy supersedes ours with nothing lost. -That is a fact about blob hashes, not a judgement. -Files we really did change match no upstream commit and are reported as diverged, which is correct: on 08-27 gguf-py/gguf/tensor_mapping.py did not match, because it genuinely carried qwen4exp additions as well. - -This only reports. -It does not resolve, stage or write anything, because "upstream superseded ours" is not the same as "we want upstream's", and holding a deliberately older vintage is a legitimate decision this script cannot see. - -Use it to decide whether a refresh should merge or simply rebuild: - - python3 scripts/unsloth/carry_vintage.py \\ - --carry <carry sha> --pr-ref refs/pull/27754/head --base refs/tags/b10639 - -When every file is SUPERSEDED, rebuilding the carry from the PR head avoids the merge, and its conflicts, entirely. -A file the PR head still has and the carry does not is reported as OMITTED and blocks that advice, because a rebuild would restore it; a file the PR DELETED is merely ABSENT and blocks nothing. -""" - -from __future__ import annotations - -import argparse -import json -import subprocess -import sys - - -def git(*args: str) -> str: - r = subprocess.run(["git", *args], capture_output=True, text=True) - if r.returncode: - raise RuntimeError(" ".join(args) + ": " + r.stderr.strip()) - return r.stdout.strip() - - -def blob(rev: str, path: str) -> str | None: - """The tree entry as "mode oid", or None if the rev has no such path. - - Mode, not just the oid: a carry that only chmods a file it took verbatim has the same content as upstream, so an oid comparison calls it superseded and a rebuild silently drops the mode change. - """ - r = subprocess.run(["git", "ls-tree", "--full-tree", "-z", rev, "--", path], - capture_output=True, text=True) - if r.returncode != 0 or not r.stdout.strip(): - return None - mode, _type, oid = r.stdout.split("\0")[0].split("\t", 1)[0].split() - return f"{mode} {oid}" - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--carry", required=True, help="carry branch commit") - ap.add_argument("--pr-ref", required=True, help="upstream PR head ref or sha") - ap.add_argument("--base", required=True, help="base tag the PR forked from") - ap.add_argument("--max-commits", type=int, default=60, - help="how far back through the PR to look for a match") - ap.add_argument("--report", metavar="PATH", help="write a JSON summary here") - a = ap.parse_args() - - head = git("rev-parse", a.pr_ref) - fork = git("merge-base", head, a.base) - # --no-renames, because rename detection hides exactly the path that matters here. - # `git diff --name-only` prints only the NEW name of a rename, so a PR moving `old` to `new` never puts `old` in this list. - # A carry that deliberately keeps `old` is then never looked at: `new` comes back SUPERSEDED, nothing diverges, and the summary says a rebuild is equivalent when a rebuild deletes the file the carry is holding. - # Without detection the rename is a delete plus an add, so `old` is classified - DIVERGED, since our copy is the fork's and matches no upstream vintage. - files = [f for f in git("diff", "--name-only", "--no-renames", - fork, head).split("\n") if f] - # fork..head, not head: `--max-count` caps the output, it does not bound the walk, so a bare `head` runs straight past the fork point into the base branch. - # A file our carry deliberately holds at the BASE version then matches a pre-fork commit and is called SUPERSEDED, and the summary says rebuilding from the PR head is equivalent - it is not, it re-adds what the carry dropped. - # Only commits of the PR itself are vintages. - history = [c for c in git("rev-list", f"--max-count={a.max_commits}", - f"{fork}..{head}").split("\n") if c] - - # Paths the CARRY changed that the PR never touched at all. - # Everything above reasons only about files in the PR's diff, so a carry-only edit is invisible to it: every PR path can be SUPERSEDED, nothing diverges, and the summary says a rebuild from the PR head is equivalent while the rebuild drops that edit. - # Same failure as OMITTED, arrived at from the other side. - # Diffed from --base rather than the fork point because a carry replays the PR onto the base tag, so that is what its own delta is against; if a carry is ever based on something else, the extra paths only ever withhold the rebuild advice, which is the safe direction to be wrong in. - touched = set(files) - carry_only = [f for f in git("diff", "--name-only", "--no-renames", - a.base, a.carry).split("\n") - if f and f not in touched] - - superseded, diverged, absent, omitted = [], [], [], [] - for path in files: - ours = blob(a.carry, path) - if ours is None: - # Two very different reasons a path is missing from the carry. - # If the PR DELETED it, the carry agrees and a rebuild reproduces that. - # If it still exists at the PR head, the carry dropped it on purpose, and a rebuild would re-add it - the same mistake as calling a file held at the base version superseded. - (absent if blob(head, path) is None else omitted).append(path) - continue - if ours == blob(head, path): - superseded.append({"path": path, "vintage": head, "current": True}) - continue - # Bounding the walk to fork..head is not enough on its own. - # A PR with several commits usually does not touch every file in its first one, so the commits BEFORE the one that first changed this path still carry the fork's blob - inside the range. - # A carry holding the file at the base version matches one of those and is called SUPERSEDED again, and the summary again says a rebuild is equivalent when it would overwrite exactly what the carry is holding. - # Our copy being the fork's copy is not evidence the PR ever produced it, so it is never a vintage; such a file falls through to DIVERGED, which is where a file needing a human decision belongs. - at_fork = blob(fork, path) - hit = None if ours == at_fork else \ - next((c for c in history if blob(c, path) == ours), None) - if hit: - superseded.append({"path": path, "vintage": hit, "current": False}) - else: - diverged.append(path) - - print(f"carry {a.carry[:10]} vs {a.pr_ref} ({head[:10]}), {len(files)} file(s) touched") - print() - for e in superseded: - note = "already at PR head" if e["current"] else f"our copy is upstream {e['vintage'][:10]}" - print(f" SUPERSEDED {e['path']}\n {note}") - for p in diverged: - print(f" DIVERGED {p}\n matches no upstream vintage; we changed it, or we are " - "holding the base version on purpose. Keep it") - for p in omitted: - print(f" OMITTED {p}\n exists at the PR head, not in the carry; " - "a rebuild would re-add it") - for p in absent: - print(f" ABSENT {p}\n deleted by the PR, not in the carry") - for p in carry_only: - print(f" CARRY ONLY {p}\n changed by the carry, untouched by the PR; " - "a rebuild would drop it") - print() - if diverged: - print(f"{len(diverged)} file(s) genuinely diverge. A refresh has to merge, " - "and those files are the only ones needing judgement.") - if omitted: - print(f"{len(omitted)} file(s) the PR head still has are missing from the " - "carry. Rebuilding would restore them, so it is NOT equivalent to " - "merging; keep or re-drop each one deliberately.") - if carry_only: - print(f"{len(carry_only)} file(s) the carry changed are outside the " - "PR entirely. Rebuilding from the PR head would drop them, so it " - "is NOT equivalent to merging; carry each one across deliberately.") - # Matching an older commit of the PR proves only that we never edited the file, not that we want the newest one: the carry may be holding that vintage on purpose, which this cannot see. - # Rebuilding moves it to head, so the unqualified "nothing is lost" below has to stand down and say so. - older = [e for e in superseded if not e["current"]] - if older: - print(f"{len(older)} file(s) sit at an OLDER vintage than the PR head. " - "We never edited them, but a rebuild still moves them to head; " - "confirm no carry is holding one of them deliberately.") - if not diverged and not omitted and not carry_only and not older: - print("Nothing diverges. Rebuilding the carry from the PR head is " - "equivalent to merging it, without the conflicts.") - - if a.report: - with open(a.report, "w") as fh: - json.dump({"head": head, "superseded": superseded, - "diverged": diverged, "omitted": omitted, - "absent": absent, "carry_only": carry_only}, fh, indent=2) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/check_workflow_scalars.py b/scripts/unsloth/check_workflow_scalars.py deleted file mode 100644 index 24f6a2e1b158..000000000000 --- a/scripts/unsloth/check_workflow_scalars.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Fail when a workflow string is close to GitHub's 21000 character cap. - -GitHub's template compiler refuses any single string in a workflow file longer than 21000 characters. -The whole file then fails to compile, and the failure is close to invisible: - - - the run has zero jobs, no annotations and an empty check suite, so there is nothing to click on; - - `gh run view` says only "This run likely failed because of a workflow file issue"; - - the run is reported against whatever event triggered it, even an event the workflow does not subscribe to, because compilation never got as far as reading `on:`; - - and nothing local catches it. yaml parses the file, actionlint passes it, and so does GitHub's own published parser (@actions/workflow-parser). The limit is enforced only server-side. - -On 08-27 a 14-line explanatory comment added inside the `resolve` job's script took it from 20503 to 21620 characters and silently disabled the entire release workflow for four pushes. -The only way to see the real error was to fire a workflow_dispatch at the ref, which returns it as a 422: - - (Line: 125, Col: 14): Exceeded max expression length 21000 - -Comments inside a `run:` block scalar are part of the string and count against the limit. -Comments in the YAML around it do not, so prose belongs above a step rather than inside it. -Past that, the fix is to split the script into more steps: a step boundary costs nothing and resets the budget. - -Measuring the parsed scalar is not exactly what GitHub measures, so the warn threshold is deliberately well below the cap rather than a character-perfect model of it. -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -import yaml - -# What GitHub enforces, and the point at which a script is big enough that the next edit to it can cross the line without anyone thinking about size. -CAP = 21000 -WARN = 20000 - - -def scalars(node, path: str = ""): - """Every string in the document, with a path that names where it lives.""" - if isinstance(node, str): - yield path, node - elif isinstance(node, dict): - for k, v in node.items(): - yield from scalars(v, f"{path}.{k}") - elif isinstance(node, list): - for i, v in enumerate(node): - yield from scalars(v, f"{path}[{i}]") - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--root", default=".", help="repository root") - ap.add_argument("--cap", type=int, default=CAP, help="hard limit, fails") - ap.add_argument("--warn", type=int, default=WARN, help="soft limit, warns") - a = ap.parse_args() - - files = sorted(Path(a.root, ".github/workflows").glob("*.y*ml")) - if not files: - print(f"no workflows under {a.root}/.github/workflows", file=sys.stderr) - return 1 - - over, near = [], [] - for f in files: - try: - doc = yaml.safe_load(f.read_text()) - except yaml.YAMLError as e: - print(f"::error file={f}::not valid YAML: {e}", file=sys.stderr) - return 1 - for path, s in scalars(doc): - if len(s) > a.cap: - over.append((len(s), f, path)) - elif len(s) > a.warn: - near.append((len(s), f, path)) - - for n, f, path in sorted(near, reverse=True): - print(f"::warning file={f}::{path} is {n} characters, within " - f"{a.cap - n} of GitHub's {a.cap} character limit. Move any " - "prose out of the block scalar into YAML comments above the " - "step, or split the script into another step.") - for n, f, path in sorted(over, reverse=True): - print(f"::error file={f}::{path} is {n} characters, over GitHub's " - f"{a.cap} character limit. GitHub will refuse to compile this " - "file and every run of it will fail with no jobs and no " - "annotation. Split the script into another step; a step " - "boundary resets the budget.", file=sys.stderr) - - biggest = max((len(s) for f in files for _, s in scalars(yaml.safe_load(f.read_text()))), - default=0) - print(f"{len(files)} workflow(s), largest string {biggest} of {a.cap}" - f"{f', {len(near)} within {a.cap - a.warn} of the limit' if near else ''}") - return 1 if over else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/feature-checks.json b/scripts/unsloth/feature-checks.json deleted file mode 100644 index 93fe8d3c18f5..000000000000 --- a/scripts/unsloth/feature-checks.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "_doc": [ - "The test that proves each shipped feature works. Read by feature_matrix.py.", - "", - "Keyed by FEATURE, with the pin that currently carries it, and NOT the other", - "way round. When upstream absorbs a feature the pin is deleted, and deleting", - "the check with it would put the blind spot back somewhere else: the feature", - "is still in the release, it just arrives through the base tag now. So an", - "entry outlives its `owner`, and `owner` becomes null rather than the entry", - "being removed.", - "", - "This is the half that cannot be derived. pin_contract.py reads a pin's own", - "diff and proves the merge kept it, which needs no upkeep but can only ever", - "prove the MERGE lost nothing -- a regression inside the pin regenerates a", - "smaller contract that passes. What a feature has to DO is a human sentence.", - "", - "Every pin in pr-set.json must appear in `features` or in `unchecked`. The", - "lint enforces that, so adding a pin forces a decision instead of a silence.", - "`unchecked` is a recorded reason, not a hole.", - "", - "kinds:", - " arch test-llama-archs -a <arch> builds a synthetic model of", - " the architecture, decodes 128", - " tokens on every device and", - " compares against CPU", - " backend-op test-backend-ops test -o <OP> runs the op against the CPU", - " reference implementation", - " mtmd test-mtmd-impl projector registry, no model", - "", - "A probe that exits 0 having run nothing is a failure, not a pass: both", - "harnesses do exactly that for an excluded arch or a misspelled op name.", - "feature_matrix.py rejects skip markers and requires a non-zero case count.", - "", - "No runner in the prebuild pipeline has a GPU, so the nightly runs this on", - "CPU and every backend-op check is DEFERRED there: named and counted, never", - "reported as passing. The kernels are exactly where a merge goes wrong", - "silently, so before accepting a carry PR that touches one, build it on a", - "GPU box and run:", - "", - " python3 scripts/unsloth/feature_matrix.py --build-dir build --gpu", - "", - "and paste the output into the PR. That is the only place those checks run." - ], - "schema": 1, - "features": { - "inkling": { - "owner": "ggml-org#25731", - "checks": [ - { "kind": "arch", "arch": "inkling" }, - { "kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED" }, - { "kind": "mtmd", "projector": "inkling" } - ] - }, - "glm5next": { - "owner": "ggml-org#27754", - "checks": [ - { "kind": "arch", "arch": "glm5next" }, - { "kind": "backend-op", "op": "LIGHTNING_INDEXER" } - ] - }, - "diffusion-gemma": { - "owner": "ggml-org#24423", - "checks": [ - { "kind": "arch", "arch": "diffusion-gemma" } - ] - }, - "kimi-k3": { - "owner": "unslothai#70", - "checks": [ - { "kind": "arch", "arch": "kimi-k3" }, - { "kind": "mtmd", "projector": "kimik3" } - ] - }, - "iq1-narrow-grids": { - "owner": "unslothai#61", - "checks": [ - { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xs" }, - { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xxs" }, - { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xxxs" } - ] - }, - "qwen4exp-mtp": { - "owner": "unslothai#144", - "checks": [ - { "kind": "arch", "arch": "qwen4exp" }, - { "kind": "backend-op", "op": "TOPK_QSA" } - ] - }, - "projector-registry": { - "owner": "unslothai#176", - "checks": [ - { "kind": "mtmd", "projector": "*" } - ] - } - }, - "unchecked": { - "unslothai#95": "sampling penalties indexed by token id; behaviour is covered by test-sampling, and there is no feature surface of its own to probe", - "unslothai#137": "batched readahead for lazily read gather tables; a throughput change with no observable output difference", - "unslothai#149": "GGML_CUDA_ENABLE_UNIFIED_MEMORY=0 env parsing; needs a CUDA or HIP host, and no runner in the pipeline has one", - "unslothai#152": "per-run mmap of a context's tensors; a memory-layout change with no observable output difference", - "unslothai#157": "cudaMemcpyDefault in the ggml_cuda_cpy 2D fast path; needs a CUDA host", - "unslothai#158": "ROCm_Host compute buffer type on HIP integrated GPUs; needs a ROCm host" - } -} diff --git a/scripts/unsloth/feature_matrix.py b/scripts/unsloth/feature_matrix.py deleted file mode 100644 index 00b392ca1ec5..000000000000 --- a/scripts/unsloth/feature_matrix.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Run the test that proves each shipped feature works, against a built tree. - -pin_contract.py proves the merge did not lose a pin's code. That is a different -question from whether the feature works, and neither one implies the other: the -Inkling banded-attention kernel merged against upstream's sparse attention is -thirteen hunks of CUDA template parameter threading, where a mistake gives -wrong attention output and every static check passes. - -Keyed by FEATURE, not by pin. When upstream absorbs a feature and the pin is -deleted, removing the check with it would put the blind spot back in a -different place -- the feature is still in the release, it just arrives through -the base tag now. So the manifest binds a feature to its current pin and -survives that pin going away. - -A PASS HAS TO BE POSITIVE EVIDENCE. Both harnesses exit 0 having done nothing: - - test-llama-archs -a diffusion-gemma # excluded -> prints SKIP, exits 0 - test-backend-ops test -o TYPO # matches nothing, exits 0 - -so every probe rejects skip markers and requires a non-zero count of cases it -actually ran. Without that this file is decoration. - -CPU only under CUDA_VISIBLE_DEVICES="" is what CI can do, since no runner in -the prebuild pipeline has a GPU. Run it with the variable unset on a GPU box to -get the comparison that matters for kernels. -""" - -from __future__ import annotations - -import argparse -import json -import re -import subprocess -import sys -from pathlib import Path - -# Output that means "this did not run" from a process that exited 0. -SKIP_RE = re.compile(r"\bSKIP\b|not supported|unsupported|no tests|0 tests", re.I) - - -class Unproven(Exception): - """The probe exited 0 without demonstrating anything.""" - - -class NeedsGPU(Exception): - """Nothing is wrong; this check cannot be answered on this machine. - - test-backend-ops compares a backend against the CPU reference, so with no - accelerator present it has nothing to compare and prints "Skipping CPU - backend". Reporting that as a pass would be a lie and reporting it as a - failure would block every nightly, since no runner in the prebuild pipeline - has a GPU. It is counted and named instead. - """ - - -def bins(build_dir: Path) -> Path: - for c in (build_dir / "bin", build_dir): - if (c / "test-backend-ops").exists() or (c / "test-llama-archs").exists(): - return c - raise SystemExit(f"no test binaries under {build_dir}") - - -def run(cmd: list[str], cwd: Path, gpu: bool) -> tuple[int, str]: - env = None - if not gpu: - import os - env = dict(os.environ, CUDA_VISIBLE_DEVICES="", HIP_VISIBLE_DEVICES="") - r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) - return r.returncode, (r.stdout or "") + (r.stderr or "") - - -def probe_arch(check: dict, b: Path, gpu: bool) -> str: - """A synthetic model of this architecture decodes, and matches CPU.""" - arch = check["arch"] - rc, out = run([str(b / "test-llama-archs"), "-a", arch, "-s", "1234"], b, gpu) - if rc != 0: - raise Unproven(f"test-llama-archs -a {arch} exited {rc}") - # The arch's own rows, not the header and not another arch's. - rows = [ln for ln in out.splitlines() if ln.strip().startswith("|") and f"|{arch:>16}|" in ln - or (ln.strip().startswith("|") and ln.split("|")[1].strip() == arch)] - if not rows: - raise Unproven(f"test-llama-archs printed no row for {arch}; it is not in the harness") - ok = [r for r in rows if "OK" in r] - if not ok: - raise Unproven(f"every {arch} row was skipped, so nothing was decoded: {rows[0].strip()}") - return f"{len(ok)}/{len(rows)} device rows decoded and matched CPU" - - -def probe_backend_op(check: dict, b: Path, gpu: bool) -> str: - """The op exists in the backend and matches the CPU reference.""" - if not gpu: - raise NeedsGPU("test-backend-ops compares against CPU, so with no " - "accelerator it skips every backend and proves nothing") - cmd = [str(b / "test-backend-ops"), "test", "-o", check["op"]] - if check.get("params"): - cmd += ["-p", check["params"]] - rc, out = run(cmd, b, gpu) - if rc != 0: - raise Unproven(f"{' '.join(cmd[1:])} exited {rc}") - m = re.search(r"(\d+)/(\d+) tests passed", out) - if not m: - raise Unproven(f"{check['op']} produced no test count; the filter matched nothing") - passed, total = int(m.group(1)), int(m.group(2)) - if total == 0: - raise Unproven(f"{check['op']} matched 0 cases; the op name is stale") - if passed != total: - raise Unproven(f"{check['op']}: {passed}/{total} passed") - return f"{passed}/{total} cases matched the CPU reference" - - -def probe_mtmd(check: dict, b: Path, gpu: bool) -> str: - """The projector registry is intact, including this projector's entry.""" - rc, out = run([str(b / "test-mtmd-impl"), "test_projector_registry"], b, gpu) - if rc != 0: - raise Unproven(f"test-mtmd-impl exited {rc}") - m = re.search(r"assertions\s*:\s*(\d+)", out) - if not m or int(m.group(1)) == 0: - raise Unproven("test_projector_registry ran no assertions; the filter matched nothing") - # The registry test walks the whole enum, so it proves the table is sound. - # That the specific projector is IN the enum is pin_contract.py's job. - return f"projector registry intact over {m.group(1)} assertions" - - -PROBES = {"arch": probe_arch, "backend-op": probe_backend_op, "mtmd": probe_mtmd} - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - ap.add_argument("--build-dir", required=True) - ap.add_argument("--feature-checks", required=True) - ap.add_argument("--only", help="one feature id") - ap.add_argument("--gpu", action="store_true", - help="let the probes see the GPU; CI has none, so the default " - "hides it and the comparison is CPU-only") - ap.add_argument("--report") - args = ap.parse_args() - - b = bins(Path(args.build_dir).resolve()) - doc = json.loads(Path(args.feature_checks).read_text()) - report: dict = {"gpu": args.gpu, "features": [], "ok": False, "deferred": 0} - failed = 0 - deferred = 0 - - for name, feat in sorted(doc["features"].items()): - if args.only and name != args.only: - continue - entry = {"feature": name, "owner": feat.get("owner"), - "results": [], "problems": [], "deferred": []} - for check in feat["checks"]: - kind = check["kind"] - label = f"{kind}:{check.get('arch') or check.get('op') or check.get('projector')}" - try: - if kind not in PROBES: - raise Unproven(f"unknown check kind {kind!r}") - entry["results"].append({"check": label, "evidence": PROBES[kind](check, b, args.gpu)}) - except NeedsGPU as e: - entry["deferred"].append(f"{label}: {e}") - deferred += 1 - except Unproven as e: - entry["problems"].append(f"{label}: {e}") - except OSError as e: - entry["problems"].append(f"{label}: cannot run: {e}") - report["features"].append(entry) - if entry["problems"]: - failed += 1 - print(f"FAIL {name}", file=sys.stderr) - for p in entry["problems"]: - print(f" {p}", file=sys.stderr) - elif entry["results"]: - print(f"ok {name}: " + "; ".join(r["evidence"] for r in entry["results"]) - + (f" [{len(entry['deferred'])} needs a GPU]" if entry["deferred"] else "")) - else: - # Nothing was shown either way. Not a failure here, but it must not - # read as one of the ok lines. - print(f"-- {name}: nothing provable without a GPU " - f"({len(entry['deferred'])} check(s) deferred)") - - for pin, why in sorted(doc.get("unchecked", {}).items()): - print(f"note {pin} has no runtime check: {why}") - - report["ok"] = failed == 0 - report["deferred"] = deferred - if args.report: - Path(args.report).write_text(json.dumps(report, indent=2)) - if failed: - print(f"\n{failed} feature(s) could not be shown to work", file=sys.stderr) - return 1 - # Say what was NOT proven in the same breath as what was. A run that only - # ever prints a success line teaches the reader that green means covered. - tail = f", {deferred} check(s) need a GPU and were not run" if deferred else "" - print(f"\nall {len(report['features'])} features demonstrated" - + (" on GPU" if args.gpu else " on CPU") + tail) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/merge_checks.py b/scripts/unsloth/merge_checks.py deleted file mode 100755 index a01a6fad574a..000000000000 --- a/scripts/unsloth/merge_checks.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Post-merge checks for the two mistakes a clean build does not catch. - -Both of these were made for real on 08-27, resolving GLM-5-Next against the qwen4exp carry, and both survived compilation: - - 1. A resolver unioned two byte-identical additions and produced the same `MODEL_ARCH.GLM5NEXT` key twice in the tensor map. - Python keeps the last definition of a duplicate key, silently, so the file imports, the build passes, and the converter reads the wrong mapping. - - 2. The same union kept both an arch in a shared fallthrough condition AND a dedicated `else if (arch == LLM_ARCH_GLM5NEXT)` arm below it. - The shared condition matches first, so the dedicated arm is dead. - It compiles, and the model runs with the indexer cache that arm was supposed to build. - -Neither is a merge resolver. -They decide nothing and rewrite nothing. -They turn a silent wrong answer into a loud one, which is the property that was missing. - -Both are deliberately narrow, because a check that fires wrongly blocks a release just as effectively as a bad merge: - - - An arch is only consumed by an EARLIER arm that is a pure disjunction of `arch == LLM_ARCH_*` terms. - A conditional arm may not run, so what follows it stays reachable. - The later arm is then dead if it is a disjunction whose alternatives are all consumed, or a plain conjunction requiring a consumed arch, since its other conditions can only narrow it further. - Anything mixing `||` and `&&` is left alone, and only a term that is exactly `arch == X` counts, so `arch != X` is never read as requiring that arch. - - - The unreachable-arm check reads one physical line, so a condition split across lines is skipped rather than analysed. - Checked against the whole of src/: a line-joining variant finds exactly the same zero findings, because every multiline arch condition there is a standalone `if` with no `else if` chain below it. - Widening the regex would add false-positive surface on the release path and buy nothing today, so it stays narrow and this is recorded as a known limitation rather than fixed. - - - Chains are grouped by BRACE DEPTH, not by indentation. - This is an accuracy fix, not a widening: measured over all 181 src/**/*.cpp, depth and indentation report the same zero findings, so nothing new fires and the good tree stays clean, but depth keeps 339 arms in chains against 260 and is right in both directions where they differ. - Indentation drops the enclosing chain at any nested `if`, which silences the check on the exact 08-27 shape, and it glues two unrelated `if`s at one indent into a single chain whenever the second opener is a skipped multiline condition, which reports a reachable arm as dead and blocks a release on good code. - - - The duplicate-key check compares keys by their source text, so it only looks at keys whose value cannot change between evaluations: literals, names, attributes and tuples of those. `{fresh(): 1, fresh(): 2}` reads as one key twice and is really two entries, and a finding here stops the nightly. - - - There is deliberately NO duplicate-C++-definition check. - The obvious version keys on function name and flags legitimate overloads: it reported `llama_model_base::create_tensor`, which is two different signatures. - A real duplicate is an ODR violation the compiler already rejects, so the only gain would be failing sooner, which does not justify a false positive on the release path. - -Exits 0 when clean, 1 when anything is found. -`--report` emits JSON. -""" - -from __future__ import annotations - -import argparse -import ast -import collections -import json -import re -import sys -from pathlib import Path - -ARCH = re.compile(r"arch == (LLM_ARCH_\w+)") -COND = re.compile(r"^\s*(?:\}\s*)?else if \((.*)\)\s*\{\s*$|^\s*if \((.*)\)\s*\{\s*$") -PURE_TERM = re.compile(r"arch == LLM_ARCH_\w+") -# Encoding prefixes a raw string may carry: LR"(...)", u8R"(...)" and so on. -_RAW_PREFIX = ("", "L", "u", "U", "u8") - - -# Node types whose value does not depend on when the expression is evaluated. -# An allowlist, not a denylist, so an expression shape nobody thought about is -# treated as unstable and simply not checked, rather than blocking a release. -_STABLE = (ast.Constant, ast.Name, ast.Attribute, ast.Tuple, ast.Load, - ast.UnaryOp, ast.USub, ast.UAdd, ast.Invert) - - -def _stable_key(node: ast.AST) -> bool: - """True when this key expression names the same object every evaluation. - - `{fresh(): 1, fresh(): 2}` unparses to the same text twice and is still two - entries, so a call anywhere in the key means the two are not comparable by - text. The keys this check exists for, `MODEL_ARCH.GLM5NEXT` and plain - literals, are all stable. - """ - return all(isinstance(n, _STABLE) for n in ast.walk(node)) - - -def duplicate_dict_keys(path: Path) -> list[str]: - """Keys defined twice in one dict literal. Always at best dead code.""" - try: - tree = ast.parse(path.read_text()) - except SyntaxError as e: - return [f"{path}:{e.lineno}: does not parse: {e.msg}"] - out = [] - for node in ast.walk(tree): - if not isinstance(node, ast.Dict): - continue - keys = [ast.unparse(k) for k in node.keys - if k is not None and _stable_key(k)] - for key, n in collections.Counter(keys).items(): - if n > 1: - out.append(f"{path}:{node.lineno}: key {key} defined {n} times " - "in one dict; Python keeps only the last") - return out - - -def _pure_arch_disjunction(cond: str) -> bool: - """True when the condition is only `arch == X` terms joined by `||`.""" - if "&&" in cond: - return False - terms = [t.strip() for t in cond.split("||")] - return bool(terms) and all(PURE_TERM.fullmatch(t) for t in terms) - - -def _raw_delim(text: str, i: int) -> str | None: - """The delimiter of a raw string starting at `i`, or None if one does not. - - `i` indexes the `R`. - The delimiter is what sits between `R"` and `(`, and the literal ends only at `)delim"`, which is the whole reason a raw string cannot be found with a plain regex. - """ - if not text.startswith('R"', i): - return None - j = text.find("(", i + 2) - if j == -1: - return None - delim = text[i + 2:j] - if len(delim) > 16 or any(c in ' ()\\\t\n' for c in delim): - return None - # An `R` glued to an identifier is part of that identifier, not a prefix. - k = i - while k > 0 and (text[k - 1].isalnum() or text[k - 1] == "_"): - k -= 1 - return delim if text[k:i] in _RAW_PREFIX else None - - -def _decommented(text: str) -> list[str]: - """The file with comments and literals blanked, line structure preserved. - - Braces inside a string literal or a comment are not braces. - src/ is full of both (llama-chat.cpp alone embeds dozens of `{` in template strings), so counting them raw would desynchronise the depth for the rest of the file. - - Scanned once, left to right, rather than by substituting one construct at a time. - Order cannot fix a substitution pass: blanking raw strings first lets an `R"(` written inside a comment swallow everything to the next `)"`, and blanking comments first lets a `//` inside a raw string end the line. - Only position decides which construct is real, and a scan is what knows it. - """ - blank = lambda s: re.sub(r"[^\n]", " ", s) # noqa: E731 - out: list[str] = [] - i, n = 0, len(text) - while i < n: - delim = _raw_delim(text, i) - if delim is not None: - close = f'){delim}"' - k = text.find(close, i + 2 + len(delim) + 1) - end = n if k == -1 else k + len(close) - elif text.startswith("//", i): - k = text.find("\n", i) - end = n if k == -1 else k - elif text.startswith("/*", i): - k = text.find("*/", i + 2) - end = n if k == -1 else k + 2 - elif text[i] in "\"'": - q, j = text[i], i + 1 - while j < n and text[j] != q and text[j] != "\n": - j += 2 if text[j] == "\\" else 1 - end = min(j + 1, n) - else: - out.append(text[i]) - i += 1 - continue - out.append(blank(text[i:end])) - i = end - return "".join(out).split("\n") - - -def _depths(line: str, start: int) -> tuple[int, int]: - """(brace depth after this line, lowest depth reached inside it).""" - d = lo = start - for ch in line: - if ch == "{": - d += 1 - elif ch == "}": - d -= 1 - lo = min(lo, d) - return d, lo - - -def if_else_chains(text: str) -> list[list[tuple[int, str]]]: - """Group `if` / `else if` conditions into chains by BRACE DEPTH. - - Indentation is not the structure. - Keying chains on it, and resetting on any change, means a nested `if` inside an arm replaces the enclosing chain, so the outer arms after it are analysed as a fresh chain and an arch the outer chain already matched looks unmatched. - That is a gate that stops gating on a shape that is ordinary C++: the arch dispatch at llama-model.cpp:2434 is exactly one nested `if` away from it. - - The same key also mis-JOINS. - Two unrelated `if`s at the same indentation become one chain whenever the second one's opener is a condition the regex skips, and then a perfectly reachable arm is reported unreachable, which blocks a release on good code. - Depth gets both right: a chain lives at the depth its `if` opened at, and ends when a brace takes the file back past it. - """ - raw = text.split("\n") - clean = _decommented(text) - open_chains: dict[int, list[tuple[int, str]]] = {} - done: list[list[tuple[int, str]]] = [] - - def flush(key: int) -> None: - c = open_chains.pop(key, None) - if c and len(c) > 1: - done.append(c) - - # A chain's closing brace and its `else if` are often on separate lines, which llama.cpp does in src/llama-quant.cpp:461 among others. - # Closing the chain the moment the brace line dedents would end it one line before the arm that continues it, and the duplicate arch arm after it would then be a fresh chain with nothing taken yet, so the gate passes it. - # Ending a chain is therefore deferred one line: the next line either continues it, or it really is over. - # Blank lines do not decide either way. - pending: set[int] = set() - - depth = 0 - for i, (line, cline) in enumerate(zip(raw, clean)): - after, lo = _depths(cline, depth) - stripped = cline.lstrip() - # Matched on the decommented line: COND anchors on the `{` ending the line, so `if (arch == X) { // shared` matched nothing on the raw line and the arm vanished from the chain. - # _decommented blanks in place, so the spans still index the raw line and the condition text below is taken from there, intact. - # A line that blanked away entirely is commented-out code and opens nothing. - m = COND.search(cline) if stripped else None - # `} else if (...) {` and a bare `else if (...) {` after its own `}` line both continue the chain that lives at the depth this line dips to; a plain `if` opens one at the depth it starts from. - cont = bool(m) and (stripped.startswith("}") or stripped.startswith("else")) - key = lo if cont else depth - if stripped: - if cont: - pending.discard(key) # this line continues it after all - for k in sorted(pending, reverse=True): - flush(k) - pending.clear() - # Any chain whose closing brace this line just passed is over, unless the next line turns out to continue it. - for k in [k for k in sorted(open_chains, reverse=True) if k >= lo]: - if not (cont and k == key): - pending.add(k) - if m: - g = 1 if m.group(1) is not None else 2 - cond = line[m.start(g):m.end(g)] - if cont and key in open_chains: - open_chains[key].append((i + 1, cond)) - else: - flush(key) - open_chains[key] = [(i + 1, cond)] - # A line that both dedents and opens a chain at the same key would otherwise leave that key pending and flush the chain it just opened on the next line. - pending.discard(key) - depth = after - for k in sorted(open_chains, reverse=True): - flush(k) - return done - - -def _blocked_by(cond: str, taken: set[str]) -> set[str]: - """Arches that make this arm dead, given what earlier arms already took. - - A pure disjunction is dead only when EVERY alternative is taken. - A plain conjunction is dead as soon as ONE of its `arch == X` conjuncts is, since the other conditions can only narrow it further: an earlier unconditional `arch == X` arm makes a later `arch == X && enabled` unreachable, and that arm is not a pure disjunction so it used to be skipped entirely. - - Anything mixing `||` and `&&` is left alone rather than guessed at, and only a term that is exactly `arch == X` counts, so `!(arch == X)` and `arch != X` cannot be read as requiring that arch. - """ - archs = set(ARCH.findall(cond)) - if not archs: - return set() - if _pure_arch_disjunction(cond): - return archs if archs <= taken else set() - if "||" in cond: - return set() - hit = set() - for term in cond.split("&&"): - m = ARCH.fullmatch(term.strip().strip("()").strip()) - if m and m.group(1) in taken: - hit.add(m.group(1)) - return hit - - -def unreachable_arch_arms(path: Path) -> list[str]: - """`else if` arms whose arch a preceding pure-disjunction arm already took.""" - out = [] - for chain in if_else_chains(path.read_text()): - taken: set[str] = set() - for lineno, cond in chain: - dead = _blocked_by(cond, taken) - if dead: - out.append(f"{path}:{lineno}: unreachable, " - f"{', '.join(sorted(dead))} already matched earlier " - "in this if/else chain") - # Only an unconditional arm consumes an arch. - # A conditional one may not run, so what follows it can still be reached. - if _pure_arch_disjunction(cond): - taken |= set(ARCH.findall(cond)) - return out - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--root", default=".", help="tree to check") - ap.add_argument("--report", metavar="PATH", help="write a JSON summary here") - a = ap.parse_args() - root = Path(a.root) - - findings: list[str] = [] - scanned = {"python": 0, "cpp": 0} - for p in sorted(root.glob("gguf-py/**/*.py")): - scanned["python"] += 1 - findings += duplicate_dict_keys(p) - for p in sorted(root.glob("src/**/*.cpp")): - scanned["cpp"] += 1 - findings += unreachable_arch_arms(p) - - print(f"merge_checks: scanned {scanned['python']} python and {scanned['cpp']} c++ files") - for f in findings: - print(f" {f}") - if a.report: - Path(a.report).write_text(json.dumps( - {"ok": not findings, "scanned": scanned, "findings": findings}, indent=2)) - if findings: - print(f"merge_checks: {len(findings)} problem(s)", file=sys.stderr) - return 1 - print("merge_checks: clean") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/package_bundle.py b/scripts/unsloth/package_bundle.py deleted file mode 100644 index dca4ccee9d11..000000000000 --- a/scripts/unsloth/package_bundle.py +++ /dev/null @@ -1,381 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Cross-platform packager for Unsloth llama.cpp prebuilt bundles. - -Curates the shipped executables, their local dynamic-library closure, and the -dynamically-loaded ggml backend modules; writes the in-bundle metadata -(BUILD_INFO.txt / UNSLOTH_PREBUILT_INFO.json); archives the result. - -The curation and archive engine is OS-generic: adding a new OS means -implementing one PlatformStrategy (its dependency-walk tool, lib-name -convention, backend glob, and archive format), not writing a new packaging -script. - -The CUDA runtime (libcudart/libcublas, cudart DLLs) is intentionally NOT -bundled: the installer pairs it with the user's PyTorch runtime, selected by -runtime_line. - -Linux is the CI-validated path. macOS/Windows strategies follow the correct -platform conventions (otool/@loader_path/tar.gz; dir-local DLLs/zip) but have -not yet been exercised on their runners. - -Configuration is read from the environment (see read_config). Runs both inside -the build workflow and standalone for local testing. -""" -from __future__ import annotations - -import json -import os -import re -import shutil -import subprocess -import sys -import tarfile -import tempfile -import zipfile -from datetime import datetime, timezone -from pathlib import Path - -# Force C locale so tool output (readelf/otool) is not localized. -_C_ENV = {**os.environ, "LC_ALL": "C", "LANG": "C"} - - -def _run(cmd: list[str]) -> str: - # Fail loudly: a missing/erroring readelf|otool would otherwise yield an - # empty closure and silently ship a bundle with missing libraries. - try: - r = subprocess.run(cmd, capture_output=True, text=True, env=_C_ENV) - except FileNotFoundError: - sys.exit(f"ERROR: required tool '{cmd[0]}' not found") - if r.returncode != 0: - sys.exit(f"ERROR: {' '.join(cmd)} failed (rc={r.returncode}): {r.stderr.strip()}") - return r.stdout - - -class PlatformStrategy: - name = "generic" - exe_suffix = "" - archive_ext = ".tar.gz" - rpath = "" - # Required core: these must exist or packaging fails loudly. Every other - # executable the build produced is discovered and shipped too (see curate). - binaries = ["llama-server", "llama-cli", "llama-quantize"] - lib_suffix_re = r"\.so(\.\d+)*$" # POSIX shared-lib suffix to exclude; Windows keys on .exe - - def shipped_binaries(self) -> list[str]: - return [b + self.exe_suffix for b in self.binaries] - - def is_executable(self, path: Path) -> bool: - """True if `path` is a program to ship (not a shared library).""" - if not path.is_file(): - return False - if self.exe_suffix: # Windows: an executable is exactly a .exe - return path.suffix.lower() == self.exe_suffix - return not re.search(self.lib_suffix_re, path.name) and os.access(path, os.X_OK) - - def local_needed(self, path: Path, bin_dir: Path) -> list[str]: - """Names of dynamic libs `path` needs that are *local* (live in bin_dir).""" - raise NotImplementedError - - def backend_patterns(self) -> list[str]: - """Globs for the dlopen'd ggml backend modules (not found via the walk).""" - raise NotImplementedError - - def supports_symlinks(self) -> bool: - return True - - def archive(self, stage: Path, out_path: Path) -> None: - raise NotImplementedError - - -class LinuxStrategy(PlatformStrategy): - name = "linux" - rpath = "$ORIGIN" - - def local_needed(self, path: Path, bin_dir: Path) -> list[str]: - # Locale-independent: key only on the (NEEDED) tag and the [name]. - needed = re.findall(r"\(NEEDED\)[^\[]*\[([^\]]+)\]", _run(["readelf", "-d", str(path)])) - return [n for n in needed if (bin_dir / n).exists() or (bin_dir / n).is_symlink()] - - def backend_patterns(self) -> list[str]: - return ["libggml-cpu-*.so*", "libggml-cuda.so*", "libggml-rpc.so*"] - - def archive(self, stage: Path, out_path: Path) -> None: - with tarfile.open(out_path, "w:gz") as tar: - tar.add(stage, arcname=".") - - -class MacOSStrategy(PlatformStrategy): - name = "macos" - rpath = "@loader_path" - lib_suffix_re = r"\.dylib$" - - def local_needed(self, path: Path, bin_dir: Path) -> list[str]: - out = _run(["otool", "-L", str(path)]) - deps: list[str] = [] - for line in out.splitlines()[1:]: # first line echoes the file path - m = re.match(r"\s+(\S+)\s+\(", line) - if not m: - continue - ref = m.group(1) - base = os.path.basename(ref) - # @rpath/@loader_path/relative refs that exist locally are "ours" - if (ref.startswith("@") or not ref.startswith("/")) and (bin_dir / base).exists(): - deps.append(base) - return deps - - def backend_patterns(self) -> list[str]: - return ["libggml-*.dylib"] - - def archive(self, stage: Path, out_path: Path) -> None: - with tarfile.open(out_path, "w:gz") as tar: - tar.add(stage, arcname=".") - - -class WindowsStrategy(PlatformStrategy): - name = "windows" - exe_suffix = ".exe" - archive_ext = ".zip" - rpath = "" # Windows resolves DLLs from the executable's directory - - # No portable readelf/otool equivalent; the project's own DLLs live beside - # the binaries in build/bin/Release, so bundle those by name convention. - LOCAL_DLL_PREFIXES = ("ggml", "llama", "mtmd") - - def local_needed(self, path: Path, bin_dir: Path) -> list[str]: - return [ - p.name for p in bin_dir.glob("*.dll") - if p.name.lower().startswith(self.LOCAL_DLL_PREFIXES) - ] - - def backend_patterns(self) -> list[str]: - return ["ggml-cpu-*.dll", "ggml-cuda.dll", "ggml-rpc.dll"] - - def supports_symlinks(self) -> bool: - return False - - def archive(self, stage: Path, out_path: Path) -> None: - with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as z: - for p in sorted(stage.rglob("*")): - if p.is_file(): - z.write(p, p.relative_to(stage).as_posix()) - - -STRATEGIES = {s.name: s for s in (LinuxStrategy(), MacOSStrategy(), WindowsStrategy())} - - -def _copy_one(strategy: PlatformStrategy, bin_dir: Path, stage: Path, name: str) -> None: - src, dst = bin_dir / name, stage / name - if dst.exists() or dst.is_symlink(): - return - if strategy.supports_symlinks() and src.is_symlink(): - target = os.readlink(src) - os.symlink(target, dst) - _copy_one(strategy, bin_dir, stage, os.path.basename(target)) - elif src.exists(): - shutil.copy2(src, dst, follow_symlinks=True) - - -def curate(strategy: PlatformStrategy, bin_dir: Path, stage: Path) -> None: - roots: list[Path] = [] - required = strategy.shipped_binaries() - for b in required: - if not (bin_dir / b).exists(): - sys.exit(f"ERROR: missing {bin_dir / b}") - shutil.copy2(bin_dir / b, stage / b) - roots.append(stage / b) - - # Ship every other executable the build produced, so a curated GPU bundle - # carries the same tool set as the full-build cpu/macos/rocm tarballs (which - # tar all of build/bin). Each becomes a root too, so any library only it - # needs is pulled into the closure. Runtime libraries that live outside - # bin_dir (e.g. the CUDA runtime) are never pulled in: the walk stays local. - required_set = set(required) - for p in sorted(bin_dir.iterdir()): - if p.name in required_set or not strategy.is_executable(p): - continue - _copy_one(strategy, bin_dir, stage, p.name) - roots.append(stage / p.name) - - # Backend modules are dlopen'd, so they never appear in the NEEDED graph; - # copy them explicitly and treat them as extra roots so their own local - # dependencies get pulled into the closure too. - for pat in strategy.backend_patterns(): - for match in sorted(bin_dir.glob(pat)): - _copy_one(strategy, bin_dir, stage, match.name) - roots.append(stage / match.name) - - # Walk the local NEEDED closure from every root, scanning each lib once. - queue = list(roots) - while queue: - for need in strategy.local_needed(queue.pop(), bin_dir): - if not (stage / need).exists() and not (stage / need).is_symlink(): - _copy_one(strategy, bin_dir, stage, need) - queue.append(stage / need) - - -def detect_nvcc_sms() -> tuple[str, list[str], str]: - if not shutil.which("nvcc"): - return "unavailable", [], "nvcc not found" - r = subprocess.run(["nvcc", "--list-gpu-arch"], capture_output=True, text=True, env=_C_ENV) - if r.returncode != 0: - return "unavailable", [], f"nvcc failed (rc={r.returncode})" - sms = sorted(set(re.findall(r"compute_(\d+)", r.stdout)), key=int) - return "available", sms, f"detected {len(sms)} SM targets" - - -def write_metadata(stage: Path, strategy: PlatformStrategy, cfg: dict, sms: list[str]) -> None: - short = cfg["commit"][:7] - min_sm, max_sm = min(map(int, sms)), max(map(int, sms)) - nvcc_status, nvcc_sms, nvcc_msg = detect_nvcc_sms() - # sm_103 (B300 / GB300 Blackwell Ultra) has no native build, but it JIT-runs - # on the bundled compute_100 PTX, so any bundle that ships sm_100 also covers - # it. Declare it in supported_sms (not the native nvcc build) so every - # platform's manifest agrees -- Windows and arm64 reuse these x64 profiles. - supported_sms = list(sms) - if "100" in supported_sms and "103" not in supported_sms: - supported_sms = sorted([*supported_sms, "103"], key=int) - note = f"CUDA {cfg['line'].removeprefix('cuda')} {cfg['klass']} bundle." - - licenses = [f"Third-party licenses bundled with this llama.cpp prebuilt ({cfg['tag']}).", - f"Source: https://github.com/{cfg['source_repo']} @ {cfg['commit']}", ""] - src = Path(cfg["src_dir"]) - if (src / "LICENSE").is_file(): - licenses += ["=== llama.cpp LICENSE ===", (src / "LICENSE").read_text(), ""] - lic_dir = src / "licenses" - if lic_dir.is_dir(): - for lic in sorted(lic_dir.glob("*")): - if lic.is_file(): - licenses += [f"=== {lic.name} ===", lic.read_text(), ""] - (stage / "THIRD_PARTY_LICENSES.txt").write_text("\n".join(licenses)) - - info = { - "upstream_tag": cfg["tag"], - "source_repo": cfg["source_repo"], - "source_repo_url": f"https://github.com/{cfg['source_repo']}", - "source_ref_kind": cfg["source_ref_kind"], - "requested_source_ref": cfg["tag"], - "resolved_source_ref": cfg["tag"], - "source_commit": cfg["commit"], - "source_commit_short": short, - "platform": f"{strategy.name}-{cfg['arch']}-cuda", - "bundle_profile": cfg["profile"], - "runtime_line": cfg["line"], - "coverage_class": cfg["klass"], - "bundle_rank": int(cfg["rank"]), - "toolkit_line": cfg["toolkit_line"], - "docker_image": cfg["docker_image"], - "supported_sms": supported_sms, - "nvcc_validation_status": nvcc_status, - "nvcc_detected_sms": nvcc_sms, - "nvcc_validation_message": nvcc_msg, - "min_sm": min_sm, - "max_sm": max_sm, - "notes": note, - "build_shared_libs": True, - "ggml_backend_dl": True, - "ggml_cpu_all_variants": True, - "rpath": strategy.rpath, - } - (stage / "UNSLOTH_PREBUILT_INFO.json").write_text(json.dumps(info, indent=2)) - - build_info = [ - f"llama.cpp version: {cfg['tag']}", - f"requested source ref: {cfg['tag']}", - f"resolved source ref: {cfg['tag']}", - f"variant: {cfg['profile']}", - f"runtime line: {cfg['line']}", - f"coverage class: {cfg['klass']}", - f"bundle rank: {cfg['rank']}", - f"docker image: {cfg['docker_image']}", - "backend: CUDA", - f"toolkit version: {cfg['toolkit_line']}", - f"supported sms: {','.join(supported_sms)}", - f"nvcc validation: {nvcc_status}", - f"min sm: {min_sm}", - f"max sm: {max_sm}", - f"os: {strategy.name}", - f"arch: {cfg['arch']}", - "build_shared_libs: ON", - "ggml_backend_dl: ON", - "ggml_cpu_all_variants: ON", - "ggml_cuda_nccl: OFF", - f"rpath: {strategy.rpath}", - "llama_openssl: ON", - "openssl_linkage: dynamic", - "cxx_runtime: dynamic", - f"source commit: {cfg['commit']}", - f"source commit short: {short}", - f"built at (UTC): {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}", - f"notes: {note}", - ] - (stage / "BUILD_INFO.txt").write_text("\n".join(build_info) + "\n") - - -def read_config() -> dict: - def need(k: str) -> str: - v = os.environ.get(k) - if not v: - sys.exit(f"ERROR: missing required env {k}") - return v - - return { - "bin_dir": need("BIN_DIR"), - "src_dir": need("SRC_DIR"), - "out_dir": need("OUT_DIR"), - "tag": need("TAG"), - "commit": need("SOURCE_COMMIT"), - "profile": need("PROFILE"), - "line": need("LINE"), - "klass": need("KLASS"), - "rank": need("RANK"), - "toolkit_line": need("TOOLKIT_LINE"), - "archs": need("ARCHS"), - # Advertised compute capabilities; optional (empty -> derived from ARCHS in main()). - "sms": os.environ.get("SMS", ""), - "platform": os.environ.get("PLATFORM", "linux"), - "arch": os.environ.get("ARCH", "x64"), - "docker_image": os.environ.get("DOCKER_IMAGE", ""), - "source_repo": os.environ.get("SOURCE_REPO", "ggml-org/llama.cpp"), - "source_ref_kind": os.environ.get("SOURCE_REF_KIND", "tag"), - } - - -def main() -> int: - cfg = read_config() - strategy = STRATEGIES.get(cfg["platform"]) - if strategy is None: - sys.exit(f"ERROR: unknown PLATFORM '{cfg['platform']}' (have {sorted(STRATEGIES)})") - - # supported_sms is the concrete coverage, which for a PTX floor (e.g. the - # cuda12-legacy "50-virtual 61-virtual") is wider than the arch int itself. - # So suffixed profiles declare it via SMS; all-real profiles fall back to - # each ARCHS entry's leading SM number. - if cfg["sms"]: - sms = [s for s in re.split(r"[ ;,]+", cfg["sms"]) if s] - else: - sms = [re.match(r"\d+", a).group() for a in re.split(r"[ ;,]+", cfg["archs"]) if a] - bin_dir = Path(cfg["bin_dir"]) - out_dir = Path(cfg["out_dir"]) - out_dir.mkdir(parents=True, exist_ok=True) - - stage = Path(tempfile.mkdtemp()) - try: - curate(strategy, bin_dir, stage) - write_metadata(stage, strategy, cfg, sms) - - asset = f"app-{cfg['tag']}-{strategy.name}-{cfg['arch']}-{cfg['profile']}{strategy.archive_ext}" - out_path = out_dir / asset - strategy.archive(stage, out_path) - - print(f"wrote {out_path}") - for p in sorted(stage.iterdir()): - print(f" {p.name}") - finally: - shutil.rmtree(stage, ignore_errors=True) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/unsloth/pin_contract.py b/scripts/unsloth/pin_contract.py deleted file mode 100644 index bf07d58408e3..000000000000 --- a/scripts/unsloth/pin_contract.py +++ /dev/null @@ -1,361 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Assert the merged tree still contains what each pin carries. - -The nightly proves the pins MERGED. That is not the same as proving they are in -the release, and the difference has cost us three outages: - - * ggml-org#28133 was squash-merged upstream. The pinned commit stopped being - an ancestor of the base tag, so the merge was not a no-op -- it re-applied - code the base already had. It happened to conflict, which is the only - reason anybody noticed. A pin in that state that merges quietly ships - nothing and nothing says so. - * an additive resolution can keep the wrong side, or a later pin can land on - top of an earlier one, and the arch registration the pin exists for is - simply not in the tree any more. It still compiles. - * a pin can rot into contributing nothing at all while its entry stays in - pr-set.json for weeks. - -So: derive from each pin's OWN diff what it puts in the tree, then check the -merged tree still has it. Nothing to maintain -- the expectation comes out of -the commit, so a repin regenerates it. - -Four assertions per pin, cheapest first: - - symbols every LLM_ARCH_/GGML_OP_/PROJECTOR_TYPE_/... name the pin - introduces, in each file it introduces it to. Per FILE, not per - tree: LLM_ARCH_INKLING surviving in llama-arch.h while its arm was - dropped from llama-model.cpp is exactly the failure being looked - for, and a tree-wide grep passes it. - files every file the pin adds still exists. - lines every non-comment code line the pin adds is still in that file. - Catches a resolution that ate a hunk without touching a symbol. - redundancy - a pin whose added lines the BASE TAG already has is work upstream - took. Reported, never fatal -- upstream landing a feature overnight - must not stop that night's release. - -What this CANNOT do, stated plainly so nobody reads more into a pass than is -there: the contract is re-derived from the pin, so it can only ever prove the -MERGE did not lose something. A regression inside the pin itself regenerates a -smaller contract that passes. Proving a feature works is feature_matrix.py's -job, and it needs a build. -""" - -from __future__ import annotations - -import argparse -import json -import re -import subprocess -import sys -from collections import defaultdict -from pathlib import Path - -PIN_RE = re.compile( - r"^https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/commits/([0-9a-f]{40})/?$" -) - -# Identifier families that name a FEATURE. Deliberately not "every new symbol": -# a helper function renamed by a later upstream commit is not a lost feature, -# but a missing LLM_ARCH_ entry always is. These are the tables that decide -# whether an architecture, an op, a projector or a quant type exists at all. -SYMBOL_FAMILIES = ( - "LLM_ARCH_", "LLM_TENSOR_", "LLM_KV_", "LLM_TYPE_", - "PROJECTOR_TYPE_", "GGML_OP_", "GGML_TYPE_", "LLAMA_FTYPE_", -) -SYMBOL_RE = re.compile(r"\b(?:" + "|".join(SYMBOL_FAMILIES) + r")[A-Z0-9_]+\b") - -# The subset that names a whole feature rather than one of its tensors. Used -# only to keep --emit readable; the check itself uses all of SYMBOL_FAMILIES. -HEADLINE = ("LLM_ARCH_", "GGML_OP_", "GGML_TYPE_", "PROJECTOR_TYPE_", "LLAMA_FTYPE_") - -# A line worth tracking for survival. Comments and short punctuation drift with -# every reformat and would make the check noise; a substantial code line does -# not move on its own. -TRIVIAL_RE = re.compile(r"^\s*(?://|/\*|\*|\*/|#\s|$)") -MIN_LINE = 12 - -# Comments are stripped before anything is read off a line. A pin that merely -# NAMES an arch in a comment has not registered it, and holding the comment's -# wording as a contract fails the moment upstream rewords it. Observed on -# unslothai#70, whose comment mentions GGML_OP_SSM_SCAN to explain why it does -# NOT use it. -COMMENT_RE = re.compile(r"//.*$|/\*.*?\*/|(?<!\S)#(?!\s*(?:include|define|if|el|endif|pragma)).*$") - -# Binary and generated paths whose "lines" are meaningless. -SKIP_SUFFIXES = (".npy", ".png", ".jpg", ".gguf", ".bin", ".safetensors", ".ico", ".pdf") - - -class Failure(Exception): - """A pin whose contract the merged tree does not satisfy.""" - - -def git(args: list[str], cwd: Path | None = None, check: bool = True) -> str: - r = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) - if check and r.returncode != 0: - raise RuntimeError(f"git {' '.join(args[:3])}...: {r.stderr.strip()[:300]}") - return r.stdout - - -def blob(rev: str, path: str, cwd: Path) -> str | None: - """The tree entry as "mode oid", or None if the rev has no such path. - - Lifted from carry_vintage.py, mode included for the same reason: a change - that only chmods a file it otherwise took verbatim has identical content, - and an oid-only comparison would call that "contributed nothing". - """ - r = subprocess.run(["git", "ls-tree", "--full-tree", "-z", rev, "--", path], - cwd=cwd, capture_output=True, text=True) - if r.returncode != 0 or not r.stdout.strip(): - return None - mode, _type, oid = r.stdout.split("\0")[0].split("\t", 1)[0].split() - return f"{mode} {oid}" - - -def load_effective(prs_json: str) -> list[dict]: - """The pin list the resolve step actually merged. - - Not the same as pr-set.json: resolve drops an optional pin once its PR is - no longer open, and re-reading the file would then check a pin that is not - in the tree and report it missing. The step already has the effective list - as an output, so take it rather than recomputing the filter here and - getting it subtly different. - """ - return [{"url": p.get("url", ""), "src": p["repo"].split("/")[0], - "num": int(p["number"]), "sha": p["sha"], "required": True} - for p in json.loads(prs_json)] - - -def load_pins(pr_set: Path) -> list[dict]: - data = json.loads(pr_set.read_text()) - pins = [] - for entry in data["prs"]: - url = entry if isinstance(entry, str) else entry["url"] - m = PIN_RE.match(url) - if not m: - raise SystemExit(f"malformed pin: {url}") - pins.append({"url": url, "src": m.group(1), "num": int(m.group(2)), - "sha": m.group(3), - "required": True if isinstance(entry, str) - else entry.get("required", True)}) - return pins - - -def derive(pin: dict, base: str, cwd: Path) -> dict: - """What this pin puts in the tree, read off its own diff against the base. - - The fork point is merge-base(pin, base), not the pin's parent: a pin that - has already had the base merged into it (which repin.py and every carry - branch produce) would otherwise look like it contributed all of upstream. - """ - fork = git(["merge-base", pin["sha"], base], cwd).strip() - diff = git(["diff", "--no-renames", fork, pin["sha"]], cwd) - - symbols: dict[str, set[str]] = defaultdict(set) - lines: dict[str, list[str]] = defaultdict(list) - cur = None - for ln in diff.split("\n"): - if ln.startswith("+++ b/"): - cur = ln[6:] - elif ln.startswith("+++ "): - cur = None # /dev/null: a deletion - elif cur and ln.startswith("+") and not ln.startswith("+++"): - text = ln[1:] - stripped = text.strip() - if len(stripped) >= MIN_LINE and not TRIVIAL_RE.match(stripped): - lines[cur].append(stripped) - code = COMMENT_RE.sub("", text).strip() - if code: - symbols[cur].update(SYMBOL_RE.findall(code)) - - # Only symbols the base does not ALREADY have in that file are evidence of - # this pin. Upstream naming an arch in a file the pin also touches is not - # something the pin is owed. - new_symbols: dict[str, list[str]] = {} - for path, names in symbols.items(): - fresh = sorted(n for n in names - if n not in git(["show", f"{base}:{path}"], cwd, check=False)) - if fresh: - new_symbols[path] = fresh - - status = git(["diff", "--name-status", "--no-renames", fork, pin["sha"]], cwd) - added, owned = [], [] - for ln in status.split("\n"): - if not ln.strip(): - continue - code, path = ln.split("\t", 1) - owned.append(path) - if code.startswith("A"): - added.append(path) - - return { - "fork": fork, - "symbols": new_symbols, - "added_files": added, - "owned_paths": owned, - "lines": {p: v for p, v in lines.items() - if not p.endswith(SKIP_SUFFIXES)}, - } - - -def redundancy(contract: dict, base: str, cwd: Path) -> tuple[int, int]: - """How much of what this pin adds the base tag already has. - - This is the pr-set.json retirement rule, mechanised: "delete the entry once - a base tag carries the work". Upstream almost always SQUASHES, so the - pinned commit never becomes an ancestor and no ancestry test will ever say - the work landed; comparing the text is the only thing that can. - - Measured on the real set at b10775, the separation is not close: the pin - that upstream had already absorbed (ggml-org#28133) scored 99%, and the - highest live pin scored 33%. - """ - total = hit = 0 - for path, wanted in contract["lines"].items(): - text = git(["show", f"{base}:{path}"], cwd, check=False) - total += len(wanted) - hit += sum(1 for w in wanted if w in text) - return hit, total - - -def check(pin: dict, contract: dict, root: Path, base: str, cwd: Path, - threshold: float) -> list[str]: - problems = [] - - for path, names in sorted(contract["symbols"].items()): - target = root / path - text = target.read_text(errors="replace") if target.is_file() else "" - for name in names: - if name not in text: - problems.append( - f"{name} is missing from {path}; the pin adds it there and " - "the merged tree does not have it") - - for path in contract["added_files"]: - if not (root / path).exists(): - problems.append(f"{path} is added by the pin and missing from the merged tree") - - for path, wanted in sorted(contract["lines"].items()): - target = root / path - if not target.is_file(): - continue # already reported, or a deletion - text = target.read_text(errors="replace") - lost = [w for w in wanted if w not in text] - if not lost: - continue - kept = len(wanted) - len(lost) - ratio = kept / len(wanted) - if ratio < threshold: - problems.append( - f"{path} kept {kept}/{len(wanted)} of the lines this pin adds " - f"({ratio:.0%}); first missing: {lost[0][:90]}") - - return problems - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - ap.add_argument("--root", default=".", help="the merged tree to check") - src = ap.add_mutually_exclusive_group(required=True) - src.add_argument("--pr-set", help="scripts/unsloth/pr-set.json") - src.add_argument("--prs-json", help="the resolve step's `prs` output: the pins it " - "actually merged, optional ones already dropped") - ap.add_argument("--base", required=True, help="upstream base tag the mix was built on") - ap.add_argument("--git-dir", help="repo the pin commits are reachable from " - "(default: --root)") - ap.add_argument("--threshold", type=float, default=1.0, - help="fraction of a pin's added lines that must survive per file") - ap.add_argument("--redundant-at", type=float, default=0.95, - help="report a pin whose added lines the base tag already has " - "at this fraction or more (never fatal)") - ap.add_argument("--report", help="write a JSON report here") - ap.add_argument("--emit", action="store_true", - help="print the derived contracts and check nothing") - args = ap.parse_args() - - root = Path(args.root).resolve() - cwd = Path(args.git_dir).resolve() if args.git_dir else root - pins = (load_pins(Path(args.pr_set)) if args.pr_set - else load_effective(args.prs_json)) - - report: dict = {"base": args.base, "ok": False, "pins": [], "notices": []} - failed = 0 - notices: list[str] = [] - - for pin in pins: - name = f"{pin['src']}#{pin['num']}" - try: - contract = derive(pin, args.base, cwd) - except RuntimeError as e: - report["pins"].append({"pin": name, "sha": pin["sha"], "problems": [str(e)]}) - print(f"ERROR {name}: {e}", file=sys.stderr) - failed += 1 - continue - - entry = { - "pin": name, - "sha": pin["sha"], - "fork": contract["fork"], - "symbols": contract["symbols"], - "added_files": contract["added_files"], - "line_count": sum(len(v) for v in contract["lines"].values()), - "problems": [], - } - - if args.emit: - report["pins"].append(entry) - # Only the families that NAME a feature are printed. Every symbol - # is still checked; a new file legitimately contributes a hundred - # LLM_TENSOR_ names and listing them buries the one that matters. - sym = sorted({s for v in contract["symbols"].values() for s in v - if s.startswith(HEADLINE)}) - print(f"{name:>18} {entry['line_count']:>5} lines, " - f"{len(contract['added_files'])} new files, symbols: " - f"{', '.join(sym) if sym else '-'}") - continue - - problems = check(pin, contract, root, args.base, cwd, args.threshold) - hit, total = redundancy(contract, args.base, cwd) - entry["problems"] = problems - entry["redundant_lines"] = [hit, total] - - if total and hit / total >= args.redundant_at: - note = (f"the base tag already has {hit}/{total} ({hit / total:.0%}) of the " - "lines this pin adds; upstream has taken this work and the entry " - "should be deleted from pr-set.json") - entry["notices"] = [note] - notices.append(f"{name}: {note}") - - report["pins"].append(entry) - if problems: - failed += 1 - print(f"FAIL {name}", file=sys.stderr) - for p in problems: - print(f" {p}", file=sys.stderr) - else: - print(f"ok {name}: {len(contract['symbols'])} file(s) with new symbols, " - f"{entry['line_count']} line(s) accounted for") - - report["ok"] = failed == 0 or args.emit - report["notices"] = notices - if args.report: - Path(args.report).write_text(json.dumps(report, indent=2)) - if args.emit: - return 0 - - # Notices after the verdict lines, never mixed into them: "upstream took - # this, drop the entry" is housekeeping and must not read as a failure. - for n in notices: - print(f"note {n}") - if failed: - print(f"\n{failed} pin(s) are not intact in the merged tree", file=sys.stderr) - return 1 - print(f"\nall {len(pins)} pins are intact in the merged tree" - + (f", {len(notices)} can be retired" if notices else "")) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/pin_merge.py b/scripts/unsloth/pin_merge.py deleted file mode 100755 index 0ebe184ccd0b..000000000000 --- a/scripts/unsloth/pin_merge.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Three-way merge pr-set.json one pin at a time, and refuse anything ambiguous. - -Two repins in flight always collide. -Both edit adjacent lines of the same JSON list, so git merges them as text and reports a conflict over lines that have nothing to do with each other. -On 08-27 that happened twice in one hour, while landing the qwen4exp, Inkling and GLM-5-Next repins: each merge invalidated the next, and each one was resolved by hand into exactly what a per-element merge would have produced. - -Pin ORDER is load-bearing. -resolve merges pins sequentially, so a later pin sees the tree the earlier ones produced, and reordering the list silently changes the composition. -This never reorders: it walks the base list positionally and takes whichever side moved each entry. -That also means a pin added or removed on one side is refused rather than aligned, because guessing where an inserted pin belongs is exactly the kind of guess that would change composition order. -A side that REORDERS the list is refused for the same reason, and for a sharper one: position i would no longer name the same pin on both sides, so merging it field-wise would splice one PR's `required` onto another PR's url. - -Usable as a git merge driver: - - git config merge.pinset.name 'pr-set.json pin-wise merge' - git config merge.pinset.driver 'python3 scripts/unsloth/pin_merge.py %O %A %B' - echo 'scripts/unsloth/pr-set.json merge=pinset' >> .gitattributes - -The driver contract is to write the result over %A (ours) and exit 0, or leave it alone and exit non-zero to fall back to a normal conflict. -That fallback is the whole safety story: a refusal costs a hand resolution, which is the status quo, and never a wrong pin set. -""" - -from __future__ import annotations - -import argparse -import json -import re -import sys -from pathlib import Path - - -class Ambiguous(Exception): - """A pin set difference this script is not allowed to decide.""" - - -MISSING = object() - -# The pin url shape repin.py already enforces. -# The owner matters: ggml-org#125 and unslothai#125 are different PRs. -PIN_ID = re.compile(r"^https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/commits/") - - -def pins(doc: dict) -> list[str]: - """The pin URLs, in order. Entries are a bare string or {url, required}.""" - return [p if isinstance(p, str) else p["url"] for p in doc["prs"]] - - -def fields(entry: str | dict) -> dict: - """An entry in dict form. A bare url string is just {"url": url}.""" - return {"url": entry} if isinstance(entry, str) else dict(entry) - - -def ident(entry: str | dict) -> str: - """What a pin IS, independent of which commit it currently points at. - - A repin changes only the sha, so (owner, PR number) is the stable identity. - Anything that does not look like a pin url is its own identity, which is the conservative reading: an unrecognised url can only ever cause a refusal. - """ - url = fields(entry).get("url") - m = PIN_ID.match(url) if isinstance(url, str) else None - return f"{m.group(1)}#{m.group(2)}" if m else repr(url) - - -def refuse_reorder(b: list, o: list, t: list) -> None: - """Refuse if either side moved an entry that base holds somewhere else. - - Merging by position assumes position i means the same pin on all three sides. - A reorder breaks that assumption silently: base [A, B] with ours making A optional and theirs reordering to [B, A] merges position 0 as "url moved to B, required moved to false" and produces B(required=false), so the release skips the wrong PR and the driver still exits 0. - - Realigning by identity instead is not safe. - A duplicated entry, or a reorder combined with a repin, leaves more than one alignment consistent with the diff, and choosing one is a guess about composition order, which is load-bearing here. - Refusing costs the hand resolution that was the status quo; guessing costs a wrong build nothing downstream can see. - """ - bid = [ident(e) for e in b] - for side, entries in (("ours", o), ("theirs", t)): - for i, e in enumerate(entries): - k = ident(e) - if k != bid[i] and k in bid: - raise Ambiguous( - f"pin {i} on {side} is {k}, which base holds at position " - f"{bid.index(k)}: the list was reordered, and merging " - "reordered entries by position would take fields from " - "different PRs") - - -def refuse_duplicates(what: str, entries: list) -> None: - """Two entries of one PR are indistinguishable, so nothing can align them.""" - ids = [ident(e) for e in entries] - dupes = sorted({k for k in ids if ids.count(k) > 1}) - if dupes: - raise Ambiguous( - f"{what} names {', '.join(dupes)} more than once; two entries of " - "one PR cannot be told apart, so a swap between them reads as no " - "change and a later merge would splice their fields together") - - -def merge_keys(what: str, bd: dict, od: dict, td: dict) -> dict: - """Three-way merge a mapping key by key, refusing only a real clash. - - A key missing on a side is MISSING rather than absent, so "theirs deleted it, ours left it alone" is a deletion both sides agree on, not a no-op. - Ours' key order is kept, then keys only theirs or only base has. - """ - out: dict = {} - for k in list(od) + [k for k in td if k not in od] + \ - [k for k in bd if k not in od and k not in td]: - bv, ov, tv = bd.get(k, MISSING), od.get(k, MISSING), td.get(k, MISSING) - if ov == tv: - v = ov - elif ov == bv: - v = tv - elif tv == bv: - v = ov - else: - raise Ambiguous(f"{what} field {k!r} changed differently on both " - f"sides:\n ours: {ov}\n theirs: {tv}") - if v is not MISSING: - out[k] = v - return out - - -def merge_entry(i: int, b, o, t): - """Three-way merge one pin ENTRY, not just its url. - - Comparing whole entries matters: an entry carries `required` as well as `url`, and comparing only urls makes a `required` flip on one side look like "no change", so rebuilding the list from ours drops it silently. - """ - if o == t: - return o # same on both sides, including untouched - if o == b: - return t # only theirs touched this entry - if t == b: - return o # only ours touched this entry - # Both sides touched it. - # Merging field-wise is only meaningful while all three sides name the SAME PR. - # A repin moves the sha and keeps the identity, which is the case this is for. - # REPLACING the pin with another PR while the other side edits a field is not: base PR100(required=true), ours PR200, theirs PR100 required=false merges url from ours and required from theirs and yields PR200(required=false), so the release skips a PR nobody made optional and the driver still exits 0. - # - # Refused rather than resolved even when both sides agree on the replacement, because base then describes a different PR and every field comparison below is against settings that were never PR200's. - named = {ident(b), ident(o), ident(t)} - if len(named) != 1: - raise Ambiguous( - f"pin {i} names different PRs across the sides (base {ident(b)}, " - f"ours {ident(o)}, theirs {ident(t)}) and both sides edited it: " - "merging their fields would attach one PR's settings to another") - out = merge_keys(f"pin {i}", fields(b), fields(o), fields(t)) - if "url" not in out: - raise Ambiguous(f"pin {i} lost its url") - # Keep the bare-string form when nothing but the url is present, so the file's shape is not rewritten by merging it. - return out["url"] if list(out) == ["url"] else out - - -def merge_pins(base: dict, ours: dict, theirs: dict) -> dict: - b, o, t = base["prs"], ours["prs"], theirs["prs"] - if not len(b) == len(o) == len(t): - raise Ambiguous( - f"pin count differs (base={len(b)} ours={len(o)} theirs={len(t)}); " - "a pin was added or removed, and placing it is an ordering decision" - ) - # Every side, and the result. - # A duplicate on one input defeats the reorder guard below; a duplicate only in the RESULT is made here, by the two sides adding the same PR at different positions. - for what, side in (("base", b), ("ours", o), ("theirs", t)): - refuse_duplicates(what, side) - refuse_reorder(b, o, t) - merged = [merge_entry(i, bx, ox, tx) - for i, (bx, ox, tx) in enumerate(zip(b, o, t))] - refuse_duplicates("the merged pin set", merged) - # Everything outside .prs is three-way merged the same way. - # Rebuilding the document from ours instead would silently drop a change theirs made to a top-level field - `_doc`, or any schema field added later - and this driver REPLACES git's text merge rather than running after it, so nothing downstream would ever notice the loss. - # A real two-sided clash refuses, which costs a hand resolution and never a wrong document. - skel = [{k: (None if k == "prs" else v) for k, v in d.items()} - for d in (base, ours, theirs)] # .prs is merged positionally - out = merge_keys("document", *skel) - out["prs"] = merged # keeps ours' key position - return out - - -def load(path: str) -> dict: - return json.loads(Path(path).read_text()) - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - # argparse %-formats help strings, so a literal percent must be doubled. - ap.add_argument("base", help="%%O, the merge base") - ap.add_argument("ours", help="%%A, our version; the result is written here") - ap.add_argument("theirs", help="%%B, their version") - ap.add_argument("--stdout", action="store_true", - help="print the result instead of writing over `ours`") - ap.add_argument("--report", metavar="PATH", help="write a JSON summary here") - a = ap.parse_args() - - report: dict = {"ok": False, "reason": None} - try: - merged = merge_pins(load(a.base), load(a.ours), load(a.theirs)) - except (Ambiguous, KeyError, json.JSONDecodeError) as e: - report["reason"] = str(e) - if a.report: - Path(a.report).write_text(json.dumps(report, indent=2)) - print(f"pin_merge: refused: {e}", file=sys.stderr) - return 1 - - text = json.dumps(merged, indent=2) + "\n" - if a.stdout: - sys.stdout.write(text) - else: - Path(a.ours).write_text(text) - report["ok"] = True - report["pins"] = pins(merged) - if a.report: - Path(a.report).write_text(json.dumps(report, indent=2)) - # stderr, so --stdout emits nothing but the merged document - print(f"pin_merge: merged {len(pins(merged))} pins", file=sys.stderr) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/pr-set.json b/scripts/unsloth/pr-set.json deleted file mode 100644 index bdca9a2dd569..000000000000 --- a/scripts/unsloth/pr-set.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "_doc": [ - "ggml-org/llama.cpp or unslothai/llama.cpp PRs to merge into the nightly prebuilds. Each entry", - "pins an exact commit -- copy the url of the commit you reviewed from the PR's commits tab:", - " https://github.com/ggml-org/llama.cpp/pull/15926/commits/59a3d0cb8f611aa3110ecea3d0afd16b1b18ee06", - "Only that commit is built, even if the author keeps pushing; update the pin to take newer code.", - "An empty list is a plain upstream build.", - "Closed and merged PRs are STILL merged in. Upstream tags lag their merges and the base is", - "aged a further UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS, so a pin dropped on merge leaves its arch", - "in neither the base nor the mix. Once the base tag contains the commit the merge is an empty", - "no-op ONLY if upstream took the PR as a merge commit. Upstream usually squashes, and a squash", - "is not an ancestor of the pinned commit, so the merge re-applies code the base already has and", - "the whole build stops on an unresolvable conflict. Delete the entry once a base tag carries the", - "work, do not wait for it to rot away. A closed-unmerged pin ships code upstream", - "declined -- the resolve log warns, but nothing else stops it, so prune those deliberately.", - "Use {\"url\": \"...\", \"required\": false} for an entry that should be skipped once it is not open.", - "Merging an unslothai PR into fork master drops it from the nightly (the tree is the upstream", - "tag + pins), so keep its pin listed until the change lands upstream." - ], - "prs": [ - "https://github.com/ggml-org/llama.cpp/pull/24423/commits/c6f8d604b67611b73f7965c0bd39d26e7365a489", - "https://github.com/ggml-org/llama.cpp/pull/25731/commits/36df1bf409c8b257689321a971a66973ee817ee1", - "https://github.com/unslothai/llama.cpp/pull/70/commits/883f2c9ba78f3847148454adf025da29385fff3e", - "https://github.com/unslothai/llama.cpp/pull/61/commits/46cbf0e95786fe8f5b7c0e86d57aaf8f8eceea7f", - "https://github.com/unslothai/llama.cpp/pull/95/commits/3db8cb5b2e9bf291057b9f19960e8601a162da81", - "https://github.com/ggml-org/llama.cpp/pull/27754/commits/629b50552801912b3e2078f9799e4d77213197d7", - "https://github.com/unslothai/llama.cpp/pull/137/commits/4e1865e34ec5f6ca39403215c89129c13731be70", - "https://github.com/unslothai/llama.cpp/pull/158/commits/abfc45b9cb21eae4848cb82196e659f42c9a8341", - "https://github.com/unslothai/llama.cpp/pull/157/commits/6c6da89266ba7839d825c9997782af4f4d26b81b", - "https://github.com/unslothai/llama.cpp/pull/149/commits/b65a2dce12c14a489e19a059cb6ee59112f1b733", - "https://github.com/unslothai/llama.cpp/pull/144/commits/a9e9c3c5fed8a0bb5cc617532d0d16b8f59c13e0", - "https://github.com/unslothai/llama.cpp/pull/152/commits/b2b5ed9ff86427a530b762a45d3fdbd453bcd4e8", - "https://github.com/unslothai/llama.cpp/pull/176/commits/09ce1a4d2939844e211f7b4d30a296f4c1aed9a8" - ] -} diff --git a/scripts/unsloth/repin.py b/scripts/unsloth/repin.py deleted file mode 100644 index ac9028592ee5..000000000000 --- a/scripts/unsloth/repin.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Merge the current base tag into each pinned PR branch we control, and repin. - -The nightly builds an upstream release tag plus a list of pinned PR commits. -Upstream moves several times a day, so a pin that merged yesterday routinely -stops merging today -- that is what broke four nightlies in a week, and every -fix was the same mechanical merge done by hand. - -This does that merge, and only where it is safe to: - - * branches we own (see OWNED). A third-party PR is reported, never pushed to. - * pins that are still their branch head. If the author has pushed past the - pin, merging into the branch would silently widen the release to include - code nobody reviewed, which is the exact property the pin file exists to - hold. Report it and let a human decide. - * conflicts that additive_merge.py can prove are pure add/add. Anything else - is left alone and reported. - -Writes the new pins back to pr-set.json and prints a markdown report. Pushing -and opening the PR is the caller's job; nothing here talks to a remote except -to fetch. -""" - -from __future__ import annotations - -import argparse -import json -import re -import subprocess -import sys -from pathlib import Path - -# Branches we may push to. Anything else gets a report line and no write. -OWNED = ("unslothai/", "danielhanchen/") - -PIN_RE = re.compile( - r"^https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/commits/([0-9a-f]{40})/?$" -) -HERE = Path(__file__).resolve().parent - - -def run(args, cwd=None, check=True, quiet=True): - r = subprocess.run(args, cwd=cwd, capture_output=True, text=True) - if check and r.returncode != 0: - raise RuntimeError(f"{' '.join(args[:4])}... failed: {r.stderr.strip()[:400]}") - if not quiet and r.stdout: - print(r.stdout.rstrip()) - return r - - -def gh_json(path): - r = run(["gh", "api", path], check=False) - if r.returncode != 0: - return None - try: - return json.loads(r.stdout) - except json.JSONDecodeError: - return None - - -def load_pins(pr_set: Path) -> tuple[dict, list[dict]]: - data = json.loads(pr_set.read_text()) - pins = [] - for entry in data["prs"]: - url = entry if isinstance(entry, str) else entry["url"] - required = True if isinstance(entry, str) else entry.get("required", True) - m = PIN_RE.match(url) - if not m: - raise SystemExit(f"malformed pin: {url}") - pins.append( - { - "url": url, - "required": required, - "src": f"{m.group(1)}/llama.cpp", - "num": int(m.group(2)), - "sha": m.group(3), - } - ) - return data, pins - - -def repin_one(pin: dict, base: str, work: Path) -> dict: - """Try to bring one pin up to `base`. Never raises for an expected refusal.""" - out = dict(pin, action="skip", note="", new_sha="", files=[], hunks=[]) - pr = gh_json(f"repos/{pin['src']}/pulls/{pin['num']}") - if pr is None: - out["note"] = "could not read the PR from the API" - return out - if pr.get("state") != "open": - out["note"] = f"PR is {pr.get('state')}; not repinning a closed PR" - return out - - head_repo = (pr.get("head", {}).get("repo") or {}).get("full_name") - head_ref = pr.get("head", {}).get("ref") - head_sha = pr.get("head", {}).get("sha") - out.update(head_repo=head_repo, head_ref=head_ref) - - if not head_repo: - out["note"] = "head repository was deleted" - return out - if not head_repo.startswith(OWNED): - out["action"] = "third-party" - out["note"] = f"`{head_repo}` is not ours; ask the author to merge master" - return out - if head_sha != pin["sha"]: - out["note"] = ( - f"branch head `{head_sha[:10]}` has moved past the pin `{pin['sha'][:10]}`; " - "repinning would pull in unreviewed commits" - ) - return out - - repo = work / f"r{pin['num']}" - run(["git", "clone", "-q", "--filter=blob:none", "--no-checkout", - f"https://github.com/{pin['src']}.git", str(repo)]) - run(["git", "fetch", "-q", "--no-tags", "origin", pin["sha"]], cwd=repo, check=False) - r = run(["git", "fetch", "-q", "--no-tags", - "https://github.com/ggml-org/llama.cpp.git", - f"refs/tags/{base}:refs/tags/{base}"], cwd=repo, check=False) - if r.returncode != 0: - out["note"] = f"could not fetch base tag {base}" - return out - if run(["git", "rev-parse", "--verify", f"{pin['sha']}^{{commit}}"], - cwd=repo, check=False).returncode != 0: - out["note"] = f"pinned commit {pin['sha'][:10]} is gone (force-pushed away)" - return out - - run(["git", "checkout", "-q", "--detach", pin["sha"]], cwd=repo) - if run(["git", "merge-base", "--is-ancestor", f"refs/tags/{base}", "HEAD"], - cwd=repo, check=False).returncode == 0: - out["note"] = f"already contains {base}" - return out - - # diff3 is what makes the add/add proof possible: without the base section - # an edit/edit conflict is indistinguishable from an add/add one. - git_id = ["-c", "user.name=unsloth-repin-bot", - "-c", "user.email=unsloth-repin-bot@users.noreply.github.com"] - m = run(["git", "-c", "merge.conflictStyle=diff3", *git_id, "merge", "--no-ff", - "--no-edit", "-m", f"Merge {base} into {head_ref}", f"refs/tags/{base}"], - cwd=repo, check=False) - - if m.returncode != 0: - report = work / f"res{pin['num']}.json" - rc = subprocess.run( - [sys.executable, str(HERE / "additive_merge.py"), - "--repo", str(repo), "--report", str(report)], - capture_output=True, text=True, - ).returncode - res = json.loads(report.read_text()) if report.exists() else {} - if rc != 0: - out["action"] = "conflict" - refused = res.get("refused", []) - out["files"] = [x["file"] for x in refused if x["file"] != "-"] - if out["files"]: - out["note"] = "; ".join(f"`{x['file']}`: {x['reason']}" for x in refused) - else: - # git refused the merge without leaving a single conflicted - # file, so the conflict report explains nothing. Its stderr is - # the only thing that does, and discarding it turns a - # diagnosable failure into "no conflicted files". - tail = ((m.stderr or "") + (m.stdout or "")).strip().splitlines() - out["note"] = ("merge failed with no conflicts: " + " / ".join(tail[-3:]) - if tail else "merge failed and git said nothing") - run(["git", "merge", "--abort"], cwd=repo, check=False) - return out - out["hunks"] = [ - {"file": f["file"], **h} for f in res.get("resolved", []) for h in f["hunks"] - ] - out["files"] = [f["file"] for f in res.get("resolved", [])] - run(["git", *git_id, "commit", "-q", "--no-edit"], cwd=repo) - - new_sha = run(["git", "rev-parse", "HEAD"], cwd=repo).stdout.strip() - out["action"] = "repin" - out["new_sha"] = new_sha - out["repo_path"] = str(repo) - out["touches_workflows"] = bool( - run(["git", "diff", "--name-only", f"{pin['sha']}..HEAD", "--", - ".github/workflows"], cwd=repo).stdout.strip() - ) - return out - - -def markdown(base: str, results: list[dict]) -> str: - L = [f"Base tag: `{base}`", ""] - L += ["| pin | branch | outcome |", "|---|---|---|"] - for r in results: - pin = f"[`{r['src']}#{r['num']}`](https://github.com/{r['src']}/pull/{r['num']})" - branch = f"`{r.get('head_repo') or '?'}:{r.get('head_ref') or '?'}`" - if r["action"] == "repin": - what = f"repinned `{r['sha'][:10]}` to `{r['new_sha'][:10]}`" - if r["hunks"]: - what += f" ({len(r['hunks'])} add/add hunk(s) resolved)" - elif r["action"] == "conflict": - what = f"**conflict, not resolvable automatically** -- {r['note']}" - elif r["action"] == "third-party": - what = f"not ours -- {r['note']}" - else: - what = r["note"] or "no change" - L.append(f"| {pin} | {branch} | {what} |") - L.append("") - - for r in results: - if not r.get("hunks"): - continue - L += [f"<details><summary>Resolutions for <code>{r['src']}#{r['num']}</code></summary>", ""] - for h in r["hunks"]: - L += [f"`{h['file']}`", "", "```diff"] - L += [f"-{x}" for x in h["ours"].rstrip("\n").split("\n")] - L += [f"-{x}" for x in h["theirs"].rstrip("\n").split("\n")] - L += [f"+{x}" for x in h["resolution"].rstrip("\n").split("\n")] - L += ["```", ""] - L += ["</details>", ""] - - if any(r.get("touches_workflows") for r in results): - L += ["Some merges carry upstream changes under `.github/workflows/`, so the " - "push needs a token with workflow write permission.", ""] - return "\n".join(L) - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--pr-set", required=True) - ap.add_argument("--base", required=True, help="upstream release tag to merge in") - ap.add_argument("--work", required=True, help="scratch directory for clones") - ap.add_argument("--report", help="write a JSON report here") - ap.add_argument("--markdown", help="write the human-readable report here") - args = ap.parse_args() - - pr_set = Path(args.pr_set) - work = Path(args.work) - work.mkdir(parents=True, exist_ok=True) - _, pins = load_pins(pr_set) - - results = [] - for pin in pins: - try: - r = repin_one(pin, args.base, work) - except RuntimeError as e: - r = dict(pin, action="skip", note=f"error: {e}", new_sha="", files=[], hunks=[]) - results.append(r) - print(f"{r['src']}#{r['num']}: {r['action']} {r['note']}".rstrip()) - - # Swap the shas in the raw text rather than re-serialising. Rewriting the - # JSON would reflow the whole file and bury a four-character change in a - # whole-file diff, which is the opposite of what a reviewer needs here. - text = pr_set.read_text() - changed = 0 - for r in results: - if r["action"] != "repin": - continue - if r["sha"] not in text: - print(f"::warning::{r['src']}#{r['num']}: pin not found verbatim; not rewritten") - continue - text = text.replace(r["sha"], r["new_sha"]) - changed += 1 - if changed: - pr_set.write_text(text) - - if args.report: - Path(args.report).write_text(json.dumps( - {"base": args.base, "changed": changed, "results": results}, indent=2)) - if args.markdown: - Path(args.markdown).write_text(markdown(args.base, results)) - - blocked = [r for r in results if r["action"] in ("conflict", "third-party")] - print(f"\n{changed} repinned, {len(blocked)} needing a human") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/sync_deletes.py b/scripts/unsloth/sync_deletes.py deleted file mode 100755 index 3b8c53ca6e5d..000000000000 --- a/scripts/unsloth/sync_deletes.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Resolve the modify/delete conflicts a fork sync produces, and only those. - -Measured over every merge commit in this fork: 149 file-level conflicts, of which 68 (45 percent) are the same one. -Upstream edits a workflow this fork deleted on purpose, git cannot know which side wins, and a human keeps the deletion. -Every single time: 68 of 68 historical instances resolved by keeping the deletion, with no exceptions. - -That is not a heuristic, it is the fork's stated invariant. -This fork owns no upstream CI. -scripts/unsloth/upstream-sync.json requires the diff from the sync point to master to touch only .github/ and scripts/unsloth/, and verify_upstream_sync.py already treats these deletions as legitimate. -The deletions are policy, so re-applying them is bookkeeping. - -Scope is deliberately tight, because the cost of being wrong is a workflow silently reappearing and firing on the fork: - - - only paths under .github/workflows/ - - only modify/delete conflicts, never content conflicts - - never a file named unsloth-*.yml, which is ours; if one of those is ever in a modify/delete conflict, something is wrong and a human should look - - the delete must be on our side; upstream deleting a file we modified is the opposite situation and is left alone - -The same policy covers a workflow upstream ADDED since the last sync. -That is not a conflict at all, so git merges it in silently and the fork acquires a workflow that starts firing on it. -Replaying sync e8735f35d3 caught exactly this: resolving only the conflicts left .github/workflows/build-wasm.yml in the tree, where the recorded human resolution had deleted it. -With --added handled too, the replay reproduces that tree exactly. - -Anything else is left conflicted. -Exits 0 if it resolved everything it was asked about, 1 if any conflict remains. -""" - -from __future__ import annotations - -import argparse -import json -import subprocess -import sys -from pathlib import Path - -PREFIX = ".github/workflows/" -OURS = "unsloth-" - - -def git(*args: str, cwd: str = ".") -> subprocess.CompletedProcess: - return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) - - -def unmerged(cwd: str) -> dict[str, set[int]]: - """path -> set of stages present (1 base, 2 ours, 3 theirs).""" - out: dict[str, set[int]] = {} - r = git("ls-files", "-u", cwd=cwd) - # A listing that could not run gives empty stdout, which reads as "no - # conflicts" and makes this script report that it resolved everything it - # was asked to. Not a repo, or an unreadable index, has to be an error. - if r.returncode != 0: - raise RuntimeError(f"git ls-files -u in {cwd}: {r.stderr.strip()}") - for line in r.stdout.split("\n"): - if not line.strip(): - continue - meta, path = line.split("\t", 1) - stage = int(meta.split()[2]) - out.setdefault(path.strip(), set()).add(stage) - return out - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--repo", default=".") - ap.add_argument("--merge-base", help="also drop upstream workflows added since this rev") - ap.add_argument("--report", metavar="PATH") - a = ap.parse_args() - - try: - stages = unmerged(a.repo) - except RuntimeError as e: - print(f"sync_deletes: {e}", file=sys.stderr) - if a.report: - Path(a.report).write_text(json.dumps( - {"ok": False, "resolved": [], "removed_added": [], - "left": [str(e)]}, indent=2)) - return 1 - resolved, left, added = [], [], [] - for path, st in sorted(stages.items()): - name = path.rsplit("/", 1)[-1] - # stage 2 missing means our side deleted it; stage 3 present means upstream still has it. - # That is the sync case, and only that. - ours_deleted = 2 not in st and 3 in st - if (path.startswith(PREFIX) and not name.startswith(OURS) and ours_deleted): - r = git("rm", "-q", "--force", "--", path, cwd=a.repo) - if r.returncode == 0: - resolved.append(path) - else: - left.append(f"{path}: git rm failed: {r.stderr.strip()}") - else: - why = ("we own this workflow" if name.startswith(OURS) - else "not an upstream workflow path" if not path.startswith(PREFIX) - else "not a delete on our side") - left.append(f"{path}: {why}") - - # Workflows upstream added since the merge base. - # No conflict, so nothing above sees them, and the fork silently gains CI that fires on its own repo. - if a.merge_base: - # Against the WORKING TREE, not HEAD: mid-merge, HEAD is still our pre-merge commit, so merge_base..HEAD describes our side rather than the merge result and finds nothing. - # --no-renames, or an upstream workflow that was RENAMED reads as R rather than A and this filter drops it. - # The fork would then carry the renamed workflow and it would start firing here, which is the exact thing this block exists to stop. - r = git("diff", "--name-status", "--diff-filter=A", "--no-renames", - a.merge_base, "--", PREFIX, cwd=a.repo) - if r.returncode != 0: - # An unusable --merge-base produces empty stdout, which is indistinguishable from "upstream added nothing" if the exit code is ignored. - # This script reporting success is what tells a sync it can proceed, so a listing that never ran has to be a failure, not a quiet zero: otherwise the sync carries every newly added upstream workflow in and they start firing on the fork. - left.append(f"{a.merge_base}: could not list workflows added since " - f"it: {r.stderr.strip()}") - else: - for line in r.stdout.split("\n"): - if not line.strip(): - continue - path = line.split("\t", 1)[1].strip() - name = path.rsplit("/", 1)[-1] - if name.startswith(OURS): - continue - rm = git("rm", "-q", "--force", "--", path, cwd=a.repo) - if rm.returncode == 0: - added.append(path) - else: - # Same reasoning as the resolve loop above: a removal that did not happen is reported, never dropped. - left.append(f"{path}: git rm failed: {rm.stderr.strip()}") - - for p in resolved: - print(f"resolved {p}: kept the fork's deletion") - for p in added: - print(f"removed {p}: upstream added it; this fork carries no upstream workflows") - for p in left: - print(f"left {p}", file=sys.stderr) - if a.report: - Path(a.report).write_text(json.dumps( - {"ok": not left, "resolved": resolved, - "removed_added": added, "left": left}, indent=2)) - return 1 if left else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/test_additive_merge.py b/scripts/unsloth/test_additive_merge.py deleted file mode 100644 index 0f91e9afd12b..000000000000 --- a/scripts/unsloth/test_additive_merge.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Tests for additive_merge.py. Run: python3 scripts/unsloth/test_additive_merge.py - -Every case builds a real git conflict rather than a hand-written one, so the -markers are exactly what git produces. -""" -import json -import subprocess -import sys -import tempfile -from pathlib import Path - -SCRIPT = Path(__file__).resolve().parent / "additive_merge.py" -FAILS = [] - - -def check(name, cond, extra=""): - print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) - if not cond: - FAILS.append(name) - - -def git(repo, *args, **kw): - return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], - cwd=repo, capture_output=True, text=True, **kw) - - -def make_conflict(base_txt, ours_txt, theirs_txt): - """Build a real git conflict, return (repo, conflicted file path).""" - d = Path(tempfile.mkdtemp(prefix="am_")) - git(d, "init", "-q", "-b", "main") - f = d / "f.c" - f.write_text(base_txt) - git(d, "add", "-A"); git(d, "commit", "-qm", "base") - git(d, "checkout", "-qb", "side") - f.write_text(theirs_txt) - git(d, "add", "-A"); git(d, "commit", "-qm", "theirs") - git(d, "checkout", "-q", "main") - f.write_text(ours_txt) - git(d, "add", "-A"); git(d, "commit", "-qm", "ours") - git(d, "-c", "merge.conflictStyle=diff3", "merge", "side") - return d, f - - -def run(repo, *extra): - rep = repo / "r.json" - p = subprocess.run([sys.executable, str(SCRIPT), "--repo", str(repo), "--report", str(rep), *extra], - capture_output=True, text=True) - return p.returncode, json.loads(rep.read_text()) if rep.exists() else {} - - -# --- 1. pure add/add: the real recurring shape ------------------------------ -base = "switch (arch) {\n case A:\n break;\n}\n" -ours = "switch (arch) {\n case A:\n case LLM_ARCH_INKLING:\n break;\n}\n" -theirs = "switch (arch) {\n case A:\n case LLM_ARCH_DEEPSEEK4:\n break;\n}\n" -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -txt = f.read_text() -check("add/add resolves", rc == 0 and rep["ok"], rep) -check("add/add unions both labels", - "LLM_ARCH_DEEPSEEK4" in txt and "LLM_ARCH_INKLING" in txt and "<<<<" not in txt, txt) -check("add/add puts upstream first (matches hand repin)", - txt.index("DEEPSEEK4") < txt.index("INKLING"), txt) -check("add/add stages the file", - git(repo, "diff", "--name-only", "--diff-filter=U").stdout.strip() == "") - -# --- 2. edit/edit on a shared line: must refuse ----------------------------- -base = "if (a == X || a == Y) {\n" -ours = "if (a == X || a == Y || a == KIMI) {\n" -theirs = "if (a == X || a == Y || a == MINIMAX) {\n" -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -check("edit/edit refuses", rc == 1 and not rep["ok"]) -check("edit/edit says why", "base is not empty" in (rep["refused"][0]["reason"] if rep["refused"] else ""), - rep) -check("edit/edit leaves markers in place", "<<<<" in f.read_text()) - -# --- 3. same line added twice: must refuse, not duplicate ------------------- -base = "a\nz\n" -ours = "a\ncase FOO:\n break;\nz\n" -theirs = "a\ncase FOO:\n break;\nz\n" -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -check("identical add/add is not a conflict at all", rc == 1 and "no conflicted files" in json.dumps(rep)) - -base = "a\nz\n" -ours = "a\nstatic void helper() {\n log(\"same\");\n}\nz\n" -theirs = "a\nstatic void helper2() {\n log(\"same\");\n}\nz\n" -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -check("overlapping add/add refuses on shared CONTENT", - rc == 1 and "made twice" in json.dumps(rep), rep) -reason = rep["refused"][0]["reason"] if rep.get("refused") else "" -check("overlapping add/add names the content line, not the braces", - reason.endswith('twice: log("same");'), reason) - -# --- 3b. two independent case arms: braces are shared, content is not ------- -# The real tools/mtmd/clip.cpp shape. Refusing this on `{` and `} break;` is -# what took the 09-02 nightly's last pin down. -base = "switch (t) {\n}\n" -ours = ("switch (t) {\n case PROJECTOR_TYPE_KIMIK3:\n {\n" - " builder = std::make_unique<clip_graph_kimik3>(ctx, img);\n" - " } break;\n}\n") -theirs = ("switch (t) {\n case PROJECTOR_TYPE_DEEPSEEK4V:\n {\n" - " builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);\n" - " } break;\n}\n") -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -txt = f.read_text() -check("independent case arms resolve despite shared braces", rc == 0 and rep["ok"], rep) -check("independent case arms keep both", "KIMIK3" in txt and "DEEPSEEK4V" in txt and "<<<<" not in txt, txt) -check("independent case arms keep both bodies once", - txt.count("} break;") == 2 and txt.count("clip_graph_kimik3") == 1, txt) - -# --- 3b2. two case arms that share a body line, which is a coincidence ------ -# The clip.cpp shape after upstream landed DEEPSEEK4V: both arms set the same -# rope_theta, and refusing on that is the shared-line check backwards. -base = "switch (t) {\n}\n" -ours = ("switch (t) {\n case PROJECTOR_TYPE_KIMIK3:\n {\n" - " hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;\n" - " hparams.rope_theta = 10000.0f;\n } break;\n}\n") -theirs = ("switch (t) {\n case PROJECTOR_TYPE_DEEPSEEK4V:\n {\n" - " hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;\n" - " hparams.rope_theta = 10000.0f;\n } break;\n}\n") -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -txt = f.read_text() -check("case arms with a coincidentally shared body line resolve", rc == 0 and rep["ok"], rep) -check("case arms with a shared body line keep both arms", - txt.count("rope_theta") == 2 and "KIMIK3" in txt and "DEEPSEEK4V" in txt, txt) - -# --- 3b3. the SAME arm added twice keeps its label, so it still refuses ----- -base = "switch (t) {\n}\n" -ours = ("switch (t) {\n case PROJECTOR_TYPE_KIMIK3:\n {\n" - " hparams.rope_theta = 10000.0f;\n } break;\n}\n") -theirs = ("switch (t) {\n case PROJECTOR_TYPE_KIMIK3:\n {\n" - " hparams.rope_theta = 50000.0f;\n } break;\n}\n") -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -check("the same case label on both sides still refuses", - rc == 1 and "made twice" in json.dumps(rep), rep) - -# --- 3b4. only one side is case arms: no label proof, ordinary rules apply -- -base = "a\nz\n" -ours = "a\ncase FOO:\n f(1);\n break;\nz\n" -theirs = "a\nstatic void helper() { f(1); }\nz\n" -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -check("one side not a case arm falls back to the shared-line check", - rc == 0 and rep["ok"], rep) - -base = "a\nz\n" -ours = "a\ncase FOO:\n f(1);\n break;\nz\n" -theirs = "a\nstatic void helper();\n f(1);\nz\n" -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -check("one side not a case arm still refuses on a shared content line", - rc == 1 and "made twice" in json.dumps(rep), rep) - -# --- 3c. one side adds only scaffolding: nothing distinguishes the two ------ -base = "a\nz\n" -ours = "a\n}\nz\n" -theirs = "a\ncase BAR:\n break;\nz\n" -repo, f = make_conflict(base, ours, theirs) -rc, rep = run(repo) -check("scaffolding-only addition refuses", - rc == 1 and "scaffolding" in json.dumps(rep), rep) - -# --- 4. one file good, one file bad: refuse the whole merge ---------------- -d = Path(tempfile.mkdtemp(prefix="am_")) -git(d, "init", "-q", "-b", "main") -(d / "good.c").write_text("x\ny\n") -(d / "bad.c").write_text("if (a || b) {\n") -git(d, "add", "-A"); git(d, "commit", "-qm", "base") -git(d, "checkout", "-qb", "side") -(d / "good.c").write_text("x\ncase UP:\ny\n") -(d / "bad.c").write_text("if (a || b || up) {\n") -git(d, "add", "-A"); git(d, "commit", "-qm", "theirs") -git(d, "checkout", "-q", "main") -(d / "good.c").write_text("x\ncase MINE:\ny\n") -(d / "bad.c").write_text("if (a || b || mine) {\n") -git(d, "add", "-A"); git(d, "commit", "-qm", "ours") -git(d, "-c", "merge.conflictStyle=diff3", "merge", "side") -rc, rep = run(d) -check("mixed: overall refuses", rc == 1 and not rep["ok"]) -check("mixed: both files still unmerged in the index", - {ln.split("\t")[-1] for ln in git(d, "ls-files", "-u").stdout.splitlines()} == {"good.c", "bad.c"}, - git(d, "ls-files", "-u").stdout) -check("mixed: the resolvable file is NOT half-written", - "<<<<" in (d / "good.c").read_text(), (d / "good.c").read_text()) - -# --- 5. dry-run writes nothing -------------------------------------------- -base = "switch (arch) {\n case A:\n break;\n}\n" -ours = "switch (arch) {\n case A:\n case MINE:\n break;\n}\n" -theirs = "switch (arch) {\n case A:\n case UP:\n break;\n}\n" -repo, f = make_conflict(base, ours, theirs) -before = f.read_text() -rc, rep = run(repo, "--dry-run") -check("dry-run reports ok", rc == 0 and rep["ok"]) -check("dry-run does not touch the file", f.read_text() == before) - -print() -print(f"{len(FAILS)} failure(s)" + (": " + ", ".join(FAILS) if FAILS else "")) -sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_carry_vintage.py b/scripts/unsloth/test_carry_vintage.py deleted file mode 100644 index f420dd48472d..000000000000 --- a/scripts/unsloth/test_carry_vintage.py +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Tests for carry_vintage.py. Run: python3 scripts/unsloth/test_carry_vintage.py - -Builds a throwaway repo shaped like a real carry: an upstream base tag, an upstream PR branch with two commits on top of it, and a carry branch that replayed the PR onto the base while deliberately dropping one file's change. - -The case that matters is that dropped file. -Its content equals the BASE version, which is reachable from the PR head, so a vintage search that walks the PR head's whole ancestry finds a "match" in a commit that is not part of the PR at all, calls the file SUPERSEDED, and concludes that rebuilding from the PR head is equivalent to merging - which would re-add exactly what the carry dropped. -""" -import json -import subprocess -import sys -import tempfile -from pathlib import Path - -SCRIPT = Path(__file__).resolve().parent / "carry_vintage.py" -FAILS = [] - - -def check(name, cond, extra=""): - print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) - if not cond: - FAILS.append(name) - - -def git(d, *args): - r = subprocess.run(["git", *args], cwd=d, capture_output=True, text=True) - if r.returncode: - raise RuntimeError(" ".join(args) + ": " + r.stderr) - return r.stdout.strip() - - -def write(d, name, text): - (Path(d) / name).write_text(text) - - -def fixture(): - d = tempfile.mkdtemp(prefix="cv_") - git(d, "init", "-q", "-b", "main") - git(d, "config", "user.email", "t@t") - git(d, "config", "user.name", "t") - write(d, "dropped.txt", "base version\n") - write(d, "taken.txt", "base version\n") - write(d, "edited.txt", "base version\n") - git(d, "add", "-A") - git(d, "commit", "-qm", "base") - base = git(d, "rev-parse", "HEAD") - - # Upstream PR: two commits, touching all three files. - git(d, "checkout", "-q", "-b", "pr") - write(d, "dropped.txt", "upstream v1\n") - write(d, "taken.txt", "upstream v1\n") - write(d, "edited.txt", "upstream v1\n") - git(d, "commit", "-qam", "pr c1") - mid = git(d, "rev-parse", "HEAD") - write(d, "taken.txt", "upstream v2\n") - write(d, "edited.txt", "upstream v2\n") - git(d, "commit", "-qam", "pr c2") - head = git(d, "rev-parse", "HEAD") - - # The carry: PR replayed onto base, with dropped.txt held at the base version on purpose, edited.txt at an older PR vintage, and taken.txt already at the PR head. - git(d, "checkout", "-q", "-b", "carry", base) - write(d, "dropped.txt", "base version\n") - write(d, "taken.txt", "upstream v2\n") - write(d, "edited.txt", "upstream v1\n") - git(d, "commit", "-qam", "carry") - carry = git(d, "rev-parse", "HEAD") - return d, base, mid, head, carry - - -d, base, mid, head, carry = fixture() -report = str(Path(d) / "report.json") -r = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry, "--pr-ref", head, - "--base", base, "--report", report], - cwd=d, capture_output=True, text=True) -check("runs clean", r.returncode == 0, r.stderr) -out = json.loads(Path(report).read_text()) -sup = {e["path"]: e["vintage"] for e in out["superseded"]} - -check("a file held at the BASE version is not called superseded", - "dropped.txt" in out["diverged"], json.dumps(out, indent=1)) -check("no vintage is a commit outside the PR", - all(v in (mid, head) for v in sup.values()), json.dumps(sup, indent=1)) -check("a file already at the PR head is superseded", - sup.get("taken.txt") == head, json.dumps(sup, indent=1)) -check("a file at an older PR commit is superseded at that vintage", - sup.get("edited.txt") == mid, json.dumps(sup, indent=1)) -check("a real divergence still forces a merge", - "has to merge" in r.stdout, r.stdout[-300:]) - -# With the deliberately dropped file removed from the picture, nothing diverges any more. -# edited.txt is still at an older vintage than the PR head, though, so the advice stays qualified: we never edited that file, but a rebuild moves it to head, and only a person knows whether the carry meant to hold it. -git(d, "checkout", "-q", "carry") -write(d, "dropped.txt", "upstream v1\n") -git(d, "commit", "-qam", "take it after all") -carry2 = git(d, "rev-parse", "HEAD") -r2 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry2, "--pr-ref", head, - "--base", base], cwd=d, capture_output=True, text=True) -check("nothing diverges once the dropped file is taken", - r2.returncode == 0 and "has to merge" not in r2.stdout, r2.stdout[-300:]) -check("an older vintage still qualifies the advice", - "OLDER vintage" in r2.stdout and "Nothing diverges" not in r2.stdout, - r2.stdout[-300:]) - -# A carry that deliberately OMITS a file the PR head still has. -# Nothing diverges, every file the carry does have is superseded, and the naive answer is "rebuild from the PR head" - which restores the omitted file and loses the omission. -# A file the PR DELETED is a different case: the carry not having it is agreement, and a rebuild reproduces it exactly. -def omission_fixture(): - d = tempfile.mkdtemp(prefix="cv_") - git(d, "init", "-q", "-b", "main") - git(d, "config", "user.email", "t@t") - git(d, "config", "user.name", "t") - write(d, "keep.txt", "base version\n") - write(d, "doomed.txt", "base version\n") - git(d, "add", "-A") - git(d, "commit", "-qm", "base") - base = git(d, "rev-parse", "HEAD") - - # The PR edits keep.txt, adds win.cmake, and deletes doomed.txt. - git(d, "checkout", "-q", "-b", "pr") - write(d, "keep.txt", "upstream v1\n") - write(d, "win.cmake", "windows-only build tweak\n") - git(d, "rm", "-q", "doomed.txt") - git(d, "add", "-A") - git(d, "commit", "-qm", "pr c1") - head = git(d, "rev-parse", "HEAD") - - # The carry replays it but never took win.cmake. - git(d, "checkout", "-q", "-b", "carry", base) - write(d, "keep.txt", "upstream v1\n") - git(d, "rm", "-q", "doomed.txt") - git(d, "add", "-A") - git(d, "commit", "-qm", "carry without win.cmake") - return d, base, head, git(d, "rev-parse", "HEAD") - - -d2, base2, head2, carry3 = omission_fixture() -# outside the repo, so the "writes nothing" check below sees a clean tree -report2 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") -r3 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry3, "--pr-ref", head2, - "--base", base2, "--report", report2], - cwd=d2, capture_output=True, text=True) -check("runs clean on an omitting carry", r3.returncode == 0, r3.stderr) -out2 = json.loads(Path(report2).read_text()) -check("a file present at the PR head but not in the carry is OMITTED, not absent", - out2.get("omitted") == ["win.cmake"] and "win.cmake" not in out2.get("absent", []), - json.dumps(out2, indent=1)) -check("a file the PR deleted stays merely ABSENT", - out2.get("absent") == ["doomed.txt"], json.dumps(out2, indent=1)) -check("an omitted file suppresses the rebuild recommendation", - "Nothing diverges" not in r3.stdout, r3.stdout[-400:]) -check("and says why rebuilding is not equivalent", - "NOT equivalent" in r3.stdout, r3.stdout[-400:]) -check("the omitting carry still writes nothing", - git(d2, "status", "--porcelain") == "", git(d2, "status", "--porcelain")) - -# Take the omitted file, and the rebuild advice is correct again. -write(d2, "win.cmake", "windows-only build tweak\n") -git(d2, "add", "-A") -git(d2, "commit", "-qm", "take win.cmake after all") -r4 = subprocess.run([sys.executable, str(SCRIPT), "--carry", git(d2, "rev-parse", "HEAD"), - "--pr-ref", head2, "--base", base2], cwd=d2, capture_output=True, text=True) -check("a PR deletion alone still allows the rebuild", - r4.returncode == 0 and "Nothing diverges" in r4.stdout, r4.stdout[-400:]) - -# A multi-commit PR that does not touch every file in its FIRST commit. -# The commits before the one that first changed a path still carry the fork's blob, and they are inside fork..head, so a carry deliberately holding that file at the base version matches one of them and is called SUPERSEDED - the same wrong "rebuilding is equivalent" answer the fork bound was meant to end, one commit further in. -def late_touch_fixture(): - d = tempfile.mkdtemp(prefix="cv_") - git(d, "init", "-q", "-b", "main") - git(d, "config", "user.email", "t@t") - git(d, "config", "user.name", "t") - write(d, "held.txt", "base version\n") - write(d, "early.txt", "base version\n") - git(d, "add", "-A") - git(d, "commit", "-qm", "base") - base = git(d, "rev-parse", "HEAD") - - # c1 touches early.txt only, so held.txt is still the base blob AT c1. - git(d, "checkout", "-q", "-b", "pr") - write(d, "early.txt", "upstream v1\n") - git(d, "commit", "-qam", "pr c1") - # c2 is the first commit to touch held.txt. - write(d, "held.txt", "upstream v2\n") - git(d, "commit", "-qam", "pr c2") - head = git(d, "rev-parse", "HEAD") - - # The carry took early.txt and holds held.txt at the base version. - git(d, "checkout", "-q", "-b", "carry", base) - write(d, "early.txt", "upstream v1\n") - git(d, "commit", "-qam", "carry") - return d, base, head, git(d, "rev-parse", "HEAD") - - -d3, base3, head3, carry4 = late_touch_fixture() -report3 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") -r5 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry4, "--pr-ref", head3, - "--base", base3, "--report", report3], - cwd=d3, capture_output=True, text=True) -check("runs clean on a late-touch PR", r5.returncode == 0, r5.stderr) -out3 = json.loads(Path(report3).read_text()) -check("a file held at base is not superseded by a PR commit that predates its first change", - "held.txt" in out3["diverged"], json.dumps(out3, indent=1)) -check("no in-range commit before the first change counts as a vintage", - all(e["path"] != "held.txt" for e in out3["superseded"]), json.dumps(out3, indent=1)) -check("and the rebuild advice is withheld", - "Nothing diverges" not in r5.stdout, r5.stdout[-400:]) -check("a file the carry really did take is still superseded", - any(e["path"] == "early.txt" for e in out3["superseded"]), json.dumps(out3, indent=1)) -check("the late-touch run still writes nothing", - git(d3, "status", "--porcelain") == "", git(d3, "status", "--porcelain")) - -# A PR that RENAMES a file while the carry deliberately keeps the old path. -# `git diff --name-only` prints only the new name of a detected rename, so the old path never entered the file list at all: the new path came back SUPERSEDED, nothing diverged, and the summary recommended a rebuild - which deletes the path the carry is holding, unreported. -def rename_fixture(): - d = tempfile.mkdtemp(prefix="cv_") - git(d, "init", "-q", "-b", "main") - git(d, "config", "user.email", "t@t") - git(d, "config", "user.name", "t") - # Long enough that git scores the move as a rename rather than add+delete. - write(d, "old.py", "".join(f"line {i}\n" for i in range(40))) - git(d, "add", "-A") - git(d, "commit", "-qm", "base") - base = git(d, "rev-parse", "HEAD") - - git(d, "checkout", "-q", "-b", "pr") - git(d, "mv", "old.py", "new.py") - git(d, "commit", "-qm", "pr renames it") - head = git(d, "rev-parse", "HEAD") - - # The carry takes the new path AND keeps the old one, on purpose. - git(d, "checkout", "-q", "-b", "carry", base) - write(d, "new.py", "".join(f"line {i}\n" for i in range(40))) - git(d, "add", "-A") - git(d, "commit", "-qm", "carry keeps both") - return d, base, head, git(d, "rev-parse", "HEAD") - - -d4, base4, head4, carry5 = rename_fixture() -report4 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") -r6 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry5, "--pr-ref", head4, - "--base", base4, "--report", report4], - cwd=d4, capture_output=True, text=True) -check("runs clean on a renaming PR", r6.returncode == 0, r6.stderr) -out4 = json.loads(Path(report4).read_text()) -check("the old side of a rename is scanned at all", - "old.py" in out4["diverged"] + out4["absent"] + out4["omitted"] - or any(e["path"] == "old.py" for e in out4["superseded"]), - json.dumps(out4, indent=1)) -check("a retained old path is reported as diverged, not silently dropped", - "old.py" in out4["diverged"], json.dumps(out4, indent=1)) -check("a rename does not license the rebuild advice", - "Nothing diverges" not in r6.stdout, r6.stdout[-400:]) -check("the renaming run still writes nothing", - git(d4, "status", "--porcelain") == "", git(d4, "status", "--porcelain")) - -# A carry that changes a file the PR never touched. -# Everything the PR did touch is superseded, so nothing diverges and nothing is omitted, and the rebuild advice was given anyway - while a rebuild from the PR head drops the carry-only change, which is invisible to a scan of the PR's own file list. -def carry_only_fixture(): - d = tempfile.mkdtemp(prefix="cv_") - git(d, "init", "-q", "-b", "main") - git(d, "config", "user.email", "t@t") - git(d, "config", "user.name", "t") - write(d, "theirs.txt", "base version\n") - write(d, "ours_only.txt", "base version\n") - git(d, "add", "-A") - git(d, "commit", "-qm", "base") - base = git(d, "rev-parse", "HEAD") - - # The PR touches theirs.txt and nothing else. - git(d, "checkout", "-q", "-b", "pr") - write(d, "theirs.txt", "upstream v1\n") - git(d, "commit", "-qam", "pr c1") - head = git(d, "rev-parse", "HEAD") - - # The carry takes the PR's change AND makes one of its own, off the PR. - git(d, "checkout", "-q", "-b", "carry", base) - write(d, "theirs.txt", "upstream v1\n") - write(d, "ours_only.txt", "a fix we carry ourselves\n") - git(d, "commit", "-qam", "carry plus our own fix") - return d, base, head, git(d, "rev-parse", "HEAD") - - -d5, base5, head5, carry6 = carry_only_fixture() -report5 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") -r7 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry6, "--pr-ref", head5, - "--base", base5, "--report", report5], - cwd=d5, capture_output=True, text=True) -check("runs clean on a carry with its own change", r7.returncode == 0, r7.stderr) -out5 = json.loads(Path(report5).read_text()) -check("a file only the carry changed is reported", - out5.get("carry_only") == ["ours_only.txt"], json.dumps(out5, indent=1)) -check("a carry-only change withholds the rebuild advice", - "Nothing diverges" not in r7.stdout, r7.stdout[-400:]) -check("and the PR's own file is still superseded", - any(e["path"] == "theirs.txt" for e in out5["superseded"]), json.dumps(out5, indent=1)) -check("the carry-only run still writes nothing", - git(d5, "status", "--porcelain") == "", git(d5, "status", "--porcelain")) - -# A carry that takes the PR's content but changes the file mode. -# Content is identical, so an oid-only comparison called it superseded and recommended a rebuild, which drops the mode change. -def mode_fixture(): - d = tempfile.mkdtemp(prefix="cv_") - git(d, "init", "-q", "-b", "main") - git(d, "config", "user.email", "t@t") - git(d, "config", "user.name", "t") - write(d, "tool.sh", "#!/bin/sh\necho base\n") - git(d, "add", "-A") - git(d, "commit", "-qm", "base") - base = git(d, "rev-parse", "HEAD") - - git(d, "checkout", "-q", "-b", "pr") - write(d, "tool.sh", "#!/bin/sh\necho upstream v1\n") - git(d, "commit", "-qam", "pr c1") - head = git(d, "rev-parse", "HEAD") - - # The carry takes that content verbatim and makes it executable. - git(d, "checkout", "-q", "-b", "carry", base) - write(d, "tool.sh", "#!/bin/sh\necho upstream v1\n") - git(d, "add", "-A") - git(d, "update-index", "--chmod=+x", "tool.sh") - git(d, "commit", "-qm", "carry makes it executable") - return d, base, head, git(d, "rev-parse", "HEAD") - - -d6, base6, head6, carry7 = mode_fixture() -report6 = str(Path(tempfile.mkdtemp(prefix="cvr_")) / "report.json") -r8 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry7, "--pr-ref", head6, - "--base", base6, "--report", report6], - cwd=d6, capture_output=True, text=True) -check("runs clean on a mode-only carry change", r8.returncode == 0, r8.stderr) -out6 = json.loads(Path(report6).read_text()) -check("a mode-only difference is not a vintage match", - "tool.sh" in out6["diverged"], json.dumps(out6, indent=1)) -check("a mode-only difference withholds the rebuild advice", - "Nothing diverges" not in r8.stdout, r8.stdout[-400:]) - -# A carry holding a file at an older commit of the PR. -# Nothing diverges, so the advice used to be an unqualified "rebuilding is equivalent" - but a rebuild moves that file to head, and the carry may be holding the older vintage on purpose, which is a decision this script cannot see. -def older_vintage_fixture(): - d = tempfile.mkdtemp(prefix="cv_") - git(d, "init", "-q", "-b", "main") - git(d, "config", "user.email", "t@t") - git(d, "config", "user.name", "t") - write(d, "f.txt", "base\n") - git(d, "add", "-A") - git(d, "commit", "-qm", "base") - base = git(d, "rev-parse", "HEAD") - - git(d, "checkout", "-q", "-b", "pr") - write(d, "f.txt", "upstream v1\n") - git(d, "commit", "-qam", "pr c1") - write(d, "f.txt", "upstream v2\n") - git(d, "commit", "-qam", "pr c2") - head = git(d, "rev-parse", "HEAD") - - # The carry stopped at v1. - git(d, "checkout", "-q", "-b", "carry", base) - write(d, "f.txt", "upstream v1\n") - git(d, "commit", "-qam", "carry holds v1") - return d, base, head, git(d, "rev-parse", "HEAD") - - -d7, base7, head7, carry8 = older_vintage_fixture() -r9 = subprocess.run([sys.executable, str(SCRIPT), "--carry", carry8, "--pr-ref", head7, - "--base", base7], cwd=d7, capture_output=True, text=True) -check("runs clean on an older-vintage carry", r9.returncode == 0, r9.stderr) -check("an older vintage is still recognised as superseded", - "SUPERSEDED" in r9.stdout, r9.stdout[-400:]) -check("an older vintage withholds the unqualified rebuild advice", - "Nothing diverges" not in r9.stdout, r9.stdout[-400:]) -check("and says a rebuild would move it to head", - "OLDER vintage" in r9.stdout, r9.stdout[-400:]) - -print() -print(f"{len(FAILS)} failure(s)" if FAILS else "all carry_vintage tests passed") -sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_feature_matrix.py b/scripts/unsloth/test_feature_matrix.py deleted file mode 100644 index ee050f7f761e..000000000000 --- a/scripts/unsloth/test_feature_matrix.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Tests for feature_matrix.py. Run: python3 scripts/unsloth/test_feature_matrix.py - -The thing worth testing here is not that a passing probe passes. It is that a -probe which exits 0 having proved NOTHING is reported as a failure, because both -real harnesses do exactly that: - - test-llama-archs -a <excluded arch> prints SKIP, exits 0 - test-backend-ops test -o <typo> matches nothing, exits 0 - -So the fakes below are the real output shapes, verbatim, and the assertions are -about what the script refuses to call a pass. -""" -import json -import os -import stat -import subprocess -import sys -import tempfile -from pathlib import Path - -SCRIPT = Path(__file__).resolve().parent / "feature_matrix.py" -FAILS = [] - - -def check(name, cond, extra=""): - print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) - if not cond: - FAILS.append(name) - - -def fake(dirp: Path, name: str, stdout: str, rc: int = 0): - p = dirp / name - p.write_text("#!/bin/sh\ncat <<'XEOF'\n" + stdout + "\nXEOF\nexit " + str(rc) + "\n") - p.chmod(p.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) - - -# Real output shapes, copied from actual runs. -ARCHS_OK = """main: using seed 1234 -| Model arch.| Device|Config| NMSE vs. CPU|Roundtrip| -|----------------|-------------------------------|------|---------------|---------| -| inkling| NVIDIA B200| MoE| OK (1.82e-11)| SKIP| -| inkling|Intel(R) Xeon(R) Platinum 8559C| MoE| OK (0.00e+00)| SKIP|""" -ARCHS_ALL_SKIP = """main: using seed 1234 -| Model arch.| Device|Config| NMSE vs. CPU|Roundtrip| -|----------------|-------------------------------|------|---------------|---------| -| inkling| NVIDIA B200| Dense|SKIP | SKIP|""" -ARCHS_ABSENT = """main: using seed 1234 -| Model arch.| Device|Config| NMSE vs. CPU|Roundtrip| -|----------------|-------------------------------|------|---------------|---------|""" -OPS_OK = """ FLASH_ATTN_EXT_BANDED(hsk=64): OK - 13/13 tests passed - Backend CUDA0: OK""" -OPS_NOTHING = """Backend 1/2: CUDA0 -Backend 2/2: CPU - Skipping CPU backend -2/2 backends passed -OK""" -MTMD_OK = """test_projector_registry (185 assertion(s)) [PASS] - -tests : 1 -assertions : 185 -failures : 0""" -MTMD_NOTHING = """tests : 0 -assertions : 0 -failures : 0""" - - -def build(archs=ARCHS_OK, ops=OPS_OK, mtmd=MTMD_OK, checks=None): - d = Path(tempfile.mkdtemp(prefix="fm_")) - (d / "bin").mkdir() - fake(d / "bin", "test-llama-archs", archs) - fake(d / "bin", "test-backend-ops", ops) - fake(d / "bin", "test-mtmd-impl", mtmd) - manifest = d / "feature-checks.json" - manifest.write_text(json.dumps({ - "schema": 1, - "features": {"inkling": {"owner": "unslothai#172", "checks": checks or [ - {"kind": "arch", "arch": "inkling"}, - {"kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED"}, - ]}}, - "unchecked": {"unslothai#95": "no feature surface"}, - })) - return d, manifest - - -def run(d, manifest, *extra): - rep = d / "r.json" - p = subprocess.run([sys.executable, str(SCRIPT), "--build-dir", str(d), - "--feature-checks", str(manifest), "--report", str(rep), *extra], - capture_output=True, text=True) - return p.returncode, (json.loads(rep.read_text()) if rep.exists() else {}), p.stdout + p.stderr - - -# --- 1. everything genuinely ran ------------------------------------------ -d, m = build() -rc, rep, out = run(d, m, "--gpu") -check("a real pass passes", rc == 0 and rep["ok"], out) -check("the evidence is recorded, not just the verdict", - "1.82e-11" not in out and "2/2 device rows" in out, out) - -# --- 2. the arch harness skipped the arch and exited 0 --------------------- -d, m = build(archs=ARCHS_ALL_SKIP) -rc, rep, out = run(d, m, "--gpu") -check("an all-SKIP arch run is a failure", rc == 1, out) -check("and says nothing was decoded", "nothing was decoded" in out, out) - -# --- 3. the arch is not in the harness at all ----------------------------- -d, m = build(archs=ARCHS_ABSENT) -rc, rep, out = run(d, m, "--gpu") -check("an arch with no row at all is a failure", rc == 1, out) -check("and says it is not in the harness", "not in the harness" in out, out) - -# --- 4. the op filter matched nothing ------------------------------------- -d, m = build(ops=OPS_NOTHING) -rc, rep, out = run(d, m, "--gpu") -check("an op filter that matched nothing is a failure", rc == 1, out) -check("and says the filter matched nothing", "matched nothing" in out, out) - -# --- 5. the op ran and failed --------------------------------------------- -d, m = build(ops=" 11/13 tests passed\n Backend CUDA0: FAIL") -rc, rep, out = run(d, m, "--gpu") -check("a failing op is a failure", rc == 1 and "11/13" in out, out) - -# --- 6. no GPU: op probes are deferred, not passed and not failed --------- -d, m = build() -rc, rep, out = run(d, m) -check("without a GPU the op probe is deferred", rc == 0 and rep["deferred"] == 1, out) -check("deferral is stated in the summary", "need a GPU" in out, out) -check("deferral is not counted as evidence", - len(rep["features"][0]["results"]) == 1, rep) - -# --- 7. a feature with nothing but GPU checks reads as unproven, not ok --- -d, m = build(checks=[{"kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED"}]) -rc, rep, out = run(d, m) -check("a wholly deferred feature does not print ok", - rc == 0 and "nothing provable without a GPU" in out and "\nok inkling" not in out, out) - -# --- 8. the mtmd probe ran no assertions ---------------------------------- -d, m = build(mtmd=MTMD_NOTHING, checks=[{"kind": "mtmd", "projector": "kimik3"}]) -rc, rep, out = run(d, m, "--gpu") -check("an mtmd run with zero assertions is a failure", rc == 1, out) - -# --- 9. unchecked pins are reported, not hidden --------------------------- -d, m = build() -rc, rep, out = run(d, m, "--gpu") -check("knowingly unchecked pins are printed", "unslothai#95 has no runtime check" in out, out) - -print() -print(f"{len(FAILS)} failure(s)" + (": " + ", ".join(FAILS) if FAILS else "")) -sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_merge_checks.py b/scripts/unsloth/test_merge_checks.py deleted file mode 100644 index c0b9be1ddd88..000000000000 --- a/scripts/unsloth/test_merge_checks.py +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Tests for merge_checks.py. Run: python3 scripts/unsloth/test_merge_checks.py - -The positive cases are reduced from the two real 08-27 mistakes. -The negative cases are the shapes that must NOT fire, because a check that blocks a good merge costs a release just as surely as a bad merge does. -""" -import subprocess -import sys -import tempfile -from pathlib import Path - -SCRIPT = Path(__file__).resolve().parent / "merge_checks.py" -FAILS = [] - - -def check(name, cond, extra=""): - print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) - if not cond: - FAILS.append(name) - - -def run(py=None, cpp=None): - d = Path(tempfile.mkdtemp(prefix="mc_")) - (d / "gguf-py" / "gguf").mkdir(parents=True) - (d / "src" / "models").mkdir(parents=True) - (d / "gguf-py" / "gguf" / "t.py").write_text(py or "x = {}\n") - (d / "src" / "t.cpp").write_text(cpp or "int main() { return 0; }\n") - r = subprocess.run([sys.executable, str(SCRIPT), "--root", str(d)], - capture_output=True, text=True) - return r.returncode, r.stdout + r.stderr - - -DUP_KEY = """ -MAP = { - ARCH.QWEN4EXP: {"a": 1}, - ARCH.GLM5NEXT: {"b": 2}, - ARCH.GLM5NEXT: {"b": 2}, -} -""" -OK_KEYS = """ -MAP = { - ARCH.QWEN4EXP: {"a": 1}, - ARCH.GLM5NEXT: {"b": 2}, -} -""" -DEAD_ARM = """ -void f() { - if (arch == LLM_ARCH_FALCON_H1) { - a(); - } else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { - b(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" -LIVE_ARM = """ -void f() { - if (arch == LLM_ARCH_GLM5NEXT && hparams.indexer_head_size > 0) { - a(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - b(); - } -} -""" -DISTINCT_ARMS = """ -void f() { - if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35) { - a(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - b(); - } -} -""" - -rc, out = run(py=DUP_KEY) -check("catches a duplicate dict key", rc == 1 and "GLM5NEXT" in out, out) -rc, out = run(py=OK_KEYS) -check("clean on distinct dict keys", rc == 0, out) - -rc, out = run(cpp=DEAD_ARM) -check("catches an unreachable arch arm", rc == 1 and "unreachable" in out, out) -rc, out = run(cpp=LIVE_ARM) -check("does NOT fire when the earlier arm has &&", rc == 0, out) -rc, out = run(cpp=DISTINCT_ARMS) -check("does NOT fire on distinct arches", rc == 0, out) - -rc, out = run() -check("clean tree exits 0", rc == 0, out) - -# A nested `if` inside an arm must not end the enclosing chain. -# Tracking chains by indentation resets on the nested arm and analyses the outer `else if` as a fresh chain, so the dedicated GLM5NEXT arm below a shared fallthrough - the exact 08-27 mistake - stops being reported. -NESTED_DEAD_ARM = """ -void f() { - if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { - if (hparams.indexer_head_size > 0) { - a(); - } else if (hparams.n_expert > 0) { - b(); - } - } else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" -# Two unrelated chains at the same indentation. -# The second one's opener is a multiline condition, which the regex deliberately skips, so an indentation key appends the reachable GLM5NEXT arm to the FIRST chain and calls it dead. -# Nothing here is unreachable, and firing would block a release on good code. -SEPARATE_CHAINS = """ -void f() { - if (arch == LLM_ARCH_GLM5NEXT) { - a(); - } - unrelated(); - if (hparams.moe_every_n_layers > 0 && - il % hparams.moe_every_n_layers == 1) { - b(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" -# A brace inside a string literal or a comment is not a brace. -# Miscounting one shifts the depth for the rest of the file, which would silence every chain after it. -BRACES_IN_LITERALS = """ -void f() { - const char * tmpl = "{% if x %}{{ y }}{% endif %}"; - // a stray } in a comment { - if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { - a(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" - -rc, out = run(cpp=NESTED_DEAD_ARM) -check("catches a dead arm across a nested if", rc == 1 and "unreachable" in out, out) -rc, out = run(cpp=SEPARATE_CHAINS) -check("does NOT glue two chains at the same indentation", rc == 0, out) -rc, out = run(cpp=BRACES_IN_LITERALS) -check("still analyses a chain after braces in a string or comment", - rc == 1 and "unreachable" in out, out) - -# llama.cpp puts `else if` on its own line as often as not, src/llama-quant.cpp among them. -# Ending the chain on the brace line loses the arm that follows, so the duplicate arch started a fresh chain with nothing taken and passed. -NEXT_LINE_ELSE = """ -void f() { - if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { - a(); - } - else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" -# Same, with a blank line in between: still one chain. -NEXT_LINE_ELSE_BLANK = """ -void f() { - if (arch == LLM_ARCH_GLM5NEXT) { - a(); - } - - else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" -# The other direction, which deferring the close could break: a chain that really has ended, followed by an unrelated chain at the same depth. -# Joining them reports a reachable arm as dead and blocks a release on good code. -CLOSED_THEN_NEW = """ -void f() { - if (arch == LLM_ARCH_GLM5NEXT) { - a(); - } else { - b(); - } - g(); - if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } else if (arch == LLM_ARCH_QWEN3NEXT) { - d(); - } -} -""" - -# COND anchors on the `{` that ends the line, so a trailing comment after the brace stopped the arm matching at all and it left the chain silently. -TRAILING_COMMENT = """ -void f() { - if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { // shared setup - a(); - } else if (arch == LLM_ARCH_GLM5NEXT) { /* dedicated */ - c(); - } -} -""" -# The condition text must survive the comment stripping, since a mangled one would fail to parse as a pure disjunction and quietly stop being checked. -COMMENTED_OUT_ARM = """ -void f() { - if (arch == LLM_ARCH_GLM5NEXT) { - a(); - //} else if (arch == LLM_ARCH_GLM5NEXT) { - } else if (arch == LLM_ARCH_QWEN3NEXT) { - c(); - } -} -""" - -# A raw string ends only at its own delimiter, so it can hold a quote and a brace that the ordinary string regex misreads. -# The stray `}` closed the chain early and the duplicate arm after it passed as clean. -RAW_STRING = """ -void f() { - if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { - const char * s = R"foo("})foo"; - a(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" -# A raw string is blanked before comments, so the `//` inside one is text, not the start of a comment, and the brace after it still counts. -RAW_WITH_SLASHES = """ -void f() { - const char * u = R"(https://example.com/{x})"; - if (arch == LLM_ARCH_GLM5NEXT) { - a(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" - -# The other ordering. -# Blanking raw strings before comments let an `R"(` written inside a comment open a literal that ran to the next `)"`, swallowing the duplicate arm in between. -# Neither order fixes this, which is why the scan is positional: whichever construct starts first wins, and here that is the comment. -RAW_INSIDE_COMMENT = """ -void f() { - if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { - // see R"( for the delimiter rules - a(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - const char * s = R"(text)"; - c(); - } -} -""" -# An R glued to an identifier is part of it, not a raw-string prefix. -# Reading CHAR"( as a literal would blank the rest of the chain. -IDENT_ENDING_IN_R = """ -void f() { - if (arch == LLM_ARCH_GLM5NEXT) { - int n = FOOR; - a(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" - -# An unconditional arm takes the arch outright, so a later arm testing the same arch with an extra condition can never run. -# It is not a pure disjunction, so it was skipped and the dead arm passed. -CONJUNCTION_AFTER_PLAIN = """ -void f() { - if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_GLM5NEXT) { - a(); - } else if (arch == LLM_ARCH_GLM5NEXT && hparams.n_expert > 0) { - c(); - } -} -""" -# The reverse must stay clean: a CONDITIONAL arm does not consume the arch, so a later arm testing it is genuinely reachable. -PLAIN_AFTER_CONJUNCTION = """ -void f() { - if (arch == LLM_ARCH_GLM5NEXT && hparams.n_expert > 0) { - a(); - } else if (arch == LLM_ARCH_GLM5NEXT) { - c(); - } -} -""" -# A negated test names the arch but does not require it, so it is reachable. -NEGATED_ARM = """ -void f() { - if (arch == LLM_ARCH_GLM5NEXT) { - a(); - } else if (arch != LLM_ARCH_GLM5NEXT && n > 0) { - c(); - } -} -""" -# Only one alternative of a disjunction was taken, so the arm can still run. -PARTIAL_DISJUNCTION = """ -void f() { - if (arch == LLM_ARCH_GLM5NEXT) { - a(); - } else if (arch == LLM_ARCH_GLM5NEXT || arch == LLM_ARCH_QWEN3NEXT) { - c(); - } -} -""" - -# A key that is a CALL is a different object each evaluation, so repeating it is -# two entries, not one. ast.unparse renders both the same, and a finding here -# blocks the nightly, so an unstable key must not be compared by text at all. -DYNAMIC_KEY = """ -MAP = {fresh(): 1, fresh(): 2} -""" -# A walrus rebinds between elements, so the same name is not the same value. -WALRUS_KEY = """ -MAP = {(n := 1): "a", (n := 2): "b"} -""" -# Stable keys that are not enum attributes still have to be caught. -DUP_LITERAL_KEY = """ -MAP = {"a": 1, "b": 2, "a": 3} -""" - -rc, out = run(py=DYNAMIC_KEY) -check("a repeated call key is not reported as a duplicate", rc == 0, out) -rc, out = run(py=WALRUS_KEY) -check("a walrus key is not reported as a duplicate", rc == 0, out) -rc, out = run(py=DUP_LITERAL_KEY) -check("a duplicate literal key is still caught", - rc == 1 and "defined 2 times" in out, out) - -rc, out = run(cpp=CONJUNCTION_AFTER_PLAIN) -check("catches a conditioned arm after an unconditional match of the same arch", - rc == 1 and "unreachable" in out, out) -rc, out = run(cpp=PLAIN_AFTER_CONJUNCTION) -check("a conditional arm does not consume the arch for what follows", - rc == 0, out) -rc, out = run(cpp=NEGATED_ARM) -check("a negated arch test is not read as requiring that arch", rc == 0, out) -rc, out = run(cpp=PARTIAL_DISJUNCTION) -check("a disjunction with one untaken alternative stays reachable", rc == 0, out) - -rc, out = run(cpp=RAW_INSIDE_COMMENT) -check("an R\"( inside a comment does not open a raw string", - rc == 1 and "unreachable" in out, out) -rc, out = run(cpp=IDENT_ENDING_IN_R) -check("an identifier ending in R is not a raw-string prefix", - rc == 1 and "unreachable" in out, out) - -rc, out = run(cpp=RAW_STRING) -check("a brace inside a raw string does not close the chain", - rc == 1 and "unreachable" in out, out) -rc, out = run(cpp=RAW_WITH_SLASHES) -check("a raw string holding // is not treated as a comment", - rc == 1 and "unreachable" in out, out) - -rc, out = run(cpp=TRAILING_COMMENT) -check("catches a dead arm despite a comment after the brace", - rc == 1 and "unreachable" in out, out) -check("and reports the arm that is actually dead", ":5:" in out, out) -rc, out = run(cpp=COMMENTED_OUT_ARM) -check("a commented-out arm is not treated as a live one", rc == 0, out) - -rc, out = run(cpp=NEXT_LINE_ELSE) -check("catches a dead arm when else if starts on the next line", - rc == 1 and "unreachable" in out, out) -rc, out = run(cpp=NEXT_LINE_ELSE_BLANK) -check("a blank line between } and else does not end the chain", - rc == 1 and "unreachable" in out, out) -rc, out = run(cpp=CLOSED_THEN_NEW) -check("a genuinely closed chain does not absorb the next one", rc == 0, out) - -print() -print(f"{len(FAILS)} failure(s)" if FAILS else "all merge_checks tests passed") -sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_pin_contract.py b/scripts/unsloth/test_pin_contract.py deleted file mode 100644 index 3c7b58d62d54..000000000000 --- a/scripts/unsloth/test_pin_contract.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Tests for pin_contract.py. Run: python3 scripts/unsloth/test_pin_contract.py - -Every case builds a real repository with a real base tag, a real pin branch and -a real merge, then damages the merged tree the way a bad resolution damages it. -A hand-written fixture would only prove the checker reads its own output format. -""" -import json -import subprocess -import sys -import tempfile -from pathlib import Path - -SCRIPT = Path(__file__).resolve().parent / "pin_contract.py" -FAILS = [] - - -def check(name, cond, extra=""): - print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) - if not cond: - FAILS.append(name) - - -def git(repo, *args): - return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], - cwd=repo, capture_output=True, text=True) - - -ARCH_H_BASE = """\ -enum llm_arch { - LLM_ARCH_LLAMA, - LLM_ARCH_UNKNOWN, -}; -""" -MODEL_CPP_BASE = """\ -void build_model(llm_arch arch) { - switch (arch) { - case LLM_ARCH_LLAMA: - build_llama(); - break; - } -} -""" - - -def make_repo(): - """A base tag `b1` plus a pin branch adding one architecture, merged.""" - d = Path(tempfile.mkdtemp(prefix="pc_")) - git(d, "init", "-q", "-b", "main") - (d / "src").mkdir() - (d / "src" / "llama-arch.h").write_text(ARCH_H_BASE) - (d / "src" / "llama-model.cpp").write_text(MODEL_CPP_BASE) - git(d, "add", "-A"); git(d, "commit", "-qm", "base") - git(d, "tag", "b1") - - git(d, "checkout", "-qb", "pin") - (d / "src" / "llama-arch.h").write_text( - ARCH_H_BASE.replace(" LLM_ARCH_UNKNOWN,", - " LLM_ARCH_INKLING,\n LLM_ARCH_UNKNOWN,")) - (d / "src" / "llama-model.cpp").write_text( - MODEL_CPP_BASE.replace(" }\n}", - " case LLM_ARCH_INKLING:\n" - " build_inkling_with_banded_bias();\n" - " break;\n }\n}")) - (d / "src" / "inkling.cpp").write_text( - "void build_inkling_with_banded_bias() { do_the_banded_thing(); }\n") - git(d, "add", "-A"); git(d, "commit", "-qm", "add inkling") - sha = git(d, "rev-parse", "HEAD").stdout.strip() - - git(d, "checkout", "-q", "main") - git(d, "merge", "-q", "--no-ff", "--no-edit", "-m", "merge pin", "pin") - - # Outside the work tree on purpose: a test that commits after this would - # otherwise sweep the pin file into the pin's own diff. - pr_set = Path(tempfile.mkdtemp(prefix="pcset_")) / "pr-set.json" - pr_set.write_text(json.dumps({"prs": [ - f"https://github.com/unslothai/llama.cpp/pull/1/commits/{sha}"]})) - return d, pr_set, sha - - -def run(repo, pr_set, *extra): - rep = repo / "r.json" - p = subprocess.run([sys.executable, str(SCRIPT), "--root", str(repo), - "--pr-set", str(pr_set), "--base", "b1", - "--report", str(rep), *extra], - capture_output=True, text=True) - return p.returncode, (json.loads(rep.read_text()) if rep.exists() else {}), p.stderr - - -# --- 1. an intact merge passes ------------------------------------------- -repo, pr_set, sha = make_repo() -rc, rep, err = run(repo, pr_set) -check("intact merge passes", rc == 0 and rep["ok"], err) -check("intact merge finds the new arch", - "LLM_ARCH_INKLING" in json.dumps(rep["pins"][0]["symbols"]), rep) -check("intact merge reports no notices", rep["notices"] == [], rep) - -# --- 2. the arm is dropped from ONE file: a tree-wide grep would pass ------ -# The real shape: LLM_ARCH_INKLING survives in the enum and the dispatch arm -# that makes it do anything is gone. -repo, pr_set, sha = make_repo() -p = repo / "src" / "llama-model.cpp" -p.write_text(MODEL_CPP_BASE) -rc, rep, err = run(repo, pr_set) -check("a dropped dispatch arm fails", rc == 1 and not rep["ok"], err) -check("the failure names the file, not just the symbol", - any("llama-model.cpp" in x for x in rep["pins"][0]["problems"]), rep) -check("the enum copy of the symbol does not rescue it", - "LLM_ARCH_INKLING" in (repo / "src" / "llama-arch.h").read_text()) - -# --- 3. a whole added file goes missing ---------------------------------- -repo, pr_set, sha = make_repo() -(repo / "src" / "inkling.cpp").unlink() -rc, rep, err = run(repo, pr_set) -check("a missing added file fails", rc == 1, err) -check("the failure names the file", - any("inkling.cpp" in x for x in rep["pins"][0]["problems"]), rep) - -# --- 4. a hunk is eaten without touching a symbol ------------------------- -repo, pr_set, sha = make_repo() -(repo / "src" / "inkling.cpp").write_text( - "void build_inkling_with_banded_bias() { }\n") # body gone, name kept -rc, rep, err = run(repo, pr_set) -check("an eaten body fails on line survival", rc == 1, err) -check("line survival names what went missing", - any("do_the_banded_thing" in x for x in rep["pins"][0]["problems"]), rep) - -# --- 5. redundancy: the base already has everything the pin adds ---------- -# Built the way it happens for real: upstream lands the same work, so the base -# tag has it and the pin is not an ancestor of anything. -d = Path(tempfile.mkdtemp(prefix="pc_")) -git(d, "init", "-q", "-b", "main") -(d / "src").mkdir() -(d / "src" / "f.cpp").write_text("int a() { return 1; }\n") -git(d, "add", "-A"); git(d, "commit", "-qm", "root") -git(d, "checkout", "-qb", "pin") -(d / "src" / "f.cpp").write_text( - "int a() { return 1; }\nint the_new_helper() { return 42; }\n") -git(d, "add", "-A"); git(d, "commit", "-qm", "pin work") -sha5 = git(d, "rev-parse", "HEAD").stdout.strip() -git(d, "checkout", "-q", "main") -(d / "src" / "f.cpp").write_text( # upstream squashed the same work - "int a() { return 1; }\nint the_new_helper() { return 42; }\n") -git(d, "add", "-A"); git(d, "commit", "-qm", "upstream squash of the same change") -git(d, "tag", "b1") -ps5 = Path(tempfile.mkdtemp(prefix="pcset_")) / "pr-set.json" -ps5.write_text(json.dumps({"prs": [ - f"https://github.com/unslothai/llama.cpp/pull/1/commits/{sha5}"]})) -rc, rep, err = run(d, ps5) -check("a pin the base already carries is reported", rep["notices"], rep) -check("redundancy says to delete the entry", - "deleted from pr-set.json" in " ".join(rep["notices"]), rep) -check("redundancy is NOT fatal", rc == 0, err) - -# --- 6. --emit checks nothing --------------------------------------------- -repo, pr_set, sha = make_repo() -(repo / "src" / "inkling.cpp").unlink() -rc, rep, err = run(repo, pr_set, "--emit") -check("--emit does not check", rc == 0 and rep["ok"], err) -check("--emit still derives the contract", - rep["pins"][0]["added_files"] == ["src/inkling.cpp"], rep) - -# --- 7. a comment is not a contract --------------------------------------- -# unslothai#70 has a comment naming GGML_OP_SSM_SCAN to say it does NOT use it. -# Holding comment wording would fail the moment upstream rewords it. -repo, pr_set, sha = make_repo() -git(repo, "checkout", "-q", "pin") -(repo / "src" / "note.cpp").write_text( - "// unlike LLM_ARCH_MISTRAL this one does its own thing\nint g() { return 0; }\n") -git(repo, "add", "-A"); git(repo, "commit", "-qm", "comment") -sha7 = git(repo, "rev-parse", "HEAD").stdout.strip() -git(repo, "checkout", "-q", "main") -git(repo, "merge", "-q", "--no-ff", "--no-edit", "-m", "m2", "pin") -(repo / "src" / "note.cpp").write_text( # comment reworded, code kept - "// this one does its own thing\nint g() { return 0; }\n") -pr_set.write_text(json.dumps({"prs": [ - f"https://github.com/unslothai/llama.cpp/pull/1/commits/{sha7}"]})) -rc, rep, err = run(repo, pr_set) -check("a reworded comment does not fail the pin", rc == 0, err) -check("no symbol was harvested from the comment", - "LLM_ARCH_MISTRAL" not in json.dumps(rep["pins"][0]["symbols"]), rep) - -print() -print(f"{len(FAILS)} failure(s)" + (": " + ", ".join(FAILS) if FAILS else "")) -sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_pin_merge.py b/scripts/unsloth/test_pin_merge.py deleted file mode 100644 index 40b227da5ce6..000000000000 --- a/scripts/unsloth/test_pin_merge.py +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Tests for pin_merge.py. Run: python3 scripts/unsloth/test_pin_merge.py - -The first case is the real 08-27 collision: master had moved qwen4exp to 950f135b28 while the GLM-5-Next branch was replacing pin 118 with 125. -It was resolved by hand at the time; this asserts the script reproduces that answer. -""" -import json -import subprocess -import sys -import tempfile -from pathlib import Path - -SCRIPT = Path(__file__).resolve().parent / "pin_merge.py" -FAILS = [] - -U = "https://github.com/unslothai/llama.cpp/pull" -BASE_PINS = [ - f"{U}/107/commits/74acc40c37ae2eb36031981feda392b793944f72", - f"{U}/108/commits/27278df7000ade4a638d044202dbe82975421df6", - f"{U}/70/commits/edfd4c1a3b7a653303a85257ddac2a1f3ce39a2f", - f"{U}/91/commits/c86ed269986f2dced6325c5c58bda966a2e2ead1", - f"{U}/95/commits/3db8cb5b2e9bf291057b9f19960e8601a162da81", - f"{U}/114/commits/c4ddc4805dbc12727897b354237bfd9225212b06", - f"{U}/118/commits/3766b41229c20249fd4d83d7ba297499d50e9b80", -] - - -def check(name, cond, extra=""): - print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) - if not cond: - FAILS.append(name) - - -def run(base, ours, theirs): - d = Path(tempfile.mkdtemp(prefix="pm_")) - paths = [] - for name, pins in (("base", base), ("ours", ours), ("theirs", theirs)): - p = d / f"{name}.json" - p.write_text(json.dumps({"prs": pins}, indent=2)) - paths.append(str(p)) - r = subprocess.run([sys.executable, str(SCRIPT), *paths, "--stdout"], - capture_output=True, text=True) - pins = json.loads(r.stdout)["prs"] if r.returncode == 0 else None - return r.returncode, pins, r.stderr.strip() - - -def sub(pins, i, sha): - out = list(pins) - out[i] = out[i].rsplit("/", 1)[0] + "/" + sha - return out - - -def replace(pins, i, url): - out = list(pins) - out[i] = url - return out - - -# 1. the real 08-27 collision -ours = replace(BASE_PINS, 6, f"{U}/125/commits/f48b99e1fd04628da2a3d4ea5acc335d9ea67f7a") -theirs = sub(BASE_PINS, 5, "950f135b28789057721a65d76de98fbbcd2f7dd6") -rc, pins, err = run(BASE_PINS, ours, theirs) -check("real 08-27 collision resolves", rc == 0, err) -if pins: - check("takes theirs for the pin only theirs moved", pins[5] == theirs[5], pins[5]) - check("takes ours for the pin only ours moved", pins[6] == ours[6], pins[6]) - check("leaves untouched pins alone", pins[:5] == BASE_PINS[:5]) - check("preserves pin order", [p.split("/pull/")[1].split("/")[0] for p in pins] - == ["107", "108", "70", "91", "95", "114", "125"]) - -# 2. both sides repin the same entry differently -rc, _, err = run(BASE_PINS, sub(BASE_PINS, 5, "a" * 40), sub(BASE_PINS, 5, "b" * 40)) -check("refuses a genuine two-sided repin", rc == 1, err) -check("says which pin was ambiguous", "pin 5" in err, err) - -# 3. a pin added on one side -rc, _, err = run(BASE_PINS, BASE_PINS + [f"{U}/999/commits/{'c' * 40}"], BASE_PINS) -check("refuses an added pin", rc == 1, err) - -# 4. a pin removed on one side -rc, _, err = run(BASE_PINS, BASE_PINS[:-1], BASE_PINS) -check("refuses a removed pin", rc == 1, err) - -# 5. both sides make the identical repin -same = sub(BASE_PINS, 5, "d" * 40) -rc, pins, err = run(BASE_PINS, same, same) -check("accepts an identical repin on both sides", rc == 0 and pins == same, err) - -# 6. neither side changed anything -rc, pins, err = run(BASE_PINS, BASE_PINS, BASE_PINS) -check("no-op merge is a no-op", rc == 0 and pins == BASE_PINS, err) - -# 7. object-form entries keep their other fields -objs = [{"url": u, "required": False} for u in BASE_PINS] -o = json.loads(json.dumps(objs)); o[5]["url"] = sub(BASE_PINS, 5, "e" * 40)[5] -d = Path(tempfile.mkdtemp(prefix="pm_")) -for name, pins_ in (("base", objs), ("ours", o), ("theirs", objs)): - (d / f"{name}.json").write_text(json.dumps({"prs": pins_}, indent=2)) -r = subprocess.run([sys.executable, str(SCRIPT), str(d / "base.json"), str(d / "ours.json"), - str(d / "theirs.json"), "--stdout"], capture_output=True, text=True) -ok = r.returncode == 0 and all(e.get("required") is False for e in json.loads(r.stdout)["prs"]) -check("object-form entries keep their other fields", ok, r.stdout[:200] + r.stderr) - - -def run_objs(base, ours, theirs): - d = Path(tempfile.mkdtemp(prefix="pm_")) - paths = [] - for name, entries in (("base", base), ("ours", ours), ("theirs", theirs)): - p = d / f"{name}.json" - p.write_text(json.dumps({"prs": entries}, indent=2)) - paths.append(str(p)) - r = subprocess.run([sys.executable, str(SCRIPT), *paths, "--stdout"], - capture_output=True, text=True) - return r.returncode, (json.loads(r.stdout)["prs"] if r.returncode == 0 else None), r.stderr.strip() - - -# 8. theirs flips `required` on one entry while ours repins a DIFFERENT one. -# Comparing only urls makes theirs' flip invisible, and the result is rebuilt from ours, so the flip is silently dropped by a merge that reports success. -objs = [{"url": u, "required": True} for u in BASE_PINS] -o = json.loads(json.dumps(objs)); o[5]["url"] = sub(BASE_PINS, 5, "e" * 40)[5] -t = json.loads(json.dumps(objs)); t[1]["required"] = False -rc, pins, err = run_objs(objs, o, t) -check("keeps theirs' non-url field change on an entry ours did not touch", - rc == 0 and pins is not None and pins[1]["required"] is False, err or json.dumps(pins)) -check("keeps ours' repin alongside theirs' field change", - rc == 0 and pins is not None and pins[5]["url"] == o[5]["url"], err) - -# 9. both sides touch the SAME entry, but different fields: still mergeable. -o = json.loads(json.dumps(objs)); o[3]["url"] = sub(BASE_PINS, 3, "f" * 40)[3] -t = json.loads(json.dumps(objs)); t[3]["required"] = False -rc, pins, err = run_objs(objs, o, t) -check("merges a repin and a field change on the same entry", - rc == 0 and pins is not None - and pins[3]["url"] == o[3]["url"] and pins[3]["required"] is False, err) - -# 10. both sides set the same field to different values: still refused. -o = json.loads(json.dumps(objs)); o[2]["required"] = False -t = json.loads(json.dumps(objs)); t[2]["required"] = "maybe" -rc, _, err = run_objs(objs, o, t) -check("refuses a two-sided change to the same field", rc == 1, err) - -def run_docs(base, ours, theirs): - """Like run(), but the caller supplies the whole document, not just pins.""" - d = Path(tempfile.mkdtemp(prefix="pm_")) - paths = [] - for name, doc in (("base", base), ("ours", ours), ("theirs", theirs)): - p = d / f"{name}.json" - p.write_text(json.dumps(doc, indent=2)) - paths.append(str(p)) - r = subprocess.run([sys.executable, str(SCRIPT), *paths, "--stdout"], - capture_output=True, text=True) - return r.returncode, (json.loads(r.stdout) if r.returncode == 0 else None), r.stderr.strip() - - -# 12. theirs edits a TOP-LEVEL field while ours repins an entry. -# Rebuilding the document from ours drops theirs' edit and still exits 0, and because a merge driver replaces git's text merge outright, nothing else ever sees the loss. -doc = {"_doc": ["old doc line"], "prs": list(BASE_PINS)} -o = json.loads(json.dumps(doc)); o["prs"] = sub(BASE_PINS, 5, "a" * 40) -t = json.loads(json.dumps(doc)); t["_doc"] = ["old doc line", "prune closed pins"] -rc, out, err = run_docs(doc, o, t) -check("keeps theirs' top-level field change alongside ours' repin", - rc == 0 and out is not None and out["_doc"] == t["_doc"], err or json.dumps(out)) -check("still takes ours' repin when theirs edited the document", - rc == 0 and out is not None and out["prs"] == o["prs"], err) -check("keeps .prs in its original key position", - rc == 0 and out is not None and list(out) == ["_doc", "prs"], json.dumps(list(out or {}))) - -# 13. theirs ADDS a top-level field ours has never seen: it has to survive. -t = json.loads(json.dumps(doc)); t["base_tag"] = "b10639" -rc, out, err = run_docs(doc, o, t) -check("keeps a top-level field only theirs added", - rc == 0 and out is not None and out.get("base_tag") == "b10639", err or json.dumps(out)) - -# 14. both sides set the same top-level field differently: refuse, never guess. -o2 = json.loads(json.dumps(doc)); o2["_doc"] = ["ours' rewrite"] -t2 = json.loads(json.dumps(doc)); t2["_doc"] = ["theirs' rewrite"] -rc, _, err = run_docs(doc, o2, t2) -check("refuses a two-sided change to the same top-level field", rc == 1, err) -check("names the clashing top-level field", "_doc" in err, err) - -# 15. a top-level field theirs deleted stays deleted. -t3 = json.loads(json.dumps(doc)); del t3["_doc"] -rc, out, err = run_docs(doc, o, t3) -check("honours a top-level field theirs deleted", - rc == 0 and out is not None and "_doc" not in out, err or json.dumps(out)) - -# 16. --help must not crash: argparse %-formats help strings, and the driver placeholders %O/%A/%B are literal percents that have to be escaped. -r = subprocess.run([sys.executable, str(SCRIPT), "--help"], capture_output=True, text=True) -check("--help does not crash on the %O/%A/%B placeholders", - r.returncode == 0 and "%O" in r.stdout, (r.stderr or r.stdout)[-200:]) - -# 17. one side REORDERS the pins while the other changes a field. -# Merging by position then combines fields belonging to different PRs: base [A(required), B(required)] with ours making A optional and theirs swapping the two produces B(required=false), so the release skips the wrong PR, and the driver exits 0 while doing it. -# A reorder must be refused instead. -two = [{"url": BASE_PINS[0], "required": True}, {"url": BASE_PINS[1], "required": True}] -o = json.loads(json.dumps(two)); o[0]["required"] = False -t = [two[1], two[0]] -rc, pins, err = run_objs(two, o, t) -check("refuses a reorder that would splice fields across PRs", rc == 1, - json.dumps(pins) if pins else err) -check("names the reordered position", "reorder" in err, err) -check("never emits a pin carrying another PR's field", - pins is None or pins[0]["required"] is not False, json.dumps(pins)) - -# 18. a reorder that also repins the moved entry still has to be refused: the url no longer matches, so only the PR number identifies the entry. -t = [dict(two[1]), dict(two[0])] -t[0]["url"] = sub(BASE_PINS, 1, "9" * 40)[1] -rc, _, err = run_objs(two, o, t) -check("refuses a reorder combined with a repin", rc == 1, err) - -# 19. a reorder on OUR side is refused too, not just on theirs. -o2 = [two[1], two[0]] -t2 = json.loads(json.dumps(two)); t2[0]["required"] = False -rc, _, err = run_objs(two, o2, t2) -check("refuses a reorder on ours", rc == 1, err) - -# 20. swapping a pin for a DIFFERENT PR at the same position is not a reorder and must keep merging, which is case 1's real 08-27 resolution. -rc, pins, err = run(BASE_PINS, - replace(BASE_PINS, 6, f"{U}/125/commits/{'a' * 40}"), - sub(BASE_PINS, 5, "b" * 40)) -check("a same-position swap to a new PR is not a reorder", rc == 0, err) - -# 21. a plain repin is not a reorder either, on either side. -rc, pins, err = run(BASE_PINS, sub(BASE_PINS, 0, "1" * 40), sub(BASE_PINS, 3, "2" * 40)) -check("two repins at different positions still merge", rc == 0, err) - -# 22. a same-position swap to a different PR while the OTHER side edits that entry's fields. -# Test 20's swap is safe only because nobody else touched the entry; here both sides did, so the field-wise merge runs and takes the url from one PR and `required` from another. -# Base A(required=true) with ours swapping in B and theirs making A optional yielded B(required=false) and exit 0, which makes the release skip a PR nobody made optional. -two = [{"url": BASE_PINS[0], "required": True}, {"url": BASE_PINS[1], "required": True}] -o = json.loads(json.dumps(two)); o[0]["url"] = f"{U}/999/commits/{'c' * 40}" -t = json.loads(json.dumps(two)); t[0]["required"] = False -rc, pins, err = run_objs(two, o, t) -check("refuses a same-position PR swap the other side also edited", rc == 1, - json.dumps(pins) if pins else err) -check("names both PRs in the refusal", "#999" in err and "#107" in err, err) -check("never emits the swapped-in PR carrying the other's field", - pins is None or pins[0].get("required") is not False, json.dumps(pins)) - -# 23. both sides swap position 0 to the SAME new PR but disagree on a field. -# Base still describes the PR that is gone, so every field comparison below is against settings that were never this PR's: refuse rather than pick one. -o = json.loads(json.dumps(two)); o[0]["url"] = f"{U}/999/commits/{'c' * 40}" -t = json.loads(json.dumps(two)) -t[0]["url"] = f"{U}/999/commits/{'c' * 40}"; t[0]["required"] = False -rc, _, err = run_objs(two, o, t) -check("refuses an agreed swap the sides disagree about", rc == 1, err) - -# 24. two entries pinning two commits of the SAME PR share one identity, so a swap between them looks like no change and the reorder guard never fires: a field theirs changed on the first entry then lands on the second. -dup = [{"url": f"{U}/107/commits/{'a' * 40}", "required": True}, - {"url": f"{U}/107/commits/{'b' * 40}", "required": True}] -o = [json.loads(json.dumps(dup[1])), json.loads(json.dumps(dup[0]))] -t = json.loads(json.dumps(dup)); t[0]["required"] = False -rc, pins, err = run_objs(dup, o, t) -check("refuses a pin set that names one PR twice", rc == 1, - json.dumps(pins) if pins else err) -check("names the duplicated PR", "#107" in err, err) - -# 25. neither side has a duplicate, but the merge makes one: ours puts a new PR at position 0 and theirs puts the SAME new PR at position 1. -# Base has it nowhere, so the reorder guard sees nothing, and the driver emitted [C, C]. -two_ab = [{"url": f"{U}/107/commits/{'a' * 40}"}, {"url": f"{U}/108/commits/{'b' * 40}"}] -c_entry = {"url": f"{U}/999/commits/{'c' * 40}"} -o = [json.loads(json.dumps(c_entry)), json.loads(json.dumps(two_ab[1]))] -t = [json.loads(json.dumps(two_ab[0])), json.loads(json.dumps(c_entry))] -rc, pins, err = run_objs(two_ab, o, t) -check("refuses a merge that would pin one PR twice", rc == 1, - json.dumps(pins) if pins else err) -check("says the duplicate is in the merged set", - "merged pin set" in err and "#999" in err, err) - -print() -print(f"{len(FAILS)} failure(s)" if FAILS else "all pin_merge tests passed") -sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_sync_deletes.py b/scripts/unsloth/test_sync_deletes.py deleted file mode 100644 index 7be889a11c1a..000000000000 --- a/scripts/unsloth/test_sync_deletes.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Tests for sync_deletes.py. Run: python3 scripts/unsloth/test_sync_deletes.py - -Every case builds a real git merge, so the index stages are the ones git actually produces rather than a hand-written approximation. -""" -import subprocess -import sys -import tempfile -from pathlib import Path - -SCRIPT = Path(__file__).resolve().parent / "sync_deletes.py" -FAILS = [] - - -def check(name, cond, extra=""): - print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) - if not cond: - FAILS.append(name) - - -def git(repo, *args): - return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], - cwd=repo, capture_output=True, text=True) - - -def scenario(path, ours_deletes=True, upstream_modifies=True, upstream_adds=None): - """Base has `path`; upstream edits it; we delete it. Returns (repo, merge_base).""" - d = Path(tempfile.mkdtemp(prefix="sd_")) - git(d, "init", "-q", "-b", "main") - f = d / path - f.parent.mkdir(parents=True, exist_ok=True) - f.write_text("name: base\n") - (d / "src").mkdir(exist_ok=True) - (d / "src" / "model.cpp").write_text("int x;\n") - git(d, "add", "-A"); git(d, "commit", "-qm", "base") - base = git(d, "rev-parse", "HEAD").stdout.strip() - - git(d, "checkout", "-qb", "upstream") - if upstream_modifies: - f.write_text("name: upstream edit\n") - if upstream_adds: - na = d / upstream_adds - na.parent.mkdir(parents=True, exist_ok=True) - na.write_text("name: new upstream workflow\n") - git(d, "add", "-A"); git(d, "commit", "-qm", "upstream") - - git(d, "checkout", "-q", "main") - if ours_deletes: - git(d, "rm", "-q", str(f.relative_to(d))) - git(d, "commit", "-qm", "fork deletes it") - git(d, "-c", "merge.conflictStyle=diff3", "merge", "--no-ff", "--no-edit", "upstream") - return d, base - - -def run(repo, base=None): - args = [sys.executable, str(SCRIPT), "--repo", str(repo)] - if base: - args += ["--merge-base", base] - r = subprocess.run(args, capture_output=True, text=True) - return r.returncode, r.stdout + r.stderr - - -# 1. the 68-of-68 historical case -d, base = scenario(".github/workflows/build-apple.yml") -rc, out = run(d) -check("resolves an upstream-workflow modify/delete", rc == 0, out) -check("the file stays deleted", not (d / ".github/workflows/build-apple.yml").exists()) -check("no unmerged paths remain", git(d, "ls-files", "-u").stdout.strip() == "") - -# 2. a workflow we own must never be touched automatically -d, base = scenario(".github/workflows/unsloth-prebuilt.yml") -rc, out = run(d) -check("refuses a fork-owned workflow", rc == 1 and "we own this workflow" in out, out) - -# 3. a source file in the same shape must never be touched -d, base = scenario("src/model.cpp") -rc, out = run(d) -check("refuses a source file", rc == 1 and "not an upstream workflow path" in out, out) - -# 4. a workflow upstream added, which is not a conflict at all -d, base = scenario(".github/workflows/build-apple.yml", - upstream_adds=".github/workflows/build-wasm.yml") -rc, out = run(d, base) -check("drops a newly added upstream workflow", rc == 0, out) -check("the added workflow is gone", not (d / ".github/workflows/build-wasm.yml").exists()) - -# 5. an upstream composite ACTION must survive; only workflows are dropped -d, base = scenario(".github/workflows/build-apple.yml", - upstream_adds=".github/actions/ccache-buckets/action.yml") -rc, out = run(d, base) -check("keeps upstream composite actions", (d / ".github/actions/ccache-buckets/action.yml").exists(), out) - -# 6. source files are never removed by the added-workflow sweep -check("source file untouched throughout", (d / "src" / "model.cpp").exists()) - -# 7. an unusable --merge-base. -# git diff exits nonzero with empty stdout, which reads exactly like "upstream added nothing" if only stdout is looked at, so the run reported success and a sync would have carried every newly added upstream workflow in. -# The listing failing has to fail the script. -d, base = scenario(".github/workflows/build-apple.yml", - upstream_adds=".github/workflows/build-wasm.yml") -rc, out = run(d, "0000000000000000000000000000000000000000") -check("fails when the added-workflow listing cannot run", rc == 1, out) -check("says which rev it could not use", - "0000000000" in out and "could not list" in out, out) -check("and leaves the added workflow in place to be dealt with", - (d / ".github/workflows/build-wasm.yml").exists(), out) - -# 8. upstream RENAMES a workflow rather than adding one. -# Rename detection calls the new path R, not A, so --diff-filter=A saw nothing and the script exited 0 with the renamed upstream workflow left live in the fork. -def rename_scenario(): - d = Path(tempfile.mkdtemp(prefix="sd_")) - git(d, "init", "-q", "-b", "main") - wf = d / ".github" / "workflows" - wf.mkdir(parents=True) - # Long enough that git scores the move as a rename rather than add+delete. - (wf / "old.yml").write_text("".join(f"# line {i}\n" for i in range(40))) - (d / "src").mkdir() - (d / "src" / "model.cpp").write_text("int x;\n") - git(d, "add", "-A"); git(d, "commit", "-qm", "base") - base = git(d, "rev-parse", "HEAD").stdout.strip() - - git(d, "checkout", "-qb", "upstream") - git(d, "mv", ".github/workflows/old.yml", ".github/workflows/new.yml") - git(d, "commit", "-qm", "upstream renames it") - - git(d, "checkout", "-q", "main") - git(d, "merge", "--no-ff", "--no-edit", "upstream") - return d, base - - -d, base = rename_scenario() -rc, out = run(d, base) -check("drops an upstream workflow that arrived by rename", rc == 0, out) -check("the renamed workflow is gone", - not (d / ".github/workflows/new.yml").exists(), out) -check("source is still untouched", (d / "src" / "model.cpp").exists(), out) - -# 9. --repo names something that is not a git repository. ls-files fails, its -# empty stdout read as "no conflicts", and the script reported that it had -# resolved everything it was asked to. -notrepo = Path(tempfile.mkdtemp(prefix="sd_notrepo_")) -rc, out = run(notrepo) -check("fails when the unmerged listing cannot run", rc == 1, out) -check("says what could not be listed", "ls-files" in out, out) - -print() -print(f"{len(FAILS)} failure(s)" if FAILS else "all sync_deletes tests passed") -sys.exit(1 if FAILS else 0) diff --git a/scripts/unsloth/test_upload_release_assets.sh b/scripts/unsloth/test_upload_release_assets.sh deleted file mode 100755 index 35d7a725f125..000000000000 --- a/scripts/unsloth/test_upload_release_assets.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Tests for upload_release_assets.sh. Run: bash scripts/unsloth/test_upload_release_assets.sh -# -# Every case runs the real uploader against a stub `gh` that reproduces the -# failure modes seen against uploads.github.com: a PUT that wedges and never -# returns, a transient 5xx, an asset committed at the wrong size, an asset left -# in a non-uploaded state, and a `gh` that exits 0 without the asset landing. -# Budgets are shrunk so a stall is killed in ~1s instead of ~500s. -set -uo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -SCRIPT="$HERE/upload_release_assets.sh" -STUBDIR="$(mktemp -d)" -trap 'rm -rf "$STUBDIR"' EXIT -mkdir -p "$STUBDIR/bin" - -cat > "$STUBDIR/bin/gh" <<'STUB' -#!/usr/bin/env bash -# Stub gh: serves `release upload` and `release view` off a file registry. -set -uo pipefail -REG="${STUB_REG:?}"; mkdir -p "$REG/assets" "$REG/attempts" - -in_list() { case " ${2:-} " in *" $1 "*) return 0 ;; *) return 1 ;; esac; } - -if [ "$1" = "release" ] && [ "$2" = "upload" ]; then - file="${*: -1}"; size="$(stat -c %s "$file")" - # GitHub sanitises the asset name server-side; mirror that here so the test - # exercises the same name mapping the uploader has to verify against. - name="$(printf '%s' "$(basename "$file")" | tr -c 'A-Za-z0-9._-' '.')" - c="$REG/attempts/$name"; n=$(( $(cat "$c" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$c" - - if in_list "$name" "${STUB_STALL_ALWAYS:-}"; then sleep 300; exit 0; fi - if in_list "$name" "${STUB_STALL_ONCE:-}" && [ "$n" -le 1 ]; then sleep 300; exit 0; fi - if in_list "$name" "${STUB_FAIL_ALWAYS:-}"; then echo "stub: HTTP 502" >&2; exit 1; fi - if in_list "$name" "${STUB_FAIL_ONCE:-}" && [ "$n" -le 1 ]; then echo "stub: HTTP 502" >&2; exit 1; fi - # Below: exits 0, but leaves the release in a state the caller must catch. - if in_list "$name" "${STUB_WRONGSIZE:-}" && [ "$n" -le 1 ]; then - printf '%s\t%s\tuploaded\n' "$name" "$(( size - 1 ))" > "$REG/assets/$name"; exit 0 - fi - if in_list "$name" "${STUB_NOTUPLOADED:-}" && [ "$n" -le 1 ]; then - printf '%s\t%s\tstarter\n' "$name" "$size" > "$REG/assets/$name"; exit 0 - fi - if in_list "$name" "${STUB_SILENT_DROP:-}"; then exit 0; fi - printf '%s\t%s\tuploaded\n' "$name" "$size" > "$REG/assets/$name" - exit 0 -fi - -if [ "$1" = "release" ] && [ "$2" = "view" ]; then - if [ "${STUB_VIEW_FAILS:-}" = 1 ]; then echo "stub: HTTP 503" >&2; exit 1; fi - jqexpr="" - for ((i=1;i<=$#;i++)); do - if [ "${!i}" = "--jq" ]; then j=$((i+1)); jqexpr="${!j}"; fi - done - { echo '{"assets":[' - first=1 - for a in "$REG"/assets/*; do - [ -e "$a" ] || continue - IFS=$'\t' read -r n s st < "$a" - if [ "$first" = 1 ]; then first=0; else echo ','; fi - printf '{"name":"%s","size":%s,"state":"%s"}' "$n" "$s" "$st" - done - echo ']}'; } | jq -r "$jqexpr" - exit 0 -fi -echo "stub: unhandled: $*" >&2; exit 64 -STUB -chmod +x "$STUBDIR/bin/gh" -export PATH="$STUBDIR/bin:$PATH" - -export UPLOAD_GRACE_SECONDS=1 UPLOAD_MIN_RATE_MB_S=1000 UPLOAD_HEARTBEAT_SECONDS=3 -export UPLOAD_JOBS=4 UPLOAD_ATTEMPTS=3 - -FAILS=() - -run_case() { # name expected_rc [env ...] - local name="$1" want="$2"; shift 2 - local d; d="$(mktemp -d)"; mkdir -p "$d/dist" - local i - for i in 01 02 03 04 05 06; do head -c 1000000 /dev/zero > "$d/dist/bundle-$i.tar.gz"; done - # A name GitHub will rewrite, so the verify path's name mapping is covered. - head -c 1000 /dev/zero > "$d/dist/has space.json" - local out rc - out="$(STUB_REG="$d/reg" env "$@" bash "$SCRIPT" --tag T --repo o/r --dist "$d/dist" 2>&1)"; rc=$? - if [ "$rc" = "$want" ]; then - printf 'PASS %s\n' "$name" - else - printf 'FAIL %s :: rc=%s want=%s\n' "$name" "$rc" "$want" - printf '%s\n' "$out" | sed 's/^/ | /' - FAILS+=("$name") - fi - rm -rf "$d" -} - -run_case "happy path" 0 IGNORED=1 -run_case "stall, recovers on retry" 0 STUB_STALL_ONCE="bundle-02.tar.gz bundle-05.tar.gz" -run_case "transient 502, recovers" 0 STUB_FAIL_ONCE="bundle-03.tar.gz" -run_case "permanent stall aborts" 1 STUB_STALL_ALWAYS="bundle-04.tar.gz" -run_case "permanent 502 aborts" 1 STUB_FAIL_ALWAYS="bundle-01.tar.gz" -run_case "wrong size caught, re-uploaded" 0 STUB_WRONGSIZE="bundle-06.tar.gz" -run_case "non-uploaded state re-uploaded" 0 STUB_NOTUPLOADED="bundle-02.tar.gz" -run_case "gh exits 0, asset never lands" 1 STUB_SILENT_DROP="bundle-03.tar.gz" -run_case "phase deadline aborts" 1 UPLOAD_DEADLINE_MINUTES=0 STUB_STALL_ALWAYS="bundle-01.tar.gz" -# The verify read must fail the script, not just its subshell: a process -# substitution would swallow this and report every asset as verified. -run_case "verification API outage aborts" 1 STUB_VIEW_FAILS=1 - -echo -echo "${#FAILS[@]} failure(s)${FAILS[*]:+: ${FAILS[*]}}" -[ "${#FAILS[@]}" -eq 0 ] diff --git a/scripts/unsloth/test_verify_upstream_sync.py b/scripts/unsloth/test_verify_upstream_sync.py deleted file mode 100644 index a7ad460d7c68..000000000000 --- a/scripts/unsloth/test_verify_upstream_sync.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Negative controls for verify_upstream_sync.py. - -A checker that only ever prints PASS is worthless, so this builds a throwaway repository -shaped like the real one (a base, an upstream that moves on, a fork that customises its own -files), then injects each failure mode one at a time and asserts the checker reports it. - -The four modes are the four ways a sync can actually eat our work: - - modify upstream reformats or reverts a file we own - delete our file is dropped in the merge - renumber a published GGML/LLAMA enum id shifts, which corrupts every GGUF that stores it - subtle a value inside our Python changes without the file being obviously touched - -Run: python3 scripts/unsloth/test_verify_upstream_sync.py -""" - -from __future__ import annotations - -import os -import subprocess -import sys -import tempfile - -HERE = os.path.dirname(os.path.abspath(__file__)) -CHECKER = os.path.join(HERE, "verify_upstream_sync.py") - - -def sh(cwd: str, *args: str) -> str: - p = subprocess.run(args, cwd=cwd, capture_output=True, text=True) - if p.returncode: - raise SystemExit(f"{' '.join(args)} failed in {cwd}:\n{p.stderr}") - return p.stdout - - -def git(repo: str, *args: str) -> str: - return sh(repo, "git", "-c", "user.email=t@t", "-c", "user.name=t", *args) - - -def write(repo: str, path: str, text: str) -> None: - full = os.path.join(repo, path) - os.makedirs(os.path.dirname(full), exist_ok=True) - with open(full, "w") as f: - f.write(text) - - -HEADER_BASE = """ - enum ggml_type { - GGML_TYPE_F32 = 0, - GGML_TYPE_F16 = 1, - GGML_TYPE_COUNT = 2, - }; -""" -HEADER_UPSTREAM = """ - enum ggml_type { - GGML_TYPE_F32 = 0, - GGML_TYPE_F16 = 1, - GGML_TYPE_Q4_0 = 2, - GGML_TYPE_COUNT = 3, - }; -""" -OURS_PY = 'OWNED = ("unslothai/", "danielhanchen/")\nLIMIT = 7\n\n\ndef pin():\n return OWNED\n' - - -def build_repo(tmp: str) -> str: - """base -> upstream advances; fork adds its own files and deletes an upstream one.""" - up = os.path.join(tmp, "upstream") - os.makedirs(up) - git(up, "init", "-q", "-b", "master") - write(up, "ggml/include/ggml.h", HEADER_BASE) - write(up, "src/model.cpp", "int main(){return 0;}\n") - write(up, ".github/workflows/ci.yml", "name: CI\n") - git(up, "add", "-A"); git(up, "commit", "-qm", "base") - - fork = os.path.join(tmp, "fork") - sh(tmp, "git", "clone", "-q", up, fork) - git(fork, "remote", "add", "upstream", up) - - # upstream moves on: a new type, so a new COUNT - write(up, "ggml/include/ggml.h", HEADER_UPSTREAM) - write(up, "src/model.cpp", "int main(){return 1;}\n") - git(up, "add", "-A"); git(up, "commit", "-qm", "upstream: add Q4_0") - - # the fork customises only its own tree, and drops upstream CI - write(fork, "scripts/unsloth/repin.py", OURS_PY) - write(fork, ".github/workflows/unsloth-prebuilt.yml", "name: prebuilt\n") - os.remove(os.path.join(fork, ".github/workflows/ci.yml")) - git(fork, "add", "-A"); git(fork, "commit", "-qm", "unsloth: our own CI and scripts") - git(fork, "branch", "-f", "forkmaster", "HEAD") - git(fork, "fetch", "-q", "upstream", "master") - return fork - - -def merge(fork: str, name: str) -> str: - git(fork, "checkout", "-q", "-B", name, "forkmaster") - p = subprocess.run(["git", "merge", "--no-commit", "--no-ff", "upstream/master"], - cwd=fork, capture_output=True, text=True) - # the fork's deletion of upstream CI conflicts as modify/delete; keep it deleted - conf = sh(fork, "git", "diff", "--name-only", "--diff-filter=U").split() - if conf: - git(fork, "rm", "-q", *conf) - git(fork, "commit", "-qm", f"merge {name}") - return sh(fork, "git", "rev-parse", "HEAD").strip() - - -def run_checker(fork: str, rev: str) -> tuple[int, str]: - p = subprocess.run([sys.executable, CHECKER, "--repo", fork, "--merge", rev, - "--fork", "forkmaster", "--upstream", "upstream/master"], - capture_output=True, text=True) - return p.returncode, p.stdout + p.stderr - - -def main() -> int: - fails = 0 - with tempfile.TemporaryDirectory() as tmp: - fork = build_repo(tmp) - - rc, out = run_checker(fork, merge(fork, "clean")) - if rc == 0 and "PASS: the sync is additive only" in out: - print("PASS clean merge is accepted") - else: - fails += 1 - print(f"FAIL clean merge was rejected (rc={rc})\n{out}") - - cases = [ - ("modify our file", "content", - lambda: write(fork, "scripts/unsloth/repin.py", OURS_PY + "# upstream reflow\n")), - ("delete our file", "content", - lambda: os.remove(os.path.join(fork, "scripts/unsloth/repin.py"))), - ("renumber a published id", "c_enums", - lambda: write(fork, "ggml/include/ggml.h", - HEADER_UPSTREAM.replace("GGML_TYPE_F16 = 1", "GGML_TYPE_F16 = 5"))), - ("subtle value change in our Python", "ast", - lambda: write(fork, "scripts/unsloth/repin.py", - OURS_PY.replace('"danielhanchen/"', '"someone-else/"'))), - ] - for i, (label, expect, mutate) in enumerate(cases): - rev = merge(fork, f"bad{i}") - mutate() - git(fork, "add", "-A") - git(fork, "commit", "-qm", f"negative control: {label}") - rev = sh(fork, "git", "rev-parse", "HEAD").strip() - rc, out = run_checker(fork, rev) - caught = rc == 1 and any(l.startswith("FAIL " + expect) for l in out.splitlines()) - if caught: - print(f"PASS caught: {label} (via {expect})") - else: - fails += 1 - print(f"FAIL MISSED: {label} (expected FAIL {expect}, rc={rc})\n{out}") - - print(f"\n{'all negative controls caught' if not fails else f'{fails} FAILURES'}") - return 1 if fails else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/unsloth/upload_release_assets.sh b/scripts/unsloth/upload_release_assets.sh deleted file mode 100755 index b6f0e8adf26c..000000000000 --- a/scripts/unsloth/upload_release_assets.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Upload a directory of files to a draft release, then check the release really -# holds them before the caller flips draft=false. -# Run: upload_release_assets.sh --tag TAG --repo OWNER/REPO --dist DIR -# -# `gh release create ... dist/*` uploads with a fixed 5-worker pool and no -# per-connection timeout, so a few wedged PUTs block everything behind them. In -# run 31335302864 six large bundles stalled at ~0.03 MB/s and held the pool for -# 3h45m, while the other 25 assets took 37s in total. The job finished 25 -# minutes short of its 350m cap. -# -# So: bound each attempt, bound the whole phase, and verify against the API -# before we publish. -set -euo pipefail - -# Defaults come from measured healthy throughput on the runners, 33-47 MB/s. -JOBS="${UPLOAD_JOBS:-4}" -ATTEMPTS="${UPLOAD_ATTEMPTS:-4}" -# Slowest rate we still call "progressing". ~17x below healthy, so a bad day -# retries nothing but a wedged PUT dies fast. -MIN_RATE_MB_S="${UPLOAD_MIN_RATE_MB_S:-2}" -# Per-attempt allowance for setup and server-side commit, which do not scale -# with file size. -GRACE_SECONDS="${UPLOAD_GRACE_SECONDS:-120}" -# Healthy is ~4 minutes for 7.2 GB, so only a real pathology trips this. -DEADLINE_MINUTES="${UPLOAD_DEADLINE_MINUTES:-90}" -HEARTBEAT_SECONDS="${UPLOAD_HEARTBEAT_SECONDS:-60}" - -log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; } -die() { printf '%s ERROR: %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; exit 1; } - -file_size() { stat -c %s "$1"; } - -# GitHub rewrites characters outside [A-Za-z0-9._-] to '.', so verify against -# the name the API will report. printf, not basename: basename's trailing -# newline is also outside the set and would become a phantom '.' on every name. -asset_name() { printf '%s' "${1##*/}" | tr -c 'A-Za-z0-9._-' '.'; } - -# Worker mode. The parent fans out with xargs by re-invoking this script, which -# is safer than `export -f`: an unexported function fails per file at runtime. -if [ "${1:-}" = "--upload-one" ]; then - f="$2" - : "${TAG:?}" "${REPO:?}" "${UPLOAD_DEADLINE_EPOCH:?}" - name="$(asset_name "$f")" - bytes="$(file_size "$f")" - mb=$(( bytes / 1000000 )) - budget=$(( GRACE_SECONDS + mb / MIN_RATE_MB_S )) - - for attempt in $(seq 1 "$ATTEMPTS"); do - now="$(date +%s)" - if [ "$now" -ge "$UPLOAD_DEADLINE_EPOCH" ]; then - die "deadline reached before uploading $name" - fi - # Keep one file's budget inside the phase deadline. Floor at 1: `timeout 0` - # means no timeout, which brings back the hang this script prevents. - remaining=$(( UPLOAD_DEADLINE_EPOCH - now )) - this_budget="$budget" - if [ "$this_budget" -gt "$remaining" ]; then this_budget="$remaining"; fi - if [ "$this_budget" -lt 1 ]; then this_budget=1; fi - - start="$now" - # --clobber keeps a retry after a killed upload idempotent, else GitHub - # 422s on the duplicate name. Take the status here, not from $? after an - # `if`: a false `if` with no else exits 0, so every stall would read clean. - rc=0 - timeout -k 30 "$this_budget" gh release upload "$TAG" --repo "$REPO" --clobber "$f" || rc=$? - elapsed=$(( $(date +%s) - start )) - if [ "$rc" -eq 0 ]; then - if [ "$elapsed" -lt 1 ]; then elapsed=1; fi - log "uploaded $name (${mb} MB in ${elapsed}s, $(( mb / elapsed )) MB/s, attempt ${attempt})" - exit 0 - fi - if [ "$rc" -ge 124 ]; then - log "STALLED $name: no completion in ${elapsed}s (budget ${this_budget}s, ${mb} MB); attempt ${attempt}/${ATTEMPTS}" - else - log "FAILED $name: gh exit ${rc} after ${elapsed}s; attempt ${attempt}/${ATTEMPTS}" - fi - if [ "$attempt" -eq "$ATTEMPTS" ]; then - die "gave up on $name after ${ATTEMPTS} attempts" - fi - sleep $(( attempt * 15 )) - done - exit 1 -fi - -# Parent mode. -TAG="" REPO="" DIST="" -while [ $# -gt 0 ]; do - case "$1" in - --tag) TAG="$2"; shift 2 ;; - --repo) REPO="$2"; shift 2 ;; - --dist) DIST="$2"; shift 2 ;; - *) die "unknown argument: $1" ;; - esac -done -[ -n "$TAG" ] || die "--tag is required" -[ -n "$REPO" ] || die "--repo is required" -[ -n "$DIST" ] || die "--dist is required" -[ -d "$DIST" ] || die "dist directory not found: $DIST" - -# NUL-delimited: a name with a space would otherwise split into two bad paths. -mapfile -d '' -t FILES < <(find "$DIST" -maxdepth 1 -type f -print0 | sort -z) -[ "${#FILES[@]}" -gt 0 ] || die "no files to upload in $DIST" - -total_bytes=0 -for f in "${FILES[@]}"; do total_bytes=$(( total_bytes + $(file_size "$f") )); done -log "uploading ${#FILES[@]} assets ($(( total_bytes / 1000000 )) MB) to draft $TAG with ${JOBS} workers" - -UPLOAD_DEADLINE_EPOCH=$(( $(date +%s) + DEADLINE_MINUTES * 60 )) -export TAG REPO UPLOAD_DEADLINE_EPOCH JOBS ATTEMPTS MIN_RATE_MB_S GRACE_SECONDS - -self="$(readlink -f "$0")" - -# The incident was 4 hours of silence, so report what the API has accepted. -heartbeat() { - while sleep "$HEARTBEAT_SECONDS"; do - n="$(gh release view "$TAG" --repo "$REPO" --json assets --jq '[.assets[]|select(.state=="uploaded")]|length' 2>/dev/null || echo '?')" - log "heartbeat: ${n}/${#FILES[@]} assets uploaded, $(( (UPLOAD_DEADLINE_EPOCH - $(date +%s)) / 60 ))m left in budget" - done -} -heartbeat & hb_pid=$! -trap 'kill "$hb_pid" 2>/dev/null || true' EXIT - -upload_pass() { - # Run through `bash`, so a lost exec bit cannot break the publish. - printf '%s\0' "$@" | xargs -0 -P "$JOBS" -n 1 bash "$self" --upload-one -} - -pass_rc=0 -upload_pass "${FILES[@]}" || pass_rc=$? -[ "$pass_rc" -eq 0 ] || log "upload pass reported failures (xargs exit ${pass_rc}); verification decides" - -# gh exiting 0 does not prove the asset is complete, so set BAD to every local -# file the release does not hold at the same size and state "uploaded". Read the -# API here, not inside `< <(...)`, where a failed read exits only the subshell -# and leaves BAD empty, i.e. publishes a release we never checked. -verify() { - local remote f - remote="$(gh release view "$TAG" --repo "$REPO" --json assets \ - --jq '.assets[] | select(.state=="uploaded") | "\(.name)\t\(.size)"')" \ - || die "could not read release assets for verification" - BAD=() - for f in "${FILES[@]}"; do - if ! grep -qxF "$(asset_name "$f") $(file_size "$f")" <<<"$remote"; then - BAD+=("$f") - fi - done -} - -verify -if [ "${#BAD[@]}" -gt 0 ]; then - log "verification found ${#BAD[@]} missing or mismatched assets; re-uploading" - for f in "${BAD[@]}"; do log " - $(asset_name "$f")"; done - upload_pass "${BAD[@]}" || true - verify -fi - -if [ "${#BAD[@]}" -gt 0 ]; then - for f in "${BAD[@]}"; do printf 'ERROR: asset never landed: %s\n' "$(asset_name "$f")" >&2; done - die "refusing to publish: ${#BAD[@]}/${#FILES[@]} assets missing after re-upload" -fi - -log "verified all ${#FILES[@]} assets present, sized and uploaded" diff --git a/scripts/unsloth/upstream-sync.json b/scripts/unsloth/upstream-sync.json deleted file mode 100644 index cc89fc2b09cb..000000000000 --- a/scripts/unsloth/upstream-sync.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "_doc": [ - "The upstream commit master was last synced to, and the invariant that keeps the sync cheap.", - "", - "Update BOTH fields in the same commit as the sync merge. unsloth-upstream-sync-guard.yml", - "reads this file and fails master if either invariant breaks:", - "", - " 1. `commit` must be an ancestor of master. This is the check that would have caught the", - " 08-07 sync (PR #80), which was squash-merged: its content landed but git never learned", - " upstream had been incorporated, so the merge base stayed at 2026-06-10 and every later", - " merge three-way merged against it. Merging b10632 conflicted in 539 files with that", - " base and in 21 with the true one. ALWAYS merge a sync PR with a merge commit.", - "", - " 2. The diff from `commit` to master must touch only .github/ and scripts/unsloth/. This", - " fork deliberately owns no llama.cpp source; that is what makes a sync provably additive", - " and lets scripts/unsloth/verify_upstream_sync.py check it exactly rather than by eye.", - " If this ever fails, the fork has acquired source divergence and syncs stop being cheap.", - "", - "Note for whoever runs the next sync: verify_upstream_sync.py derives its base from", - "merge-base(--fork, --upstream). Pass a --fork ref whose ancestry is already correct, or it", - "measures the stale set and reports failures that are artefacts of the bad base." - ], - "tag": "b10632", - "commit": "11cd98842874cc1b87ac274bd2d5cceb38102bb2", - "synced_at": "2026-08-26" -} diff --git a/scripts/unsloth/verify_upstream_sync.py b/scripts/unsloth/verify_upstream_sync.py deleted file mode 100755 index 7fff2b605f5f..000000000000 --- a/scripts/unsloth/verify_upstream_sync.py +++ /dev/null @@ -1,345 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -"""Prove an upstream-sync merge is additive only: it may bring in upstream content, but it must -never modify, revert or delete anything this fork authored. - -The invariant, stated precisely. Let - - B = merge-base(fork_master, upstream_master) - F = fork_master (the fork before the sync) - U = upstream_master - M = the merge commit under test - -For every path P that the fork touched in B..F, the merge must satisfy M:P == F:P, byte for -byte. That is the whole rule, and it is checked exactly rather than approximated by reading a -diff. Three things can break it and each is reported separately: - - MODIFIED M:P exists but differs from F:P. Upstream edited a file we own, or a conflict - was resolved in upstream's favour. - DELETED P is in F and gone in M. Our work was dropped. - LOSTCOMMIT a commit reachable from F is not reachable from M. History was rewritten. - -Deletions of upstream files are legitimate and are checked in the other direction: every path -missing from M must ALSO have been missing from F. A file the fork already deleted staying -deleted preserves our state; a file the fork had that vanishes is a violation. - -Two extra layers beyond the byte comparison, because a byte-identical file can still be -semantically wrong if a neighbouring definition moved: - - * AST check on the Python surface (gguf-py and convert scripts). Every top-level class, - function and assignment name the fork defines must still be defined in the merged tree, - and every enum member the fork added must still carry the same value. A renumbered - GGML_TYPE_* would be caught here even if the file "looks" additive. - * enum-value check on the C headers, by regex over the id assignments that matter, since a - silently renumbered type id is the failure mode that would corrupt published GGUFs. - -Usage: - verify_upstream_sync.py --repo <path> --merge <rev> [--fork origin/master] - [--upstream upstream/master] [--json out.json] - -Exit 0 only if every check passes. -""" - -from __future__ import annotations - -import argparse -import ast -import json -import os -import re -import subprocess -import sys -from collections import defaultdict - - -def git(repo: str, *args: str, ok_fail: bool = False) -> str: - p = subprocess.run(("git", "-C", repo) + args, capture_output=True, text=True) - if p.returncode and not ok_fail: - raise SystemExit(f"git {' '.join(args)} failed:\n{p.stderr.strip()}") - return p.stdout - -def lines(s: str) -> list[str]: - return [x for x in s.splitlines() if x.strip()] - -def blob(repo: str, rev: str, path: str) -> bytes | None: - """File content at a revision, or None if the path does not exist there.""" - p = subprocess.run(("git", "-C", repo, "show", f"{rev}:{path}"), - capture_output=True) - return p.stdout if p.returncode == 0 else None - - -# ---------------------------------------------------------------- AST surface - -def py_surface(src: bytes) -> dict[str, str]: - """Top-level names a Python file defines, plus every enum-ish member and its literal value. - - Keys are dotted so a member cannot collide across classes. Values are a repr of the - assigned constant where there is one, else the node type, so a renumber shows up as a - changed value rather than a missing name. - """ - try: - tree = ast.parse(src.decode("utf-8", "replace")) - except SyntaxError: - return {} - - out: dict[str, str] = {} - - def const(node: ast.AST) -> str: - """A value fingerprint that survives reformatting but not a real change. - - ast.unparse normalises whitespace, quote style and line breaks, so a reflow is - invisible while an edited element is not. Falling back to the node type name would - make every tuple, list and dict compare equal, which is how a dropped entry such as - OWNED = ("a", "b") -> ("a",) slips past. - """ - if isinstance(node, ast.Constant): - return repr(node.value) - try: - return ast.unparse(node) - except Exception: - return type(node).__name__ - - def walk(body: list[ast.stmt], prefix: str) -> None: - for n in body: - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)): - out[f"{prefix}{n.name}"] = "def" - elif isinstance(n, ast.ClassDef): - out[f"{prefix}{n.name}"] = "class" - walk(n.body, f"{prefix}{n.name}.") - elif isinstance(n, ast.Assign): - for t in n.targets: - if isinstance(t, ast.Name): - out[f"{prefix}{t.id}"] = const(n.value) - elif isinstance(n, ast.AnnAssign) and isinstance(n.target, ast.Name): - out[f"{prefix}{n.target.id}"] = const(n.value) if n.value else "ann" - walk(tree.body, "") - return out - - -# --------------------------------------------------------- C enum value check - -C_ENUM = re.compile( - rb"^\s*(GGML_TYPE_[A-Z0-9_]+|GGML_FTYPE_[A-Z0-9_]+|LLAMA_FTYPE_[A-Z0-9_]+)\s*=\s*(-?\d+)", - re.M) - -def c_enum_values(src: bytes) -> dict[str, int]: - return {m.group(1).decode(): int(m.group(2)) for m in C_ENUM.finditer(src)} - - -# --------------------------------------------------------------------- checks - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--repo", required=True) - ap.add_argument("--merge", required=True, help="the merge commit under test") - ap.add_argument("--fork", default="origin/master", help="fork state BEFORE the sync") - ap.add_argument("--upstream", default="upstream/master") - ap.add_argument("--json", help="write the full report here") - ap.add_argument("--show", type=int, default=15, help="max paths to print per category") - a = ap.parse_args() - - R = a.repo - if git(R, "rev-parse", "--is-shallow-repository").strip() == "true": - print("REFUSING: shallow clone, history checks would be meaningless. " - "Re-clone without --depth.", file=sys.stderr) - return 2 - - M = git(R, "rev-parse", a.merge).strip() - F = git(R, "rev-parse", a.fork).strip() - U = git(R, "rev-parse", a.upstream).strip() - B = git(R, "merge-base", a.fork, a.upstream).strip() - print(f"merge {M[:12]}\nfork {F[:12]} ({a.fork})\n" - f"upstream {U[:12]} ({a.upstream})\nmerge-base {B[:12]}\n") - - rep: dict = {"merge": M, "fork": F, "upstream": U, "base": B, "checks": {}} - fail = False - - # --- 1. no fork commit may be dropped ----------------------------------- - lost = lines(git(R, "rev-list", a.fork, f"^{M}")) - fork_commits = len(lines(git(R, "rev-list", f"{B}..{a.fork}"))) - rep["checks"]["history"] = {"fork_commits_since_base": fork_commits, - "lost": lost} - if lost: - fail = True - print(f"FAIL history: {len(lost)} fork commits are NOT ancestors of the merge") - for c in lost[:a.show]: - print(f" {c[:12]} {git(R, 'log', '-1', '--format=%s', c).strip()[:70]}") - else: - print(f"PASS history: all {fork_commits} fork commits since the base are ancestors " - f"of the merge, none dropped") - - # --- 2. every path the fork touched must survive byte-identical --------- - # --diff-filter with -M off: a rename upstream must not silently "move" our file. - touched = sorted(set(lines(git(R, "diff", "--name-only", "--no-renames", f"{B}..{a.fork}")))) - modified, deleted, ok = [], [], 0 - for p in touched: - fb = blob(R, a.fork, p) - mb = blob(R, M, p) - if fb is None: - # the fork itself deleted it; it must still be absent - if mb is not None: - modified.append((p, "fork deleted it, merge resurrected it")) - else: - ok += 1 - continue - if mb is None: - deleted.append(p) - elif mb != fb: - modified.append((p, f"{len(fb)} B -> {len(mb)} B")) - else: - ok += 1 - rep["checks"]["content"] = {"fork_touched_paths": len(touched), "identical": ok, - "modified": modified, "deleted": deleted} - if modified or deleted: - fail = True - print(f"FAIL content: of {len(touched)} fork-touched paths, " - f"{len(modified)} modified and {len(deleted)} deleted") - for p, why in modified[:a.show]: - print(f" MODIFIED {p} ({why})") - for p in deleted[:a.show]: - print(f" DELETED {p}") - else: - print(f"PASS content: all {len(touched)} paths the fork touched are byte-identical " - f"in the merge") - - # --- 3. nothing the fork had may vanish, even if it never touched it ----- - fork_tree = set(lines(git(R, "ls-tree", "-r", "--name-only", a.fork))) - merge_tree = set(lines(git(R, "ls-tree", "-r", "--name-only", M))) - vanished = sorted(fork_tree - merge_tree) - # a vanished path is only acceptable if upstream deleted it AND the fork never touched it - unexplained = [p for p in vanished if p in set(touched)] - rep["checks"]["tree"] = {"fork_files": len(fork_tree), "merge_files": len(merge_tree), - "vanished": vanished, "unexplained": unexplained} - if unexplained: - fail = True - print(f"FAIL tree: {len(unexplained)} fork-authored files vanished from the merge") - for p in unexplained[:a.show]: - print(f" {p}") - else: - print(f"PASS tree: {len(fork_tree)} fork files -> {len(merge_tree)} merged files, " - f"{len(vanished)} vanished and none of them fork-authored") - - # --- 4. every deletion must be upstream's own, never ours --------------- - # Upstream retires its own files and a sync has to carry that through, so a deletion is - # only a violation if the file is one WE own. Two independent tests, both must hold: - # the path is absent from upstream's tree (upstream really did delete it), and the fork - # never touched it. - gone = sorted(set(lines(git(R, "diff", "--name-only", "--diff-filter=D", - f"{a.fork}..{M}")))) - touched_set = set(touched) - ours, not_upstream = [], [] - for p in gone: - if p in touched_set: - ours.append(p) - elif blob(R, a.upstream, p) is not None: - not_upstream.append(p) # still exists upstream, so nobody asked us to drop it - rep["checks"]["deletions"] = {"deleted_vs_fork": gone, "fork_authored": ours, - "still_present_upstream": not_upstream} - if ours or not_upstream: - fail = True - print(f"FAIL deletions: {len(gone)} deletions, {len(ours)} fork-authored, " - f"{len(not_upstream)} not deleted upstream either") - for p in (ours + not_upstream)[:a.show]: - print(f" {p}") - else: - print(f"PASS deletions: all {len(gone)} deletions are upstream retiring its own " - f"files, none fork-authored") - - # --- 5. AST surface on Python the fork owns ----------------------------- - pyfiles = [p for p in touched if p.endswith(".py")] - lost_names: list[tuple[str, str, str, str]] = [] - for p in pyfiles: - fb, mb = blob(R, a.fork, p), blob(R, M, p) - if fb is None or mb is None: - continue - fs, ms = py_surface(fb), py_surface(mb) - for name, val in fs.items(): - if name not in ms: - lost_names.append((p, name, val, "MISSING")) - elif ms[name] != val: - lost_names.append((p, name, val, ms[name])) - rep["checks"]["ast"] = {"python_files": len(pyfiles), "regressions": lost_names} - if lost_names: - fail = True - print(f"FAIL ast: {len(lost_names)} Python definitions lost or changed value") - for p, n, was, now in lost_names[:a.show]: - print(f" {p}:{n} was {was} now {now}") - else: - print(f"PASS ast: every top-level definition and constant in the {len(pyfiles)} " - f"fork-touched Python files survives with the same value") - - # --- 6. C enum ids, the failure that would corrupt published GGUFs ------ - # Deliberately NOT limited to fork-touched headers. A published GGUF stores these ids, so - # an id that shifts or collides is a data-corruption bug no matter who moved it, and on a - # fork whose master carries no source changes the fork-touched set would be empty. - # The reference to compare against depends on who owns the header. For one the fork - # customises, our values must survive. For one the fork does not, the merged copy must - # match UPSTREAM exactly, and upstream growing an enum (a new type, so a new _COUNT) is - # the sync working, not a violation. Comparing an unowned header against the fork's stale - # copy would flag every legitimate upstream addition. - ID_HEADERS = ["ggml/include/ggml.h", "include/llama.h"] - hdrs = sorted(set([p for p in touched if p.endswith((".h", ".hpp"))]) | - {p for p in ID_HEADERS if blob(R, M, p) is not None}) - enum_bad: list[tuple[str, str, int, object]] = [] - dupes: list[tuple[str, int, list[str]]] = [] - owned = set(touched) - for p in hdrs: - ref = a.fork if p in owned else a.upstream - rb, mb = blob(R, ref, p), blob(R, M, p) - if rb is None or mb is None: - continue - fe, me = c_enum_values(rb), c_enum_values(mb) - for name, v in fe.items(): - now = me.get(name, "MISSING") - if now == v: - continue - # A _COUNT sentinel is not an id, it is one past the last one, so a branch that - # legitimately adds types must move it. Allow it to grow and require it to still - # bound every real id; anything else, including a shrink, is a violation. - if name.endswith("_COUNT") and isinstance(now, int) and now > v: - fam = name[:-len("_COUNT")] - real = [x for n2, x in me.items() if n2.startswith(fam) and n2 != name] - if real and now > max(real): - continue - enum_bad.append((f"{p} [vs {ref}]", name, v, now)) - # two names sharing one id in the same enum family is a collision - byfam: dict[str, dict[int, list[str]]] = defaultdict(lambda: defaultdict(list)) - for name, v in me.items(): - fam = name.split("_")[0] + ("_FTYPE" if "_FTYPE_" in name else "_TYPE") - byfam[fam][v].append(name) - for fam, vals in byfam.items(): - for v, names in vals.items(): - if len(names) > 1 and not any(n.endswith("_COUNT") for n in names): - dupes.append((f"{p}:{fam}", v, sorted(names))) - rep["checks"]["c_enums"] = {"headers": len(hdrs), "changed": enum_bad, "collisions": dupes} - if enum_bad or dupes: - fail = True - print(f"FAIL c_enums: {len(enum_bad)} ids changed, {len(dupes)} id collisions") - for p, n, was, now in enum_bad[:a.show]: - print(f" {p}:{n} was {was} now {now}") - for where, v, names in dupes[:a.show]: - print(f" COLLISION {where} = {v}: {', '.join(names)}") - else: - print(f"PASS c_enums: every GGML_TYPE/GGML_FTYPE/LLAMA_FTYPE id the fork defines " - f"keeps its value across {len(hdrs)} headers, no collisions") - - # --- context: what the merge actually brought in ------------------------ - added = len(lines(git(R, "diff", "--name-only", "--diff-filter=A", f"{a.fork}..{M}"))) - changed = len(lines(git(R, "diff", "--name-only", f"{a.fork}..{M}"))) - rep["summary"] = {"files_added_by_merge": added, "files_changed_by_merge": changed, - "pass": not fail} - print(f"\nmerge brings in {changed} changed files, {added} of them new") - - if a.json: - with open(a.json, "w") as f: - json.dump(rep, f, indent=1) - print(f"report: {a.json}") - - print("\n" + ("VIOLATION: the sync is not additive-only" if fail - else "PASS: the sync is additive only, nothing fork-authored was altered")) - return 1 if fail else 0 - - -if __name__ == "__main__": - sys.exit(main()) From 1cfb42bf597adc6d2bc4c65721e75c9fa31390a7 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Wed, 9 Sep 2026 16:27:46 +0000 Subject: [PATCH 03/22] Pin that an exact-mode cross-sequence copy leaves both sequences as they were --- tests/test-exact-pages.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test-exact-pages.cpp b/tests/test-exact-pages.cpp index 8ea77e7ea3d9..cee2188bdf70 100644 --- a/tests/test-exact-pages.cpp +++ b/tests/test-exact-pages.cpp @@ -89,6 +89,16 @@ int main(int argc, char ** argv) { return 1; } + // a page belongs to one sequence, so a cross-sequence copy is refused whole rather than half + // applied: the pool logs the refusal and leaves the destination empty and the source as it was + llama_memory_seq_cp(mem, 0, 1, -1, -1); + + if (llama_memory_seq_pos_max(mem, 1) != -1 || llama_memory_seq_pos_max(mem, 0) != 599) { + fprintf(stderr, "%s : a refused copy left sequence 1 at %d and sequence 0 at %d\n", __func__, + llama_memory_seq_pos_max(mem, 1), llama_memory_seq_pos_max(mem, 0)); + return 1; + } + // the removal every accepted speculative step makes: a rejected tail that is not there. It // must leave the pool alone, ownership included if (!llama_memory_seq_rm(mem, 0, 600, -1) || llama_memory_seq_pos_max(mem, 0) != 599) { From 17a0f3b9678618ccbaa3887e3d179edd4a32fd57 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Wed, 9 Sep 2026 16:27:46 +0000 Subject: [PATCH 04/22] Condense the allocation granularity comment to two lines --- include/llama.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/include/llama.h b/include/llama.h index dfb19eae8da4..5cba47653298 100644 --- a/include/llama.h +++ b/include/llama.h @@ -804,13 +804,8 @@ extern "C" { // Check if the memory supports shifting LLAMA_API bool llama_memory_can_shift(llama_memory_t mem); - // [TAG_EXACT_CONCURRENCY] cells the memory allocates in one indivisible unit: 1 ordinarily, - // larger where a mode places cells in blocks, and then n contiguous tokens occupy - // round_up(n, granularity) cells. A caller deciding whether the pool has room must round the - // same way. round_up is the contiguous case only: a block is held for as long as any cell in - // it is live, so a sequence left with holes by a partial llama_memory_seq_rm still holds every - // block that has one, which can be far more than round_up of what it has left. Removing - // positions 1 to 510 of a 512-token sequence leaves two live cells holding two whole blocks. + // [TAG_EXACT_CONCURRENCY] cells the memory allocates in one indivisible block: 1 ordinarily, larger where a mode places cells in blocks + // n contiguous tokens then occupy round_up(n, granularity) cells; a sequence left with holes still holds every block one live cell is in LLAMA_API uint32_t llama_memory_alloc_granularity(llama_memory_t mem); // [TAG_PREEMPT] run the in-place update a seq_add() recorded, which llama_decode() would otherwise run at the start of the next batch From 1d5548978df86d27a3099f463120f0c30e1aca16 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 05:53:23 +0000 Subject: [PATCH 05/22] Park only when asked: --preempt-ram defaults to 0, so a server without the flag behaves as upstream --- common/arg.cpp | 2 +- common/common.h | 2 +- tools/server/README.md | 2 +- tools/server/server-context.cpp | 7 +++-- tools/server/tests/unit/test_preempt.py | 28 +++++++++++++++++++ .../server/tests/unit/test_preempt_notify.py | 2 ++ 6 files changed, 37 insertions(+), 6 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 5d58afb3005f..1b7e477c72c3 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1736,7 +1736,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex add_opt(common_arg( {"--preempt-ram"}, "N", string_format("with a unified KV cache, park a slot in host RAM instead of failing every slot when the cache fills; " - "N is the maximum host RAM for parked sequences in MiB (default: %d, -1 - no limit, 0 - disable)", params.preempt_ram_mib), + "N is the maximum host RAM for parked sequences in MiB (default: %d - disabled, -1 - no limit)", params.preempt_ram_mib), [](common_params & params, int value) { params.preempt_ram_mib = value; } diff --git a/common/common.h b/common/common.h index 7eb1c059d341..60bda08d74fd 100644 --- a/common/common.h +++ b/common/common.h @@ -630,7 +630,7 @@ struct common_params { int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unset int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. - int32_t preempt_ram_mib = 8192; // host RAM for parked (preempted) sequences: -1 = no limit, 0 = disable preemption + int32_t preempt_ram_mib = 0; // host RAM for parked (preempted) sequences: 0 = preemption off (the default), -1 = no limit bool preempt_async = true; // park and restore on a stream of their own, off the decode loop std::string hostname = "127.0.0.1"; diff --git a/tools/server/README.md b/tools/server/README.md index 036cf1be006e..f5ca9f30f296 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -168,7 +168,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `-ctxcp, --ctx-checkpoints, --swa-checkpoints N` | max number of context checkpoints to create per slot (default: 32)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293)<br/>(env: LLAMA_ARG_CTX_CHECKPOINTS) | | `-cms, --checkpoint-min-step N` | minimum spacing between context checkpoints in tokens (default: 8192, 0 = no minimum)<br/>(env: LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT) | | `-cram, --cache-ram N` | set the maximum cache size in MiB (default: 8192, -1 - no limit, 0 - disable)[(more info)](https://github.com/ggml-org/llama.cpp/pull/16391)<br/>(env: LLAMA_ARG_CACHE_RAM) | -| `--preempt-ram N` | with a unified KV cache, park a slot in host RAM instead of failing every slot when the cache fills; N is the maximum host RAM for parked sequences in MiB (default: 8192, -1 - no limit, 0 - disable)<br/>(env: LLAMA_ARG_PREEMPT_RAM) | +| `--preempt-ram N` | with a unified KV cache, park a slot in host RAM instead of failing every slot when the cache fills; N is the maximum host RAM for parked sequences in MiB (default: 0 - disabled, -1 - no limit)<br/>(env: LLAMA_ARG_PREEMPT_RAM) | | `--preempt-async`, `--no-preempt-async` | copy a parked sequence out of and back into the KV cache on a stream of its own: the copy out overlaps with the slots that keep decoding, while a copy back in, and a kv-full retry behind a copy out that has not landed, wait for it (default: enabled, needs a backend that can copy asynchronously, otherwise the copies are synchronous as before)<br/>(env: LLAMA_ARG_PREEMPT_ASYNC) | | `-kvu, --kv-unified, -no-kvu, --no-kv-unified` | use single unified KV buffer shared across all sequences (default: enabled if number of slots is auto)<br/>(env: LLAMA_ARG_KV_UNIFIED) | | `--cache-idle-slots, --no-cache-idle-slots` | save idle slots to the prompt cache on new task, and clear them when using unified KV (default: enabled, requires cache-ram)<br/>(env: LLAMA_ARG_CACHE_IDLE_SLOTS) | diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 55cfef08eaf4..a4d11f55f163 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1880,7 +1880,8 @@ struct server_context_impl { } else { SRV_WRN("%s", "preemption: this backend cannot copy asynchronously, parking and resuming synchronously\n"); } - } else if (params_base.preempt_async && !llama_model_is_recurrent(model_tgt) && preempt_state_relocates()) { + } else if (params_base.preempt_ram_mib != 0 && params_base.preempt_async && + !llama_model_is_recurrent(model_tgt) && preempt_state_relocates()) { SRV_WRN("%s", "preemption: a recurrent state does not stay in one row, so a copy running beside the decode could read another sequence; parking and resuming synchronously\n"); } @@ -1903,7 +1904,7 @@ struct server_context_impl { SRV_WRN("LLAMA_SERVER_PREEMPT_GRANULARITY = %d (test knob: planning the kv pool in blocks of %d cells)\n", preempt_alloc_granularity, preempt_alloc_granularity); - } else if (preempt_alloc_granularity > 1) { + } else if (preempt_alloc_granularity > 1 && params_base.preempt_ram_mib != 0) { SRV_INF("preemption: the kv pool allocates %d cells at a time, planning in pages\n", preempt_alloc_granularity); } @@ -1949,7 +1950,7 @@ struct server_context_impl { // assigned, not only set: a context reloaded with an attention model after a recurrent one gets preemption back preempt_recurrent = llama_model_is_recurrent(model_tgt); - if (preempt_recurrent) { + if (preempt_recurrent && params_base.preempt_ram_mib != 0) { SRV_WRN("%s", "preemption: off, the recurrent cache holds one state per sequence whatever its length, so there is no cell pool to run out of\n"); } } diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index 06776d0bbf25..7377b040228d 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -28,6 +28,8 @@ def create_server(): server = ServerPreset.tinyllama2() server.n_slots = 2 server.kv_unified = True + # the server parks only when asked: --preempt-ram defaults to 0, and this suite is about parking + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "8192" server.server_slots = True server.server_metrics = True server.temperature = 0.0 @@ -211,6 +213,32 @@ def test_a_request_that_cannot_be_helped_gets_the_context_error_and_the_server_l assert after.body["timings"]["predicted_n"] == 8 +def test_a_server_that_never_asked_for_parking_behaves_as_upstream(): + """--preempt-ram defaults to 0, so a unified-cache server started without it parks nothing.""" + os.environ.pop("LLAMA_ARG_PREEMPT_RAM", None) + _start(n_ctx=256) + + text = _log() + assert "preemption:" not in text, "a server that did not ask for parking announced it" + assert _ASYNC_BANNER not in text, "the async park path was set up without being asked for" + + assert any(res.status_code != 200 for res in _complete_all(160)) + + text = _log() + assert "Context size has been exceeded" in text + assert "preempted" not in text + assert "last resort" not in text, "the retry ladder consulted the planner" + assert "GGML_ASSERT" not in text + + metrics = _metrics() + assert metrics["n_preempt_total"] == 0 + assert metrics["preempt_ram_bytes"] == 0 + + after = _complete(8) + assert after.status_code == 200 + assert after.body["timings"]["predicted_n"] == 8 + + @pytest.mark.parametrize("planner", ["on", "off"]) def test_a_late_prompt_and_a_generating_slot_both_finish(planner): if planner == "off": diff --git a/tools/server/tests/unit/test_preempt_notify.py b/tools/server/tests/unit/test_preempt_notify.py index 84c9983259fd..5591b0b5a830 100644 --- a/tools/server/tests/unit/test_preempt_notify.py +++ b/tools/server/tests/unit/test_preempt_notify.py @@ -21,6 +21,8 @@ def create_server(): server = ServerPreset.tinyllama2() server.n_slots = 2 server.kv_unified = True + # the server parks only when asked: --preempt-ram defaults to 0, and this suite is about parking + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "8192" server.temperature = 0.0 server.seed = 42 fd, server.log_path = tempfile.mkstemp(suffix=".log") From 0b9caab1189cef2e31ca87c9610b9fe80a3ce495 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 07:00:59 +0000 Subject: [PATCH 06/22] Read src[5] as a page table only for the op that has one --- ggml/src/ggml-cuda/fattn-common.cuh | 12 +++++++++--- ggml/src/ggml-cuda/fattn.cu | 8 ++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index e2689c4f0032..5ab83a1f7c89 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -10,6 +10,12 @@ #define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction. #define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs. +// [TAG_EXACT_CONCURRENCY] the page table of the paged path, which only the ordinary flash +// attention op carries: another op is free to keep a tensor of its own in the same slot +static __forceinline__ const ggml_tensor * ggml_cuda_fattn_pages(const ggml_tensor * dst) { + return dst->op == GGML_OP_FLASH_ATTN_EXT ? dst->src[5] : nullptr; +} + // log(2) = 0.6931, by adding this to the KQ maximum used for the softmax the numerical range representable // by the VKQ accumulators is effectively being shifted up by a factor of 2. // This reduces issues with numerical overflow but also causes larger values to be flushed to zero. @@ -1108,7 +1114,7 @@ void launch_fattn( // multiple sequences of possibly different lengths. // [TAG_BATCH_INVARIANT] without this scan the KV loop runs to K->ne[1], which grows with the other sequences; the mask bounds it by the sequence's own extent const bool batch_invariant_KV_max = ggml_cuda_batch_invariant() != 0; - if (!use_sparse && !dst->src[5] && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || batch_invariant_KV_max)) { + if (!use_sparse && !ggml_cuda_fattn_pages(dst) && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || batch_invariant_KV_max)) { const int64_t s31 = mask->nb[1] / sizeof(half2); const int64_t s33 = mask->nb[3] / sizeof(half2); @@ -1166,7 +1172,7 @@ void launch_fattn( if (ntiles_dst % blocks_num.x != 0) { // Fixup is only needed if the SMs work on fractional tiles. dst_tmp_meta.alloc((size_t(blocks_num.x) * ncols * (2 + DV/2))); } - } else if (dst->src[5] || ggml_cuda_batch_invariant()) { + } else if (ggml_cuda_fattn_pages(dst) || ggml_cuda_batch_invariant()) { // [TAG_BATCH_INVARIANT] the KV split between blocks, and so the order the partials combine in, follows K->ne[1]: pin it to one block per tile parallel_blocks = 1; @@ -1247,7 +1253,7 @@ void launch_fattn( V_data, mask ? ((const char *) mask->data) : nullptr, sinks ? ((const char *) sinks->data) : nullptr, - dst->src[5] ? (const int *) dst->src[5]->data : KV_max.ptr, + ggml_cuda_fattn_pages(dst) ? (const int *) ggml_cuda_fattn_pages(dst)->data : KV_max.ptr, !stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr, scale, max_bias, m0, m1, n_head_log2, logit_softcap, Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 0182c031959a..9b12ef629655 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -744,13 +744,13 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ggml_cuda_set_device(ctx.device); - if (dst->src[5]) { + if (const ggml_tensor * pages = ggml_cuda_fattn_pages(dst)) { GGML_ASSERT(dst->src[0]->ne[0] == 256 && dst->src[2]->ne[0] == 256); GGML_ASSERT(dst->src[1]->type == GGML_TYPE_F16 && dst->src[2]->type == GGML_TYPE_F16); GGML_ASSERT(dst->src[3] && dst->src[0]->ne[3] == 1); - GGML_ASSERT(dst->src[5]->type == GGML_TYPE_I32 && ggml_is_contiguous(dst->src[5])); - GGML_ASSERT(dst->src[5]->ne[0] == 1 + dst->src[1]->ne[1]/FATTN_KQ_STRIDE); - GGML_ASSERT(dst->src[5]->ne[1] == dst->src[0]->ne[1]); + GGML_ASSERT(pages->type == GGML_TYPE_I32 && ggml_is_contiguous(pages)); + GGML_ASSERT(pages->ne[0] == 1 + dst->src[1]->ne[1]/FATTN_KQ_STRIDE); + GGML_ASSERT(pages->ne[1] == dst->src[0]->ne[1]); float softcap; memcpy(&softcap, (const float *) dst->op_params + 2, sizeof(softcap)); GGML_ASSERT(softcap == 0.0f); From 15bcc6c7a8bb5bbb732b904f28449c8be1fed2b9 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 07:01:00 +0000 Subject: [PATCH 07/22] Normalize sequence planes in every batch-invariant mode, not only under exact concurrency --- ggml/src/ggml-cuda/ggml-cuda.cu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index e3059965906b..ff5ce6c5fbf0 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -2002,7 +2002,8 @@ static bool ggml_cuda_mul_mat_split_columns( ggml_backend_cuda_context & ctx, int cc, int warp_size, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { // recurrent output projections broadcast one weight matrix over sequence planes, so normalize each plane before applying the column policy - if (ggml_cuda_exact_concurrency() && src0->ne[2] == 1 && src0->ne[3] == 1 && + // every mode owes the caller the batch-of-one column policy, and a plane the policy never sees is left batched + if (ggml_cuda_batch_invariant() && src0->ne[2] == 1 && src0->ne[3] == 1 && (dst->ne[2] > 1 || dst->ne[3] > 1) && src1->ne[2] == dst->ne[2] && src1->ne[3] == dst->ne[3]) { for (int64_t i3 = 0; i3 < dst->ne[3]; ++i3) { From 3e7d9530ac7cf9c21b30e97e6ca2901659cfda6c Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 12:40:03 +0000 Subject: [PATCH 08/22] Fix the CI legs that broke on the last rebase: a portable setenv in test-exact-pages, the OpenVINO support probe's struct return, and park assertions that match the park log lines rather than the slot JSON (cherry picked from commit 025286e3ee219d9477e2782ff79e28abdd199bd5) --- ggml/src/ggml-openvino/ggml-openvino.cpp | 2 +- tests/test-exact-pages.cpp | 16 ++++++++++++++-- tools/server/tests/unit/test_preempt.py | 16 ++++++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index e640e666138d..51cc2168ce72 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -1173,7 +1173,7 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { case GGML_OP_FLASH_ATTN_EXT: { // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads if (op->src[5]) { - return true; + return {false, "FLASH_ATTN_EXT with a page table is CUDA only"}; } float scale = 1.0f; float max_bias = 0.0f; diff --git a/tests/test-exact-pages.cpp b/tests/test-exact-pages.cpp index cee2188bdf70..1ac8829da52a 100644 --- a/tests/test-exact-pages.cpp +++ b/tests/test-exact-pages.cpp @@ -42,10 +42,22 @@ static bool decode_range(llama_context * ctx, llama_seq_id seq, llama_pos first, return ok; } +// Windows has no setenv +static void set_env_default(const char * name, const char * value) { + if (getenv(name)) { + return; + } +#ifdef _WIN32 + _putenv_s(name, value); +#else + setenv(name, value, 0); +#endif +} + int main(int argc, char ** argv) { // read before the model is loaded: both are latched on first use - setenv("LLAMA_EXACT_CONCURRENCY", "1", 0); - setenv("LLAMA_KV_CACHE_DEBUG", "1", 0); + set_env_default("LLAMA_EXACT_CONCURRENCY", "1"); + set_env_default("LLAMA_KV_CACHE_DEBUG", "1"); common_params params; diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index 7377b040228d..d4689a69bc96 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -100,6 +100,14 @@ def _assert_completed(results, n_predict: int): assert res.body["timings"]["predicted_n"] == n_predict +_PARK_MARKERS = ("preempted:", "preempted as a last resort", "preempted on request") + + +def _assert_nothing_parked(text: str): + # the park log lines, not the bare word: a verbose log prints every slot's "is_preempted" + assert not any(m in text for m in _PARK_MARKERS), "nothing could be parked here" + + def _assert_recovered(text: str, parked: str = "preempted:"): """Nothing was ended for want of cells: a slot was parked and came back.""" assert "Context size has been exceeded" not in text @@ -206,7 +214,7 @@ def test_a_request_that_cannot_be_helped_gets_the_context_error_and_the_server_l text = _log() assert "Context size has been exceeded" in text - assert "preempted" not in text, "nothing could be parked here" + _assert_nothing_parked(text) assert "GGML_ASSERT" not in text after = _complete(8) assert after.status_code == 200 @@ -226,7 +234,7 @@ def test_a_server_that_never_asked_for_parking_behaves_as_upstream(): text = _log() assert "Context size has been exceeded" in text - assert "preempted" not in text + _assert_nothing_parked(text) assert "last resort" not in text, "the retry ladder consulted the planner" assert "GGML_ASSERT" not in text @@ -418,7 +426,7 @@ def test_a_recurrent_model_is_served_without_preemption(): text = _log() assert "preemption: off, the recurrent cache holds one state per sequence" in text - assert "preempted" not in text + _assert_nothing_parked(text) assert "Context size has been exceeded" not in text @@ -754,7 +762,7 @@ def test_a_sibling_prompt_with_an_invalid_token_is_refused_before_anything_strea assert "invalid tokens" in str(res.body) text = _log() - assert "preempted" not in text + _assert_nothing_parked(text) def test_a_recompute_park_bounds_its_draft_by_the_tokens_it_comes_back_with(): From 2176de1adffe533396a1359d03fe8dd4a9ce1310 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 13:23:15 +0000 Subject: [PATCH 09/22] Build test-exact-buft only where llama-impl.h links, queue the started-slot follower behind a slot that is seen busy, and give the notify resident time to decode on a loaded runner (cherry picked from commit 752993177548be6d06f2f3a30a989b524db08f3d) --- tests/CMakeLists.txt | 5 ++-- tools/server/tests/unit/test_preempt.py | 28 ++++++++++++++----- .../server/tests/unit/test_preempt_notify.py | 4 ++- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b14f648c30fa..4508c0c002d4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -160,6 +160,8 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) llama_build_and_test(test-grammar-integration.cpp) llama_build_and_test(test-llama-grammar.cpp) llama_build_and_test(test-batch-alloc.cpp) + # [TAG_EXACT_CONCURRENCY] the buffer types the mode treats as invariant, through llama-impl.h + llama_build_and_test(test-exact-buft.cpp) llama_build_and_test(test-chat.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) target_include_directories(test-chat PRIVATE ${PROJECT_SOURCE_DIR}/tools/server) target_link_libraries(test-chat PRIVATE server-context) @@ -364,9 +366,8 @@ unset(LLAMA_TEST_NAME) llama_build_and_test(test-mtmd-impl.cpp) target_link_libraries(test-mtmd-impl PRIVATE mtmd) -# [TAG_EXACT_CONCURRENCY] the batch shape and the buffer types the mode requires, checked without a model +# [TAG_EXACT_CONCURRENCY] the batch shape the mode requires, checked without a model llama_build_and_test(test-exact-geometry.cpp) -llama_build_and_test(test-exact-buft.cpp) # server helpers that need no model if (LLAMA_BUILD_TOOLS) diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index d4689a69bc96..9e63ce79cbde 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -5,6 +5,7 @@ import struct import subprocess import threading +from concurrent.futures import ThreadPoolExecutor import time import tempfile import pytest @@ -87,6 +88,17 @@ def _complete_all(n_predict: int, prompts=(_PROMPT_A, _PROMPT_B)): return parallel_function_calls([(_complete, (n_predict, prompt)) for prompt in prompts]) +def _wait_processing(slot_ids, timeout: float = 30.0): + """Return once every one of these slots is processing; a request that ended first is a failure, not a hang.""" + deadline = time.time() + timeout + while time.time() < deadline: + slots = server.make_request("GET", "/slots").body + if all(any(s["id"] == i and s["is_processing"] for s in slots) for i in slot_ids): + return + time.sleep(0.005) + pytest.fail(f"slots {slot_ids} never showed as processing") + + def _prompt_of(n_tokens: int, text: str) -> list: """A prompt of exactly n_tokens tokens, as ids: no BOS is added to one of those.""" base = server.make_request("POST", "/tokenize", data={"content": text}).body["tokens"] @@ -392,18 +404,20 @@ def test_a_started_slot_is_counted_by_the_cells_it_holds_not_by_the_prompt_it_ke # the last request waits for slot 0 and is started on it holding the first request's cells; counted by the prompt it keeps instead, the pool looks free and a parked slot is restored into cells that are still taken _start(n_ctx=256, n_slots=3) - results = parallel_function_calls([ - (_complete, (60, _prompt_of(115, _PROMPT_C), 0)), - (_complete, (100, _PROMPT_A, 1)), - (_complete, (100, _PROMPT_B, 2)), - (_complete, (8, _PROMPT_C, 0, 0.0, 0)), - ]) + # queued behind a busy slot 0, so it starts on the cells the first request keeps while the two long ones still want the pool; polling for a busy slot 0 with all four in flight missed a short first request on a Windows runner and sent the follower into an idle pool + with ThreadPoolExecutor(3) as pool: + long_ones = [pool.submit(_complete, 200, _PROMPT_A, 1), pool.submit(_complete, 200, _PROMPT_B, 2)] + _wait_processing([1, 2]) + first = pool.submit(_complete, 100, _prompt_of(115, _PROMPT_C), 0) + _wait_processing([0]) + follower = _complete(8, _PROMPT_C, 0) + results = [first.result(), long_ones[0].result(), long_ones[1].result(), follower] text = _log() assert "trimmed to the" in text, "the started slot kept the cells of the request before it" assert "resume failed" not in text assert "Context size has been exceeded" not in text - for res, n_predict in zip(results, (60, 100, 100, 8)): + for res, n_predict in zip(results, (100, 200, 200, 8)): assert res.status_code == 200, res.body assert res.body["timings"]["predicted_n"] == n_predict diff --git a/tools/server/tests/unit/test_preempt_notify.py b/tools/server/tests/unit/test_preempt_notify.py index 5591b0b5a830..938e538018a9 100644 --- a/tools/server/tests/unit/test_preempt_notify.py +++ b/tools/server/tests/unit/test_preempt_notify.py @@ -172,7 +172,9 @@ def _resident(): t.start() # the pool has to be nearly full before the second prompt starts, so that its prefill is what runs out of cells - for _ in range(600): + # the resident grows by decoding: 1400 tokens took over 12 s on a loaded CI runner + deadline = time.time() + 90 + while t.is_alive() and time.time() < deadline: slots = requests.get(f"http://{server.server_host}:{server.server_port}/slots").json() if any(slot.get("n_prompt_tokens", 0) >= 1400 for slot in slots): break From d0202ea4e41059cd4b814f3f1e00b2ebff7b51c9 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 13:59:08 +0000 Subject: [PATCH 10/22] Condense the exact-mode weight check comment so it is not hard-wrapped (cherry picked from commit 8ec97d690c169d18e0d8532b8b3075266076c03d) --- src/llama-context.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index a58983ff81e3..430740bf9a8a 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -33,17 +33,8 @@ static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) { throw std::runtime_error("Unsupported ctx type"); } -// [TAG_EXACT_CONCURRENCY] the caches check where the KV lives; this checks where the weights that -// produce the tokens live. Every per-layer weight and the output head must be on a backend with -// the mode's kernels, otherwise a sequence's own matmuls change with the width of the step it -// shares, while the mode still reports itself as on. -// -// token_embd is deliberately not required to move: it feeds get_rows, a per-row copy, and -// GET_ROWS reports a batch size of 0 to the offload test, so it stays on the same backend at -// every width. A model that ties its head to the embedding uses that same tensor for the output -// matmul, and model.output points at it, so the head check below still covers that case. A lora -// that adapts it is applied with a mul_mat instead, and MUL_MAT reports the ubatch width, so -// llama_adapter_lora_init_impl() refuses one that inherits a host buffer. +// [TAG_EXACT_CONCURRENCY] the caches check where the KV lives; this checks the weights. Every per-layer weight and the output head must sit on a backend with the mode's kernels, else a sequence's own matmuls change with the width of the step it shares. +// token_embd is exempt: it feeds GET_ROWS, a per-row copy that reports a batch size of 0 to the offload test, so it stays put at every width. A tied head is that same tensor and model.output points at it, so the head check covers it; a lora on it runs as MUL_MAT, which llama_adapter_lora_init_impl() refuses on a host buffer. static void llama_exact_check_weights(const llama_model & model) { auto host_buft = [](const ggml_tensor * t) -> ggml_backend_buffer_type_t { if (!t || !t->buffer) { From a05a31816180ecbb5a98fc273f0af0cc50610d8c Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 14:09:35 +0000 Subject: [PATCH 11/22] tests : give the rotation test time to finish and stop the deliberate disconnect from raising test_a_resident_cycling_through_context_shifts_is_rotated_out_for_a_parked_head ran into the 600 s default request timeout on the ubuntu Server leg: the server was healthy and the third request came back with all 9000 tokens, the other two were still generating when the client gave up. The rotation is triggered by a 2 s wait, not by a token count, so the generation cannot be shortened without losing the rotation on a fast host; the requests get a longer timeout instead. test_a_resident_that_cannot_be_swapped_out_is_rotated_by_recompute stops the server with a request still in flight on purpose. Its thread raised the resulting ConnectionError as an unhandled thread exception, which is the noise that made the timeout above look like a crash. (cherry picked from commit 9bcd8ab21e66df43a7628f73f0263d731d8722bb) --- tools/server/tests/unit/test_preempt.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index 9e63ce79cbde..7792b087420e 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -69,7 +69,8 @@ def _require_async(text: str): pytest.skip("this backend cannot copy asynchronously, the async park path is not exercised") -def _complete(n_predict: int, prompt="Hi how are you", id_slot: int = -1, delay: float = 0.0, after_slot_busy=None): +def _complete(n_predict: int, prompt="Hi how are you", id_slot: int = -1, delay: float = 0.0, after_slot_busy=None, + timeout: float = DEFAULT_REQUEST_TIMEOUT): time.sleep(delay) if after_slot_busy is not None: # sent once that slot is processing, so the request queues behind it whatever the host's speed @@ -81,11 +82,11 @@ def _complete(n_predict: int, prompt="Hi how are you", id_slot: int = -1, delay: return server.make_request("POST", "/completion", data={ "n_predict": n_predict, "prompt": prompt, "id_slot": id_slot, "ignore_eos": True, "return_tokens": True, "temperature": 0.0, "seed": 42, - }) + }, timeout=timeout) -def _complete_all(n_predict: int, prompts=(_PROMPT_A, _PROMPT_B)): - return parallel_function_calls([(_complete, (n_predict, prompt)) for prompt in prompts]) +def _complete_all(n_predict: int, prompts=(_PROMPT_A, _PROMPT_B), timeout: float = DEFAULT_REQUEST_TIMEOUT): + return parallel_function_calls([(_complete, (n_predict, prompt, -1, 0.0, None, timeout)) for prompt in prompts]) def _wait_processing(slot_ids, timeout: float = 30.0): @@ -303,8 +304,9 @@ def test_a_resident_cycling_through_context_shifts_is_rotated_out_for_a_parked_h # with context shift on a resident would hold its cells for as long as it generates, so once the head has waited its turn the resident is parked and the two take turns _start(n_slots=3, n_ctx=384, enable_ctx_shift=True) + # the rotation waits on a clock, not on a token count, so the generation has to be long enough on a fast host; that is a lot of tokens for a slow one, and it takes turns with two others, so it is given more than the usual wait n_predict = 9000 - _assert_completed(_complete_all(n_predict, (_PROMPT_A, _PROMPT_B, _PROMPT_C)), n_predict) + _assert_completed(_complete_all(n_predict, (_PROMPT_A, _PROMPT_B, _PROMPT_C), timeout=1800), n_predict) text = _log() _assert_recovered(text, "rotated out after") @@ -338,8 +340,16 @@ def unending_request(): "ignore_eos": True, "temperature": 0.0, "seed": 42, }, timeout=600) + # the server is stopped below with this request still in flight, so the disconnect it then sees is expected and must not surface as an unhandled thread exception unending = [] - t = threading.Thread(target=lambda: unending.append(unending_request())) + + def run_unending(): + try: + unending.append(unending_request()) + except requests.exceptions.RequestException: + pass + + t = threading.Thread(target=run_unending) t.start() try: From f8b7514d494c58492d9d65625790c13f7761d491 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 14:11:19 +0000 Subject: [PATCH 12/22] server : keep the tokens a recompute restore replays out of the request's prompt stats A recompute restore re-enters prompt processing to put back the cells the park dropped. Those tokens are submitted with is_prompt set, so every one of them was added to n_prompt_processed and pushed t_prompt_last to the end of the replay, while n_gen deliberately carries across the park. The reported prompt length grew with every park, and the generation time covered only the tokens after the last re-prefill, so predicted_per_second was inflated by the ratio of the two. The replay is still counted in the server-wide prompt metrics, where it is real compute; it just no longer moves the slot's prompt count or the prompt/generation boundary. (cherry picked from commit d3833b025ddca8b73cf78abbd418f8c7b9537900) --- tools/server/server-context.cpp | 14 ++++++++++---- tools/server/tests/unit/test_preempt.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a4d11f55f163..c02f858f8168 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -5177,8 +5177,12 @@ struct server_context_impl { } metrics_queue_prompt(n_tokens_out); - slot.stats.n_prompt_processed += n_tokens_out; - slot.stats.update_prompt_last(); + + // [TAG_PREEMPT] a re-prefill puts back what a park dropped: real compute, counted above, but it is not the request's prompt + if (!slot.preempt_reprefill) { + slot.stats.n_prompt_processed += n_tokens_out; + slot.stats.update_prompt_last(); + } // add the mtmd chunk to cache { @@ -5918,7 +5922,8 @@ struct server_context_impl { n_prompt_tokens++; auto & slot = slots[t.id_slot]; - if (slot.stats.is_set()) { + // [TAG_PREEMPT] replayed tokens stay out of the slot's prompt count, they were counted when the request first processed its prompt + if (slot.stats.is_set() && !slot.preempt_reprefill) { slot.stats.n_prompt_processed++; } } @@ -5936,7 +5941,8 @@ struct server_context_impl { for (int i = off; i < off + n_tokens; ++i) { const auto & t = batch.tokens[i]; auto & slot = slots[t.id_slot]; - if (t.is_prompt && slot.stats.is_set()) { + // [TAG_PREEMPT] a re-prefill must not move the prompt/generation boundary: n_gen carries across the park, so the generation time would then cover only the tokens after it + if (t.is_prompt && slot.stats.is_set() && !slot.preempt_reprefill) { slot.stats.set_prompt_last(t_now); } } diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index 7792b087420e..48828e72a9b8 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -878,6 +878,26 @@ def test_a_recompute_park_is_reported_to_the_client_and_to_metrics(): assert plain.body["preempt"] == {"parks": 0, "recomputes": 0} +def test_a_recompute_restore_does_not_count_its_replay_as_prompt(): + # the re-prefill puts back what the park dropped: counted as prompt it would move the prompt/generation boundary, and since n_gen carries across the park the generation time would then cover only the tokens after the last re-prefill + os.environ["LLAMA_ARG_PREEMPT_RAM"] = "1" + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + _start(n_ctx=2048, n_batch=2048) + + n_prompt = 1950 + n_predict = 24 + res = _complete(n_predict, _prompt_of(n_prompt, _PROMPT_C)) + + assert res.status_code == 200, res.body + assert res.body["preempt"]["recomputes"] >= 1, res.body["preempt"] + + timings = res.body["timings"] + assert timings["prompt_n"] == n_prompt, timings + assert timings["predicted_n"] == n_predict, timings + # every re-prefill happens inside the generation, so the generation holds the longer time of the two + assert timings["predicted_ms"] > timings["prompt_ms"], timings + + def test_slots_reports_the_recomputes_of_the_current_task(): # a reader watching the slots sees the same count the request is given at the end os.environ["LLAMA_ARG_PREEMPT_RAM"] = "1" From 0beed5117d54d7f99f7b3ec41fe8248b018ae4c6 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 14:11:32 +0000 Subject: [PATCH 13/22] server : put the preemption record inside the completed response of a streamed /v1/responses A non-streamed /v1/responses carries preempt in the response object, next to usage. The streamed one wrote it on the SSE data beside the response object, so a client that keeps the response of the response.completed event, which is the object the OpenAI SDK hands back, never saw it. It now sits in the same place either way, and unconditionally, as the non-streamed body already did. (cherry picked from commit e4168eb6a5eae4620535b1f1c353af40ba740e48) --- tools/server/README.md | 2 +- tools/server/server-task.cpp | 5 ++-- tools/server/tests/unit/test_preempt.py | 36 +++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/tools/server/README.md b/tools/server/README.md index f5ca9f30f296..923d380af2b9 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -677,7 +677,7 @@ These words will not be included in the completion, so make sure to add them to - `tokens_cached`: Number of tokens from the prompt which could be re-used from previous completion - `tokens_evaluated`: Number of tokens evaluated in total from the prompt - `truncated`: Boolean indicating if the context size was exceeded during generation, i.e. the number of tokens provided in the prompt (`tokens_evaluated`) plus tokens generated (`tokens predicted`) exceeded the context size (`n_ctx`) -- `preempt`: How the request was served while the unified KV cache was full (see `--preempt-ram`). `parks` is how often the request was parked to make room for another, and `recomputes` is how many of those parks dropped the sequence's cells because `--preempt-ram` was spent, so that the resume re-prefilled its tokens instead of restoring the bytes that were saved. A re-prefilled sequence continues from the same tokens, but its numerics are not guaranteed identical to the sequence that left, `LLAMA_EXACT_CONCURRENCY` included: raise `--preempt-ram` until `recomputes` stays 0 where that matters. Both fields are present in the final response of a streamed completion as well. +- `preempt`: How the request was served while the unified KV cache was full (see `--preempt-ram`). `parks` is how often the request was parked to make room for another, and `recomputes` is how many of those parks dropped the sequence's cells because `--preempt-ram` was spent, so that the resume re-prefilled its tokens instead of restoring the bytes that were saved. A re-prefilled sequence continues from the same tokens, but its numerics are not guaranteed identical to the sequence that left, `LLAMA_EXACT_CONCURRENCY` included: raise `--preempt-ram` until `recomputes` stays 0 where that matters. Both fields are present in the final response of a streamed completion as well, and on the OpenAI-compatible endpoints: the final chunk of a streamed `/v1/chat/completions`, the `message_delta` event of `/v1/messages`, and the response object of `/v1/responses` streamed or not, which in a stream is the `response` of the `response.completed` event. While a request is streaming, the server sends SSE comment lines that a client reading raw lines can act on and every SSE event consumer ignores: diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index e9fc854c7961..5c2376b1fc2d 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -714,14 +714,15 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() { {"output_tokens", n_decoded}, {"total_tokens", n_decoded + n_prompt_tokens}, {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }}, - }} + }}, + // [TAG_PREEMPT] inside the response object, where the non-streaming body carries it: that object is what a client keeps from the stream + {"preempt", preempt_to_json()}, }}, }} }); if (stats.is_set()) { server_sent_events.back().at("data")["timings"] = stats.to_json(); - server_sent_events.back().at("data")["preempt"] = preempt_to_json(); } return server_sent_events; diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index 48828e72a9b8..daef40f99ccb 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -940,6 +940,42 @@ def test_a_swap_park_is_not_reported_as_a_recompute(): assert _metrics()["preempt_recompute_total"] == 0 +def _stream_responses(n_predict: int, prompt: str) -> dict: + """One streaming /v1/responses request: the data of its response.completed event.""" + url = f"http://{server.server_host}:{server.server_port}/v1/responses" + res = requests.post(url, json={ + "model": "test", "input": prompt, "max_output_tokens": n_predict, + "temperature": 0.0, "stream": True, + }, stream=True, timeout=600) + assert res.status_code == 200, res.text + completed = None + for raw in res.iter_lines(): + line = raw.decode("utf-8") + if line.startswith("data: "): + data = json.loads(line[6:]) + if data.get("type") == "response.completed": + completed = data + assert completed is not None, "the stream never reached response.completed" + return completed + + +def test_a_streamed_response_carries_the_preempt_record_where_a_plain_one_does(): + # what a client keeps from a streamed /v1/responses is data["response"], so the record has to be in that object, the same place the non-streamed body carries it + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + _start(n_ctx=512) + + completed = _stream_responses(24, _PROMPT_A) + + assert completed["response"]["preempt"]["parks"] >= 1, completed["response"] + assert completed["response"]["preempt"]["recomputes"] == 0, completed["response"] + + plain = server.make_request("POST", "/v1/responses", data={ + "model": "test", "input": _PROMPT_B, "max_output_tokens": 4, "temperature": 0.0, + }) + assert plain.status_code == 200, plain.body + assert sorted(plain.body["preempt"]) == sorted(completed["response"]["preempt"]) == ["parks", "recomputes"] + + def test_two_image_chats_that_outgrow_the_parking_budget_both_finish(): # a media chunk could not be parked by recompute, so with the host budget spent nothing could be parked at all and the pool overflowing ended both chats. The chunk comes back the way it went in: re-encoded off the task, its cells reserved whole os.environ["LLAMA_MEDIA_MARKER"] = "<__media__>" From d040ae7c7fd5656291008ccaddbf2964b0f22076 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 15:17:57 +0000 Subject: [PATCH 14/22] tests : condense the exact-pages preamble so it is not hard-wrapped (cherry picked from commit 8f2a95dca5065a00ba4b574517e6714a720d2c32) --- tests/test-exact-pages.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/test-exact-pages.cpp b/tests/test-exact-pages.cpp index 1ac8829da52a..e9cdef82ffe7 100644 --- a/tests/test-exact-pages.cpp +++ b/tests/test-exact-pages.cpp @@ -1,13 +1,5 @@ -// [TAG_EXACT_CONCURRENCY] page bookkeeping of the paged KV pool: a removal that empties nothing, -// a removal that leaves holes, and the pages those holes keep reserved. -// -// LLAMA_KV_CACHE_DEBUG=1 makes the pool rebuild its page ownership from the live cells on every -// ubatch and assert that it says what the incrementally maintained one says, so this test drives -// the removal paths and lets that oracle check them. -// -// The mode needs a CUDA (or ROCm/MUSA) build, 256-wide K and V heads and a fully offloaded F16 KV -// cache. Where the context cannot be created the test reports what it skipped and passes: it has -// nothing to say about a build without those. +// [TAG_EXACT_CONCURRENCY] drives the removal paths of the paged KV pool - a removal that empties nothing, one that leaves holes, and the pages those holes keep reserved - while LLAMA_KV_CACHE_DEBUG=1 makes the pool cross-check page ownership against the live cells every ubatch. +// Needs a CUDA build with 256-wide heads and a fully offloaded F16 cache; without one the test reports what it skipped and passes. #include "arg.h" #include "common.h" From c5a89f7f7f00624a6fd74614c6e55e1dda88fb13 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 15:28:08 +0000 Subject: [PATCH 15/22] llama : refuse an asynchronous state transfer for a model whose states move between rows (cherry picked from commit 6d62ad77b3bec02af15a74d2d0e6d0f12746a957) --- include/llama.h | 1 + src/llama-context.cpp | 6 ++++++ tests/test-state-seq-copy.cpp | 7 +++++++ 3 files changed, 14 insertions(+) diff --git a/include/llama.h b/include/llama.h index 5cba47653298..fc5c2a2abac3 100644 --- a/include/llama.h +++ b/include/llama.h @@ -957,6 +957,7 @@ extern "C" { struct llama_state_seq_copy; // NULL when the backends cannot copy asynchronously, or cannot say whether a copy has finished without waiting for it; the caller then uses the synchronous calls + // Also NULL for a recurrent or hybrid model: its states move between rows on every decode, so a transfer beside a decode can read another sequence LLAMA_API struct llama_state_seq_copy * llama_state_seq_copy_init(struct llama_context * ctx); LLAMA_API void llama_state_seq_copy_free(struct llama_state_seq_copy * cpy); diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 430740bf9a8a..34d3db0ccea4 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -3701,6 +3701,12 @@ void llama_context::state_seq_copy_fence() { } llama_state_seq_copy * llama_context::state_seq_copy_init() { + // [TAG_STATE_ASYNC] a recurrent state keeps no fixed row: find_slot() gathers the live rows together, so a decode beside a transfer moves or overwrites the row the transfer reads. A hybrid carries that half too + if (llm_arch_is_recurrent(model.arch) || llm_arch_is_hybrid(model.arch)) { + LLAMA_LOG_INFO("%s: this model moves sequence states between rows, so they are copied synchronously\n", __func__); + return nullptr; + } + std::unique_ptr<llama_state_seq_copy> cpy(new llama_state_seq_copy()); cpy->ctx = this; diff --git a/tests/test-state-seq-copy.cpp b/tests/test-state-seq-copy.cpp index bd3cd9cc30a2..7a322fd20543 100644 --- a/tests/test-state-seq-copy.cpp +++ b/tests/test-state-seq-copy.cpp @@ -63,6 +63,13 @@ int main(int argc, char ** argv) { llama_state_seq_copy * cpy = llama_state_seq_copy_init(ctx); + // a recurrent state does not stay in one row, so these models are refused a transfer whatever the backend can do + if (llama_model_is_recurrent(llama_init->model()) || llama_model_is_hybrid(llama_init->model())) { + CHECK(cpy == nullptr); + fprintf(stderr, "%s : a recurrent or hybrid model is refused a transfer, as it must be\n", __func__); + return 0; + } + if (cpy == nullptr) { fprintf(stderr, "%s : this backend cannot copy sequence states asynchronously, skipping\n", __func__); return 0; From 7eb9cb780741dda641c321ccf16daa84725ee4e9 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 15:28:08 +0000 Subject: [PATCH 16/22] tests : drive the started-slot trim from the prompt lengths instead of the host's speed (cherry picked from commit 9aabc85448bf2c76bd423185601016802e117e57) --- tools/server/tests/unit/test_preempt.py | 34 ++++++++++++++++++------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index daef40f99ccb..44c4df3ce0a3 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -100,6 +100,17 @@ def _wait_processing(slot_ids, timeout: float = 30.0): pytest.fail(f"slots {slot_ids} never showed as processing") +def _wait_preempted(timeout: float = 30.0) -> bool: + """True once some slot is parked: its cells are in host RAM and it wants them back.""" + deadline = time.time() + timeout + while time.time() < deadline: + slots = server.make_request("GET", "/slots").body + if any(s["is_preempted"] for s in slots): + return True + time.sleep(0.005) + return False + + def _prompt_of(n_tokens: int, text: str) -> list: """A prompt of exactly n_tokens tokens, as ids: no BOS is added to one of those.""" base = server.make_request("POST", "/tokenize", data={"content": text}).body["tokens"] @@ -412,22 +423,27 @@ def test_cancel_while_a_copy_is_in_flight_frees_the_slot(): def test_a_started_slot_is_counted_by_the_cells_it_holds_not_by_the_prompt_it_keeps(): # the last request waits for slot 0 and is started on it holding the first request's cells; counted by the prompt it keeps instead, the pool looks free and a parked slot is restored into cells that are still taken - _start(n_ctx=256, n_slots=3) + _start(n_ctx=1024, n_slots=3) + + # the lengths, not the host's speed, decide who is parked: the three prompts (500 + 200 + 200) fit the 1024 cells, so both of the others are parked holding at least their whole prompt once slot 0 grows into the rest, and slot 0 is the largest slot throughout, which the planner never picks as a victim. Slot 0 ends holding 960 of the 1024 cells, too few left for either parked slot to come back + ids = _prompt_of(500, _PROMPT_C) - # queued behind a busy slot 0, so it starts on the cells the first request keeps while the two long ones still want the pool; polling for a busy slot 0 with all four in flight missed a short first request on a Windows runner and sent the follower into an idle pool - with ThreadPoolExecutor(3) as pool: - long_ones = [pool.submit(_complete, 200, _PROMPT_A, 1), pool.submit(_complete, 200, _PROMPT_B, 2)] - _wait_processing([1, 2]) - first = pool.submit(_complete, 100, _prompt_of(115, _PROMPT_C), 0) + with ThreadPoolExecutor(4) as pool: + first = pool.submit(_complete, 460, ids, 0) _wait_processing([0]) - follower = _complete(8, _PROMPT_C, 0) - results = [first.result(), long_ones[0].result(), long_ones[1].result(), follower] + # queued behind slot 0 whatever the host's speed, and a real prefix of what slot 0 holds: it starts on 960 cells while keeping 8 of them + follower = pool.submit(_complete, 8, ids[:8], 0) + long_ones = [pool.submit(_complete, 400, _prompt_of(200, _PROMPT_A), 1), + pool.submit(_complete, 400, _prompt_of(200, _PROMPT_B), 2)] + parked = _wait_preempted() + results = [first.result(), follower.result(), long_ones[0].result(), long_ones[1].result()] text = _log() + assert parked, "the pool never came under pressure, so no slot was waiting for the cells slot 0 keeps" assert "trimmed to the" in text, "the started slot kept the cells of the request before it" assert "resume failed" not in text assert "Context size has been exceeded" not in text - for res, n_predict in zip(results, (100, 200, 200, 8)): + for res, n_predict in zip(results, (460, 8, 400, 400)): assert res.status_code == 200, res.body assert res.body["timings"]["predicted_n"] == n_predict From d8daa8fc514ac964eff151bff299c9fe8bfda131 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 16:29:13 +0000 Subject: [PATCH 17/22] tests : make the failed-save park a matter of lengths rather than of two requests overlapping (cherry picked from commit d14f07b8d3929a448de3e76f331affeb3d4d0f10) --- tools/server/tests/unit/test_preempt.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index 44c4df3ce0a3..46b8218d604f 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -327,16 +327,24 @@ def test_a_resident_cycling_through_context_shifts_is_rotated_out_for_a_parked_h def test_a_park_whose_host_allocation_fails_is_parked_by_recompute(): # the budget grants permission to allocate, not a successful allocation: a failed save used to stop the planner and leave the pool to overflow, although the same victim could be parked by dropping its cells os.environ["LLAMA_SERVER_PREEMPT_FAIL_SAVE"] = "1" - _start(n_ctx=256) + _start(n_ctx=1024, n_slots=2) - n_predict = 160 - results = _complete_all(n_predict) + # lengths decide the overlap, not the host's speed: two 171-cell requests fired together did not overlap on a Windows runner, so nothing was parked. The leader ends at 960 of 1024 cells, so the second is parked whatever the client's lag + leader = _prompt_of(500, _PROMPT_A) + other = _prompt_of(200, _PROMPT_B) + with ThreadPoolExecutor(1) as pool: + first = pool.submit(_complete, 460, leader, 0) + _wait_processing([0]) + second = _complete(400, other, 1) + results = [first.result(), second] text = _log() assert "could not take the host memory" in text, "the injected allocation failure never fired" assert "tokens to re-prefill" in text, "the failed save did not fall back to recompute" assert "Context size has been exceeded" not in text - _assert_completed(results, n_predict) + for res, n_predict in zip(results, (460, 400)): + assert res.status_code == 200, res.body + assert res.body["timings"]["predicted_n"] == n_predict def test_a_resident_that_cannot_be_swapped_out_is_rotated_by_recompute(): From 5b66acb4d7904f231d264f685fca13cc3c840ea1 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 17:40:53 +0000 Subject: [PATCH 18/22] server : reset the prompt counters when a recompute park restarts a prefill (cherry picked from commit 535671031558a71c4a4c0fcde581fbc37301e320) --- tools/server/server-context.cpp | 4 +++ tools/server/tests/unit/test_preempt.py | 40 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c02f858f8168..dfbb687737db 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -593,6 +593,10 @@ struct server_slot { if (preempt_tokens.empty()) { state = state_before_preempt; // its prompt had not been processed yet, so it is processed again from the start + // the park dropped the cells the prompt step had already filled, and the restart does not pass through SLOT_STATE_STARTED, where these two are set: left alone they would count the dropped prefix a second time + stats.n_prompt_cached = 0; + stats.n_prompt_processed = 0; + return true; } diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index 46b8218d604f..ee691ee71173 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -1025,3 +1025,43 @@ def test_two_image_chats_that_outgrow_the_parking_budget_both_finish(): for res in results: assert res.status_code == 200, res.body assert res.body["tokens_predicted"] == n_predict + + +def test_a_park_during_the_prefill_does_not_count_the_restarted_prompt_twice(): + # a park with no budget for the state drops the cells of a prompt that is still being processed, and the resume starts that prefill again from nothing: what it had counted before the park has to go with the cells, or the request reports more prompt tokens than it has + # the park is made to fail its host allocation, so it drops the cells instead: the state of a half processed prompt is small enough to fit any budget + os.environ["LLAMA_SERVER_PREEMPT_FAIL_SAVE"] = "1" + _start(n_ctx=2048, n_batch=256) + + resident = [] + t = threading.Thread(target=lambda: resident.append(_complete(1900, _PROMPT_A)), daemon=True) + t.start() + + # the pool has to be nearly full before the second prompt starts, so that it is that prefill which runs out of cells + deadline = time.time() + 90 + while t.is_alive() and time.time() < deadline: + slots = server.make_request("GET", "/slots").body + if any(slot.get("n_prompt_tokens", 0) >= 1600 for slot in slots): + break + time.sleep(0.005) + else: + pytest.fail("the resident never grew into the pool") + + n_prompt = 500 + n_predict = 8 + comments, final = _stream_completion(n_predict, _prompt_of(n_prompt, _PROMPT_B)) + t.join(120) + + assert resident and resident[0].status_code == 200, resident + assert "error" not in final, final + assert comments and comments[0] == ": preempted", comments + + # a park that dropped fewer cells than the resume has tokens to put back is a park taken mid-prefill, which is the case this test is about + parks = [(int(cells), int(again)) for cells, again in re.findall( + r"preempted: (\d+) cells dropped .*? (\d+) tokens to re-prefill", _log())] + assert any(0 < cells < again for cells, again in parks), parks + + timings = final["timings"] + assert timings["prompt_n"] == n_prompt, timings + assert timings["cache_n"] == 0, timings + assert timings["predicted_n"] == n_predict, timings From b4433f5665f5c2fcc1a217454560a8d43b31a445 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 18:53:42 +0000 Subject: [PATCH 19/22] server : number the preempt notices of every task of a batched request, index 0 included (cherry picked from commit d36213ce31c9500d914f66c14a3edab627092800) --- tools/server/README.md | 2 +- tools/server/server-context.cpp | 3 ++- tools/server/server-queue.cpp | 8 ++++++-- tools/server/server-task.h | 4 ++++ tools/server/tests/unit/test_preempt_notify.py | 15 +++++++++++++++ 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tools/server/README.md b/tools/server/README.md index 923d380af2b9..383aafbfd739 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -688,7 +688,7 @@ While a request is streaming, the server sends SSE comment lines that a client r A park can happen while the prompt is still being processed, before the request has produced a token. The notice is not held back for the first chunk in that case: the response headers and the `: preempted` line go out at the moment the slot is parked, on every streaming surface (`/completion`, `/v1/chat/completions`, `/v1/responses`, `/v1/messages`), so a client never has to tell that silence from a stall. `: resumed`, and `: recomputed` where it applies, follow when the slot runs again. -With more than one prompt in the request, the index of the prompt follows the word, for example `: resumed 1`. +When the request asks for more than one completion, either several prompts or `n` above one, the index of the completion follows the word, for example `: resumed 1`, including index `0`. A request with a single completion carries no index. ### POST `/tokenize`: Tokenize a given text diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index dfbb687737db..9d13145138b5 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -117,7 +117,7 @@ constexpr int64_t PREEMPT_KEEPALIVE_MS = 2000; // SSE keepalive period while a s constexpr int32_t PREEMPT_N_STARVED = 3; // preemptions after which a slot is protected static std::string preempt_notice_comment(const server_task_result_preempt_notice & notice) { - const std::string suffix = (notice.index > 0 ? " " + std::to_string(notice.index) : "") + "\n\n"; + const std::string suffix = (notice.batched ? " " + std::to_string(notice.index) : "") + "\n\n"; std::string res = (notice.parked ? ": preempted" : ": resumed") + suffix; @@ -2684,6 +2684,7 @@ struct server_context_impl { res->parked = parked; res->recomputed = recomputed; res->n_preempt = slot.n_preempt; + res->batched = slot.task->batched; queue_results.send(std::move(res)); } diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp index b5c8ab4a8ace..c555e1856c2c 100644 --- a/tools/server/server-queue.cpp +++ b/tools/server/server-queue.cpp @@ -530,12 +530,16 @@ void server_response_reader::post_tasks(std::vector<server_task> && tasks, bool id_tasks = server_task::get_list_id(tasks); states.reserve(tasks.size()); size_t index = 0; + // [TAG_PREEMPT] several prompts, or several completions of one prompt, all number their results, and their preempt notices have to say which one they belong to + const bool batched = id_tasks.size() > 1; for (auto & task : tasks) { - task.index = index++; + task.index = index++; + task.batched = batched; states.push_back(task.create_state()); // for child tasks for (auto & child_task : task.child_tasks) { - child_task.index = index++; + child_task.index = index++; + child_task.batched = batched; states.push_back(child_task.create_state()); } } diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 0d852757eca7..c5dd2206108e 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -139,6 +139,9 @@ struct server_task { // TODO @ngxson : remove this field and implement a mapping task_id -> idx in the response_reader size_t index = 0; // used when there are multiple prompts (batch request) + // [TAG_PREEMPT] this request yielded more than one task, so index tells its results apart and the preempt notices carry it + bool batched = false; + // used by SERVER_TASK_TYPE_CANCEL int id_target = -1; int id_slot = -1; @@ -403,6 +406,7 @@ struct server_task_result_preempt_notice : server_task_result { bool parked = false; // true when the slot was just parked, false when restored bool recomputed = false; // this resume re-prefilled its tokens instead of restoring saved bytes int32_t n_preempt = 0; // how many times this task has been parked so far + bool batched = false; // one of several tasks of its request, so the notice names which one by index virtual bool is_stop() override { return false; diff --git a/tools/server/tests/unit/test_preempt_notify.py b/tools/server/tests/unit/test_preempt_notify.py index 938e538018a9..d28a640260b8 100644 --- a/tools/server/tests/unit/test_preempt_notify.py +++ b/tools/server/tests/unit/test_preempt_notify.py @@ -1,5 +1,6 @@ import json import os +import re import tempfile import threading import pytest @@ -259,6 +260,20 @@ def test_a_rotation_tells_both_streams_and_a_head_parked_past_the_budget_is_kept assert "Context size has been exceeded" not in text +def test_every_notice_of_a_multi_prompt_stream_names_the_prompt_it_is_about(): + # one request, two prompts: a client reading the shared stream can only tell the notices apart by their index, so index 0 has to be spelled out like any other + os.environ["LLAMA_SERVER_PREEMPT_EVERY"] = "8" + _start(n_ctx=512) + + comments, datas = _stream_raw("/completion", _completion_payload(32) | {"prompt": [_PROMPT_A, _PROMPT_B]}) + notices = [c for c in comments if c.startswith(": preempted") or c.startswith(": resumed")] + assert notices, comments + assert all(re.fullmatch(r": (preempted|resumed) [01]", c) for c in notices), notices + for index in (0, 1): + assert f": preempted {index}" in notices, notices + assert f": resumed {index}" in notices, notices + + def test_an_oversized_sibling_prompt_is_errored_before_a_valid_one_is_parked(): # a request can carry several prompts; a valid one can be parked and its notice opens the stream, so the sibling that does not fit has to be found before any of them is queued os.environ["LLAMA_ARG_PREEMPT_RAM"] = "8192" From 41abdfb23111fbcbf1bace38fb4655ed8cc7cc72 Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 19:09:29 +0000 Subject: [PATCH 20/22] tests : let the server log settle before asserting on its text (cherry picked from commit 7a32e3b45c98d0286bc241d5b4bdcc0c93f150cb) --- tools/server/tests/unit/test_preempt.py | 15 +++++++++++--- .../server/tests/unit/test_preempt_notify.py | 20 +++++++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index ee691ee71173..c4809b724716 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -61,7 +61,16 @@ def _start_async(**kwargs): def _log() -> str: - return open(server.log_path).read() + """The server log once its writer thread has stopped growing it: on a loaded host the log lags the response that came from it.""" + deadline = time.time() + 5.0 + last = -1 + while time.time() < deadline: + size = os.path.getsize(server.log_path) + if size == last: + break + last = size + time.sleep(0.1) + return open(server.log_path, errors="replace").read() def _require_async(text: str): @@ -688,14 +697,14 @@ def test_exact_concurrency_prefills_a_prompt_in_the_ubatches_it_would_get_alone( _start(n_ctx=16384, n_slots=4, n_batch=2048, n_ubatch=512, fa="on", n_gpu_layer=99, cache_ram=0) def prefill(first_token: int) -> list: - mark = len(open(server.log_path, errors="replace").read()) + mark = len(_log()) res = server.make_request("POST", "/completion", data={ "prompt": list(range(first_token, first_token + 3500)), "n_predict": 1, "cache_prompt": False, "temperature": 0.0, "seed": 42, }, timeout=600) assert res.status_code == 200, res.body assert res.body["timings"]["prompt_n"] == 3500 - text = open(server.log_path, errors="replace").read()[mark:] + text = _log()[mark:] # a decode step is one token per slot, so the prompt's own ubatches are the wide ones return [w for w in _ubatch_widths(text) if w > 3] diff --git a/tools/server/tests/unit/test_preempt_notify.py b/tools/server/tests/unit/test_preempt_notify.py index d28a640260b8..420f557fa009 100644 --- a/tools/server/tests/unit/test_preempt_notify.py +++ b/tools/server/tests/unit/test_preempt_notify.py @@ -2,6 +2,7 @@ import os import re import tempfile +import time import threading import pytest import requests @@ -49,6 +50,19 @@ def _chat_payload(n_predict: int) -> dict: "temperature": 0.0, "seed": 42, "stream": True} +def _log() -> str: + """The server log once its writer thread has stopped growing it: on a loaded host the log lags the response that came from it.""" + deadline = time.time() + 5.0 + last = -1 + while time.time() < deadline: + size = os.path.getsize(server.log_path) + if size == last: + break + last = size + time.sleep(0.1) + return open(server.log_path, errors="replace").read() + + def _post(path: str, data: dict): return requests.post(f"http://{server.server_host}:{server.server_port}{path}", json=data, stream=True) @@ -159,8 +173,6 @@ def _prefill_payload(path: str, prompt: str, n_predict: int) -> dict: @pytest.mark.parametrize("path", ["/completion", "/v1/chat/completions", "/v1/responses", "/v1/messages"]) def test_a_park_during_prompt_processing_opens_the_stream_with_the_notice(path): # a park before the first token is the case a client cannot tell from a stall, so the notice goes out with the response headers rather than waiting for a chunk that is not coming - import time - server.server_slots = True _start(n_ctx=2048, n_batch=256) @@ -194,7 +206,7 @@ def _resident(): seen.append((time.time() - t0, line)) t.join(120) - text = open(server.log_path).read() + text = _log() assert "preempted:" in text, "nothing was parked while the prompt was being processed" comments = [(at, line) for at, line in seen if line.startswith(":")] @@ -253,7 +265,7 @@ def test_a_rotation_tells_both_streams_and_a_head_parked_past_the_budget_is_kept assert n_parked >= 2, [r[0] for r in results] assert n_keepalive >= 1, "a parked stream was left silent past its keepalive interval" - text = open(server.log_path).read() + text = _log() assert "rotated out after" in text assert "no rotation: --preempt-ram 2 MiB" in text assert "resumed after" in text From c5dc0f92a9ba12d82151b4f6eaea49573458672d Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 20:08:53 +0000 Subject: [PATCH 21/22] llama : keep the exact-mode host buffer comment to whole sentences (cherry picked from commit 5066bfd35a1fd984909c24ba57a76859b0156106) --- src/llama-impl.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/llama-impl.cpp b/src/llama-impl.cpp index c6e6d356655a..923845e1ecd6 100644 --- a/src/llama-impl.cpp +++ b/src/llama-impl.cpp @@ -179,10 +179,8 @@ bool llama_exact_backend_name(const char * reg_name) { return reg_name && (strcmp(reg_name, "CUDA") == 0 || strcmp(reg_name, "ROCm") == 0 || strcmp(reg_name, "MUSA") == 0); } -// [TAG_EXACT_CONCURRENCY] a host buffer is the interesting case: the scheduler runs an operation on -// the backend holding its weight, and moves a host weight's operation to the GPU only once the batch -// is wide enough (ggml_backend_cuda_device_offload_op), while the CPU matmul picks between its SGEMM -// and its vector dot by the batch width too. +// [TAG_EXACT_CONCURRENCY] a host buffer is the case that matters: the scheduler runs an op on the backend that holds its weight, and moves a host weight's op to the GPU only once the batch is wide enough (ggml_backend_cuda_device_offload_op). +// The CPU matmul picks between its SGEMM and its vector dot by the batch width too. bool llama_exact_buft_invariant(ggml_backend_buffer_type_t buft) { if (!buft || ggml_backend_buft_is_host(buft)) { return false; From ae195390599e1ea8c4dbcc6b1d8ec77f5db90c3a Mon Sep 17 00:00:00 2001 From: Daniel Han <danielhanchen@gmail.com> Date: Thu, 10 Sep 2026 20:35:49 +0000 Subject: [PATCH 22/22] tests : make the two generations that do not fit overlap by their lengths, not by the client's speed (cherry picked from commit dcdb691e8bb269836f7f491d58ec9db52b970dc2) --- tools/server/tests/unit/test_preempt.py | 38 ++++++++++++++++++------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/tools/server/tests/unit/test_preempt.py b/tools/server/tests/unit/test_preempt.py index c4809b724716..c47cda62b17f 100644 --- a/tools/server/tests/unit/test_preempt.py +++ b/tools/server/tests/unit/test_preempt.py @@ -109,6 +109,17 @@ def _wait_processing(slot_ids, timeout: float = 30.0): pytest.fail(f"slots {slot_ids} never showed as processing") +def _complete_overlapping(n_predict, n_prompt, timeout: float = DEFAULT_REQUEST_TIMEOUT): + """A leader on slot 0 and a follower on slot 1 that certainly overlap: the follower is sent once the leader is seen processing, so the lengths and not the client's speed decide what the pool has to hold.""" + leader = _prompt_of(n_prompt[0], _PROMPT_A) + other = _prompt_of(n_prompt[1], _PROMPT_B) + with ThreadPoolExecutor(1) as pool: + first = pool.submit(_complete, n_predict[0], leader, 0, 0.0, None, timeout) + _wait_processing([0]) + second = _complete(n_predict[1], other, 1, 0.0, None, timeout) + return [first.result(), second] + + def _wait_preempted(timeout: float = 30.0) -> bool: """True once some slot is parked: its cells are in host RAM and it wants them back.""" deadline = time.time() + timeout @@ -191,7 +202,7 @@ def test_forced_parks_do_not_change_the_output(mode): @pytest.mark.parametrize("knob", ["planner", "pages", "async", "last-resort", "last-resort-unlimited"]) def test_two_generations_that_do_not_fit_together_both_finish(knob): - # each request fits the pool alone (168 of 256 cells) but not together; without preemption both end with "Context size has been exceeded" + # each request fits the pool alone (960 and 600 of 1024 cells) but not together; without preemption both end with "Context size has been exceeded" if knob == "pages": # a block allocator gives a whole block to one sequence, so the planner has to count cells: counting tokens it sees room the allocator cannot find os.environ["LLAMA_SERVER_PREEMPT_GRANULARITY"] = "64" @@ -199,19 +210,24 @@ def test_two_generations_that_do_not_fit_together_both_finish(knob): os.environ["LLAMA_SERVER_PREEMPT_PLANNER"] = "off" if knob == "last-resort-unlimited": os.environ["LLAMA_ARG_PREEMPT_RAM"] = "-1" - (_start_async if knob == "async" else _start)(n_ctx=256) + n_ctx = 1024 + (_start_async if knob == "async" else _start)(n_ctx=n_ctx) + + # the lengths, not the client's speed, decide the overlap: two equal requests fired together did not overlap on a Windows runner, the first finished before the second arrived, and the last resort never saw the two residents it needs. + # the follower is sent once the leader is seen processing, so it holds its cells while the leader grows into the rest of the pool + n_predict = (460, 400) + results = _complete_overlapping(n_predict, (500, 200)) - n_predict = 160 - results = _complete_all(n_predict) text = _log() _assert_recovered(text, "preempted as a last resort" if knob.startswith("last-resort") else "preempted:") - _assert_completed(results, n_predict) - for res in results: + for res, n_wanted in zip(results, n_predict): + assert res.status_code == 200, res.body + assert res.body["timings"]["predicted_n"] == n_wanted assert res.body["truncated"] is False - assert len(res.body["tokens"]) == n_predict + assert len(res.body["tokens"]) == n_wanted if knob == "pages": - held = [int(n) for n in re.findall(r"kv (\d+)/256", text)] + held = [int(n) for n in re.findall(rf"kv (\d+)/{n_ctx}", text)] wanted = [int(n) for n in re.findall(r"\(wanted (\d+)\)", text)] assert held and wanted, f"the planner logged no figures:\n{text}" assert all(n % 64 == 0 for n in held + wanted), f"not whole blocks: {held} {wanted}" @@ -236,7 +252,8 @@ def test_a_request_that_cannot_be_helped_gets_the_context_error_and_the_server_l _start(n_ctx=256) if knob == "ram-0": - assert any(res.status_code != 200 for res in _complete_all(160)) + # the overflow has to be a matter of lengths: two equal requests fired together did not overlap on a Windows runner, and each one fits the pool alone + assert any(res.status_code != 200 for res in _complete_overlapping((110, 100), (120, 60))) else: res = server.make_request("POST", "/completion", data={ "n_predict": 160, "n_cmpl": 2, "prompt": _PROMPT_A, @@ -263,7 +280,8 @@ def test_a_server_that_never_asked_for_parking_behaves_as_upstream(): assert "preemption:" not in text, "a server that did not ask for parking announced it" assert _ASYNC_BANNER not in text, "the async park path was set up without being asked for" - assert any(res.status_code != 200 for res in _complete_all(160)) + # as above, the two have to be resident together for the pool to overflow at all + assert any(res.status_code != 200 for res in _complete_overlapping((110, 100), (120, 60))) text = _log() assert "Context size has been exceeded" in text