From acbfbc8100646b55099eaaa41561118f770a6db3 Mon Sep 17 00:00:00 2001 From: Daniele Zannotti Date: Tue, 29 Sep 2026 19:42:50 +0100 Subject: [PATCH 1/3] ci : nightly ROCm release builds in the lemonade-sdk/llamacpp-rocm layout Port of lemonade-sdk/llamacpp-rocm's build, test and nightly-alert workflows (MIT). Same targets, same TheRock nightly ROCm, same bundled runtime libraries, same llama-bNNNN-{os}-rocm-{target}-x64.zip assets and bNNNN release numbering. Differences from the original: it builds this repository instead of cloning upstream, falls back to the job token when there is no GH_TOKEN secret, runs the PR trigger only for changes to the pipeline, and gates the self-hosted Strix Halo test jobs behind repository variables so missing runners cannot hold a run in the queue. Assisted-by: Claude Opus 5.5 Co-Authored-By: Claude Opus 5.5 --- .../actions/test-llamacpp-build/action.yml | 238 ++++ .github/workflows/build-llamacpp-rocm.yml | 1107 +++++++++++++++++ .github/workflows/nightly-failure-alert.yml | 66 + .github/workflows/test-llamacpp-rocm.yml | 394 ++++++ docs/rocm-releases.md | 42 + scripts/rocm-release/gather_required_libs.py | 57 + 6 files changed, 1904 insertions(+) create mode 100644 .github/actions/test-llamacpp-build/action.yml create mode 100644 .github/workflows/build-llamacpp-rocm.yml create mode 100644 .github/workflows/nightly-failure-alert.yml create mode 100644 .github/workflows/test-llamacpp-rocm.yml create mode 100644 docs/rocm-releases.md create mode 100644 scripts/rocm-release/gather_required_libs.py diff --git a/.github/actions/test-llamacpp-build/action.yml b/.github/actions/test-llamacpp-build/action.yml new file mode 100644 index 000000000000..3152939708c2 --- /dev/null +++ b/.github/actions/test-llamacpp-build/action.yml @@ -0,0 +1,238 @@ +# Same as lemonade-sdk/llamacpp-rocm (MIT). +name: 'Test Llama.cpp Build' +description: 'Download and test llama.cpp build artifacts' + +inputs: + os_type: + description: 'Operating system type (Windows or Linux)' + required: true + gfx_target: + description: 'GPU target (e.g., gfx1151, gfx1150)' + required: true + artifact_name: + description: 'Full artifact name to download' + required: true + +runs: + using: "composite" + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + path: llama-binaries + + - name: Download test model (Windows) + if: inputs.os_type == 'Windows' + shell: powershell + run: | + $modelUrl = "https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_0.gguf?download=true" + $modelPath = "Qwen3-0.6B-Q4_0.gguf" + + Write-Host "Downloading test model from: $modelUrl" + curl.exe -L -o $modelPath $modelUrl + + # Verify download + if (Test-Path $modelPath) { + $fileSize = (Get-Item $modelPath).Length + Write-Host "Model downloaded successfully. Size: $fileSize bytes" + } else { + Write-Error "Failed to download model" + exit 1 + } + + - name: Download test model (Linux) + if: inputs.os_type == 'Linux' + shell: bash + run: | + model_url="https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_0.gguf?download=true" + model_path="Qwen3-0.6B-Q4_0.gguf" + + echo "Downloading test model from: $model_url" + curl -L -o "$model_path" "$model_url" + + # Verify download + if [ -f "$model_path" ]; then + file_size=$(stat -c%s "$model_path") + echo "Model downloaded successfully. Size: $file_size bytes" + else + echo "Failed to download model" + exit 1 + fi + + - name: Set up library path (Linux) + if: inputs.os_type == 'Linux' + shell: bash + run: | + # Add the llama-binaries directory to LD_LIBRARY_PATH + echo "LD_LIBRARY_PATH=$(pwd)/llama-binaries:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Run llama-cli test (Windows) + if: inputs.os_type == 'Windows' + shell: powershell + run: | + $llamaCliPath = "llama-binaries\llama-cli.exe" + $modelPath = "Qwen3-0.6B-Q4_0.gguf" + + # Verify llama-cli exists + if (-not (Test-Path $llamaCliPath)) { + Write-Error "llama-cli.exe not found at: $llamaCliPath" + Write-Host "Available files in llama-binaries:" + Get-ChildItem -Recurse "llama-binaries" + exit 1 + } + + Write-Host "Running llama-cli test for ${{ inputs.gfx_target }}..." + Write-Host "Command: $llamaCliPath -m `"$modelPath`" -ngl 99 -p `"I believe the meaning of life is`" -st -v" + + # Run the command and capture output + try { + $process = Start-Process -FilePath $llamaCliPath -ArgumentList @("-m", "`"$modelPath`"", "-ngl", "99", "-p", "`"I believe the meaning of life is`"", "-st", "-v") -NoNewWindow -Wait -PassThru -RedirectStandardOutput "stdout.txt" -RedirectStandardError "stderr.txt" + + # Wait for the process to complete + $process.WaitForExit() + + # Read both stdout and stderr + $stdout = Get-Content "stdout.txt" -Raw -ErrorAction SilentlyContinue + $stderr = Get-Content "stderr.txt" -Raw -ErrorAction SilentlyContinue + + # Always print the output (both stdout and stderr) + Write-Host "=== LLAMA-CLI STDOUT ===" + if ($stdout) { + Write-Host $stdout + } else { + Write-Host "(empty)" + } + Write-Host "=== LLAMA-CLI STDERR ===" + if ($stderr) { + Write-Host $stderr + } else { + Write-Host "(empty)" + } + Write-Host "=== END OUTPUT ===" + Write-Host "Process exit code: $($process.ExitCode)" + + # Combine output for pattern matching (stderr often contains important CUDA/GPU info) + $output = $stdout + $stderr + + # Clean up temp files + Remove-Item "stdout.txt" -ErrorAction SilentlyContinue + Remove-Item "stderr.txt" -ErrorAction SilentlyContinue + } + catch { + Write-Error "Failed to start llama-cli process: $_" + Write-Error "This usually indicates a missing dependency (DLL) or permission issue" + # Try to get more details about dependencies + Write-Host "Checking for dependency issues..." + Write-Host "Files in llama-binaries directory:" + Get-ChildItem "llama-binaries" | Format-Table Name, Length + exit 1 + } + + # Check if the process exited successfully + if ($process.ExitCode -ne 0) { + Write-Host "❌ llama-cli exited with error code: $($process.ExitCode)" + Write-Host "This error code typically indicates:" + Write-Host " - Exit code 127: Command or library not found (missing DLL)" + Write-Host " - Exit code 1: General error" + Write-Host " - Other codes: Application-specific errors" + Write-Host "" + Write-Host "Please check the STDERR output above for specific error messages." + exit 1 + } + + # Check for expected outputs + $foundGpuOffload = $output -match "offloaded 29/29 layers to GPU" + $foundThinkTag = $output -match "" + + Write-Host "=== TEST RESULTS ===" + Write-Host "Expected 'offloaded 29/29 layers to GPU': $(if ($foundGpuOffload) { 'FOUND' } else { 'NOT FOUND' })" + Write-Host "Expected '' tag: $(if ($foundThinkTag) { 'FOUND' } else { 'NOT FOUND' })" + + if ($foundGpuOffload -and $foundThinkTag) { + Write-Host "✅ Test PASSED - Both expected outputs found" + } else { + Write-Host "❌ Test FAILED - Missing expected outputs" + exit 1 + } + + - name: Run llama-cli test (Linux) + if: inputs.os_type == 'Linux' + shell: bash + run: | + llama_cli_path="./llama-binaries/llama-cli" + model_path="Qwen3-0.6B-Q4_0.gguf" + output_file="llama_output.txt" + + # Make llama-cli executable + chmod +x "$llama_cli_path" + + # Verify llama-cli exists + if [ ! -f "$llama_cli_path" ]; then + echo "llama-cli not found at: $llama_cli_path" + echo "Available files in llama-binaries:" + find llama-binaries -type f + exit 1 + fi + + echo "Running llama-cli test for ${{ inputs.gfx_target }}..." + echo "Command: $llama_cli_path -m \"$model_path\" -ngl 99 -p \"I believe the meaning of life is\" -st -v" + + # Run the command and capture output to file (more robust than variable) + set +e # Don't exit on error, we want to capture the output and check exit code manually + "$llama_cli_path" -m "$model_path" -ngl 99 -p "I believe the meaning of life is" -st -v > "$output_file" 2>&1 + exit_code=$? + set -e # Re-enable exit on error + + echo "=== LLAMA-CLI OUTPUT ===" + if [ -f "$output_file" ] && [ -s "$output_file" ]; then + cat "$output_file" + else + echo "(empty)" + fi + echo "=== END OUTPUT ===" + echo "Process exit code: $exit_code" + + # Check if the process exited successfully + if [ $exit_code -ne 0 ]; then + echo "❌ llama-cli exited with error code: $exit_code" + echo "This error code typically indicates:" + echo " - Exit code 127: Command or library not found (missing shared library)" + echo " - Exit code 1: General error" + echo " - Other codes: Application-specific errors" + echo "" + echo "Checking for missing library dependencies..." + ldd "$llama_cli_path" || echo "ldd command failed" + echo "" + echo "Please check the output above for specific error messages." + rm -f "$output_file" + exit 1 + fi + + # Check for expected outputs using grep -F (fixed strings, no regex interpretation) + # This handles special characters like angle brackets more reliably + found_gpu_offload=false + found_think_tag=false + + if grep -F "offloaded 29/29 layers to GPU" "$output_file" > /dev/null 2>&1; then + found_gpu_offload=true + fi + + if grep -F "" "$output_file" > /dev/null 2>&1; then + found_think_tag=true + fi + + echo "=== TEST RESULTS ===" + echo "Expected 'offloaded 29/29 layers to GPU': $(if [ "$found_gpu_offload" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)" + echo "Expected '' tag: $(if [ "$found_think_tag" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)" + + # Clean up output file + rm -f "$output_file" + + if [ "$found_gpu_offload" = true ] && [ "$found_think_tag" = true ]; then + echo "✅ Test PASSED - Both expected outputs found" + else + echo "❌ Test FAILED - Missing expected outputs" + exit 1 + fi + diff --git a/.github/workflows/build-llamacpp-rocm.yml b/.github/workflows/build-llamacpp-rocm.yml new file mode 100644 index 000000000000..cba4cfa1c5f4 --- /dev/null +++ b/.github/workflows/build-llamacpp-rocm.yml @@ -0,0 +1,1107 @@ +# Nightly ROCm release builds of this fork, same layout as lemonade-sdk/llamacpp-rocm +# (https://github.com/lemonade-sdk/llamacpp-rocm, MIT) so the zips drop into the same tooling. +# Differences: builds this repository instead of cloning upstream llama.cpp, uses the job token when no +# GH_TOKEN secret exists, and runs the self-hosted Strix Halo tests only when the runners are registered. +name: Build Llama.cpp + ROCm +on: + workflow_dispatch: + inputs: + operating_systems: + description: 'Operating systems to build for (comma-separated: windows,ubuntu)' + required: false + default: 'windows,ubuntu' + gfx_target: + description: 'AMD GPU targets (comma-separated)' + required: false + default: 'gfx1151,gfx1150,gfx120X,gfx110X,gfx103X,gfx90a,gfx908' + rocm_version: + description: 'ROCm version to use (e.g., 7.13.0a20260318) or "latest" to auto-detect' + required: false + default: 'latest' + llamacpp_version: + description: 'Branch or tag of this repository to build, or "latest" for master' + required: false + default: 'latest' + create_release: + description: 'Create a GitHub release after successful build' + required: false + default: true + type: boolean + + # Trigger on pushes to pull requests that change the release pipeline + pull_request: + types: [opened, synchronize, reopened] + paths: + - '.github/workflows/build-llamacpp-rocm.yml' + - '.github/actions/test-llamacpp-build/**' + - 'scripts/rocm-release/**' + + # Trigger on a schedule to build nightly releases + schedule: + # Runs at 1:00 PM UTC, which is 5:00 AM PST (UTC-8) + # This is two hours after the ROCm nightly tarball is triggered + - cron: '0 13 * * *' + +env: + OPERATING_SYSTEMS: ${{ github.event.inputs.operating_systems || 'windows,ubuntu' }} + GFX_TARGETS: ${{ github.event.inputs.gfx_target || 'gfx1151,gfx1150,gfx120X,gfx110X,gfx103X,gfx90a,gfx908' }} + ROCM_VERSION: ${{ github.event.inputs.rocm_version || 'latest' }} + LLAMACPP_VERSION: ${{ github.event.inputs.llamacpp_version || 'latest' }} + +jobs: + prepare-matrix: + runs-on: ubuntu-22.04 + outputs: + windows_matrix: ${{ steps.set-matrix.outputs.windows_matrix }} + ubuntu_matrix: ${{ steps.set-matrix.outputs.ubuntu_matrix }} + should_build_windows: ${{ steps.set-matrix.outputs.should_build_windows }} + should_build_ubuntu: ${{ steps.set-matrix.outputs.should_build_ubuntu }} + steps: + - name: Set matrix + id: set-matrix + run: | + targets="${{ env.GFX_TARGETS }}" + operating_systems="${{ env.OPERATING_SYSTEMS }}" + + echo "Input targets: $targets" + echo "Input operating systems: $operating_systems" + + # Convert targets to JSON array + matrix_targets=$(echo "$targets" \ + | tr ',' '\n' \ + | sed 's/^ *//;s/ *$//' \ + | sed 's/^"//;s/"$//' \ + | jq -R . \ + | jq -s '{gfx_target: .}' \ + | jq -c) + + # Check which operating systems to build + should_build_windows="false" + should_build_ubuntu="false" + + if [[ "$operating_systems" == *"windows"* ]]; then + should_build_windows="true" + echo "windows_matrix=$matrix_targets" >> $GITHUB_OUTPUT + fi + + if [[ "$operating_systems" == *"ubuntu"* ]]; then + should_build_ubuntu="true" + echo "ubuntu_matrix=$matrix_targets" >> $GITHUB_OUTPUT + fi + + echo "should_build_windows=$should_build_windows" >> $GITHUB_OUTPUT + echo "should_build_ubuntu=$should_build_ubuntu" >> $GITHUB_OUTPUT + + echo "Windows build: $should_build_windows" + echo "Ubuntu build: $should_build_ubuntu" + echo "Generated matrix: $matrix_targets" + + build-windows: + # Pinned to windows-2022 (VS 2022 / MSVC 14.4x). The windows-latest image moved to + # VS 2026 (MSVC 14.51) on 2026-06-08, whose declares isgreater/isless/etc. as + # constexpr (implicitly __host__ __device__), which collides with ROCm clang's + # __device__-only HIP math wrappers and breaks the HIP build. See llama.cpp#22570. + runs-on: windows-2022 + needs: prepare-matrix + if: needs.prepare-matrix.outputs.should_build_windows == 'true' + strategy: + matrix: ${{fromJson(needs.prepare-matrix.outputs.windows_matrix)}} + fail-fast: false + outputs: + rocm_version: ${{ steps.set-outputs.outputs.rocm_version }} + llamacpp_commit_hash: ${{ steps.set-outputs.outputs.llamacpp_commit_hash }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Clean up existing directories (safety precaution) + run: | + # Remove existing llama.cpp directory if it exists + if (Test-Path "llama.cpp") { + Write-Host "Removing existing llama.cpp directory..." + Remove-Item -Recurse -Force "llama.cpp" + } + + # Remove existing C:\opt\rocm directory if it exists + if (Test-Path "C:\opt\rocm") { + Write-Host "Removing existing C:\opt\rocm directory..." + Remove-Item -Recurse -Force "C:\opt\rocm" + } + + # Remove any existing ROCm tarball + if (Test-Path "rocm.tar.gz") { + Write-Host "Removing existing rocm.tar.gz..." + Remove-Item -Force "rocm.tar.gz" + } + + Write-Host "Cleanup completed successfully" + + - name: Install Visual Studio Build Tools + run: | + # Download and install Visual Studio Build Tools + $vsInstallerUrl = "https://aka.ms/vs/17/release/vs_buildtools.exe" + $vsInstallerPath = "$env:TEMP\vs_buildtools.exe" + + Write-Host "Downloading Visual Studio Build Tools..." + Invoke-WebRequest -Uri $vsInstallerUrl -OutFile $vsInstallerPath + + Write-Host "Installing Visual Studio Build Tools..." + Start-Process -FilePath $vsInstallerPath -ArgumentList "--quiet", "--wait", "--norestart", "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "--add", "Microsoft.VisualStudio.Component.VC.CMake.Project", "--add", "Microsoft.VisualStudio.Component.VC.ATL", "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621" -Wait + + # Clean up installer + Remove-Item $vsInstallerPath -Force + + - name: Install build dependencies + run: | + Write-Host "Installing build dependencies using manual methods..." + + # Install Ninja + Write-Host "Installing Ninja..." + $ninjaUrl = "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip" + $ninjaPath = "$env:TEMP\ninja-win.zip" + $ninjaDir = "C:\ninja" + New-Item -ItemType Directory -Force -Path $ninjaDir + Invoke-WebRequest -Uri $ninjaUrl -OutFile $ninjaPath + Expand-Archive -Path $ninjaPath -DestinationPath $ninjaDir -Force + + # Install Strawberry Perl + Write-Host "Installing Strawberry Perl..." + $perlUrl = "http://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.msi" + $perlPath = "$env:TEMP\strawberry-perl-5.32.1.1-64bit.msi" + Invoke-WebRequest -Uri $perlUrl -OutFile $perlPath + Start-Process msiexec.exe -ArgumentList "/i $perlPath /quiet /norestart" -Wait + + # Verify installations + $env:PATH = "C:\ninja;C:\Strawberry\perl\bin;$env:PATH" + Write-Host "Verifying installations..." + ninja --version + perl --version + + Write-Host "Manual installation of build dependencies completed" + + - name: Download ROCm nightly tarball + run: | + # Determine ROCm version to use + $rocmVersion = "${{ env.ROCM_VERSION }}" + $currentTarget = "${{ matrix.gfx_target }}" + + # Add appropriate suffixes for different GPU targets + $s3Target = $currentTarget + if ($currentTarget -eq "gfx103X" -or $currentTarget -eq "gfx110X" -or $currentTarget -eq "gfx120X") { + $s3Target = "$currentTarget-all" + Write-Host "Using target with -all suffix: $s3Target" + } + + # TheRock now publishes nightlies to the ROCm core tarball index. That + # index is a static HTML page with an embedded JSON `files` array (name + + # mtime), so we parse that instead of the retired S3 XML bucket listing. + $baseUrl = "https://nightly.repo.amd.com/rocm/core/tarball" + if ($rocmVersion -eq "latest") { + Write-Host "Auto-detecting latest ROCm version for target: $currentTarget" + $indexHtml = (Invoke-WebRequest "$baseUrl/" -UseBasicParsing).Content + $filesMatch = [regex]::Match($indexHtml, 'const files = (\[.*?\]);', [System.Text.RegularExpressions.RegexOptions]::Singleline) + if (-not $filesMatch.Success) { + Write-Error "Failed to parse file index from $baseUrl/" + exit 1 + } + + # Pick the newest by the YYYYMMDD build date embedded in the filename. + # Assign before piping so the parsed array enumerates element-by-element. + $allFiles = $filesMatch.Groups[1].Value | ConvertFrom-Json + $prefix = "therock-dist-windows-$s3Target-" + # Anchor the match so the version comes immediately after the prefix. + # This excludes variants like "...-all-tests-.tar.gz" that + # TheRock also publishes and share the same prefix/date. + $versionPattern = "^$([regex]::Escape($prefix))\d+\.\d+\.\d+(a|rc)\d+\.tar\.gz$" + $latest = $allFiles | + Where-Object { $_.name -match $versionPattern } | + Sort-Object { [regex]::Match($_.name, '(\d{8})\.tar\.gz$').Groups[1].Value } | + Select-Object -Last 1 + if (-not $latest) { + Write-Error "No tarball found for prefix '$prefix' at $baseUrl/" + exit 1 + } + $latestFile = $latest.name + Write-Host "Found latest file: $latestFile" + + # Extract version from the filename for environment variable + if ($latestFile -match "therock-dist-windows-$s3Target-(\d+\.\d+\.\d+(?:a|rc)\d+)\.tar\.gz") { + $rocmVersion = $matches[1] + Write-Host "Detected latest ROCm version: $rocmVersion" + } else { + Write-Error "Failed to extract ROCm version from latest file: $latestFile" + Write-Error "Expected pattern: therock-dist-windows-$s3Target-.tar.gz" + exit 1 + } + + $rocmUrl = "$baseUrl/$latestFile" + } else { + # For specific versions, construct the URL directly + $rocmUrl = "$baseUrl/therock-dist-windows-$s3Target-$rocmVersion.tar.gz" + } + + # Store the version for use in other steps + echo "DETECTED_ROCM_VERSION=$rocmVersion" >> $env:GITHUB_ENV + + Write-Host "Downloading ROCm from: $rocmUrl" + Invoke-WebRequest -Uri $rocmUrl -OutFile "rocm.tar.gz" + + - name: Extract ROCm to C:\opt\rocm + run: | + # Create directory if it doesn't exist + New-Item -ItemType Directory -Force -Path "C:\opt\rocm" + + # Extract the tarball + tar -xzf rocm.tar.gz -C C:\opt\rocm --strip-components=1 + + - name: Clone llama.cpp + run: | + $llamacppVersion = "${{ env.LLAMACPP_VERSION }}" + if ($llamacppVersion -eq "latest") { + Write-Host "Cloning master branch (latest) with blobless clone" + git clone --filter=blob:none --single-branch --branch master https://github.com/${{ github.repository }}.git llama.cpp + } else { + Write-Host "Cloning llama.cpp version: $llamacppVersion with blobless clone" + git clone --filter=blob:none --single-branch --branch $llamacppVersion https://github.com/${{ github.repository }}.git llama.cpp + } + cd llama.cpp + + # Get commit hash (5 digits) and store it as environment variable + $commitHash = git rev-parse --short=5 HEAD + echo "LLAMACPP_COMMIT_HASH=$commitHash" >> $env:GITHUB_ENV + Write-Host "llama.cpp commit hash (5 digits): $commitHash" + + # Show current commit info + Write-Host "Current llama.cpp commit:" + git log --oneline -1 + + - name: Build Llama.cpp + ROCm + shell: cmd + run: | + + REM Map GPU targets + set "current_target=${{ matrix.gfx_target }}" + echo Input target: %current_target% + + if "%current_target%"=="gfx110X" ( + set "mapped_target=gfx1100;gfx1101;gfx1102;gfx1103" + ) else if "%current_target%"=="gfx103X" ( + set "mapped_target=gfx1030;gfx1031;gfx1032;gfx1034" + ) else if "%current_target%"=="gfx1151" ( + set "mapped_target=gfx1151" + ) else if "%current_target%"=="gfx1150" ( + set "mapped_target=gfx1150" + ) else if "%current_target%"=="gfx120X" ( + set "mapped_target=gfx1200;gfx1201" + ) else ( + set "mapped_target=%current_target%" + ) + echo Mapped target: %mapped_target% + + REM Set up environment variables and PATH + set HIP_PATH=C:\opt\rocm + set HIP_PLATFORM=amd + set PATH=%HIP_PATH%\lib\llvm\bin;%HIP_PATH%\bin;%PATH% + + REM Set up x64 Native Tools Command Prompt environment + call "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath > vs_path.txt + set /p VS_PATH=> $env:GITHUB_OUTPUT + echo "llamacpp_commit_hash=$env:LLAMACPP_COMMIT_HASH" >> $env:GITHUB_OUTPUT + + Write-Host "Final rocm_version: $rocmVersion" + Write-Host "Final llamacpp_commit_hash: $env:LLAMACPP_COMMIT_HASH" + + - name: Create release summary + run: | + $summary = @" + # Llama.cpp + ROCm Build Summary (Windows - ${{ matrix.gfx_target }}) + + ## Build Configuration + - **GPU Target**: ${{ matrix.gfx_target }} + - **ROCm Version**: ${{ env.DETECTED_ROCM_VERSION || env.ROCM_VERSION }} + - **llama.cpp Commit Hash**: ${{ env.LLAMACPP_COMMIT_HASH }} + - **Build Type**: Release + - **Platform**: Windows + + ## Build Artifacts + The compiled binaries are available in the build artifacts. + + "@ + + Set-Content -Path "build-summary-windows-${{ matrix.gfx_target }}.md" -Value $summary + + - name: Upload build summary + id: upload-summary + uses: actions/upload-artifact@v4 + with: + name: build-summary-windows-${{ matrix.gfx_target }} + path: build-summary-windows-${{ matrix.gfx_target }}.md + retention-days: 30 + + - name: Clean up build artifacts and temporary files + if: always() + run: | + Write-Host "Starting cleanup process..." + + # Remove ROCm tarball + if (Test-Path "rocm.tar.gz") { + Write-Host "Removing rocm.tar.gz..." + Remove-Item -Force "rocm.tar.gz" + } + + # Remove ROCm installation directory + if (Test-Path "C:\opt\rocm") { + Write-Host "Removing C:\opt\rocm directory..." + Remove-Item -Recurse -Force "C:\opt\rocm" + } + # Remove llama.cpp source directory (keeping only the build artifacts) + if (Test-Path "llama.cpp") { + Write-Host "Removing llama.cpp source directory..." + Remove-Item -Recurse -Force "llama.cpp" + } + + # Clean up any temporary build files + if (Test-Path "build-summary-windows-${{ matrix.gfx_target }}.md") { + Write-Host "Removing build-summary-windows-${{ matrix.gfx_target }}.md..." + Remove-Item -Force "build-summary-windows-${{ matrix.gfx_target }}.md" + } + + Write-Host "Cleanup completed successfully" + + build-ubuntu: + runs-on: ubuntu-22.04 + needs: prepare-matrix + if: needs.prepare-matrix.outputs.should_build_ubuntu == 'true' + strategy: + matrix: ${{fromJson(needs.prepare-matrix.outputs.ubuntu_matrix)}} + fail-fast: false + outputs: + rocm_version: ${{ steps.set-outputs.outputs.rocm_version }} + llamacpp_commit_hash: ${{ steps.set-outputs.outputs.llamacpp_commit_hash }} + + + steps: + - name: Free disk space + # Remove unused runner files to free up disk space + run: curl -fsSL https://raw.githubusercontent.com/kou/arrow/e49d8ae15583ceff03237571569099a6ad62be32/ci/scripts/util_free_space.sh | bash + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Clean up existing directories (safety precaution) + run: | + # Remove existing llama.cpp directory if it exists + if [ -d "llama.cpp" ]; then + echo "Removing existing llama.cpp directory..." + rm -rf llama.cpp + fi + + # Remove existing /opt/rocm directory if it exists + if [ -d "/opt/rocm" ]; then + echo "Removing existing /opt/rocm directory..." + sudo rm -rf /opt/rocm + fi + + # Remove any existing ROCm tarball + if [ -f "rocm.tar.gz" ]; then + echo "Removing existing rocm.tar.gz..." + rm -f rocm.tar.gz + fi + + echo "Cleanup completed successfully" + + - name: Install build dependencies + run: | + echo "Installing build dependencies..." + sudo apt update + sudo apt install -y cmake ninja-build unzip curl + + # Verify installations + echo "Verifying installations..." + cmake --version + ninja --version + echo "Build dependencies installation completed" + + - name: Download and extract ROCm directly to /opt/rocm + run: | + # Determine ROCm version to use + rocm_version="${{ env.ROCM_VERSION }}" + current_target="${{ matrix.gfx_target }}" + + # Add appropriate suffixes for different GPU targets + s3_target="$current_target" + if [[ "$current_target" = "gfx103X" || "$current_target" = "gfx110X" || "$current_target" = "gfx120X" ]]; then + s3_target="${current_target}-all" + echo "Using target with -all suffix: $s3_target" + fi + + # TheRock now publishes nightlies to the ROCm core tarball index. That + # index is a static HTML page with an embedded JSON `files` array (name + + # mtime), so we parse that instead of the retired S3 XML bucket listing. + base_url="https://nightly.repo.amd.com/rocm/core/tarball" + if [ "$rocm_version" = "latest" ]; then + echo "Auto-detecting latest ROCm version for target: $current_target" + prefix="therock-dist-linux-${s3_target}-" + + files_json=$(curl -s "$base_url/" | tr '\n' ' ' | grep -oP 'const files = \K\[.*?\](?=\s*;)') + if [ -z "$files_json" ]; then + echo "Failed to parse file index from $base_url/" + exit 1 + fi + + # Pick the newest by the YYYYMMDD build date embedded in the filename. + # Anchor the match so the version comes immediately after the prefix. + # This excludes variants like "...-all-tests-.tar.gz" that + # TheRock also publishes and share the same prefix/date. + latest_file=$(echo "$files_json" | jq -r --arg p "$prefix" \ + '[.[] | select(.name | test("^" + $p + "[0-9]+\\.[0-9]+\\.[0-9]+(a|rc)[0-9]+\\.tar\\.gz$"))] | sort_by(.name | capture("(?[0-9]{8})\\.tar\\.gz$").d) | last | .name // empty') + if [ -z "$latest_file" ]; then + echo "No tarball found for prefix '$prefix' at $base_url/" + exit 1 + fi + echo "Found latest file: $latest_file" + + # Extract version from the filename for environment variable + if [[ "$latest_file" =~ therock-dist-linux-${s3_target}-([0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+)\.tar\.gz ]]; then + rocm_version="${BASH_REMATCH[1]}" + echo "Detected latest ROCm version: $rocm_version" + else + echo "Failed to extract ROCm version from latest file: $latest_file" + echo "Expected pattern: therock-dist-linux-${s3_target}-.tar.gz" + exit 1 + fi + + rocm_url="$base_url/$latest_file" + else + # For specific versions, construct the URL directly + rocm_url="$base_url/therock-dist-linux-${s3_target}-${rocm_version}.tar.gz" + fi + + # Store the version for use in other steps + echo "DETECTED_ROCM_VERSION=$rocm_version" >> $GITHUB_ENV + + echo "Streaming ROCm from: $rocm_url directly to extraction" + + # Create directory if it doesn't exist + sudo mkdir -p /opt/rocm + + # Stream download directly into tar extraction (no intermediate file) + curl -sL "$rocm_url" | sudo tar --use-compress-program=gzip -xf - -C /opt/rocm --strip-components=1 + + - name: Set ROCm environment variables + run: | + echo "Setting ROCm environment variables..." + + # Set environment variables for this step and subsequent steps + echo "HIP_PATH=/opt/rocm" >> $GITHUB_ENV + echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV + echo "HIP_PLATFORM=amd" >> $GITHUB_ENV + echo "HIP_CLANG_PATH=/opt/rocm/llvm/bin" >> $GITHUB_ENV + echo "HIP_INCLUDE_PATH=/opt/rocm/include" >> $GITHUB_ENV + echo "HIP_LIB_PATH=/opt/rocm/lib" >> $GITHUB_ENV + echo "HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode" >> $GITHUB_ENV + + # Update PATH + echo "/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH" >> $GITHUB_PATH + + # Set library paths + echo "LD_LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV + echo "LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:${LIBRARY_PATH:-}" >> $GITHUB_ENV + echo "CPATH=/opt/rocm/include:${CPATH:-}" >> $GITHUB_ENV + echo "PKG_CONFIG_PATH=/opt/rocm/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> $GITHUB_ENV + + echo "ROCm environment variables set successfully" + + + - name: Clone llama.cpp + run: | + llamacpp_version="${{ env.LLAMACPP_VERSION }}" + if [ "$llamacpp_version" = "latest" ]; then + echo "Cloning master branch (latest) with blobless clone" + git clone --filter=blob:none --single-branch --branch master https://github.com/${{ github.repository }}.git llama.cpp + else + echo "Cloning llama.cpp version: $llamacpp_version with blobless clone" + git clone --filter=blob:none --single-branch --branch "$llamacpp_version" https://github.com/${{ github.repository }}.git llama.cpp + fi + cd llama.cpp + + # Get commit hash (5 digits) and store it as environment variable + commit_hash=$(git rev-parse --short=5 HEAD) + echo "LLAMACPP_COMMIT_HASH=$commit_hash" >> $GITHUB_ENV + echo "llama.cpp commit hash (5 digits): $commit_hash" + + # Show current commit info + echo "Current llama.cpp commit:" + git log --oneline -1 + + + - name: Build Llama.cpp + ROCm + run: | + # Map GPU targets + current_target="${{ matrix.gfx_target }}" + echo "Input target: $current_target" + + if [ "$current_target" = "gfx110X" ]; then + mapped_target="gfx1100;gfx1101;gfx1102;gfx1103" + elif [ "$current_target" = "gfx103X" ]; then + mapped_target="gfx1030;gfx1031;gfx1032;gfx1034" + elif [ "$current_target" = "gfx1151" ]; then + mapped_target="gfx1151" + elif [ "$current_target" = "gfx1150" ]; then + mapped_target="gfx1150" + elif [ "$current_target" = "gfx120X" ]; then + mapped_target="gfx1200;gfx1201" + else + mapped_target="$current_target" + fi + echo "Mapped target: $mapped_target" + + # Create build directory + cd llama.cpp + mkdir build + cd build + + # Configure the project + cmake .. -G Ninja \ + -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang \ + -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ + -DCMAKE_CXX_FLAGS="-I/opt/rocm/include" \ + -DCMAKE_CROSSCOMPILING=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DGPU_TARGETS="$mapped_target" \ + -DBUILD_SHARED_LIBS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DGGML_HIP=ON \ + -DGGML_OPENMP=OFF \ + -DGGML_CUDA_FORCE_CUBLAS=OFF \ + -DGGML_RPC=ON \ + -DGGML_HIP_ROCWMMA_FATTN=OFF \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_STATIC=OFF \ + -DCMAKE_SYSTEM_NAME=Linux + + # Build the project + cmake --build . -j $(nproc) + + - name: Copy ROCm core libs to build directory + run: | + build_bin_path="llama.cpp/build/bin" + rocm_bin_path="/opt/rocm/bin" + + # Copy the rocblas/library folder and all its contents + rocblas_lib_path="/opt/rocm/lib/rocblas/library" + if [ -d "$rocblas_lib_path" ]; then + echo "Copying rocblas/library folder and all contents..." + dest_rocblas_path="$build_bin_path/rocblas/library" + mkdir -p "$(dirname "$dest_rocblas_path")" + cp -r "$rocblas_lib_path" "$(dirname "$dest_rocblas_path")/" + echo "Copied: rocblas/library folder with all contents" + + # List the contents of the copied rocblas/library folder + echo "Contents of rocblas/library:" + find "$dest_rocblas_path" -type f -exec ls -la {} \; | head -20 + else + echo "Warning: rocblas/library folder not found at: $rocblas_lib_path" + fi + + # Copy the hipblaslt/library folder and all its contents + hipblaslt_lib_path="/opt/rocm/lib/hipblaslt/library" + if [ -d "$hipblaslt_lib_path" ]; then + echo "Copying hipblaslt/library folder and all contents..." + dest_hipblaslt_path="$build_bin_path/hipblaslt/library" + mkdir -p "$(dirname "$dest_hipblaslt_path")" + cp -r "$hipblaslt_lib_path" "$(dirname "$dest_hipblaslt_path")/" + echo "Copied: hipblaslt/library folder with all contents" + + # List the contents of the copied hipblaslt/library folder + echo "Contents of hipblaslt/library:" + find "$dest_hipblaslt_path" -type f -exec ls -la {} \; | head -20 + else + echo "Warning: hipblaslt/library folder not found at: $hipblaslt_lib_path" + fi + + # Copy required ROCm libraries to build directory + # If artifacts from ROCm or Llama.cpp change, you may need to update this list + # To get a new list of all libraries, run: + # scripts/rocm-release/gather_required_libs.py --rocm-dir /opt/rocm --dest-dir llama.cpp/build/bin + echo "Copying required ROCm libraries to build directory..." + cp -v /opt/rocm/lib/libhipblas.so* "$build_bin_path/" 2>/dev/null || echo "libhipblas.so* not found" + cp -v /opt/rocm/lib/librocblas.so* "$build_bin_path/" 2>/dev/null || echo "librocblas.so* not found" + cp -v /opt/rocm/lib/libamdhip64.so* "$build_bin_path/" 2>/dev/null || echo "libamdhip64.so* not found" + cp -v /opt/rocm/lib/librocsolver.so* "$build_bin_path/" 2>/dev/null || echo "librocsolver.so* not found" + cp -v /opt/rocm/lib/libroctx64.so* "$build_bin_path/" 2>/dev/null || echo "libroctx64.so* not found" + cp -v /opt/rocm/lib/libhipblaslt.so* "$build_bin_path/" 2>/dev/null || echo "libhipblaslt.so* not found" + cp -v /opt/rocm/lib/libtensilelite-host.so* "$build_bin_path/" 2>/dev/null || echo "libtensilelite-host.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_liblzma.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_liblzma.so* not found" + cp -v /opt/rocm/lib/librocprofiler-register.so* "$build_bin_path/" 2>/dev/null || echo "librocprofiler-register.so* not found" + cp -v /opt/rocm/lib/libamd_comgr.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr.so* not found" + cp -v /opt/rocm/lib/libamd_comgr_loader.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr_loader.so* not found" + cp -v /opt/rocm/lib/libhsa-runtime64.so* "$build_bin_path/" 2>/dev/null || echo "libhsa-runtime64.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_numa.so* not found" + cp -v /opt/rocm/lib/librocroller.so* "$build_bin_path/" 2>/dev/null || echo "librocroller.so* not found" + cp -v /opt/rocm/lib/liborigami.so* "$build_bin_path/" 2>/dev/null || echo "liborigami.so* not found" + cp -v /opt/rocm/lib/librocm_kpack.so* "$build_bin_path/" 2>/dev/null || echo "librocm_kpack.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_z.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_zstd.so* not found" + cp -v /opt/rocm/lib/llvm/lib/libLLVM.so* "$build_bin_path/" 2>/dev/null || echo "libLLVM.so* not found" + cp -v /opt/rocm/lib/llvm/lib/libclang-cpp.so* "$build_bin_path/" 2>/dev/null || echo "libclang-cpp.so* not found" + cp -v /opt/rocm/lib/llvm/lib/libomp.so* "$build_bin_path/" 2>/dev/null || echo "libomp.so* not found" + + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_elf.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm_amdgpu.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm_amdgpu.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_bz2.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_bz2.so* not found" + + echo "Finished copying required ROCm libraries" + + - name: Set RPATH for portable distribution + run: | + sudo apt-get install -y patchelf + cd llama.cpp/build/bin + # Set RPATH to $ORIGIN so all libraries (including the comgr stub loader) find deps locally + for file in *.so* llama-*; do + [ -f "$file" ] && [ ! -L "$file" ] && patchelf --set-rpath '$ORIGIN' "$file" 2>/dev/null || true + done + + - name: List build artifacts (including ROCm files) + run: | + cd llama.cpp/build/bin + echo "Final build artifacts (including ROCm library files):" + ls -la + + - name: Upload build artifacts + id: upload-artifacts + uses: actions/upload-artifact@v4 + with: + name: llama-ubuntu-rocm-${{ matrix.gfx_target }}-x64 + path: llama.cpp/build/bin/ + retention-days: 30 + + - name: Set job outputs + id: set-outputs + run: | + echo "Setting job outputs..." + echo "Current target: ${{ matrix.gfx_target }}" + echo "DETECTED_ROCM_VERSION: ${DETECTED_ROCM_VERSION}" + echo "ROCM_VERSION: ${{ env.ROCM_VERSION }}" + echo "LLAMACPP_COMMIT_HASH: ${LLAMACPP_COMMIT_HASH}" + + rocm_version="${DETECTED_ROCM_VERSION:-${{ env.ROCM_VERSION }}}" + echo "rocm_version=$rocm_version" >> $GITHUB_OUTPUT + echo "llamacpp_commit_hash=${LLAMACPP_COMMIT_HASH}" >> $GITHUB_OUTPUT + + echo "Final rocm_version: $rocm_version" + echo "Final llamacpp_commit_hash: ${LLAMACPP_COMMIT_HASH}" + + - name: Create release summary + run: | + cat > "build-summary-ubuntu-${{ matrix.gfx_target }}.md" << 'EOF' + # Llama.cpp + ROCm Build Summary (Ubuntu - ${{ matrix.gfx_target }}) + + ## Build Configuration + - **GPU Target**: ${{ matrix.gfx_target }} + - **ROCm Version**: ${{ env.DETECTED_ROCM_VERSION || env.ROCM_VERSION }} + - **llama.cpp Commit Hash**: ${{ env.LLAMACPP_COMMIT_HASH }} + - **Build Type**: Release + - **Platform**: Ubuntu + + ## Build Artifacts + The compiled binaries are available in the build artifacts. + EOF + + - name: Upload build summary + id: upload-summary + uses: actions/upload-artifact@v4 + with: + name: build-summary-ubuntu-${{ matrix.gfx_target }} + path: build-summary-ubuntu-${{ matrix.gfx_target }}.md + retention-days: 30 + + - name: Clean up build artifacts and temporary files + if: always() + run: | + echo "Starting cleanup process..." + + # Remove ROCm tarball + if [ -f "rocm.tar.gz" ]; then + echo "Removing rocm.tar.gz..." + rm -f rocm.tar.gz + fi + + # Remove ROCm installation directory + if [ -d "/opt/rocm" ]; then + echo "Removing /opt/rocm directory..." + sudo rm -rf /opt/rocm + fi + + # Remove llama.cpp source directory (keeping only the build artifacts) + if [ -d "llama.cpp" ]; then + echo "Removing llama.cpp source directory..." + rm -rf llama.cpp + fi + + # Clean up any temporary build files + if [ -f "build-summary-ubuntu-${{ matrix.gfx_target }}.md" ]; then + echo "Removing build-summary-ubuntu-${{ matrix.gfx_target }}.md..." + rm -f "build-summary-ubuntu-${{ matrix.gfx_target }}.md" + fi + + echo "Cleanup completed successfully" + + test-stx-halo: + runs-on: ${{ matrix.runner }} + needs: [prepare-matrix, build-windows, build-ubuntu] + if: | + vars.STX_HALO_RUNNERS == 'true' && + (needs.build-windows.result == 'success' || needs.build-ubuntu.result == 'success') && + contains(github.event.inputs.gfx_target || 'gfx1151,gfx1150,gfx120X,gfx110X', 'gfx1151') + strategy: + matrix: + include: + - os: Windows + gfx_target: gfx1151 + artifact_prefix: windows + runner: ["stx-halo", "Windows"] + - os: Linux + gfx_target: gfx1151 + artifact_prefix: ubuntu + runner: ["stx-halo", "Linux"] + fail-fast: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Test llama.cpp build + uses: ./.github/actions/test-llamacpp-build + with: + os_type: ${{ matrix.os }} + gfx_target: ${{ matrix.gfx_target }} + artifact_name: llama-${{ matrix.artifact_prefix }}-rocm-${{ matrix.gfx_target }}-x64 + + test-stx: + runs-on: ${{ matrix.runner }} + needs: [prepare-matrix, build-windows, build-ubuntu] + if: | + vars.STX_RUNNERS == 'true' && + (needs.build-windows.result == 'success' || needs.build-ubuntu.result == 'success') && + contains(github.event.inputs.gfx_target || 'gfx1151,gfx1150,gfx120X,gfx110X', 'gfx1150') + strategy: + matrix: + include: + - os: Windows + gfx_target: gfx1150 + artifact_prefix: windows + runner: ["stx", "Windows"] + fail-fast: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Test llama.cpp build + uses: ./.github/actions/test-llamacpp-build + with: + os_type: ${{ matrix.os }} + gfx_target: ${{ matrix.gfx_target }} + artifact_name: llama-${{ matrix.artifact_prefix }}-rocm-${{ matrix.gfx_target }}-x64 + + create-release: + needs: [prepare-matrix, build-windows, build-ubuntu, test-stx-halo, test-stx] + runs-on: ubuntu-22.04 + permissions: + contents: write + pull-requests: write + if: | + always() && + (needs.build-windows.result == 'success' || needs.build-windows.result == 'skipped') && + (needs.build-ubuntu.result == 'success' || needs.build-ubuntu.result == 'skipped') && + (needs.build-windows.result == 'success' || needs.build-ubuntu.result == 'success') && + (needs.test-stx-halo.result == 'success' || needs.test-stx-halo.result == 'skipped') && + (needs.test-stx.result == 'success' || needs.test-stx.result == 'skipped') && + github.event_name != 'pull_request' && + (github.event_name == 'workflow_dispatch' && + (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == null) || + github.event_name == 'schedule') + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download all build artifacts + uses: actions/download-artifact@v4 + with: + path: ./all-artifacts + + - name: Generate release tag + id: generate-tag + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }} + run: | + # Get all existing release tags that match the pattern b#### + existing_tags=$(gh release list --limit 1000 --json tagName --jq '.[].tagName' | grep -E '^b[0-9]{4}$' | sort -V || echo "") + + if [ -z "$existing_tags" ]; then + # No existing sequential tags found, start with b1000 + next_number=1000 + echo "No existing sequential release tags found, starting with b1000" + else + # Find the highest existing number + highest_tag=$(echo "$existing_tags" | tail -n 1) + highest_number=$(echo "$highest_tag" | sed 's/^b//') + next_number=$((highest_number + 1)) + echo "Highest existing tag: $highest_tag (number: $highest_number)" + echo "Next number will be: $next_number" + fi + + # Format as b#### (4-digit number with leading zeros) + TAG=$(printf "b%04d" $next_number) + + echo "tag=${TAG}" >> $GITHUB_OUTPUT + echo "Generated release tag: ${TAG}" + + - name: Check if tag already exists + id: check-tag + run: | + TAG="${{ steps.generate-tag.outputs.tag }}" + + # Check if the tag already exists + if git ls-remote --tags origin "$TAG" | grep -q "$TAG"; then + echo "Tag $TAG already exists, skipping release creation" + echo "tag_exists=true" >> $GITHUB_OUTPUT + else + echo "Tag $TAG does not exist, proceeding with release creation" + echo "tag_exists=false" >> $GITHUB_OUTPUT + fi + + - name: Create archives for all target artifacts + if: steps.check-tag.outputs.tag_exists == 'false' + run: | + # Parse targets and operating systems from environment + targets="${{ env.GFX_TARGETS }}" + operating_systems="${{ env.OPERATING_SYSTEMS }}" + TAG="${{ steps.generate-tag.outputs.tag }}" + + echo "Processing targets: $targets" + echo "Processing operating systems: $operating_systems" + echo "Using release tag: $TAG" + + # Create individual archives for each target and OS combination + IFS=',' read -ra TARGET_ARRAY <<< "$targets" + IFS=',' read -ra OS_ARRAY <<< "$operating_systems" + + for os in "${OS_ARRAY[@]}"; do + os=$(echo "$os" | xargs) # trim whitespace + for target in "${TARGET_ARRAY[@]}"; do + target=$(echo "$target" | xargs) # trim whitespace + + echo "Processing OS: $os, target: $target" + + # Use artifact name to find the directory + artifact_name="llama-${os}-rocm-${target}-x64" + artifact_dir="./all-artifacts/${artifact_name}" + + # Create final archive with release tag + final_archive_name="llama-${TAG}-${os}-rocm-${target}-x64" + + if [ -d "$artifact_dir" ]; then + echo "Creating archive: ${final_archive_name}.zip" + cd "$artifact_dir" + zip -r "../../${final_archive_name}.zip" * + cd ../../ + else + echo "Warning: Artifact directory not found: $artifact_dir" + ls -la ./all-artifacts/ + fi + done + done + + echo "Created archives:" + ls -la *.zip + + - name: Create Release + if: steps.check-tag.outputs.tag_exists == 'false' + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }} + run: | + TAG="${{ steps.generate-tag.outputs.tag }}" + # Get ROCm version from either Windows or Ubuntu build (whichever succeeded) + ROCM_VERSION="${{ needs.build-windows.outputs.rocm_version || needs.build-ubuntu.outputs.rocm_version }}" + # Get llama.cpp commit hash from either Windows or Ubuntu build (whichever succeeded) + LLAMACPP_COMMIT_HASH="${{ needs.build-windows.outputs.llamacpp_commit_hash || needs.build-ubuntu.outputs.llamacpp_commit_hash }}" + targets="${{ env.GFX_TARGETS }}" + operating_systems="${{ env.OPERATING_SYSTEMS }}" + + echo "Creating release with tag: $TAG" + echo "GPU Targets: $targets" + echo "Operating Systems: $operating_systems" + echo "ROCm Version: $ROCM_VERSION" + echo "Llama.cpp Commit: $LLAMACPP_COMMIT_HASH" + + # Verify archives exist + ls -la *.zip + + # Prepare upload files list + upload_files="" + IFS=',' read -ra TARGET_ARRAY <<< "$targets" + IFS=',' read -ra OS_ARRAY <<< "$operating_systems" + + for os in "${OS_ARRAY[@]}"; do + os=$(echo "$os" | xargs) # trim whitespace + for target in "${TARGET_ARRAY[@]}"; do + target=$(echo "$target" | xargs) # trim whitespace + + final_archive_name="llama-${TAG}-${os}-rocm-${target}-x64" + if [ -f "${final_archive_name}.zip" ]; then + upload_files="${upload_files} ${final_archive_name}.zip" + fi + done + done + + echo "Files to upload: $upload_files" + + # Create release with GitHub CLI + gh release create "$TAG" \ + --title "$TAG" \ + --notes "**Build Number**: $TAG + **Operating System(s)**: $operating_systems + **GPU Target(s)**: $targets + **ROCm Version**: $ROCM_VERSION + **Llama.cpp Commit Hash**: $LLAMACPP_COMMIT_HASH + **Build Date**: $(date -u '+%Y-%m-%d %H:%M:%S UTC') + + This release includes compiled llama.cpp binaries with ROCm support for multiple GPU targets and operating systems, with all essential ROCm runtime libraries included." \ + $upload_files diff --git a/.github/workflows/nightly-failure-alert.yml b/.github/workflows/nightly-failure-alert.yml new file mode 100644 index 000000000000..3b4ab66736a3 --- /dev/null +++ b/.github/workflows/nightly-failure-alert.yml @@ -0,0 +1,66 @@ +# Same as lemonade-sdk/llamacpp-rocm (MIT). +name: Nightly Failure Alert +on: + workflow_run: + workflows: ["Build Llama.cpp + ROCm"] + types: [completed] + +env: + # GitHub user(s) to alert, space-separated + ALERT_USERS: dzannotti + ALERT_LABEL: nightly-failure + +jobs: + alert: + if: | + github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.conclusion != 'success' + runs-on: ubuntu-22.04 + permissions: + issues: write + actions: read + steps: + - name: Open or update failure issue + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + CONCLUSION: ${{ github.event.workflow_run.conclusion }} + RUN_DATE: ${{ github.event.workflow_run.run_started_at }} + run: | + failed_jobs=$(gh api --paginate "repos/$GH_REPO/actions/runs/$RUN_ID/jobs?per_page=100" \ + --jq '.jobs[] | select(.conclusion != "success" and .conclusion != "skipped") | "- [\(.name)](\(.html_url)) — \(.conclusion)"') + [ -z "$failed_jobs" ] && failed_jobs="- (no individual job failures reported)" + + mentions=$(for u in $ALERT_USERS; do printf '@%s ' "$u"; done) + assignees=$(echo $ALERT_USERS | tr ' ' ',') + + body=$(cat <> $GITHUB_OUTPUT + echo "Final release tag: $release_tag" + + - name: Set matrix + id: set-matrix + run: | + targets="${{ env.GFX_TARGETS }}" + operating_systems="${{ env.OPERATING_SYSTEMS }}" + + echo "Input targets: $targets" + echo "Input operating systems: $operating_systems" + # Convert targets to JSON array + matrix_targets=$(echo "$targets" \ + | tr ',' '\n' \ + | sed 's/^ *//;s/ *$//' \ + | sed 's/^"//;s/"$//' \ + | jq -R . \ + | jq -s '{gfx_target: .}' \ + | jq -c) + # Check which operating systems to test + should_test_windows="false" + should_test_ubuntu="false" + + if [[ "$operating_systems" == *"windows"* ]]; then + should_test_windows="true" + echo "windows_matrix=$matrix_targets" >> $GITHUB_OUTPUT + fi + + if [[ "$operating_systems" == *"ubuntu"* ]]; then + should_test_ubuntu="true" + echo "ubuntu_matrix=$matrix_targets" >> $GITHUB_OUTPUT + fi + + echo "should_test_windows=$should_test_windows" >> $GITHUB_OUTPUT + echo "should_test_ubuntu=$should_test_ubuntu" >> $GITHUB_OUTPUT + + echo "Windows test: $should_test_windows" + echo "Ubuntu test: $should_test_ubuntu" + echo "Generated matrix: $matrix_targets" + test-windows: + runs-on: [stx-halo, Windows] + needs: prepare-matrix + if: vars.STX_HALO_RUNNERS == 'true' && needs.prepare-matrix.outputs.should_test_windows == 'true' + strategy: + matrix: ${{fromJson(needs.prepare-matrix.outputs.windows_matrix)}} + fail-fast: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download release artifacts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + $releaseTag = "${{ needs.prepare-matrix.outputs.latest_release_tag }}" + $target = "${{ matrix.gfx_target }}" + $assetName = "llama-${releaseTag}-windows-rocm-${target}-x64.zip" + $repo = "${{ github.repository }}" + + Write-Host "Downloading release asset: $assetName" + Write-Host "From release: $releaseTag" + + # Get release information using GitHub REST API + $headers = @{ "Authorization" = "token $env:GITHUB_TOKEN" } + $releaseUrl = "https://api.github.com/repos/$repo/releases/tags/$releaseTag" + + try { + $release = Invoke-RestMethod -Uri $releaseUrl -Headers $headers + $asset = $release.assets | Where-Object { $_.name -eq $assetName } + + if (-not $asset) { + Write-Error "Asset '$assetName' not found in release '$releaseTag'" + exit 1 + } + + Write-Host "Found asset: $($asset.name) (Size: $($asset.size) bytes)" + + # Download the specific asset + Invoke-WebRequest -Uri $asset.browser_download_url -OutFile $assetName -Headers $headers + Write-Host "Downloaded $assetName successfully" + } + catch { + Write-Error "Failed to download release asset: $_" + exit 1 + } + + # Extract the downloaded zip file + Write-Host "Extracting $assetName..." + Expand-Archive -Path $assetName -DestinationPath "llama-binaries" -Force + + # List contents + Write-Host "Contents of llama-binaries:" + Get-ChildItem -Recurse "llama-binaries" | Format-Table Name, Length + + - name: Download test model + run: | + $modelUrl = "https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_0.gguf?download=true" + $modelPath = "Qwen3-0.6B-Q4_0.gguf" + + Write-Host "Downloading test model from: $modelUrl" + curl.exe -L -o $modelPath $modelUrl + + # Verify download + if (Test-Path $modelPath) { + $fileSize = (Get-Item $modelPath).Length + Write-Host "Model downloaded successfully. Size: $fileSize bytes" + } else { + Write-Error "Failed to download model" + exit 1 + } + + - name: Run llama-cli test + run: | + $llamaCliPath = "llama-binaries\llama-cli.exe" + $modelPath = "Qwen3-0.6B-Q4_0.gguf" + + # Verify llama-cli exists + if (-not (Test-Path $llamaCliPath)) { + Write-Error "llama-cli.exe not found at: $llamaCliPath" + Write-Host "Available files in llama-binaries:" + Get-ChildItem -Recurse "llama-binaries" + exit 1 + } + + Write-Host "Running llama-cli test..." + Write-Host "Command: $llamaCliPath -m `"$modelPath`" -ngl 99 -p `"I believe the meaning of life is`" -st" + + # Run the command and capture output + try { + $process = Start-Process -FilePath $llamaCliPath -ArgumentList @("-m", "`"$modelPath`"", "-ngl", "99", "-p", "`"I believe the meaning of life is`"", "-st") -NoNewWindow -Wait -PassThru -RedirectStandardOutput "stdout.txt" -RedirectStandardError "stderr.txt" + + # Wait for the process to complete + $process.WaitForExit() + + # Read both stdout and stderr + $stdout = Get-Content "stdout.txt" -Raw -ErrorAction SilentlyContinue + $stderr = Get-Content "stderr.txt" -Raw -ErrorAction SilentlyContinue + + # Combine output (stderr often contains important CUDA/GPU info) + $output = $stdout + $stderr + + # Clean up temp files + Remove-Item "stdout.txt" -ErrorAction SilentlyContinue + Remove-Item "stderr.txt" -ErrorAction SilentlyContinue + + Write-Host "Process exit code: $($process.ExitCode)" + } + catch { + Write-Error "Failed to run llama-cli: $_" + exit 1 + } + + Write-Host "=== LLAMA-CLI OUTPUT ===" + Write-Host $output + Write-Host "=== END OUTPUT ===" + + # Check if the process exited successfully + if ($process.ExitCode -ne 0) { + Write-Host "❌ llama-cli exited with error code: $($process.ExitCode)" + exit 1 + } + + # Check for expected outputs + $foundGpuOffload = $output -match "offloaded 29/29 layers to GPU" + $foundThinkTag = $output -match "" + + Write-Host "=== TEST RESULTS ===" + Write-Host "Expected 'offloaded 29/29 layers to GPU': $(if ($foundGpuOffload) { 'FOUND' } else { 'NOT FOUND' })" + Write-Host "Expected '' tag: $(if ($foundThinkTag) { 'FOUND' } else { 'NOT FOUND' })" + + if ($foundGpuOffload -and $foundThinkTag) { + Write-Host "✅ Test PASSED - Both expected outputs found" + } else { + Write-Host "❌ Test FAILED - Missing expected outputs" + exit 1 + } + test-ubuntu: + runs-on: [stx-halo, Linux] + needs: prepare-matrix + if: vars.STX_HALO_RUNNERS == 'true' && needs.prepare-matrix.outputs.should_test_ubuntu == 'true' + strategy: + matrix: ${{fromJson(needs.prepare-matrix.outputs.ubuntu_matrix)}} + fail-fast: false + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install jq + run: | + if ! command -v jq &> /dev/null; then + echo "Installing jq..." + # Download and install jq binary directly + mkdir -p ~/bin + curl -L -o ~/bin/jq https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64 + chmod +x ~/bin/jq + export PATH="$HOME/bin:$PATH" + echo "PATH=$HOME/bin:$PATH" >> $GITHUB_ENV + echo "jq installed to ~/bin/jq" + else + echo "jq is already installed" + fi + + - name: Download release artifacts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + release_tag="${{ needs.prepare-matrix.outputs.latest_release_tag }}" + target="${{ matrix.gfx_target }}" + asset_name="llama-${release_tag}-ubuntu-rocm-${target}-x64.zip" + repo="${{ github.repository }}" + + echo "Downloading release asset: $asset_name" + echo "From release: $release_tag" + + # Get release information using GitHub REST API + release_url="https://api.github.com/repos/$repo/releases/tags/$release_tag" + + # Get release data and extract asset download URL + release_data=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "$release_url") + + if [ $? -ne 0 ]; then + echo "Error: Failed to fetch release information" + exit 1 + fi + + # Extract the browser_download_url for the specific asset + download_url=$(echo "$release_data" | jq -r ".assets[] | select(.name == \"$asset_name\") | .browser_download_url") + + if [ "$download_url" = "null" ] || [ -z "$download_url" ]; then + echo "Error: Asset '$asset_name' not found in release '$release_tag'" + echo "Available assets:" + echo "$release_data" | jq -r '.assets[].name' + exit 1 + fi + + asset_size=$(echo "$release_data" | jq -r ".assets[] | select(.name == \"$asset_name\") | .size") + echo "Found asset: $asset_name (Size: $asset_size bytes)" + + # Download the specific asset + curl -L -H "Authorization: token $GITHUB_TOKEN" -o "$asset_name" "$download_url" + + if [ $? -eq 0 ]; then + echo "Downloaded $asset_name successfully" + else + echo "Error: Failed to download $asset_name" + exit 1 + fi + + # Extract the downloaded zip file + echo "Extracting $asset_name..." + unzip -q "$asset_name" -d llama-binaries + + # List contents + echo "Contents of llama-binaries:" + find llama-binaries -type f -exec ls -la {} \; + + - name: Download test model + run: | + model_url="https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_0.gguf?download=true" + model_path="Qwen3-0.6B-Q4_0.gguf" + + echo "Downloading test model from: $model_url" + curl -L -o "$model_path" "$model_url" + + # Verify download + if [ -f "$model_path" ]; then + file_size=$(stat -c%s "$model_path") + echo "Model downloaded successfully. Size: $file_size bytes" + else + echo "Failed to download model" + exit 1 + fi + + - name: Set up library path + run: | + # Add the llama-binaries directory to LD_LIBRARY_PATH + echo "LD_LIBRARY_PATH=$(pwd)/llama-binaries:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Run llama-cli test + run: | + llama_cli_path="./llama-binaries/llama-cli" + model_path="Qwen3-0.6B-Q4_0.gguf" + + # Make llama-cli executable + chmod +x "$llama_cli_path" + + # Verify llama-cli exists + if [ ! -f "$llama_cli_path" ]; then + echo "llama-cli not found at: $llama_cli_path" + echo "Available files in llama-binaries:" + find llama-binaries -type f + exit 1 + fi + + echo "Running llama-cli test..." + echo "Command: $llama_cli_path -m \"$model_path\" -ngl 99 -p \"I believe the meaning of life is\" -st" + + # Run the command and capture output + output=$("$llama_cli_path" -m "$model_path" -ngl 99 -p "I believe the meaning of life is" -st 2>&1) + + echo "=== LLAMA-CLI OUTPUT ===" + echo "$output" + echo "=== END OUTPUT ===" + + # Check for expected outputs + if echo "$output" | grep -q "offloaded 29/29 layers to GPU"; then + found_gpu_offload=true + else + found_gpu_offload=false + fi + + if echo "$output" | grep -q ""; then + found_think_tag=true + else + found_think_tag=false + fi + + echo "=== TEST RESULTS ===" + echo "Expected 'offloaded 29/29 layers to GPU': $(if [ "$found_gpu_offload" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)" + echo "Expected '' tag: $(if [ "$found_think_tag" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)" + + if [ "$found_gpu_offload" = true ] && [ "$found_think_tag" = true ]; then + echo "✅ Test PASSED - Both expected outputs found" + else + echo "❌ Test FAILED - Missing expected outputs" + exit 1 + fi \ No newline at end of file diff --git a/docs/rocm-releases.md b/docs/rocm-releases.md new file mode 100644 index 000000000000..a12c55f82284 --- /dev/null +++ b/docs/rocm-releases.md @@ -0,0 +1,42 @@ +# ROCm release builds + +`.github/workflows/build-llamacpp-rocm.yml` publishes prebuilt ROCm binaries of this fork. It copies the +pipeline of [lemonade-sdk/llamacpp-rocm](https://github.com/lemonade-sdk/llamacpp-rocm) (MIT) and uses the same +artifact layout, so tools that consume their releases can consume these ones too. + +## What a release contains + +- One zip per OS and GPU target: `llama-bNNNN-{windows,ubuntu}-rocm-{target}-x64.zip`. +- Targets: `gfx1151`, `gfx1150`, `gfx120X`, `gfx110X`, `gfx103X`, `gfx90a`, `gfx908`. +- Each zip holds the llama.cpp binaries and shared libraries, built with `GGML_HIP=ON`, `GGML_RPC=ON` and + `BUILD_SHARED_LIBS=ON`. +- Each zip also holds the ROCm runtime libraries it needs (hipBLAS, rocBLAS, hipBLASLt and their kernel + libraries). On Linux, RPATH is set to `$ORIGIN`, so the zip runs without a system ROCm install. +- ROCm comes from the newest TheRock nightly tarball on `nightly.repo.amd.com` unless one is pinned. +- Releases are tagged `b1000`, `b1001` and so on, counting up. This numbering does not collide with the `b1xxxx` + tags carried over from upstream llama.cpp. +- The release notes record the build number, targets, ROCm version, source commit (5 characters) and build date. + +## When it runs + +- Nightly at 13:00 UTC, two hours after TheRock publishes its nightly. This builds `master` and publishes a release. +- On demand with `workflow_dispatch`. Inputs: OS list, targets, ROCm version, branch or tag, and whether to + publish. +- On pull requests that change the pipeline itself. These runs build but do not publish. + +If a nightly run fails, `nightly-failure-alert.yml` opens or updates an issue labelled `nightly-failure`. + +## Hardware tests + +`test-stx-halo` and `test-stx` run the built zip on real hardware. They are skipped unless the repository +variables `STX_HALO_RUNNERS` and `STX_RUNNERS` are set to `true`, because jobs whose self-hosted runners are +missing sit in the queue until they time out. To enable them: + +1. Register runners with the labels `stx-halo` + `Windows` / `Linux` (and `stx` + `Windows` for gfx1150). +2. Set the matching variable to `true`. + +`test-llamacpp-rocm.yml` tests an already published release on the same runners. + +## Tokens + +The workflow uses the `GH_TOKEN` secret when it exists. Otherwise it uses the job token with `contents: write`. diff --git a/scripts/rocm-release/gather_required_libs.py b/scripts/rocm-release/gather_required_libs.py new file mode 100644 index 000000000000..2a7d09585c1b --- /dev/null +++ b/scripts/rocm-release/gather_required_libs.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# From lemonade-sdk/llamacpp-rocm (MIT): lists the ROCm libraries a build links against, see build-llamacpp-rocm.yml + +import subprocess +import os +import shutil +import argparse + + +def find_lib_in_rocm(libname, rocm_dir): + # Simple linear search for libname inside rocm_dir + for root, _, files in os.walk(rocm_dir): + if libname in files: + return os.path.join(root, libname) + + # Exit if the library is not found in rocm_dir + raise RuntimeError(f"Could not find {libname} in {rocm_dir}") + + +def main(): + parser = argparse.ArgumentParser( + description="Gather required libraries for llama-server" + ) + parser.add_argument( + "--rocm-dir", + default="/opt/rocm", + help="Path to ROCm installation directory (default: /opt/rocm)", + ) + parser.add_argument( + "--dest-dir", + default=os.path.expanduser("~/llama.cpp/build/bin"), + help="Destination directory for libraries and binary (default: ~/llama.cpp/build/bin)", + ) + + args = parser.parse_args() + + rocm_dir = args.rocm_dir + dest_dir = args.dest_dir + binary = os.path.join(dest_dir, "llama-server") + + # Create the destination directory and run the binary + os.makedirs(dest_dir, exist_ok=True) + result = subprocess.run([binary], capture_output=True, text=True) + print(f"Error Found: {result.stderr}") + + # Copy the missing libraries to the destination directory + while "error while loading shared libraries" in result.stderr: + so_file = result.stderr.split("shared libraries: ")[1].split(": ")[0] + so_file_path = find_lib_in_rocm(so_file, rocm_dir) + shutil.copy2(so_file_path, dest_dir) + print(f"Copied {so_file_path} -> {dest_dir}") + result = subprocess.run([binary], capture_output=True, text=True) + print(f"Error Found: {result.stderr}") + + +if __name__ == "__main__": + main() From 7e4f9161a528d8eb0ea450929feda026228ca176 Mon Sep 17 00:00:00 2001 From: Daniele Zannotti Date: Tue, 29 Sep 2026 19:58:29 +0100 Subject: [PATCH 2/3] ci : release every merge to master, Linux zips by default Release on push to master instead of a nightly cron and build the merged commit. Default to ubuntu; Windows remains available through workflow_dispatch. Strip the trailing whitespace the editorconfig check rejects. Assisted-by: Claude Opus 5.5 Co-Authored-By: Claude Opus 5.5 --- .../actions/test-llamacpp-build/action.yml | 66 ++--- .github/workflows/build-llamacpp-rocm.yml | 267 +++++++++--------- .github/workflows/nightly-failure-alert.yml | 8 +- .github/workflows/test-llamacpp-rocm.yml | 124 ++++---- docs/rocm-releases.md | 12 +- 5 files changed, 244 insertions(+), 233 deletions(-) diff --git a/.github/actions/test-llamacpp-build/action.yml b/.github/actions/test-llamacpp-build/action.yml index 3152939708c2..e3cc124ceb53 100644 --- a/.github/actions/test-llamacpp-build/action.yml +++ b/.github/actions/test-llamacpp-build/action.yml @@ -21,17 +21,17 @@ runs: with: name: ${{ inputs.artifact_name }} path: llama-binaries - + - name: Download test model (Windows) if: inputs.os_type == 'Windows' shell: powershell run: | $modelUrl = "https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_0.gguf?download=true" $modelPath = "Qwen3-0.6B-Q4_0.gguf" - + Write-Host "Downloading test model from: $modelUrl" curl.exe -L -o $modelPath $modelUrl - + # Verify download if (Test-Path $modelPath) { $fileSize = (Get-Item $modelPath).Length @@ -40,17 +40,17 @@ runs: Write-Error "Failed to download model" exit 1 } - + - name: Download test model (Linux) if: inputs.os_type == 'Linux' shell: bash run: | model_url="https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_0.gguf?download=true" model_path="Qwen3-0.6B-Q4_0.gguf" - + echo "Downloading test model from: $model_url" curl -L -o "$model_path" "$model_url" - + # Verify download if [ -f "$model_path" ]; then file_size=$(stat -c%s "$model_path") @@ -59,21 +59,21 @@ runs: echo "Failed to download model" exit 1 fi - + - name: Set up library path (Linux) if: inputs.os_type == 'Linux' shell: bash run: | # Add the llama-binaries directory to LD_LIBRARY_PATH echo "LD_LIBRARY_PATH=$(pwd)/llama-binaries:$LD_LIBRARY_PATH" >> $GITHUB_ENV - + - name: Run llama-cli test (Windows) if: inputs.os_type == 'Windows' shell: powershell run: | $llamaCliPath = "llama-binaries\llama-cli.exe" $modelPath = "Qwen3-0.6B-Q4_0.gguf" - + # Verify llama-cli exists if (-not (Test-Path $llamaCliPath)) { Write-Error "llama-cli.exe not found at: $llamaCliPath" @@ -81,21 +81,21 @@ runs: Get-ChildItem -Recurse "llama-binaries" exit 1 } - + Write-Host "Running llama-cli test for ${{ inputs.gfx_target }}..." Write-Host "Command: $llamaCliPath -m `"$modelPath`" -ngl 99 -p `"I believe the meaning of life is`" -st -v" - + # Run the command and capture output try { $process = Start-Process -FilePath $llamaCliPath -ArgumentList @("-m", "`"$modelPath`"", "-ngl", "99", "-p", "`"I believe the meaning of life is`"", "-st", "-v") -NoNewWindow -Wait -PassThru -RedirectStandardOutput "stdout.txt" -RedirectStandardError "stderr.txt" - + # Wait for the process to complete $process.WaitForExit() - + # Read both stdout and stderr $stdout = Get-Content "stdout.txt" -Raw -ErrorAction SilentlyContinue $stderr = Get-Content "stderr.txt" -Raw -ErrorAction SilentlyContinue - + # Always print the output (both stdout and stderr) Write-Host "=== LLAMA-CLI STDOUT ===" if ($stdout) { @@ -111,10 +111,10 @@ runs: } Write-Host "=== END OUTPUT ===" Write-Host "Process exit code: $($process.ExitCode)" - + # Combine output for pattern matching (stderr often contains important CUDA/GPU info) $output = $stdout + $stderr - + # Clean up temp files Remove-Item "stdout.txt" -ErrorAction SilentlyContinue Remove-Item "stderr.txt" -ErrorAction SilentlyContinue @@ -128,7 +128,7 @@ runs: Get-ChildItem "llama-binaries" | Format-Table Name, Length exit 1 } - + # Check if the process exited successfully if ($process.ExitCode -ne 0) { Write-Host "❌ llama-cli exited with error code: $($process.ExitCode)" @@ -140,22 +140,22 @@ runs: Write-Host "Please check the STDERR output above for specific error messages." exit 1 } - + # Check for expected outputs $foundGpuOffload = $output -match "offloaded 29/29 layers to GPU" $foundThinkTag = $output -match "" - + Write-Host "=== TEST RESULTS ===" Write-Host "Expected 'offloaded 29/29 layers to GPU': $(if ($foundGpuOffload) { 'FOUND' } else { 'NOT FOUND' })" Write-Host "Expected '' tag: $(if ($foundThinkTag) { 'FOUND' } else { 'NOT FOUND' })" - + if ($foundGpuOffload -and $foundThinkTag) { Write-Host "✅ Test PASSED - Both expected outputs found" } else { Write-Host "❌ Test FAILED - Missing expected outputs" exit 1 } - + - name: Run llama-cli test (Linux) if: inputs.os_type == 'Linux' shell: bash @@ -163,10 +163,10 @@ runs: llama_cli_path="./llama-binaries/llama-cli" model_path="Qwen3-0.6B-Q4_0.gguf" output_file="llama_output.txt" - + # Make llama-cli executable chmod +x "$llama_cli_path" - + # Verify llama-cli exists if [ ! -f "$llama_cli_path" ]; then echo "llama-cli not found at: $llama_cli_path" @@ -174,16 +174,16 @@ runs: find llama-binaries -type f exit 1 fi - + echo "Running llama-cli test for ${{ inputs.gfx_target }}..." echo "Command: $llama_cli_path -m \"$model_path\" -ngl 99 -p \"I believe the meaning of life is\" -st -v" - + # Run the command and capture output to file (more robust than variable) set +e # Don't exit on error, we want to capture the output and check exit code manually "$llama_cli_path" -m "$model_path" -ngl 99 -p "I believe the meaning of life is" -st -v > "$output_file" 2>&1 exit_code=$? set -e # Re-enable exit on error - + echo "=== LLAMA-CLI OUTPUT ===" if [ -f "$output_file" ] && [ -s "$output_file" ]; then cat "$output_file" @@ -192,7 +192,7 @@ runs: fi echo "=== END OUTPUT ===" echo "Process exit code: $exit_code" - + # Check if the process exited successfully if [ $exit_code -ne 0 ]; then echo "❌ llama-cli exited with error code: $exit_code" @@ -208,27 +208,27 @@ runs: rm -f "$output_file" exit 1 fi - + # Check for expected outputs using grep -F (fixed strings, no regex interpretation) # This handles special characters like angle brackets more reliably found_gpu_offload=false found_think_tag=false - + if grep -F "offloaded 29/29 layers to GPU" "$output_file" > /dev/null 2>&1; then found_gpu_offload=true fi - + if grep -F "" "$output_file" > /dev/null 2>&1; then found_think_tag=true fi - + echo "=== TEST RESULTS ===" echo "Expected 'offloaded 29/29 layers to GPU': $(if [ "$found_gpu_offload" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)" echo "Expected '' tag: $(if [ "$found_think_tag" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)" - + # Clean up output file rm -f "$output_file" - + if [ "$found_gpu_offload" = true ] && [ "$found_think_tag" = true ]; then echo "✅ Test PASSED - Both expected outputs found" else diff --git a/.github/workflows/build-llamacpp-rocm.yml b/.github/workflows/build-llamacpp-rocm.yml index cba4cfa1c5f4..ed08088fb9dd 100644 --- a/.github/workflows/build-llamacpp-rocm.yml +++ b/.github/workflows/build-llamacpp-rocm.yml @@ -1,7 +1,9 @@ # Nightly ROCm release builds of this fork, same layout as lemonade-sdk/llamacpp-rocm # (https://github.com/lemonade-sdk/llamacpp-rocm, MIT) so the zips drop into the same tooling. -# Differences: builds this repository instead of cloning upstream llama.cpp, uses the job token when no -# GH_TOKEN secret exists, and runs the self-hosted Strix Halo tests only when the runners are registered. +# Differences: builds this repository instead of cloning upstream llama.cpp, releases on every merge to +# master instead of nightly, builds Linux only by default (Windows still selectable by dispatch), uses the +# job token when no GH_TOKEN secret exists, and runs the self-hosted Strix Halo tests only when the runners +# are registered. name: Build Llama.cpp + ROCm on: workflow_dispatch: @@ -9,7 +11,7 @@ on: operating_systems: description: 'Operating systems to build for (comma-separated: windows,ubuntu)' required: false - default: 'windows,ubuntu' + default: 'ubuntu' gfx_target: description: 'AMD GPU targets (comma-separated)' required: false @@ -36,17 +38,22 @@ on: - '.github/actions/test-llamacpp-build/**' - 'scripts/rocm-release/**' - # Trigger on a schedule to build nightly releases - schedule: - # Runs at 1:00 PM UTC, which is 5:00 AM PST (UTC-8) - # This is two hours after the ROCm nightly tarball is triggered - - cron: '0 13 * * *' + # Release every merge to master + push: + branches: [master] + +# a newer merge supersedes a release that is still building +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true env: - OPERATING_SYSTEMS: ${{ github.event.inputs.operating_systems || 'windows,ubuntu' }} + OPERATING_SYSTEMS: ${{ github.event.inputs.operating_systems || 'ubuntu' }} GFX_TARGETS: ${{ github.event.inputs.gfx_target || 'gfx1151,gfx1150,gfx120X,gfx110X,gfx103X,gfx90a,gfx908' }} ROCM_VERSION: ${{ github.event.inputs.rocm_version || 'latest' }} LLAMACPP_VERSION: ${{ github.event.inputs.llamacpp_version || 'latest' }} + # on a push, build exactly the merged commit + BUILD_SHA: ${{ github.event_name == 'push' && github.sha || '' }} jobs: prepare-matrix: @@ -62,7 +69,7 @@ jobs: run: | targets="${{ env.GFX_TARGETS }}" operating_systems="${{ env.OPERATING_SYSTEMS }}" - + echo "Input targets: $targets" echo "Input operating systems: $operating_systems" @@ -78,20 +85,20 @@ jobs: # Check which operating systems to build should_build_windows="false" should_build_ubuntu="false" - + if [[ "$operating_systems" == *"windows"* ]]; then should_build_windows="true" echo "windows_matrix=$matrix_targets" >> $GITHUB_OUTPUT fi - + if [[ "$operating_systems" == *"ubuntu"* ]]; then should_build_ubuntu="true" echo "ubuntu_matrix=$matrix_targets" >> $GITHUB_OUTPUT fi - + echo "should_build_windows=$should_build_windows" >> $GITHUB_OUTPUT echo "should_build_ubuntu=$should_build_ubuntu" >> $GITHUB_OUTPUT - + echo "Windows build: $should_build_windows" echo "Ubuntu build: $should_build_ubuntu" echo "Generated matrix: $matrix_targets" @@ -114,7 +121,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - + - name: Clean up existing directories (safety precaution) run: | # Remove existing llama.cpp directory if it exists @@ -122,19 +129,19 @@ jobs: Write-Host "Removing existing llama.cpp directory..." Remove-Item -Recurse -Force "llama.cpp" } - + # Remove existing C:\opt\rocm directory if it exists if (Test-Path "C:\opt\rocm") { Write-Host "Removing existing C:\opt\rocm directory..." Remove-Item -Recurse -Force "C:\opt\rocm" } - + # Remove any existing ROCm tarball if (Test-Path "rocm.tar.gz") { Write-Host "Removing existing rocm.tar.gz..." Remove-Item -Force "rocm.tar.gz" } - + Write-Host "Cleanup completed successfully" - name: Install Visual Studio Build Tools @@ -142,20 +149,20 @@ jobs: # Download and install Visual Studio Build Tools $vsInstallerUrl = "https://aka.ms/vs/17/release/vs_buildtools.exe" $vsInstallerPath = "$env:TEMP\vs_buildtools.exe" - + Write-Host "Downloading Visual Studio Build Tools..." Invoke-WebRequest -Uri $vsInstallerUrl -OutFile $vsInstallerPath - + Write-Host "Installing Visual Studio Build Tools..." Start-Process -FilePath $vsInstallerPath -ArgumentList "--quiet", "--wait", "--norestart", "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "--add", "Microsoft.VisualStudio.Component.VC.CMake.Project", "--add", "Microsoft.VisualStudio.Component.VC.ATL", "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621" -Wait - + # Clean up installer Remove-Item $vsInstallerPath -Force - name: Install build dependencies run: | Write-Host "Installing build dependencies using manual methods..." - + # Install Ninja Write-Host "Installing Ninja..." $ninjaUrl = "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip" @@ -164,14 +171,14 @@ jobs: New-Item -ItemType Directory -Force -Path $ninjaDir Invoke-WebRequest -Uri $ninjaUrl -OutFile $ninjaPath Expand-Archive -Path $ninjaPath -DestinationPath $ninjaDir -Force - + # Install Strawberry Perl Write-Host "Installing Strawberry Perl..." $perlUrl = "http://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.msi" $perlPath = "$env:TEMP\strawberry-perl-5.32.1.1-64bit.msi" Invoke-WebRequest -Uri $perlUrl -OutFile $perlPath Start-Process msiexec.exe -ArgumentList "/i $perlPath /quiet /norestart" -Wait - + # Verify installations $env:PATH = "C:\ninja;C:\Strawberry\perl\bin;$env:PATH" Write-Host "Verifying installations..." @@ -179,20 +186,20 @@ jobs: perl --version Write-Host "Manual installation of build dependencies completed" - + - name: Download ROCm nightly tarball run: | # Determine ROCm version to use $rocmVersion = "${{ env.ROCM_VERSION }}" $currentTarget = "${{ matrix.gfx_target }}" - + # Add appropriate suffixes for different GPU targets $s3Target = $currentTarget if ($currentTarget -eq "gfx103X" -or $currentTarget -eq "gfx110X" -or $currentTarget -eq "gfx120X") { $s3Target = "$currentTarget-all" Write-Host "Using target with -all suffix: $s3Target" } - + # TheRock now publishes nightlies to the ROCm core tarball index. That # index is a static HTML page with an embedded JSON `files` array (name + # mtime), so we parse that instead of the retired S3 XML bucket listing. @@ -240,21 +247,21 @@ jobs: # For specific versions, construct the URL directly $rocmUrl = "$baseUrl/therock-dist-windows-$s3Target-$rocmVersion.tar.gz" } - + # Store the version for use in other steps echo "DETECTED_ROCM_VERSION=$rocmVersion" >> $env:GITHUB_ENV - + Write-Host "Downloading ROCm from: $rocmUrl" Invoke-WebRequest -Uri $rocmUrl -OutFile "rocm.tar.gz" - + - name: Extract ROCm to C:\opt\rocm run: | # Create directory if it doesn't exist New-Item -ItemType Directory -Force -Path "C:\opt\rocm" - + # Extract the tarball tar -xzf rocm.tar.gz -C C:\opt\rocm --strip-components=1 - + - name: Clone llama.cpp run: | $llamacppVersion = "${{ env.LLAMACPP_VERSION }}" @@ -266,12 +273,13 @@ jobs: git clone --filter=blob:none --single-branch --branch $llamacppVersion https://github.com/${{ github.repository }}.git llama.cpp } cd llama.cpp - + if ("${{ env.BUILD_SHA }}" -ne "") { git checkout --detach "${{ env.BUILD_SHA }}" } + # Get commit hash (5 digits) and store it as environment variable $commitHash = git rev-parse --short=5 HEAD echo "LLAMACPP_COMMIT_HASH=$commitHash" >> $env:GITHUB_ENV Write-Host "llama.cpp commit hash (5 digits): $commitHash" - + # Show current commit info Write-Host "Current llama.cpp commit:" git log --oneline -1 @@ -283,7 +291,7 @@ jobs: REM Map GPU targets set "current_target=${{ matrix.gfx_target }}" echo Input target: %current_target% - + if "%current_target%"=="gfx110X" ( set "mapped_target=gfx1100;gfx1101;gfx1102;gfx1103" ) else if "%current_target%"=="gfx103X" ( @@ -336,23 +344,23 @@ jobs: REM Build the project cmake --build . -j %NUMBER_OF_PROCESSORS% - + - name: Copy ROCm core DLLs to build directory run: | $rocmVersion = if ($env:DETECTED_ROCM_VERSION) { $env:DETECTED_ROCM_VERSION } else { $env:ROCM_VERSION } $buildBinPath = "llama.cpp\build\bin" $rocmBinPath = "C:\opt\rocm\bin" - + Write-Host "Copying ROCm core DLL files to build directory..." Write-Host "Source: $rocmBinPath" Write-Host "Destination: $buildBinPath" - + if (Test-Path $rocmBinPath) { # Copy files matching patterns and specific names. $filesToCopy = @( "amdhip64_*.dll", "rocm_kpack.dll", - "amd_comgr*.dll", + "amd_comgr*.dll", "libhipblas.dll", "rocblas.dll", "rocsolver.dll", @@ -362,7 +370,7 @@ jobs: "origami.dll", "libtensilelite-host.dll" ) - + foreach ($pattern in $filesToCopy) { $matchingFiles = Get-ChildItem -Path $rocmBinPath -Name $pattern -ErrorAction SilentlyContinue if ($matchingFiles) { @@ -376,7 +384,7 @@ jobs: Write-Host "Warning: No files found matching pattern: $pattern" } } - + # Copy the rocblas\library folder and all its contents $rocblasLibPath = Join-Path $rocmBinPath "rocblas\library" if (Test-Path $rocblasLibPath) { @@ -384,7 +392,7 @@ jobs: $destRocblasPath = Join-Path $buildBinPath "rocblas\library" Copy-Item -Path $rocblasLibPath -Destination $destRocblasPath -Recurse -Force Write-Host "Copied: rocblas\library folder with all contents" - + # List the contents of the copied rocblas\library folder Write-Host "Contents of rocblas\library:" Get-ChildItem $destRocblasPath -Recurse | Select-Object FullName, Length | Format-Table -AutoSize @@ -406,19 +414,19 @@ jobs: } else { Write-Host "Warning: rocblas\library folder not found at: $rocblasLibPath" } - + Write-Host "ROCm core files successfully copied to build directory" } else { Write-Error "ROCm bin directory not found: $rocmBinPath" exit 1 } - + - name: List build artifacts (including ROCm files) run: | cd llama.cpp\build\bin Write-Host "Final build artifacts (including ROCm core files):" Get-ChildItem -Recurse | Format-Table Name, Length, LastWriteTime - + - name: Upload build artifacts id: upload-artifacts uses: actions/upload-artifact@v4 @@ -426,7 +434,7 @@ jobs: name: llama-windows-rocm-${{ matrix.gfx_target }}-x64 path: llama.cpp/build/bin/ retention-days: 30 - + - name: Set job outputs id: set-outputs run: | @@ -435,33 +443,33 @@ jobs: Write-Host "DETECTED_ROCM_VERSION: $env:DETECTED_ROCM_VERSION" Write-Host "ROCM_VERSION: $env:ROCM_VERSION" Write-Host "LLAMACPP_COMMIT_HASH: $env:LLAMACPP_COMMIT_HASH" - + $rocmVersion = if ($env:DETECTED_ROCM_VERSION) { $env:DETECTED_ROCM_VERSION } else { $env:ROCM_VERSION } echo "rocm_version=$rocmVersion" >> $env:GITHUB_OUTPUT echo "llamacpp_commit_hash=$env:LLAMACPP_COMMIT_HASH" >> $env:GITHUB_OUTPUT - + Write-Host "Final rocm_version: $rocmVersion" Write-Host "Final llamacpp_commit_hash: $env:LLAMACPP_COMMIT_HASH" - + - name: Create release summary run: | $summary = @" # Llama.cpp + ROCm Build Summary (Windows - ${{ matrix.gfx_target }}) - + ## Build Configuration - **GPU Target**: ${{ matrix.gfx_target }} - **ROCm Version**: ${{ env.DETECTED_ROCM_VERSION || env.ROCM_VERSION }} - **llama.cpp Commit Hash**: ${{ env.LLAMACPP_COMMIT_HASH }} - **Build Type**: Release - **Platform**: Windows - + ## Build Artifacts The compiled binaries are available in the build artifacts. "@ - + Set-Content -Path "build-summary-windows-${{ matrix.gfx_target }}.md" -Value $summary - + - name: Upload build summary id: upload-summary uses: actions/upload-artifact@v4 @@ -469,18 +477,18 @@ jobs: name: build-summary-windows-${{ matrix.gfx_target }} path: build-summary-windows-${{ matrix.gfx_target }}.md retention-days: 30 - + - name: Clean up build artifacts and temporary files if: always() run: | Write-Host "Starting cleanup process..." - + # Remove ROCm tarball if (Test-Path "rocm.tar.gz") { Write-Host "Removing rocm.tar.gz..." Remove-Item -Force "rocm.tar.gz" } - + # Remove ROCm installation directory if (Test-Path "C:\opt\rocm") { Write-Host "Removing C:\opt\rocm directory..." @@ -491,13 +499,13 @@ jobs: Write-Host "Removing llama.cpp source directory..." Remove-Item -Recurse -Force "llama.cpp" } - + # Clean up any temporary build files if (Test-Path "build-summary-windows-${{ matrix.gfx_target }}.md") { Write-Host "Removing build-summary-windows-${{ matrix.gfx_target }}.md..." Remove-Item -Force "build-summary-windows-${{ matrix.gfx_target }}.md" } - + Write-Host "Cleanup completed successfully" build-ubuntu: @@ -516,10 +524,10 @@ jobs: - name: Free disk space # Remove unused runner files to free up disk space run: curl -fsSL https://raw.githubusercontent.com/kou/arrow/e49d8ae15583ceff03237571569099a6ad62be32/ci/scripts/util_free_space.sh | bash - + - name: Checkout repository uses: actions/checkout@v4 - + - name: Clean up existing directories (safety precaution) run: | # Remove existing llama.cpp directory if it exists @@ -527,19 +535,19 @@ jobs: echo "Removing existing llama.cpp directory..." rm -rf llama.cpp fi - + # Remove existing /opt/rocm directory if it exists if [ -d "/opt/rocm" ]; then echo "Removing existing /opt/rocm directory..." sudo rm -rf /opt/rocm fi - + # Remove any existing ROCm tarball if [ -f "rocm.tar.gz" ]; then echo "Removing existing rocm.tar.gz..." rm -f rocm.tar.gz fi - + echo "Cleanup completed successfully" - name: Install build dependencies @@ -547,26 +555,26 @@ jobs: echo "Installing build dependencies..." sudo apt update sudo apt install -y cmake ninja-build unzip curl - + # Verify installations echo "Verifying installations..." cmake --version ninja --version echo "Build dependencies installation completed" - + - name: Download and extract ROCm directly to /opt/rocm run: | # Determine ROCm version to use rocm_version="${{ env.ROCM_VERSION }}" current_target="${{ matrix.gfx_target }}" - + # Add appropriate suffixes for different GPU targets s3_target="$current_target" if [[ "$current_target" = "gfx103X" || "$current_target" = "gfx110X" || "$current_target" = "gfx120X" ]]; then s3_target="${current_target}-all" echo "Using target with -all suffix: $s3_target" fi - + # TheRock now publishes nightlies to the ROCm core tarball index. That # index is a static HTML page with an embedded JSON `files` array (name + # mtime), so we parse that instead of the retired S3 XML bucket listing. @@ -608,22 +616,22 @@ jobs: # For specific versions, construct the URL directly rocm_url="$base_url/therock-dist-linux-${s3_target}-${rocm_version}.tar.gz" fi - + # Store the version for use in other steps echo "DETECTED_ROCM_VERSION=$rocm_version" >> $GITHUB_ENV - + echo "Streaming ROCm from: $rocm_url directly to extraction" - + # Create directory if it doesn't exist sudo mkdir -p /opt/rocm - + # Stream download directly into tar extraction (no intermediate file) curl -sL "$rocm_url" | sudo tar --use-compress-program=gzip -xf - -C /opt/rocm --strip-components=1 - + - name: Set ROCm environment variables run: | echo "Setting ROCm environment variables..." - + # Set environment variables for this step and subsequent steps echo "HIP_PATH=/opt/rocm" >> $GITHUB_ENV echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV @@ -632,19 +640,19 @@ jobs: echo "HIP_INCLUDE_PATH=/opt/rocm/include" >> $GITHUB_ENV echo "HIP_LIB_PATH=/opt/rocm/lib" >> $GITHUB_ENV echo "HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode" >> $GITHUB_ENV - + # Update PATH echo "/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH" >> $GITHUB_PATH - + # Set library paths echo "LD_LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV echo "LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:${LIBRARY_PATH:-}" >> $GITHUB_ENV echo "CPATH=/opt/rocm/include:${CPATH:-}" >> $GITHUB_ENV echo "PKG_CONFIG_PATH=/opt/rocm/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> $GITHUB_ENV - + echo "ROCm environment variables set successfully" - + - name: Clone llama.cpp run: | llamacpp_version="${{ env.LLAMACPP_VERSION }}" @@ -656,23 +664,24 @@ jobs: git clone --filter=blob:none --single-branch --branch "$llamacpp_version" https://github.com/${{ github.repository }}.git llama.cpp fi cd llama.cpp - + if [ -n "${{ env.BUILD_SHA }}" ]; then git checkout --detach "${{ env.BUILD_SHA }}"; fi + # Get commit hash (5 digits) and store it as environment variable commit_hash=$(git rev-parse --short=5 HEAD) echo "LLAMACPP_COMMIT_HASH=$commit_hash" >> $GITHUB_ENV echo "llama.cpp commit hash (5 digits): $commit_hash" - + # Show current commit info echo "Current llama.cpp commit:" git log --oneline -1 - + - name: Build Llama.cpp + ROCm run: | # Map GPU targets current_target="${{ matrix.gfx_target }}" echo "Input target: $current_target" - + if [ "$current_target" = "gfx110X" ]; then mapped_target="gfx1100;gfx1101;gfx1102;gfx1103" elif [ "$current_target" = "gfx103X" ]; then @@ -715,12 +724,12 @@ jobs: # Build the project cmake --build . -j $(nproc) - + - name: Copy ROCm core libs to build directory run: | build_bin_path="llama.cpp/build/bin" rocm_bin_path="/opt/rocm/bin" - + # Copy the rocblas/library folder and all its contents rocblas_lib_path="/opt/rocm/lib/rocblas/library" if [ -d "$rocblas_lib_path" ]; then @@ -729,7 +738,7 @@ jobs: mkdir -p "$(dirname "$dest_rocblas_path")" cp -r "$rocblas_lib_path" "$(dirname "$dest_rocblas_path")/" echo "Copied: rocblas/library folder with all contents" - + # List the contents of the copied rocblas/library folder echo "Contents of rocblas/library:" find "$dest_rocblas_path" -type f -exec ls -la {} \; | head -20 @@ -752,7 +761,7 @@ jobs: else echo "Warning: hipblaslt/library folder not found at: $hipblaslt_lib_path" fi - + # Copy required ROCm libraries to build directory # If artifacts from ROCm or Llama.cpp change, you may need to update this list # To get a new list of all libraries, run: @@ -779,7 +788,7 @@ jobs: cp -v /opt/rocm/lib/llvm/lib/libLLVM.so* "$build_bin_path/" 2>/dev/null || echo "libLLVM.so* not found" cp -v /opt/rocm/lib/llvm/lib/libclang-cpp.so* "$build_bin_path/" 2>/dev/null || echo "libclang-cpp.so* not found" cp -v /opt/rocm/lib/llvm/lib/libomp.so* "$build_bin_path/" 2>/dev/null || echo "libomp.so* not found" - + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_elf.so* not found" cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm.so* not found" cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm_amdgpu.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm_amdgpu.so* not found" @@ -801,7 +810,7 @@ jobs: cd llama.cpp/build/bin echo "Final build artifacts (including ROCm library files):" ls -la - + - name: Upload build artifacts id: upload-artifacts uses: actions/upload-artifact@v4 @@ -809,7 +818,7 @@ jobs: name: llama-ubuntu-rocm-${{ matrix.gfx_target }}-x64 path: llama.cpp/build/bin/ retention-days: 30 - + - name: Set job outputs id: set-outputs run: | @@ -818,30 +827,30 @@ jobs: echo "DETECTED_ROCM_VERSION: ${DETECTED_ROCM_VERSION}" echo "ROCM_VERSION: ${{ env.ROCM_VERSION }}" echo "LLAMACPP_COMMIT_HASH: ${LLAMACPP_COMMIT_HASH}" - + rocm_version="${DETECTED_ROCM_VERSION:-${{ env.ROCM_VERSION }}}" echo "rocm_version=$rocm_version" >> $GITHUB_OUTPUT echo "llamacpp_commit_hash=${LLAMACPP_COMMIT_HASH}" >> $GITHUB_OUTPUT - + echo "Final rocm_version: $rocm_version" echo "Final llamacpp_commit_hash: ${LLAMACPP_COMMIT_HASH}" - + - name: Create release summary run: | cat > "build-summary-ubuntu-${{ matrix.gfx_target }}.md" << 'EOF' # Llama.cpp + ROCm Build Summary (Ubuntu - ${{ matrix.gfx_target }}) - + ## Build Configuration - **GPU Target**: ${{ matrix.gfx_target }} - **ROCm Version**: ${{ env.DETECTED_ROCM_VERSION || env.ROCM_VERSION }} - **llama.cpp Commit Hash**: ${{ env.LLAMACPP_COMMIT_HASH }} - **Build Type**: Release - **Platform**: Ubuntu - + ## Build Artifacts The compiled binaries are available in the build artifacts. EOF - + - name: Upload build summary id: upload-summary uses: actions/upload-artifact@v4 @@ -849,36 +858,36 @@ jobs: name: build-summary-ubuntu-${{ matrix.gfx_target }} path: build-summary-ubuntu-${{ matrix.gfx_target }}.md retention-days: 30 - + - name: Clean up build artifacts and temporary files if: always() run: | echo "Starting cleanup process..." - + # Remove ROCm tarball if [ -f "rocm.tar.gz" ]; then echo "Removing rocm.tar.gz..." rm -f rocm.tar.gz fi - + # Remove ROCm installation directory if [ -d "/opt/rocm" ]; then echo "Removing /opt/rocm directory..." sudo rm -rf /opt/rocm fi - + # Remove llama.cpp source directory (keeping only the build artifacts) if [ -d "llama.cpp" ]; then echo "Removing llama.cpp source directory..." rm -rf llama.cpp fi - + # Clean up any temporary build files if [ -f "build-summary-ubuntu-${{ matrix.gfx_target }}.md" ]; then echo "Removing build-summary-ubuntu-${{ matrix.gfx_target }}.md..." rm -f "build-summary-ubuntu-${{ matrix.gfx_target }}.md" fi - + echo "Cleanup completed successfully" test-stx-halo: @@ -904,7 +913,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - + - name: Test llama.cpp build uses: ./.github/actions/test-llamacpp-build with: @@ -931,7 +940,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - + - name: Test llama.cpp build uses: ./.github/actions/test-llamacpp-build with: @@ -953,19 +962,19 @@ jobs: (needs.test-stx-halo.result == 'success' || needs.test-stx-halo.result == 'skipped') && (needs.test-stx.result == 'success' || needs.test-stx.result == 'skipped') && github.event_name != 'pull_request' && - (github.event_name == 'workflow_dispatch' && + (github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == null) || - github.event_name == 'schedule') - + github.event_name == 'push') + steps: - name: Checkout repository uses: actions/checkout@v4 - + - name: Download all build artifacts uses: actions/download-artifact@v4 with: path: ./all-artifacts - + - name: Generate release tag id: generate-tag env: @@ -973,7 +982,7 @@ jobs: run: | # Get all existing release tags that match the pattern b#### existing_tags=$(gh release list --limit 1000 --json tagName --jq '.[].tagName' | grep -E '^b[0-9]{4}$' | sort -V || echo "") - + if [ -z "$existing_tags" ]; then # No existing sequential tags found, start with b1000 next_number=1000 @@ -986,18 +995,18 @@ jobs: echo "Highest existing tag: $highest_tag (number: $highest_number)" echo "Next number will be: $next_number" fi - + # Format as b#### (4-digit number with leading zeros) TAG=$(printf "b%04d" $next_number) - + echo "tag=${TAG}" >> $GITHUB_OUTPUT echo "Generated release tag: ${TAG}" - + - name: Check if tag already exists id: check-tag run: | TAG="${{ steps.generate-tag.outputs.tag }}" - + # Check if the tag already exists if git ls-remote --tags origin "$TAG" | grep -q "$TAG"; then echo "Tag $TAG already exists, skipping release creation" @@ -1006,7 +1015,7 @@ jobs: echo "Tag $TAG does not exist, proceeding with release creation" echo "tag_exists=false" >> $GITHUB_OUTPUT fi - + - name: Create archives for all target artifacts if: steps.check-tag.outputs.tag_exists == 'false' run: | @@ -1014,29 +1023,29 @@ jobs: targets="${{ env.GFX_TARGETS }}" operating_systems="${{ env.OPERATING_SYSTEMS }}" TAG="${{ steps.generate-tag.outputs.tag }}" - + echo "Processing targets: $targets" echo "Processing operating systems: $operating_systems" echo "Using release tag: $TAG" - + # Create individual archives for each target and OS combination IFS=',' read -ra TARGET_ARRAY <<< "$targets" IFS=',' read -ra OS_ARRAY <<< "$operating_systems" - + for os in "${OS_ARRAY[@]}"; do os=$(echo "$os" | xargs) # trim whitespace for target in "${TARGET_ARRAY[@]}"; do target=$(echo "$target" | xargs) # trim whitespace - + echo "Processing OS: $os, target: $target" - + # Use artifact name to find the directory artifact_name="llama-${os}-rocm-${target}-x64" artifact_dir="./all-artifacts/${artifact_name}" - + # Create final archive with release tag final_archive_name="llama-${TAG}-${os}-rocm-${target}-x64" - + if [ -d "$artifact_dir" ]; then echo "Creating archive: ${final_archive_name}.zip" cd "$artifact_dir" @@ -1048,10 +1057,10 @@ jobs: fi done done - + echo "Created archives:" ls -la *.zip - + - name: Create Release if: steps.check-tag.outputs.tag_exists == 'false' env: @@ -1064,35 +1073,35 @@ jobs: LLAMACPP_COMMIT_HASH="${{ needs.build-windows.outputs.llamacpp_commit_hash || needs.build-ubuntu.outputs.llamacpp_commit_hash }}" targets="${{ env.GFX_TARGETS }}" operating_systems="${{ env.OPERATING_SYSTEMS }}" - + echo "Creating release with tag: $TAG" echo "GPU Targets: $targets" echo "Operating Systems: $operating_systems" echo "ROCm Version: $ROCM_VERSION" echo "Llama.cpp Commit: $LLAMACPP_COMMIT_HASH" - + # Verify archives exist ls -la *.zip - + # Prepare upload files list upload_files="" IFS=',' read -ra TARGET_ARRAY <<< "$targets" IFS=',' read -ra OS_ARRAY <<< "$operating_systems" - + for os in "${OS_ARRAY[@]}"; do os=$(echo "$os" | xargs) # trim whitespace for target in "${TARGET_ARRAY[@]}"; do target=$(echo "$target" | xargs) # trim whitespace - + final_archive_name="llama-${TAG}-${os}-rocm-${target}-x64" if [ -f "${final_archive_name}.zip" ]; then upload_files="${upload_files} ${final_archive_name}.zip" fi done done - + echo "Files to upload: $upload_files" - + # Create release with GitHub CLI gh release create "$TAG" \ --title "$TAG" \ diff --git a/.github/workflows/nightly-failure-alert.yml b/.github/workflows/nightly-failure-alert.yml index 3b4ab66736a3..e26dc76a3ed3 100644 --- a/.github/workflows/nightly-failure-alert.yml +++ b/.github/workflows/nightly-failure-alert.yml @@ -13,7 +13,7 @@ env: jobs: alert: if: | - github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.event == 'push' && github.event.workflow_run.conclusion != 'success' runs-on: ubuntu-22.04 permissions: @@ -37,7 +37,7 @@ jobs: assignees=$(echo $ALERT_USERS | tr ' ' ',') body=$(cat <> $GITHUB_OUTPUT echo "Final release tag: $release_tag" - + - name: Set matrix id: set-matrix run: | targets="${{ env.GFX_TARGETS }}" operating_systems="${{ env.OPERATING_SYSTEMS }}" - + echo "Input targets: $targets" echo "Input operating systems: $operating_systems" # Convert targets to JSON array @@ -85,20 +85,20 @@ jobs: # Check which operating systems to test should_test_windows="false" should_test_ubuntu="false" - + if [[ "$operating_systems" == *"windows"* ]]; then should_test_windows="true" echo "windows_matrix=$matrix_targets" >> $GITHUB_OUTPUT fi - + if [[ "$operating_systems" == *"ubuntu"* ]]; then should_test_ubuntu="true" echo "ubuntu_matrix=$matrix_targets" >> $GITHUB_OUTPUT fi - + echo "should_test_windows=$should_test_windows" >> $GITHUB_OUTPUT echo "should_test_ubuntu=$should_test_ubuntu" >> $GITHUB_OUTPUT - + echo "Windows test: $should_test_windows" echo "Ubuntu test: $should_test_ubuntu" echo "Generated matrix: $matrix_targets" @@ -122,25 +122,25 @@ jobs: $target = "${{ matrix.gfx_target }}" $assetName = "llama-${releaseTag}-windows-rocm-${target}-x64.zip" $repo = "${{ github.repository }}" - + Write-Host "Downloading release asset: $assetName" Write-Host "From release: $releaseTag" - + # Get release information using GitHub REST API $headers = @{ "Authorization" = "token $env:GITHUB_TOKEN" } $releaseUrl = "https://api.github.com/repos/$repo/releases/tags/$releaseTag" - + try { $release = Invoke-RestMethod -Uri $releaseUrl -Headers $headers $asset = $release.assets | Where-Object { $_.name -eq $assetName } - + if (-not $asset) { Write-Error "Asset '$assetName' not found in release '$releaseTag'" exit 1 } - + Write-Host "Found asset: $($asset.name) (Size: $($asset.size) bytes)" - + # Download the specific asset Invoke-WebRequest -Uri $asset.browser_download_url -OutFile $assetName -Headers $headers Write-Host "Downloaded $assetName successfully" @@ -149,23 +149,23 @@ jobs: Write-Error "Failed to download release asset: $_" exit 1 } - + # Extract the downloaded zip file Write-Host "Extracting $assetName..." Expand-Archive -Path $assetName -DestinationPath "llama-binaries" -Force - + # List contents Write-Host "Contents of llama-binaries:" Get-ChildItem -Recurse "llama-binaries" | Format-Table Name, Length - + - name: Download test model run: | $modelUrl = "https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_0.gguf?download=true" $modelPath = "Qwen3-0.6B-Q4_0.gguf" - + Write-Host "Downloading test model from: $modelUrl" curl.exe -L -o $modelPath $modelUrl - + # Verify download if (Test-Path $modelPath) { $fileSize = (Get-Item $modelPath).Length @@ -174,12 +174,12 @@ jobs: Write-Error "Failed to download model" exit 1 } - + - name: Run llama-cli test run: | $llamaCliPath = "llama-binaries\llama-cli.exe" $modelPath = "Qwen3-0.6B-Q4_0.gguf" - + # Verify llama-cli exists if (-not (Test-Path $llamaCliPath)) { Write-Error "llama-cli.exe not found at: $llamaCliPath" @@ -187,53 +187,53 @@ jobs: Get-ChildItem -Recurse "llama-binaries" exit 1 } - + Write-Host "Running llama-cli test..." Write-Host "Command: $llamaCliPath -m `"$modelPath`" -ngl 99 -p `"I believe the meaning of life is`" -st" - + # Run the command and capture output try { $process = Start-Process -FilePath $llamaCliPath -ArgumentList @("-m", "`"$modelPath`"", "-ngl", "99", "-p", "`"I believe the meaning of life is`"", "-st") -NoNewWindow -Wait -PassThru -RedirectStandardOutput "stdout.txt" -RedirectStandardError "stderr.txt" - + # Wait for the process to complete $process.WaitForExit() - + # Read both stdout and stderr $stdout = Get-Content "stdout.txt" -Raw -ErrorAction SilentlyContinue $stderr = Get-Content "stderr.txt" -Raw -ErrorAction SilentlyContinue - + # Combine output (stderr often contains important CUDA/GPU info) $output = $stdout + $stderr - + # Clean up temp files Remove-Item "stdout.txt" -ErrorAction SilentlyContinue Remove-Item "stderr.txt" -ErrorAction SilentlyContinue - + Write-Host "Process exit code: $($process.ExitCode)" } catch { Write-Error "Failed to run llama-cli: $_" exit 1 } - + Write-Host "=== LLAMA-CLI OUTPUT ===" Write-Host $output Write-Host "=== END OUTPUT ===" - + # Check if the process exited successfully if ($process.ExitCode -ne 0) { Write-Host "❌ llama-cli exited with error code: $($process.ExitCode)" exit 1 } - + # Check for expected outputs $foundGpuOffload = $output -match "offloaded 29/29 layers to GPU" $foundThinkTag = $output -match "" - + Write-Host "=== TEST RESULTS ===" Write-Host "Expected 'offloaded 29/29 layers to GPU': $(if ($foundGpuOffload) { 'FOUND' } else { 'NOT FOUND' })" Write-Host "Expected '' tag: $(if ($foundThinkTag) { 'FOUND' } else { 'NOT FOUND' })" - + if ($foundGpuOffload -and $foundThinkTag) { Write-Host "✅ Test PASSED - Both expected outputs found" } else { @@ -266,7 +266,7 @@ jobs: else echo "jq is already installed" fi - + - name: Download release artifacts env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -275,60 +275,60 @@ jobs: target="${{ matrix.gfx_target }}" asset_name="llama-${release_tag}-ubuntu-rocm-${target}-x64.zip" repo="${{ github.repository }}" - + echo "Downloading release asset: $asset_name" echo "From release: $release_tag" - + # Get release information using GitHub REST API release_url="https://api.github.com/repos/$repo/releases/tags/$release_tag" - + # Get release data and extract asset download URL release_data=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "$release_url") - + if [ $? -ne 0 ]; then echo "Error: Failed to fetch release information" exit 1 fi - + # Extract the browser_download_url for the specific asset download_url=$(echo "$release_data" | jq -r ".assets[] | select(.name == \"$asset_name\") | .browser_download_url") - + if [ "$download_url" = "null" ] || [ -z "$download_url" ]; then echo "Error: Asset '$asset_name' not found in release '$release_tag'" echo "Available assets:" echo "$release_data" | jq -r '.assets[].name' exit 1 fi - + asset_size=$(echo "$release_data" | jq -r ".assets[] | select(.name == \"$asset_name\") | .size") echo "Found asset: $asset_name (Size: $asset_size bytes)" - + # Download the specific asset curl -L -H "Authorization: token $GITHUB_TOKEN" -o "$asset_name" "$download_url" - + if [ $? -eq 0 ]; then echo "Downloaded $asset_name successfully" else echo "Error: Failed to download $asset_name" exit 1 fi - + # Extract the downloaded zip file echo "Extracting $asset_name..." unzip -q "$asset_name" -d llama-binaries - + # List contents echo "Contents of llama-binaries:" find llama-binaries -type f -exec ls -la {} \; - + - name: Download test model run: | model_url="https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_0.gguf?download=true" model_path="Qwen3-0.6B-Q4_0.gguf" - + echo "Downloading test model from: $model_url" curl -L -o "$model_path" "$model_url" - + # Verify download if [ -f "$model_path" ]; then file_size=$(stat -c%s "$model_path") @@ -337,20 +337,20 @@ jobs: echo "Failed to download model" exit 1 fi - + - name: Set up library path run: | # Add the llama-binaries directory to LD_LIBRARY_PATH echo "LD_LIBRARY_PATH=$(pwd)/llama-binaries:$LD_LIBRARY_PATH" >> $GITHUB_ENV - + - name: Run llama-cli test run: | llama_cli_path="./llama-binaries/llama-cli" model_path="Qwen3-0.6B-Q4_0.gguf" - + # Make llama-cli executable chmod +x "$llama_cli_path" - + # Verify llama-cli exists if [ ! -f "$llama_cli_path" ]; then echo "llama-cli not found at: $llama_cli_path" @@ -358,37 +358,37 @@ jobs: find llama-binaries -type f exit 1 fi - + echo "Running llama-cli test..." echo "Command: $llama_cli_path -m \"$model_path\" -ngl 99 -p \"I believe the meaning of life is\" -st" - + # Run the command and capture output output=$("$llama_cli_path" -m "$model_path" -ngl 99 -p "I believe the meaning of life is" -st 2>&1) - + echo "=== LLAMA-CLI OUTPUT ===" echo "$output" echo "=== END OUTPUT ===" - + # Check for expected outputs if echo "$output" | grep -q "offloaded 29/29 layers to GPU"; then found_gpu_offload=true else found_gpu_offload=false fi - + if echo "$output" | grep -q ""; then found_think_tag=true else found_think_tag=false fi - + echo "=== TEST RESULTS ===" echo "Expected 'offloaded 29/29 layers to GPU': $(if [ "$found_gpu_offload" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)" echo "Expected '' tag: $(if [ "$found_think_tag" = true ]; then echo 'FOUND'; else echo 'NOT FOUND'; fi)" - + if [ "$found_gpu_offload" = true ] && [ "$found_think_tag" = true ]; then echo "✅ Test PASSED - Both expected outputs found" else echo "❌ Test FAILED - Missing expected outputs" exit 1 - fi \ No newline at end of file + fi diff --git a/docs/rocm-releases.md b/docs/rocm-releases.md index a12c55f82284..6b9807452772 100644 --- a/docs/rocm-releases.md +++ b/docs/rocm-releases.md @@ -6,7 +6,8 @@ artifact layout, so tools that consume their releases can consume these ones too ## What a release contains -- One zip per OS and GPU target: `llama-bNNNN-{windows,ubuntu}-rocm-{target}-x64.zip`. +- One zip per GPU target: `llama-bNNNN-ubuntu-rocm-{target}-x64.zip`. Windows zips + (`llama-bNNNN-windows-rocm-{target}-x64.zip`) are built only on a manual run that asks for them. - Targets: `gfx1151`, `gfx1150`, `gfx120X`, `gfx110X`, `gfx103X`, `gfx90a`, `gfx908`. - Each zip holds the llama.cpp binaries and shared libraries, built with `GGML_HIP=ON`, `GGML_RPC=ON` and `BUILD_SHARED_LIBS=ON`. @@ -19,12 +20,13 @@ artifact layout, so tools that consume their releases can consume these ones too ## When it runs -- Nightly at 13:00 UTC, two hours after TheRock publishes its nightly. This builds `master` and publishes a release. -- On demand with `workflow_dispatch`. Inputs: OS list, targets, ROCm version, branch or tag, and whether to - publish. +- On every merge to `master`. This builds the merged commit and publishes a release. A newer merge cancels a + release that is still building. +- On demand with `workflow_dispatch`. Inputs: OS list (`ubuntu` by default, `windows,ubuntu` for both), targets, + ROCm version, branch or tag, and whether to publish. - On pull requests that change the pipeline itself. These runs build but do not publish. -If a nightly run fails, `nightly-failure-alert.yml` opens or updates an issue labelled `nightly-failure`. +If a release build for a merge fails, `nightly-failure-alert.yml` opens or updates an issue labelled `nightly-failure`. ## Hardware tests From ea79e754bf32d43f0da749a063de728e08284746 Mon Sep 17 00:00:00 2001 From: Daniele Zannotti Date: Tue, 29 Sep 2026 20:13:27 +0100 Subject: [PATCH 3/3] ci : add a Linux Vulkan zip to each release build-ubuntu-vulkan builds the same commit with GGML_VULKAN=ON, portable CPU variants and $ORIGIN RPATH, and create-release adds llama-bNNNN-ubuntu-vulkan-x64.zip to the release. The vulkan dispatch input turns it off. Assisted-by: Claude Opus 5.5 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build-llamacpp-rocm.yml | 75 ++++++++++++++++++++++- docs/rocm-releases.md | 8 ++- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-llamacpp-rocm.yml b/.github/workflows/build-llamacpp-rocm.yml index ed08088fb9dd..4606bb8e0db9 100644 --- a/.github/workflows/build-llamacpp-rocm.yml +++ b/.github/workflows/build-llamacpp-rocm.yml @@ -24,6 +24,11 @@ on: description: 'Branch or tag of this repository to build, or "latest" for master' required: false default: 'latest' + vulkan: + description: 'Also build the Linux Vulkan zip' + required: false + default: true + type: boolean create_release: description: 'Create a GitHub release after successful build' required: false @@ -890,6 +895,61 @@ jobs: echo "Cleanup completed successfully" + build-ubuntu-vulkan: + # Linux Vulkan build of the same commit, packaged next to the ROCm zips in the same release. + # Portable: CPU variants and backends are loaded at runtime, RPATH is $ORIGIN, and the system + # Vulkan loader and driver (Mesa RADV) are used. + runs-on: ubuntu-22.04 + if: github.event.inputs.vulkan != 'false' + outputs: + llamacpp_commit_hash: ${{ steps.set-outputs.outputs.llamacpp_commit_hash }} + + steps: + - name: Checkout source + uses: actions/checkout@v4 + with: + ref: ${{ env.BUILD_SHA != '' && env.BUILD_SHA || (env.LLAMACPP_VERSION == 'latest' && 'master' || env.LLAMACPP_VERSION) }} + path: llama.cpp + + - name: Install build dependencies + run: | + wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add - + sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list + sudo apt-get update -y + sudo apt-get install -y build-essential cmake ninja-build vulkan-sdk zip + + - name: Build Llama.cpp + Vulkan + run: | + cd llama.cpp + echo "LLAMACPP_COMMIT_HASH=$(git rev-parse --short=5 HEAD)" >> $GITHUB_ENV + git log --oneline -1 + cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DGGML_VULKAN=ON \ + -DGGML_RPC=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DLLAMA_BUILD_BORINGSSL=ON + cmake --build build -j $(nproc) + cp LICENSE build/bin/ + ls -la build/bin + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: llama-ubuntu-vulkan-x64 + path: llama.cpp/build/bin/ + retention-days: 30 + + - name: Set job outputs + id: set-outputs + run: echo "llamacpp_commit_hash=${LLAMACPP_COMMIT_HASH}" >> $GITHUB_OUTPUT + test-stx-halo: runs-on: ${{ matrix.runner }} needs: [prepare-matrix, build-windows, build-ubuntu] @@ -949,7 +1009,7 @@ jobs: artifact_name: llama-${{ matrix.artifact_prefix }}-rocm-${{ matrix.gfx_target }}-x64 create-release: - needs: [prepare-matrix, build-windows, build-ubuntu, test-stx-halo, test-stx] + needs: [prepare-matrix, build-windows, build-ubuntu, build-ubuntu-vulkan, test-stx-halo, test-stx] runs-on: ubuntu-22.04 permissions: contents: write @@ -958,6 +1018,7 @@ jobs: always() && (needs.build-windows.result == 'success' || needs.build-windows.result == 'skipped') && (needs.build-ubuntu.result == 'success' || needs.build-ubuntu.result == 'skipped') && + (needs.build-ubuntu-vulkan.result == 'success' || needs.build-ubuntu-vulkan.result == 'skipped') && (needs.build-windows.result == 'success' || needs.build-ubuntu.result == 'success') && (needs.test-stx-halo.result == 'success' || needs.test-stx-halo.result == 'skipped') && (needs.test-stx.result == 'success' || needs.test-stx.result == 'skipped') && @@ -1058,6 +1119,11 @@ jobs: done done + if [ -d "./all-artifacts/llama-ubuntu-vulkan-x64" ]; then + echo "Creating archive: llama-${TAG}-ubuntu-vulkan-x64.zip" + (cd ./all-artifacts/llama-ubuntu-vulkan-x64 && zip -r "../../llama-${TAG}-ubuntu-vulkan-x64.zip" *) + fi + echo "Created archives:" ls -la *.zip @@ -1100,6 +1166,12 @@ jobs: done done + vulkan_line="" + if [ -f "llama-${TAG}-ubuntu-vulkan-x64.zip" ]; then + upload_files="${upload_files} llama-${TAG}-ubuntu-vulkan-x64.zip" + vulkan_line="**Vulkan**: ubuntu x64" + fi + echo "Files to upload: $upload_files" # Create release with GitHub CLI @@ -1109,6 +1181,7 @@ jobs: **Operating System(s)**: $operating_systems **GPU Target(s)**: $targets **ROCm Version**: $ROCM_VERSION + $vulkan_line **Llama.cpp Commit Hash**: $LLAMACPP_COMMIT_HASH **Build Date**: $(date -u '+%Y-%m-%d %H:%M:%S UTC') diff --git a/docs/rocm-releases.md b/docs/rocm-releases.md index 6b9807452772..d26230a1848c 100644 --- a/docs/rocm-releases.md +++ b/docs/rocm-releases.md @@ -1,4 +1,4 @@ -# ROCm release builds +# ROCm and Vulkan release builds `.github/workflows/build-llamacpp-rocm.yml` publishes prebuilt ROCm binaries of this fork. It copies the pipeline of [lemonade-sdk/llamacpp-rocm](https://github.com/lemonade-sdk/llamacpp-rocm) (MIT) and uses the same @@ -9,7 +9,11 @@ artifact layout, so tools that consume their releases can consume these ones too - One zip per GPU target: `llama-bNNNN-ubuntu-rocm-{target}-x64.zip`. Windows zips (`llama-bNNNN-windows-rocm-{target}-x64.zip`) are built only on a manual run that asks for them. - Targets: `gfx1151`, `gfx1150`, `gfx120X`, `gfx110X`, `gfx103X`, `gfx90a`, `gfx908`. -- Each zip holds the llama.cpp binaries and shared libraries, built with `GGML_HIP=ON`, `GGML_RPC=ON` and +- One Linux Vulkan zip per release: `llama-bNNNN-ubuntu-vulkan-x64.zip`. It is built from the same commit with + `GGML_VULKAN=ON`, loads the CPU variants and backends at runtime (`GGML_BACKEND_DL`, + `GGML_CPU_ALL_VARIANTS`), sets RPATH to `$ORIGIN`, and uses the system Vulkan loader and driver + (Mesa RADV on Strix Halo). A manual run can turn it off with the `vulkan` input. +- Each ROCm zip holds the llama.cpp binaries and shared libraries, built with `GGML_HIP=ON`, `GGML_RPC=ON` and `BUILD_SHARED_LIBS=ON`. - Each zip also holds the ROCm runtime libraries it needs (hipBLAS, rocBLAS, hipBLASLt and their kernel libraries). On Linux, RPATH is set to `$ORIGIN`, so the zip runs without a system ROCm install.