From 2e734877ebc8668b10b6a404db90f9be3295b2ce Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sat, 26 Sep 2026 16:13:00 -0300 Subject: [PATCH 01/21] Add opt-in local-model E2E pilot (Refs #3764) --- .github/workflows/e2e.yml | 147 ++++++++++++++++ dev-docs/integration-tests.md | 142 +++++++++++++++ integration-tests/replace.test.ts | 19 +- integration-tests/run_shell_command.test.ts | 98 ++++++++++- .../sandbox-git-ceiling.real.test.ts | 136 ++++++++++++++ packages/cli/src/utils/sandbox-containers.ts | 6 + .../cli/src/utils/sandbox-git-ceiling.test.ts | 46 +++++ scripts/tests/e2e-local-model.test.ts | 166 ++++++++++++++++++ .../pr-mergeability-workflow-wiring-b.test.ts | 18 ++ 9 files changed, 773 insertions(+), 5 deletions(-) create mode 100644 integration-tests/sandbox-git-ceiling.real.test.ts create mode 100644 packages/cli/src/utils/sandbox-git-ceiling.test.ts create mode 100644 scripts/tests/e2e-local-model.test.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4924faf13d..2b205968f1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -23,6 +23,11 @@ on: required: true default: 'main' type: 'string' + pilot_local_model: + description: 'Run local Qwen3.5 canaries instead of credentialed E2E' + required: false + default: false + type: 'boolean' # Event-type isolation prevents pull_request_target from cancelling pull_request # work and keeps merge_group out of PR-keyed groups. Label isolation prevents an @@ -171,6 +176,7 @@ jobs: needs.skip_check.result == 'success' && !(needs.e2e_doc_change_filter.result == 'success' && needs.e2e_doc_change_filter.outputs.docs_only == 'true') && needs.skip_check.outputs.should_skip != 'true' && + !(github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true) && ( ( ( @@ -342,3 +348,144 @@ jobs: shell: 'bash' run: |- bun scripts/check-e2e-model-budget.ts --ledger "$LLXPRT_E2E_MODEL_LEDGER" + + local_model_canaries: + name: 'Local Qwen3.5 pilot - ${{ matrix.sandbox }}' + if: github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true + permissions: + contents: read + runs-on: 'ubuntu-latest' + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + sandbox: + - 'sandbox:none' + - 'sandbox:docker' + steps: + - name: 'Checkout' + uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # ratchet:actions/checkout@v7 + with: + persist-credentials: false + + - name: 'Set up Node.js' + uses: 'actions/setup-node@820762786026740c76f36085b0efc47a31fe5020' # ratchet:actions/setup-node@v7 + with: + node-version: '24.x' + + - name: 'Setup Bun' + uses: 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' # ratchet:oven-sh/setup-bun@v2 + with: + bun-version-file: '.bun-version' + + - name: 'Install dependencies' + run: 'npm ci' + + - name: 'Build project' + run: 'npm run build' + + - name: 'Set up Docker' + if: "matrix.sandbox == 'sandbox:docker'" + uses: 'docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c' # ratchet:docker/setup-buildx-action@v4 + + - name: 'Install Ollama CPU runtime' + shell: 'bash' + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/ollama" + curl --fail --show-error --location --retry 3 \ + 'https://ollama.com/download/ollama-linux-amd64.tar.zst?version=0.31.1' \ + -o "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" + printf '%s %s\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check - + tar --zstd \ + --exclude='lib/ollama/cuda_v12/*' \ + --exclude='lib/ollama/cuda_v13/*' \ + --exclude='lib/ollama/vulkan/*' \ + -xf "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" -C "$RUNNER_TEMP/ollama" + rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" + echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" + "$RUNNER_TEMP/ollama/bin/ollama" --version + + - name: 'Start local Qwen3.5 model' + shell: 'bash' + env: + OLLAMA_HOST: '127.0.0.1:12644' + OLLAMA_CONTEXT_LENGTH: '262144' + OLLAMA_NUM_PARALLEL: '1' + OLLAMA_MODELS: '${{ runner.temp }}/ollama-models' + run: | + set -euo pipefail + mkdir -p "$OLLAMA_MODELS" + nohup ollama serve >"$RUNNER_TEMP/ollama-server.log" 2>&1 & + for _ in {1..60}; do + if curl --fail --silent "http://$OLLAMA_HOST/api/version"; then + break + fi + sleep 2 + done + curl --fail --silent "http://$OLLAMA_HOST/api/version" | jq -e '.version == "0.31.1"' + ollama pull qwen3.5:2b + curl --fail --silent "http://$OLLAMA_HOST/api/tags" | + jq -e '.models[] | select(.name == "qwen3.5:2b" and .digest == "324d162be6ca5629ae4517c8710434d0bd2d665bc94dbad46e9af8fbf8a2f0df")' + df -h "$RUNNER_TEMP" + + - name: 'Run local-model canaries' + env: + OPENAI_API_KEY: 'ollama-local-only' + OPENAI_BASE_URL: 'http://127.0.0.1:12644/v1' + LLXPRT_DEFAULT_MODEL: 'qwen3.5:2b' + LLXPRT_DEFAULT_PROVIDER: 'openai' + LLXPRT_AUTH_TYPE: 'provider' + GIT_CEILING_DIRECTORIES: '${{ github.workspace }}/.integration-tests' + SANDBOX_FLAGS: "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}" + KEEP_OUTPUT: 'true' + VERBOSE: 'true' + LLXPRT_FORCE_FILE_STORAGE: 'true' + LLXPRT_E2E_MODEL_LEDGER: '${{ runner.temp }}/e2e-model-ledger.jsonl' + shell: 'bash' + run: | + set -euo pipefail + if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then + npm run test:integration:sandbox:docker -- \ + integration-tests/run_shell_command.test.ts \ + integration-tests/replace.test.ts \ + --testNamePattern="should be able to run a shell command$|should be able to replace content in a file$" + else + npm run test:integration:sandbox:none -- \ + integration-tests/run_shell_command.test.ts \ + integration-tests/replace.test.ts \ + --testNamePattern="should be able to run a shell command$|should be able to replace content in a file$" + fi + + - name: 'Check E2E real-model budget (issue #2278)' + if: success() + env: + LLXPRT_E2E_MODEL_LEDGER: '${{ runner.temp }}/e2e-model-ledger.jsonl' + shell: 'bash' + run: |- + bun scripts/check-e2e-model-budget.ts --ledger "$LLXPRT_E2E_MODEL_LEDGER" + + - name: 'Report local model resources' + if: always() + shell: 'bash' + run: | + if curl --fail --silent http://127.0.0.1:12644/api/ps; then + echo + else + echo 'Local Ollama server is unavailable' + fi + free -h + df -h "$RUNNER_TEMP" + du -sh "$RUNNER_TEMP/ollama-models" "$RUNNER_TEMP/ollama" 2>/dev/null || true + if [[ "${{ matrix.sandbox }}" == 'sandbox:docker' ]]; then + docker info --format 'Docker server: {{.ServerVersion}}' + fi + + - name: 'Upload local model diagnostics' + if: always() + uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # ratchet:actions/upload-artifact@v7 + with: + name: 'local-qwen35-pilot-${{ matrix.sandbox }}' + path: | + ${{ runner.temp }}/ollama-server.log + ${{ runner.temp }}/e2e-model-ledger.jsonl diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index 70c4c84f2f..76f3408b25 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -172,3 +172,145 @@ The workflow runs the tests in different sandboxing environments to ensure LLxpr - `sandbox:none`: Runs the tests without any sandboxing. - `sandbox:docker`: Runs the tests in a Docker container. - `sandbox:podman`: Runs the tests in a Podman container. + +### Manual local-model E2E pilot (issue #3764) + +The existing `.github/workflows/e2e.yml` runs credentialed provider E2E for +internal PRs, main pushes, merge groups and approved labeled reruns. Its required +job names, quota selection, event filtering, full test coverage and budget guard +remain unchanged. The local-model experiment is a separate job in that same +workflow. It runs only on `workflow_dispatch` with the boolean +`pilot_local_model=true`; that dispatch skips the credentialed E2E matrix. +Dispatch without the pilot flag still runs the regular credentialed matrix. +Pilot jobs have distinct, non-required check names and cannot block ordinary +required PR CI. The pilot does not replace any third-party-model E2E coverage. +The credentialed full-platform E2E matrix also remains in +`.github/workflows/nightly.yml`. + +With `pilot_local_model=true`, the pilot starts Ollama 0.31.1 on +`127.0.0.1:12644` and pulls `qwen3.5:2b` (Q8_0, digest +`324d162be6ca5629ae4517c8710434d0bd2d665bc94dbad46e9af8fbf8a2f0df`). +It verifies the downloaded Linux runtime archive's SHA-256 digest +(`d297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4`) +before extraction, then checks the runtime version and model digest before testing. +The OpenAI provider uses Ollama's `/v1` compatibility API and a local placeholder +key, with no remote inference service or provider secret. Only the shell and +replace canaries run: both are real `TestRig` tests with tool-choice assertions, +the shell test checks command and tool success, and the replace test checks the +file's exact new contents. The shell step exits on a failed suite. The pilot +retains the four-request ledger budget check for two distinct real-model tests; +the ledger reports retries separately without counting their actual model +requests. + +On Linux, the Docker test leg sets `SANDBOX_FLAGS='--network host'` so the +sandboxed CLI reaches the host's loopback Ollama endpoint. This networking choice +applies only to the Docker leg. The pilot job supplies no provider credentials. +`GIT_CEILING_DIRECTORIES` prevents Git's implicit parent-repository search from +finding the source checkout from a TestRig test directory. The Docker launcher +passes the ceiling into the container, where Git tests verify its effect across +a real bind mount; the host Git test checks the same boundary without Docker. +The shell canary permits only a literal echo shell command in non-interactive +mode. The replace canary excludes `run_shell_command` and rejects unintended +tool logs. These controls prevent the +observed incidental Git discovery and shell calls in these canaries, but they +are not a security boundary against a model supplied an explicit repository +path or a generally enabled shell tool. During local evaluation, before this exclusion, the 2B model +ran unrelated Git commands and made an unintended local commit; it was removed +without discarding file changes. The runtime archive is pinned to 0.31.1; +its CUDA and Vulkan libraries are excluded during extraction, and the downloaded +archive is removed afterward. Ollama is limited to one concurrent context. The +262,144-token context uses more memory than a smaller window; local 32,768-token +runs did not reliably complete the replace task. + +To reproduce the two real-model canaries without using an existing Ollama daemon, +run these commands from the repository root. Choose a free port if 12644 is in +use and change both URLs accordingly. Install Ollama 0.31.1 first and ensure the +`qwen3.5:2b` digest above matches. Store all evidence in the repository's +ignored `tmp/` tree: + +```bash +mkdir -p tmp/verify3764/models +OLLAMA_HOST=127.0.0.1:12644 \ + OLLAMA_MODELS="$PWD/tmp/verify3764/models" \ + OLLAMA_CONTEXT_LENGTH=262144 OLLAMA_NUM_PARALLEL=1 \ + ollama serve >tmp/verify3764/ollama-local.log 2>&1 & +OLLAMA_HOST=127.0.0.1:12644 \ + OLLAMA_MODELS="$PWD/tmp/verify3764/models" ollama pull qwen3.5:2b +curl -fsS http://127.0.0.1:12644/api/tags | jq '.models[] | select(.name == "qwen3.5:2b") | {digest, size, details}' +CI=true KEEP_OUTPUT=true VERBOSE=true \ + LLXPRT_DEFAULT_PROVIDER=openai LLXPRT_DEFAULT_MODEL=qwen3.5:2b \ + OPENAI_API_KEY=ollama-local-only \ + OPENAI_BASE_URL=http://127.0.0.1:12644/v1 LLXPRT_AUTH_TYPE=provider \ + LLXPRT_FORCE_FILE_STORAGE=true \ + LLXPRT_E2E_MODEL_LEDGER="$PWD/tmp/verify3764/ledger.jsonl" \ + GIT_CEILING_DIRECTORIES="$PWD/.integration-tests" \ + bun scripts/run_bun_tests.ts --root integration-tests \ + integration-tests/run_shell_command.test.ts integration-tests/replace.test.ts \ + --testNamePattern='should be able to run a shell command$|should be able to replace content in a file' +bun scripts/check-e2e-model-budget.ts --ledger "$PWD/tmp/verify3764/ledger.jsonl" +curl -fsS http://127.0.0.1:12644/api/ps | jq '.models[] | {name, size, context_length}' +``` + +Qwen lists the 2B and 4B releases on March 2, 2026 in its +[release list](https://github.com/QwenLM/Qwen3.5); Ollama publishes the +[quantized tags](https://ollama.com/library/qwen3.5). The +[GitHub-hosted runner specifications](https://docs.github.com/en/actions/reference/runners/github-hosted-runners) +list 4 CPU, 16 GB RAM and 14 GB SSD for public `ubuntu-latest` jobs. The +macOS Apple Silicon measurements below do not establish CPU-only x64 speed, +Linux container networking, or the total runtime and disk use of a hosted +Docker E2E leg. Those require an actual Actions run before treating this change +as validated on GitHub's runners. The existing E2E workflow is registered on +the default branch, so after pushing the pilot branch it can be dispatched with +`gh workflow run e2e.yml --ref issue3764 -f branch_ref=issue3764 -f pilot_local_model=true`. +The `--ref` selects the branch version of the workflow and the pilot's checkout; +`branch_ref` identifies the branch for dispatch grouping (and is retained for +normal credentialed reruns). Do not dispatch the new workflow file: GitHub does +not register a new `workflow_dispatch` workflow until it reaches the default +branch. An ordinary `gh workflow run e2e.yml --ref issue3764 -f branch_ref=issue3764` +still runs the credentialed matrix. Local tests validate workflow structure, +but cannot establish hosted-runner performance. Inspect the pilot jobs for both +`sandbox:none` and `sandbox:docker`: job +conclusions, step durations (build, archive download, model pull, E2E), the +`Report local model resources` step (RAM, free disk, model/runtime disk usage, +Docker server), and the uploaded Ollama server log and real-model ledger. Check +the model digest and SHA-256 result in each job log. Both canaries and the budget +check need repeated passes before considering any required-coverage change. +A local Apple Silicon result cannot substitute for Linux CPU and Docker measurements. + +#### Local measurements (September 26, 2026) + +With Ollama 0.31.1 on an M4 Max, the 2B tag downloaded 2,741,192,820 bytes. +The pinned Linux x64 runtime archive downloaded 1,408,625,102 bytes and a +CPU-only extraction occupied 64 MiB locally. The first three real `TestRig` +suites with the original shell assertion passed 3/3 shell and 3/3 replace after +two replace retries (one timeout, one incorrect tool path). Those runs used the +model's 262,144-token context. At a 32,768-token limit, the first CI-style +replace attempt failed after 205 seconds with malformed file paths; the repeat +experiment was stopped rather than counted as a passing suite. The 4B model +(digest `2a654d98e6fba55d452b7043684e9b57a947e393bbffa62485a7aac05ee4eefd`) +passed only 1/3 complete suites at 32,768 tokens, so it was not selected. +With shell-command validation and exact file-content assertions, two initial +2B suites passed after retries; a third suite repeatedly searched outside the +TestRig workspace and was stopped during its second attempt. After giving the +model the absolute test-file path and naming the required tools, two of three +further suites passed (`tmp/verify3764/canaries-2b-tuned-{1,2,3}.log`). The +remaining suite failed all three replace attempts: the model wrote incorrect +content, including conversational text. The shell canary passed after retries. +At the 262,144-token context, `/api/ps` reported 6,816,498,972 bytes loaded for +2B on the M4 Max (`tmp/verify3764/ps-2b-tuned.json`). The budget guard passed: +four declared requests across two distinct tests; six recorded runs of each +include retries and failed suites (`tmp/verify3764/budget-2b-tuned.log`). A +final combined suite with the shell validator accepting only equivalent literal +`echo hello-world` commands passed both canaries, with one shell retry +(`tmp/verify3764/canaries-2b-final.log`). These are local GPU results. The +failed suite and unmeasured hosted-runner CPU/Docker performance mean this +workflow is not yet validated for PR gating. During remediation, the first two +repeat runs failed because the non-interactive shell canary was initially passed +an option array without stdin; that test setup was corrected before the third +run. In that third genuine suite, the shell canary passed, but replace failed +three of three file-level attempts (`tmp/verify3764/canaries-remediation-2b-3.log`). +The budget guard still passed with four distinct-test requests and separately +reported five replace and seven shell invocations including retries. No local +model/configuration tested here passed repeat suites consistently, so the pilot +is non-required. The three 4B suite logs likewise show only one complete passing +suite, with replace exhausting all retries in two. diff --git a/integration-tests/replace.test.ts b/integration-tests/replace.test.ts index 12234b9d7a..346e030549 100644 --- a/integration-tests/replace.test.ts +++ b/integration-tests/replace.test.ts @@ -17,21 +17,34 @@ describe('replace', () => { afterEach(async () => await rig.cleanup()); it('should be able to replace content in a file', async () => { await rig.setup('should be able to replace content in a file', { - settings: { tools: { core: ['replace', 'read_file'] } }, + settings: { + tools: { + core: ['replace', 'read_file'], + exclude: ['run_shell_command'], + }, + }, }); const fileName = 'file_to_replace.txt'; const originalContent = 'foo content'; const expectedContent = 'bar content'; - rig.createFile(fileName, originalContent); + const filePath = rig.createFile(fileName, originalContent); await rig.run({ - args: `Replace 'foo' with 'bar' in the file 'file_to_replace.txt'`, + args: `Use the replace tool on '${filePath}' to replace the exact text 'foo content' with 'bar content'. Do not add any whitespace.`, }); const foundToolCall = await rig.waitForToolCall('replace'); expect(foundToolCall, 'Expected to find a replace tool call').toBeTruthy(); + expect( + rig + .readToolLogs() + .filter( + (log) => !['replace', 'read_file'].includes(log.toolRequest.name), + ), + 'Text replacement must not invoke unrelated tools', + ).toHaveLength(0); expect(rig.readFile(fileName)).toBe(expectedContent); }); diff --git a/integration-tests/run_shell_command.test.ts b/integration-tests/run_shell_command.test.ts index 87dff22db9..8685bdc3b6 100644 --- a/integration-tests/run_shell_command.test.ts +++ b/integration-tests/run_shell_command.test.ts @@ -22,6 +22,91 @@ function getLineCountCommand(): { command: string; tool: string } { } } +function isExpectedEchoCommand(args: string): boolean { + const request: unknown = JSON.parse(args); + if ( + typeof request !== 'object' || + request === null || + !('command' in request) + ) { + return false; + } + const command = request.command; + return ( + typeof command === 'string' && + new Set([ + 'echo hello-world', + 'echo "hello-world"', + "echo 'hello-world'", + ]).has(command) + ); +} + +function isExpectedShellTrace( + logs: ReadonlyArray<{ + toolRequest: { name: string; args: string; success?: boolean }; + }>, +): boolean { + return ( + logs.length > 0 && + logs.every( + ({ toolRequest }) => + toolRequest.name === 'run_shell_command' && + isExpectedEchoCommand(toolRequest.args) && + toolRequest.success === true, + ) + ); +} + +describe('shell canary command validation', () => { + it('accepts equivalent literal echo commands but not other shell expressions', () => { + for (const command of [ + 'echo hello-world', + 'echo "hello-world"', + "echo 'hello-world'", + ]) { + expect(isExpectedEchoCommand(JSON.stringify({ command }))).toBe(true); + } + expect( + isExpectedEchoCommand(JSON.stringify({ command: 'echo goodbye' })), + ).toBe(false); + expect( + isExpectedEchoCommand( + JSON.stringify({ command: 'echo hello-world; git commit -am oops' }), + ), + ).toBe(false); + }); + + it('rejects wrong or additional tools even when the echo succeeded', () => { + const echo = { + toolRequest: { + name: 'run_shell_command', + args: JSON.stringify({ command: 'echo hello-world' }), + success: true, + }, + }; + expect(isExpectedShellTrace([echo])).toBe(true); + expect( + isExpectedShellTrace([ + echo, + { toolRequest: { name: 'read_file', args: '{}', success: true } }, + ]), + ).toBe(false); + expect( + isExpectedShellTrace([ + echo, + { + toolRequest: { + name: 'run_shell_command', + args: JSON.stringify({ command: 'git status' }), + success: true, + }, + }, + ]), + ).toBe(false); + }); +}); + describe('run_shell_command', () => { let rig: TestRig; @@ -35,9 +120,13 @@ describe('run_shell_command', () => { settings: { tools: { core: ['run_shell_command'] } }, }); - const prompt = `Please run the command "echo hello-world" and show me the output`; + const prompt = `Use the run_shell_command tool to execute exactly "echo hello-world". Show me the output.`; - const result = await rig.run({ args: prompt }); + const result = await rig.run({ + args: ['--allowed-tools=run_shell_command(echo)'], + stdin: prompt, + yolo: false, + }); const foundToolCall = await rig.waitForToolCall('run_shell_command'); @@ -53,6 +142,11 @@ describe('run_shell_command', () => { foundToolCall, 'Expected to find a run_shell_command tool call', ).toBeTruthy(); + expect( + isExpectedShellTrace(rig.readToolLogs()), + 'Expected only successful echo hello-world shell invocations', + ).toBe(true); + expect(result).toContain('hello-world'); // Validate model output - will throw if no output, warn if missing expected content // Model often reports exit code instead of showing output diff --git a/integration-tests/sandbox-git-ceiling.real.test.ts b/integration-tests/sandbox-git-ceiling.real.test.ts new file mode 100644 index 0000000000..466fd20c1d --- /dev/null +++ b/integration-tests/sandbox-git-ceiling.real.test.ts @@ -0,0 +1,136 @@ +/** + * @license + * Copyright 2026 Vybestack LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'bun:test'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, realpathSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + addContainerEnvVars, + buildContainerRunArgs, +} from '../packages/cli/src/utils/sandbox-containers.js'; + +const image = 'ghcr.io/vybestack/llxprt-code/sandbox:0.11.0'; +const runtime = process.env.LLXPRT_SANDBOX; + +function dockerReady(): boolean { + if (runtime !== 'docker') return false; + try { + execFileSync('docker', ['info'], { stdio: 'ignore', timeout: 20_000 }); + execFileSync('docker', ['image', 'inspect', image], { + stdio: 'ignore', + timeout: 20_000, + }); + return true; + } catch { + return false; + } +} + +function gitRoot(dir: string, env: NodeJS.ProcessEnv): string { + return execFileSync('git', ['-C', dir, 'rev-parse', '--show-toplevel'], { + env, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); +} + +describe('TestRig Git discovery boundary', () => { + it('blocks discovery of the host checkout from a nested test directory', () => { + const root = realpathSync( + mkdtempSync(join(tmpdir(), 'llxprt-git-ceiling-')), + ); + const ceiling = join(root, '.integration-tests'); + const testDir = join(ceiling, 'run', 'test'); + try { + mkdirSync(testDir, { recursive: true }); + execFileSync('git', ['init', '-q', root]); + const env = { ...process.env }; + delete env.GIT_CEILING_DIRECTORIES; + expect(gitRoot(testDir, env)).toBe(root); + expect(() => + gitRoot(testDir, { ...env, GIT_CEILING_DIRECTORIES: ceiling }), + ).toThrow(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.skipIf(!dockerReady())( + 'blocks discovery of a bind-mounted parent repo inside the real Docker sandbox', + () => { + const root = realpathSync( + mkdtempSync(join(tmpdir(), 'llxprt-docker-git-ceiling-')), + ); + const ceiling = join(root, '.integration-tests'); + const testDir = join(ceiling, 'run', 'test'); + const sessionDir = join(root, 'session'); + const originalCeiling = process.env.GIT_CEILING_DIRECTORIES; + try { + mkdirSync(testDir, { recursive: true }); + mkdirSync(sessionDir); + execFileSync('git', ['init', '-q', root]); + const config = { command: 'docker', image } as const; + const containerArgs = buildContainerRunArgs( + config, + image, + root, + root, + sessionDir, + ).filter((arg) => arg !== '-t'); + containerArgs[containerArgs.indexOf('--workdir') + 1] = testDir; + delete process.env.GIT_CEILING_DIRECTORIES; + const baselineArgs = [...containerArgs]; + addContainerEnvVars(baselineArgs, config, 'git-ceiling-test', [], root); + expect( + execFileSync( + 'docker', + [ + ...baselineArgs, + '--entrypoint', + 'sh', + image, + '-c', + 'git rev-parse --show-toplevel', + ], + { encoding: 'utf8', timeout: 30_000 }, + ).trim(), + ).toBe(root); + process.env.GIT_CEILING_DIRECTORIES = ceiling; + addContainerEnvVars( + containerArgs, + config, + 'git-ceiling-test', + [], + root, + ); + const args = [ + ...containerArgs, + '--entrypoint', + 'sh', + image, + '-c', + 'git rev-parse --show-toplevel', + ]; + expect(() => + execFileSync('docker', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }), + ).toThrow(); + } finally { + if (originalCeiling === undefined) { + delete process.env.GIT_CEILING_DIRECTORIES; + } else { + process.env.GIT_CEILING_DIRECTORIES = originalCeiling; + } + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/packages/cli/src/utils/sandbox-containers.ts b/packages/cli/src/utils/sandbox-containers.ts index 02802ece9c..fe015dc0b4 100644 --- a/packages/cli/src/utils/sandbox-containers.ts +++ b/packages/cli/src/utils/sandbox-containers.ts @@ -272,6 +272,7 @@ const RESERVED_SANDBOX_ENV_KEYS = new Set([ // marker gate) at attacker-chosen paths inside the container. 'LLXPRT_SANDBOX_PROJECT_KEY', 'LLXPRT_SANDBOX_CHECKPOINT_STORE', + 'GIT_CEILING_DIRECTORIES', ]); function parseSandboxEnvVars(): string[] { @@ -340,6 +341,11 @@ export function addContainerEnvVars( GEMINI_MODEL: process.env.GEMINI_MODEL, TERM: process.env.TERM, COLORTERM: process.env.COLORTERM, + GIT_CEILING_DIRECTORIES: process.env.GIT_CEILING_DIRECTORIES?.split( + path.delimiter, + ) + .map(getContainerPath) + .join(':'), }; for (const [key, val] of Object.entries(envMap)) { if (val !== undefined) { diff --git a/packages/cli/src/utils/sandbox-git-ceiling.test.ts b/packages/cli/src/utils/sandbox-git-ceiling.test.ts new file mode 100644 index 0000000000..e284641388 --- /dev/null +++ b/packages/cli/src/utils/sandbox-git-ceiling.test.ts @@ -0,0 +1,46 @@ +/** + * @license + * Copyright 2026 Vybestack LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, describe, expect, it } from 'bun:test'; +import { FatalSandboxError } from '@vybestack/llxprt-code-core'; +import { addContainerEnvVars } from './sandbox-containers.js'; + +const config = { command: 'docker', image: 'test' } as const; +const originalCeiling = process.env.GIT_CEILING_DIRECTORIES; +const originalSandboxEnv = process.env.SANDBOX_ENV; + +describe('container Git discovery ceiling', () => { + afterEach(() => { + if (originalCeiling === undefined) { + delete process.env.GIT_CEILING_DIRECTORIES; + } else { + process.env.GIT_CEILING_DIRECTORIES = originalCeiling; + } + if (originalSandboxEnv === undefined) { + delete process.env.SANDBOX_ENV; + } else { + process.env.SANDBOX_ENV = originalSandboxEnv; + } + }); + + it('passes the host ceiling into the container', () => { + process.env.GIT_CEILING_DIRECTORIES = '/workspace/.integration-tests'; + delete process.env.SANDBOX_ENV; + const args: string[] = []; + addContainerEnvVars(args, config, 'git-ceiling-test', [], '/workspace'); + expect(args).toContain( + 'GIT_CEILING_DIRECTORIES=/workspace/.integration-tests', + ); + }); + + it('rejects an override that could detach the pinned Git ceiling', () => { + process.env.GIT_CEILING_DIRECTORIES = '/workspace/.integration-tests'; + process.env.SANDBOX_ENV = 'GIT_CEILING_DIRECTORIES=/'; + expect(() => + addContainerEnvVars([], config, 'git-ceiling-test', [], '/workspace'), + ).toThrow(FatalSandboxError); + }); +}); diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts new file mode 100644 index 0000000000..b5637db8c2 --- /dev/null +++ b/scripts/tests/e2e-local-model.test.ts @@ -0,0 +1,166 @@ +/** + * @license + * Copyright 2026 Vybestack LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { + asOptionalRecord, + asRecord, + asString, + parseWorkflowYaml, + workflowJob, +} from './typed-test-helpers.ts'; + +const root = resolve(import.meta.dirname, '../..'); +const requiredText = readFileSync( + resolve(root, '.github/workflows/e2e.yml'), + 'utf8', +); +const required = parseWorkflowYaml(requiredText); +const pilot = required; +const steps = workflowJob(pilot, 'local_model_canaries').steps ?? []; + +function step(name: string) { + const found = steps.find((candidate) => candidate.name === name); + if (!found) throw new Error(`Missing pilot step: ${name}`); + return found; +} + +describe('optional local-model E2E pilot', () => { + it('dispatches the default-branch workflow against a selected ref with explicit opt-in', () => { + const dispatch = asRecord(required.on?.workflow_dispatch); + const inputs = asRecord(dispatch.inputs); + expect(asRecord(inputs.branch_ref)).toEqual({ + description: 'Branch to run on', + required: true, + default: 'main', + type: 'string', + }); + expect(asRecord(inputs.pilot_local_model)).toMatchObject({ + required: false, + default: false, + type: 'boolean', + }); + expect(asString(workflowJob(pilot, 'local_model_canaries').if)).toContain( + "github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true", + ); + expect(asString(workflowJob(required, 'e2e_linux').if)).toContain( + "!(github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true)", + ); + }); + + it('leaves required provider E2E events, check names, quota selection and budget unchanged', () => { + expect(required.on).toHaveProperty('pull_request'); + expect(required.on).toHaveProperty('push'); + expect(required.on).toHaveProperty('merge_group'); + expect(required.on).toHaveProperty('pull_request_target'); + expect(workflowJob(required, 'e2e_linux').name).toBe( + 'E2E Test (Linux) - ${{ matrix.sandbox }}', + ); + expect(requiredText).not.toContain('Block unqualified local-model gating'); + const requiredSteps = workflowJob(required, 'e2e_linux').steps ?? []; + expect(requiredSteps.map((candidate) => candidate.name)).toContain( + 'Check API quota and select optimal key', + ); + expect(requiredSteps.map((candidate) => candidate.name)).toContain( + 'Check E2E real-model budget (issue #2278)', + ); + }); + + it('only runs on explicit dispatch and uses distinct non-required check names', () => { + expect(Object.keys(pilot.jobs ?? {})).toEqual([ + 'skip_check', + 'mergeability-gate', + 'e2e_doc_change_filter', + 'e2e_linux', + 'local_model_canaries', + ]); + const job = workflowJob(pilot, 'local_model_canaries'); + expect(job.name).toContain('Local Qwen3.5 pilot'); + expect(job.name).not.toContain('E2E Test (Linux)'); + expect(asString(job.if ?? '')).toContain( + 'inputs.pilot_local_model == true', + ); + expect(job.permissions).toEqual({ contents: 'read' }); + expect(asRecord(job.strategy).matrix).toEqual({ + sandbox: ['sandbox:none', 'sandbox:docker'], + }); + }); + + it('downloads a pinned Ollama runtime and verifies its archive before extraction', () => { + const install = asString(step('Install Ollama CPU runtime').run); + expect(install).toContain('ollama-linux-amd64.tar.zst?version=0.31.1'); + expect(install).toContain( + `printf '%s %s\\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check -`, + ); + expect(install.indexOf('sha256sum --check')).toBeLessThan( + install.indexOf('tar --zstd'), + ); + expect(install).toContain("--exclude='lib/ollama/cuda_v12/*'"); + expect(install).toContain("--exclude='lib/ollama/cuda_v13/*'"); + expect(install).toContain("--exclude='lib/ollama/vulkan/*'"); + expect(install).toContain('rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst"'); + }); + + it('checks model version and digest before running either canary', () => { + const server = step('Start local Qwen3.5 model'); + const start = asString(server.run); + const env = asOptionalRecord(server.env); + expect(env?.OLLAMA_CONTEXT_LENGTH).toBe('262144'); + expect(env?.OLLAMA_NUM_PARALLEL).toBe('1'); + expect(env?.OLLAMA_HOST).toBe('127.0.0.1:12644'); + expect(start).toContain('jq -e \'.version == "0.31.1"\''); + expect(start).toContain('ollama pull qwen3.5:2b'); + expect(start).toContain( + '324d162be6ca5629ae4517c8710434d0bd2d665bc94dbad46e9af8fbf8a2f0df', + ); + expect(steps.indexOf(server)).toBeLessThan( + steps.indexOf(step('Run local-model canaries')), + ); + }); + + it('runs only real shell and replace TestRig canaries without provider secrets', () => { + const job = workflowJob(pilot, 'local_model_canaries'); + const run = step('Run local-model canaries'); + const env = asOptionalRecord(run.env); + expect(env?.OPENAI_BASE_URL).toBe('http://127.0.0.1:12644/v1'); + expect(env?.LLXPRT_DEFAULT_PROVIDER).toBe('openai'); + expect(env?.LLXPRT_DEFAULT_MODEL).toBe('qwen3.5:2b'); + expect(env?.OPENAI_API_KEY).toBe('ollama-local-only'); + expect(env?.GIT_CEILING_DIRECTORIES).toBe( + '${{ github.workspace }}/.integration-tests', + ); + expect(asString(run.run).startsWith('set -euo pipefail')).toBe(true); + expect(asString(run.run)).toContain( + 'integration-tests/run_shell_command.test.ts', + ); + expect(asString(run.run)).toContain('integration-tests/replace.test.ts'); + expect(asString(run.run)).toContain( + 'should be able to replace content in a file', + ); + expect(asString(run.run)).toContain( + 'should be able to run a shell command', + ); + expect(asString(run.run)).not.toContain('--exclude='); + expect(JSON.stringify(job)).not.toMatch(/\bsecrets(?:\.|\[)/); + }); + + it('uses host networking only for the Docker leg, and retains diagnostics on failure', () => { + const env = asOptionalRecord(step('Run local-model canaries').env); + expect(env?.SANDBOX_FLAGS).toBe( + "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}", + ); + const report = step('Report local model resources'); + const upload = step('Upload local model diagnostics'); + expect(report.if).toBe('always()'); + expect(asString(report.run)).toContain('/api/ps'); + expect(asString(report.run)).toContain('free -h'); + expect(asString(report.run)).toContain('df -h'); + expect(upload.if).toBe('always()'); + expect(asOptionalRecord(upload.with)?.path).toContain('ollama-server.log'); + }); +}); diff --git a/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts b/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts index 68b5ba3fd3..90981445d9 100644 --- a/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts +++ b/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts @@ -69,6 +69,7 @@ interface E2eContextParams { gateResult?: string; shouldRun?: string; cancelled?: boolean; + pilotLocalModel?: boolean; } function e2eContext({ @@ -84,9 +85,11 @@ function e2eContext({ gateResult = 'skipped', shouldRun, cancelled = false, + pilotLocalModel = false, }: E2eContextParams) { return { cancelled, + inputs: { pilot_local_model: pilotLocalModel }, github: { event_name: eventName, event: { @@ -113,6 +116,7 @@ function e2eContext({ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { let parsed: Record; let linuxJob: Record | undefined; + let pilotJob: Record | undefined; let gateJob: Record | undefined; let docFilterJob: Record | undefined; @@ -121,6 +125,7 @@ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { parsed = wf.parsed; const jobs = wf.jobs; linuxJob = asRecord(jobs?.e2e_linux ?? undefined); + pilotJob = asRecord(jobs?.local_model_canaries ?? undefined); gateJob = asRecord(jobs?.['mergeability-gate'] ?? undefined); docFilterJob = asRecord(jobs?.e2e_doc_change_filter ?? undefined); }); @@ -175,6 +180,12 @@ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { eventName: 'workflow_dispatch', linux: true, }, + { + name: 'explicitly opted-in pilot dispatch skips credentialed E2E', + eventName: 'workflow_dispatch', + pilotLocalModel: true, + linux: false, + }, { name: 'internal pull request with intentionally skipped gate', eventName: 'pull_request', @@ -297,6 +308,13 @@ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { expect( Boolean(evaluateE2ECondition(asString(linuxJob?.if), context)), ).toBe(scenario.linux); + expect( + Boolean(evaluateE2ECondition(asString(pilotJob?.if), context)), + ).toBe( + scenario.eventName === 'workflow_dispatch' && + 'pilotLocalModel' in scenario && + scenario.pilotLocalModel === true, + ); }); } From 567888f7d347525bc7589b99865fc8c084726219 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sat, 26 Sep 2026 17:42:39 -0300 Subject: [PATCH 02/21] Keep hosted pilot failure traces accessible (Refs #3764) --- .github/workflows/e2e.yml | 11 ++++++- scripts/tests/e2e-local-model.test.ts | 47 ++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2b205968f1..7fe27da8c9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -362,6 +362,11 @@ jobs: sandbox: - 'sandbox:none' - 'sandbox:docker' + include: + - sandbox: 'sandbox:none' + artifact_id: 'none' + - sandbox: 'sandbox:docker' + artifact_id: 'docker' steps: - name: 'Checkout' uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # ratchet:actions/checkout@v7 @@ -485,7 +490,11 @@ jobs: if: always() uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # ratchet:actions/upload-artifact@v7 with: - name: 'local-qwen35-pilot-${{ matrix.sandbox }}' + name: 'local-qwen35-pilot-${{ matrix.artifact_id }}' + if-no-files-found: error + include-hidden-files: true path: | ${{ runner.temp }}/ollama-server.log ${{ runner.temp }}/e2e-model-ledger.jsonl + ${{ github.workspace }}/.integration-tests/*/*/telemetry.log + ${{ github.workspace }}/.integration-tests/*/*/harness-diagnostics.log diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index b5637db8c2..0352085763 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -88,9 +88,40 @@ describe('optional local-model E2E pilot', () => { expect(job.permissions).toEqual({ contents: 'read' }); expect(asRecord(job.strategy).matrix).toEqual({ sandbox: ['sandbox:none', 'sandbox:docker'], + include: [ + { sandbox: 'sandbox:none', artifact_id: 'none' }, + { sandbox: 'sandbox:docker', artifact_id: 'docker' }, + ], }); }); + it('gives every sandbox a distinct artifact name without forbidden characters', () => { + const matrix = asRecord( + asRecord(workflowJob(pilot, 'local_model_canaries').strategy).matrix, + ); + const sandboxes = matrix.sandbox; + expect(sandboxes).toEqual(['sandbox:none', 'sandbox:docker']); + const entries = matrix.include; + expect(Array.isArray(entries)).toBe(true); + if (!Array.isArray(entries)) + throw new Error('Pilot matrix include is required'); + const name = asString( + asOptionalRecord(step('Upload local model diagnostics').with)?.name, + ); + expect(name).toBe('local-qwen35-pilot-${{ matrix.artifact_id }}'); + const resolvedNames = entries.map((entry) => { + const row = asRecord(entry); + expect(sandboxes).toContain(row.sandbox); + const id = asString(row.artifact_id); + expect(id).toMatch(/^[a-z0-9-]+$/); + return name.replace('${{ matrix.artifact_id }}', id); + }); + expect(new Set(resolvedNames).size).toBe(sandboxes.length); + expect( + resolvedNames.every((resolved) => !/[\\/:*?"<>|\r\n]/.test(resolved)), + ).toBe(true); + }); + it('downloads a pinned Ollama runtime and verifies its archive before extraction', () => { const install = asString(step('Install Ollama CPU runtime').run); expect(install).toContain('ollama-linux-amd64.tar.zst?version=0.31.1'); @@ -154,6 +185,10 @@ describe('optional local-model E2E pilot', () => { expect(env?.SANDBOX_FLAGS).toBe( "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}", ); + expect(env?.KEEP_OUTPUT).toBe('true'); + expect(env?.LLXPRT_E2E_MODEL_LEDGER).toBe( + '${{ runner.temp }}/e2e-model-ledger.jsonl', + ); const report = step('Report local model resources'); const upload = step('Upload local model diagnostics'); expect(report.if).toBe('always()'); @@ -161,6 +196,16 @@ describe('optional local-model E2E pilot', () => { expect(asString(report.run)).toContain('free -h'); expect(asString(report.run)).toContain('df -h'); expect(upload.if).toBe('always()'); - expect(asOptionalRecord(upload.with)?.path).toContain('ollama-server.log'); + const paths = asString(asOptionalRecord(upload.with)?.path).split('\n'); + expect(paths).toContain('${{ runner.temp }}/ollama-server.log'); + expect(paths).toContain('${{ runner.temp }}/e2e-model-ledger.jsonl'); + expect(paths).toContain( + '${{ github.workspace }}/.integration-tests/*/*/telemetry.log', + ); + expect(paths).toContain( + '${{ github.workspace }}/.integration-tests/*/*/harness-diagnostics.log', + ); + expect(asOptionalRecord(upload.with)?.['if-no-files-found']).toBe('error'); + expect(asOptionalRecord(upload.with)?.['include-hidden-files']).toBe(true); }); }); From 12481f8c13971639e7d69f29c7a3d328f2005f0d Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sat, 26 Sep 2026 19:21:47 -0300 Subject: [PATCH 03/21] Bound local-model pilot context and CPU replace deadline (Refs #3764) --- .github/workflows/e2e.yml | 8 +- dev-docs/integration-tests.md | 11 +- integration-tests/replace.test.ts | 76 +++++++------ packages/test-utils/src/test-rig-setup.ts | 8 ++ packages/test-utils/src/test-rig.test.ts | 131 +++++++++++++++++++++- packages/test-utils/src/test-rig.ts | 21 +++- scripts/bun-test-roots.ts | 10 ++ scripts/tests/bun-test-roots.bun.test.ts | 20 ++++ scripts/tests/e2e-local-model.test.ts | 13 ++- 9 files changed, 253 insertions(+), 45 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7fe27da8c9..5184b51d0d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -355,7 +355,7 @@ jobs: permissions: contents: read runs-on: 'ubuntu-latest' - timeout-minutes: 60 + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -415,7 +415,7 @@ jobs: shell: 'bash' env: OLLAMA_HOST: '127.0.0.1:12644' - OLLAMA_CONTEXT_LENGTH: '262144' + OLLAMA_CONTEXT_LENGTH: '32768' OLLAMA_NUM_PARALLEL: '1' OLLAMA_MODELS: '${{ runner.temp }}/ollama-models' run: | @@ -441,6 +441,10 @@ jobs: LLXPRT_DEFAULT_MODEL: 'qwen3.5:2b' LLXPRT_DEFAULT_PROVIDER: 'openai' LLXPRT_AUTH_TYPE: 'provider' + LLXPRT_TEST_PROFILE: 'local-qwen35-pilot' + LLXPRT_CONTEXT_LIMIT: '32768' + LLXPRT_MAX_OUTPUT_TOKENS: '8192' + LLXPRT_LOCAL_MODEL_PILOT: 'true' GIT_CEILING_DIRECTORIES: '${{ github.workspace }}/.integration-tests' SANDBOX_FLAGS: "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}" KEEP_OUTPUT: 'true' diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index 76f3408b25..839128e277 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -219,8 +219,11 @@ ran unrelated Git commands and made an unintended local commit; it was removed without discarding file changes. The runtime archive is pinned to 0.31.1; its CUDA and Vulkan libraries are excluded during extraction, and the downloaded archive is removed afterward. Ollama is limited to one concurrent context. The -262,144-token context uses more memory than a smaller window; local 32,768-token -runs did not reliably complete the replace task. +manual pilot configures both Ollama and LLxprt with a 32,768-token context and +reserves 8,192 tokens for model output. The real replace invocation has a +900,000 ms `TestRig` deadline, its Bun file has a 1,200,000 ms timeout, and each +sandbox job has a 90-minute bound. These larger deadlines apply only when +`LLXPRT_LOCAL_MODEL_PILOT=true`; normal integration-test deadlines are unchanged. To reproduce the two real-model canaries without using an existing Ollama daemon, run these commands from the repository root. Choose a free port if 12644 is in @@ -232,7 +235,7 @@ ignored `tmp/` tree: mkdir -p tmp/verify3764/models OLLAMA_HOST=127.0.0.1:12644 \ OLLAMA_MODELS="$PWD/tmp/verify3764/models" \ - OLLAMA_CONTEXT_LENGTH=262144 OLLAMA_NUM_PARALLEL=1 \ + OLLAMA_CONTEXT_LENGTH=32768 OLLAMA_NUM_PARALLEL=1 \ ollama serve >tmp/verify3764/ollama-local.log 2>&1 & OLLAMA_HOST=127.0.0.1:12644 \ OLLAMA_MODELS="$PWD/tmp/verify3764/models" ollama pull qwen3.5:2b @@ -241,6 +244,8 @@ CI=true KEEP_OUTPUT=true VERBOSE=true \ LLXPRT_DEFAULT_PROVIDER=openai LLXPRT_DEFAULT_MODEL=qwen3.5:2b \ OPENAI_API_KEY=ollama-local-only \ OPENAI_BASE_URL=http://127.0.0.1:12644/v1 LLXPRT_AUTH_TYPE=provider \ + LLXPRT_TEST_PROFILE=local-qwen35-pilot LLXPRT_CONTEXT_LIMIT=32768 \ + LLXPRT_MAX_OUTPUT_TOKENS=8192 LLXPRT_LOCAL_MODEL_PILOT=true \ LLXPRT_FORCE_FILE_STORAGE=true \ LLXPRT_E2E_MODEL_LEDGER="$PWD/tmp/verify3764/ledger.jsonl" \ GIT_CEILING_DIRECTORIES="$PWD/.integration-tests" \ diff --git a/integration-tests/replace.test.ts b/integration-tests/replace.test.ts index 346e030549..9bea4a5c67 100644 --- a/integration-tests/replace.test.ts +++ b/integration-tests/replace.test.ts @@ -5,8 +5,12 @@ */ import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; +import { env } from 'node:process'; import { TestRig } from './test-helper.js'; +const localModelPilot = env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true'; +const replaceTestOptions = localModelPilot ? { timeout: 1_200_000 } : {}; + describe('replace', () => { let rig: TestRig; @@ -15,37 +19,45 @@ describe('replace', () => { }); afterEach(async () => await rig.cleanup()); - it('should be able to replace content in a file', async () => { - await rig.setup('should be able to replace content in a file', { - settings: { - tools: { - core: ['replace', 'read_file'], - exclude: ['run_shell_command'], + it( + 'should be able to replace content in a file', + replaceTestOptions, + async () => { + await rig.setup('should be able to replace content in a file', { + settings: { + tools: { + core: ['replace', 'read_file'], + exclude: ['run_shell_command'], + }, }, - }, - }); - - const fileName = 'file_to_replace.txt'; - const originalContent = 'foo content'; - const expectedContent = 'bar content'; - - const filePath = rig.createFile(fileName, originalContent); - - await rig.run({ - args: `Use the replace tool on '${filePath}' to replace the exact text 'foo content' with 'bar content'. Do not add any whitespace.`, - }); - - const foundToolCall = await rig.waitForToolCall('replace'); - expect(foundToolCall, 'Expected to find a replace tool call').toBeTruthy(); - expect( - rig - .readToolLogs() - .filter( - (log) => !['replace', 'read_file'].includes(log.toolRequest.name), - ), - 'Text replacement must not invoke unrelated tools', - ).toHaveLength(0); - - expect(rig.readFile(fileName)).toBe(expectedContent); - }); + }); + + const fileName = 'file_to_replace.txt'; + const originalContent = 'foo content'; + const expectedContent = 'bar content'; + + const filePath = rig.createFile(fileName, originalContent); + + await rig.run({ + args: `Use the replace tool on '${filePath}' to replace the exact text 'foo content' with 'bar content'. Do not add any whitespace.`, + timeoutMs: localModelPilot ? 900_000 : undefined, + }); + + const foundToolCall = await rig.waitForToolCall('replace'); + expect( + foundToolCall, + 'Expected to find a replace tool call', + ).toBeTruthy(); + expect( + rig + .readToolLogs() + .filter( + (log) => !['replace', 'read_file'].includes(log.toolRequest.name), + ), + 'Text replacement must not invoke unrelated tools', + ).toHaveLength(0); + + expect(rig.readFile(fileName)).toBe(expectedContent); + }, + ); }); diff --git a/packages/test-utils/src/test-rig-setup.ts b/packages/test-utils/src/test-rig-setup.ts index 0e519bb6db..7331a50d18 100644 --- a/packages/test-utils/src/test-rig-setup.ts +++ b/packages/test-utils/src/test-rig-setup.ts @@ -216,6 +216,14 @@ function collectEphemeralEntries(): Array<[string, unknown]> { } } + const maxOutputTokens = env['LLXPRT_MAX_OUTPUT_TOKENS']; + if (maxOutputTokens !== undefined) { + const parsedMaxOutputTokens = Number(maxOutputTokens); + if (Number.isFinite(parsedMaxOutputTokens) && parsedMaxOutputTokens > 0) { + entries.push(['maxOutputTokens', parsedMaxOutputTokens]); + } + } + return entries; } diff --git a/packages/test-utils/src/test-rig.test.ts b/packages/test-utils/src/test-rig.test.ts index 51f7689611..fee4c4c59f 100644 --- a/packages/test-utils/src/test-rig.test.ts +++ b/packages/test-utils/src/test-rig.test.ts @@ -4,8 +4,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; +import { createServer } from 'node:http'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'bun:test'; import { restoreEnv, setEnv } from './env-test-helpers.js'; @@ -51,6 +58,100 @@ describe('TestRig setup and cleanup behavior', () => { expect(rig.originalFakeResponsesPath).toBe(fakeResponsesPath); }); + it('writes bounded context and output settings into an opt-in test profile', () => { + createRoot(); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:2b'); + setEnv('LLXPRT_CONTEXT_LIMIT', '32768'); + setEnv('LLXPRT_MAX_OUTPUT_TOKENS', '8192'); + const rig = new TestRig(); + + rig.setup('local model pilot profile'); + + const testDir = requireTestDir(rig.testDir); + const profile = JSON.parse( + readFileSync( + join(testDir, '.llxprt', 'profiles', 'local-model-pilot.json'), + 'utf8', + ), + ); + expect(profile).toMatchObject({ + provider: 'openai', + model: 'qwen3.5:2b', + ephemeralSettings: { + 'context-limit': 32768, + maxOutputTokens: 8192, + }, + }); + }); + + it('loads the generated local pilot profile in the real CLI without a global profile', async () => { + const root = createRoot(); + setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); + setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:2b'); + setEnv('OPENAI_API_KEY', 'local-test-only'); + setEnv('LLXPRT_CONTEXT_LIMIT', '32768'); + setEnv('LLXPRT_MAX_OUTPUT_TOKENS', '8192'); + + const requests: Array<{ path: string | undefined; body: string }> = []; + const server = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on('data', (chunk: Buffer) => chunks.push(chunk)); + request.on('end', () => { + requests.push({ + path: request.url, + body: Buffer.concat(chunks).toString(), + }); + const chunk = JSON.stringify({ + id: 'chatcmpl-local-profile-test', + object: 'chat.completion.chunk', + choices: [ + { + delta: { content: 'OK' }, + index: 0, + finish_reason: 'stop', + }, + ], + }); + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end(`data: ${chunk}\n\ndata: [DONE]\n\n`); + }); + }); + server.listen(0, '127.0.0.1'); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Local test server has no TCP port'); + } + setEnv('OPENAI_BASE_URL', `http://127.0.0.1:${address.port}/v1`); + const rig = new TestRig(); + rig.setup('real CLI local pilot profile'); + + await rig.run({ args: 'Respond with OK', timeoutMs: 20_000 }); + + const completionRequest = requests.find( + (request) => request.path === '/v1/chat/completions', + ); + expect({ + path: completionRequest?.path, + model: + completionRequest === undefined + ? undefined + : JSON.parse(completionRequest.body).model, + }).toStrictEqual({ + path: '/v1/chat/completions', + model: 'qwen3.5:2b', + }); + } finally { + server.closeAllConnections(); + server.close(); + } + }, 30_000); + it('cleans test directories when KEEP_OUTPUT is unset or empty', async () => { createRoot(); const rig = new TestRig(); @@ -171,4 +272,32 @@ describe('TestRig setup and cleanup behavior', () => { expect(existsSync(ledgerPath)).toBe(false); }); + + it('uses a per-run deadline without changing the default TestRig timeout', async () => { + const root = createRoot(); + const fixturePath = join(root, 'fake.jsonl'); + const fixture = JSON.stringify({ + chunks: [ + { + speaker: 'ai', + blocks: [{ type: 'text', text: 'OK' }], + metadata: { + usage: { + promptTokens: 1, + completionTokens: 1, + totalTokens: 2, + }, + }, + }, + ], + }); + writeFileSync(fixturePath, `${fixture}\n`); + setEnv('LLXPRT_TEST_PROFILE', undefined); + const rig = new TestRig(); + rig.setup('scoped run timeout', { fakeResponsesPath: fixturePath }); + + await expect( + rig.run({ args: 'test prompt', timeoutMs: 1 }), + ).rejects.toThrow('TestRig.run() timed out after 1ms'); + }); }); diff --git a/packages/test-utils/src/test-rig.ts b/packages/test-utils/src/test-rig.ts index 8766d58b47..c400a18d6d 100644 --- a/packages/test-utils/src/test-rig.ts +++ b/packages/test-utils/src/test-rig.ts @@ -87,6 +87,7 @@ interface RunMethodOptions { stdin?: string; stdinDoesNotEnd?: boolean; yolo?: boolean; + timeoutMs?: number; } /** @@ -226,7 +227,7 @@ export class TestRig { const commandArgs = [...initialArgs]; appendUserArgs(commandArgs, options.args); - appendProfileFlag(commandArgs); + appendProfileFlag(commandArgs, this.testDir as string); const childEnv = buildChildEnv( this.testDir as string, @@ -254,7 +255,7 @@ export class TestRig { options, isJsonOutput, transform, - getDefaultTimeout() * 4, + options.timeoutMs ?? getDefaultTimeout() * 4, (capture) => { this._lastRunCapture = capture; this._lastRunStdout = capture.stdout; @@ -508,7 +509,7 @@ export class TestRig { const commandArgs = [...initialArgs]; appendInteractiveArgs(commandArgs, options?.args); - appendProfileFlag(commandArgs); + appendProfileFlag(commandArgs, this.testDir as string); const childEnv = buildChildEnv( this.testDir as string, @@ -598,16 +599,24 @@ function appendUserArgs( } /** - * Append the `--profile-load` flag when a test profile is configured. + * Append the configured test profile, using inline JSON for the local pilot. */ -function appendProfileFlag(commandArgs: string[]): void { +function appendProfileFlag(commandArgs: string[], testDir: string): void { const profileName = getProfileName(); if (profileName === undefined) { return; } const ideFlagIndex = commandArgs.indexOf('--ide-mode'); const insertionIndex = ideFlagIndex >= 0 ? ideFlagIndex : commandArgs.length; - commandArgs.splice(insertionIndex, 0, '--profile-load', profileName); + if (env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true') { + const profileJson = readFileSync( + join(testDir, '.llxprt', 'profiles', `${profileName}.json`), + 'utf8', + ); + commandArgs.splice(insertionIndex, 0, '--profile', profileJson); + } else { + commandArgs.splice(insertionIndex, 0, '--profile-load', profileName); + } } /** diff --git a/scripts/bun-test-roots.ts b/scripts/bun-test-roots.ts index 1ec052c627..41e928261d 100644 --- a/scripts/bun-test-roots.ts +++ b/scripts/bun-test-roots.ts @@ -147,6 +147,15 @@ export function getErrorCode(error: unknown): string | undefined { // Root table // --------------------------------------------------------------------------- +export function integrationTestTimeoutOverrides( + environment: Readonly, +): readonly BunTestTimeoutOverride[] { + if (environment['LLXPRT_LOCAL_MODEL_PILOT'] !== 'true') { + return []; + } + return [{ pattern: /replace\.test\.ts$/, timeout: 1_200_000 }]; +} + export const BUN_TEST_ROOTS: readonly BunTestRoot[] = [ { root: 'a2a-server', @@ -258,6 +267,7 @@ export const BUN_TEST_ROOTS: readonly BunTestRoot[] = [ timeout: 300_000, retries: 2, credentialed: true, + timeoutOverrides: integrationTestTimeoutOverrides(process.env), }, ]; diff --git a/scripts/tests/bun-test-roots.bun.test.ts b/scripts/tests/bun-test-roots.bun.test.ts index 8fb9c8d0a0..53129ee8c0 100644 --- a/scripts/tests/bun-test-roots.bun.test.ts +++ b/scripts/tests/bun-test-roots.bun.test.ts @@ -26,6 +26,7 @@ import { discoverTestFilesInDirectory, getErrorCode, isTestFileName, + integrationTestTimeoutOverrides, resolveBunTestFiles, resolveRoot, resolveRootCwd, @@ -84,6 +85,25 @@ describe('BUN_TEST_ROOTS structural guarantees', () => { } }); + it('extends only the replace file timeout for the opt-in local-model pilot', () => { + expect(integrationTestTimeoutOverrides({})).toEqual([]); + + const overrides = integrationTestTimeoutOverrides({ + LLXPRT_LOCAL_MODEL_PILOT: 'true', + }); + + expect(overrides).toHaveLength(1); + expect( + overrides[0]?.pattern.test('/repo/integration-tests/replace.test.ts'), + ).toBe(true); + expect( + overrides[0]?.pattern.test( + '/repo/integration-tests/run_shell_command.test.ts', + ), + ).toBe(false); + expect(overrides[0]?.timeout).toBe(1_200_000); + }); + it('has exactly the expected set of root tokens', () => { const tokens = BUN_TEST_ROOTS.map((r) => r.root); expect(tokens).toEqual([ diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 0352085763..7e5b56d4af 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -86,6 +86,7 @@ describe('optional local-model E2E pilot', () => { 'inputs.pilot_local_model == true', ); expect(job.permissions).toEqual({ contents: 'read' }); + expect(job['timeout-minutes']).toBe(90); expect(asRecord(job.strategy).matrix).toEqual({ sandbox: ['sandbox:none', 'sandbox:docker'], include: [ @@ -141,7 +142,7 @@ describe('optional local-model E2E pilot', () => { const server = step('Start local Qwen3.5 model'); const start = asString(server.run); const env = asOptionalRecord(server.env); - expect(env?.OLLAMA_CONTEXT_LENGTH).toBe('262144'); + expect(env?.OLLAMA_CONTEXT_LENGTH).toBe('32768'); expect(env?.OLLAMA_NUM_PARALLEL).toBe('1'); expect(env?.OLLAMA_HOST).toBe('127.0.0.1:12644'); expect(start).toContain('jq -e \'.version == "0.31.1"\''); @@ -162,6 +163,16 @@ describe('optional local-model E2E pilot', () => { expect(env?.LLXPRT_DEFAULT_PROVIDER).toBe('openai'); expect(env?.LLXPRT_DEFAULT_MODEL).toBe('qwen3.5:2b'); expect(env?.OPENAI_API_KEY).toBe('ollama-local-only'); + expect(env?.LLXPRT_TEST_PROFILE).toBe('local-qwen35-pilot'); + expect(env?.LLXPRT_CONTEXT_LIMIT).toBe('32768'); + expect(env?.LLXPRT_MAX_OUTPUT_TOKENS).toBe('8192'); + expect(env?.LLXPRT_LOCAL_MODEL_PILOT).toBe('true'); + expect(Number(env?.LLXPRT_CONTEXT_LIMIT)).toBe( + Number(step('Start local Qwen3.5 model').env?.OLLAMA_CONTEXT_LENGTH), + ); + expect(Number(env?.LLXPRT_MAX_OUTPUT_TOKENS)).toBeLessThan( + Number(env?.LLXPRT_CONTEXT_LIMIT), + ); expect(env?.GIT_CEILING_DIRECTORIES).toBe( '${{ github.workspace }}/.integration-tests', ); From 8260cd94f998dd448b66576a1d966a6003bc8567 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sat, 26 Sep 2026 20:01:43 -0300 Subject: [PATCH 04/21] Allow CPU pilot first response before replace deadline (Refs #3764) --- dev-docs/integration-tests.md | 14 +++++++++++--- integration-tests/replace.test.ts | 4 ++-- packages/test-utils/src/test-rig-setup.ts | 4 ++++ packages/test-utils/src/test-rig.test.ts | 22 ++++++++++++++++++++++ scripts/bun-test-roots.ts | 2 +- scripts/tests/bun-test-roots.bun.test.ts | 2 +- 6 files changed, 41 insertions(+), 7 deletions(-) diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index 839128e277..81f8a14322 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -220,9 +220,10 @@ without discarding file changes. The runtime archive is pinned to 0.31.1; its CUDA and Vulkan libraries are excluded during extraction, and the downloaded archive is removed afterward. Ollama is limited to one concurrent context. The manual pilot configures both Ollama and LLxprt with a 32,768-token context and -reserves 8,192 tokens for model output. The real replace invocation has a -900,000 ms `TestRig` deadline, its Bun file has a 1,200,000 ms timeout, and each -sandbox job has a 90-minute bound. These larger deadlines apply only when +reserves 8,192 tokens for model output. The pilot profile allows 600,000 ms +for the first model response. The real replace invocation has a 1,200,000 ms +`TestRig` deadline, its Bun file has a 1,500,000 ms timeout, and each sandbox +job has a 90-minute bound. These larger deadlines apply only when `LLXPRT_LOCAL_MODEL_PILOT=true`; normal integration-test deadlines are unchanged. To reproduce the two real-model canaries without using an existing Ollama daemon, @@ -282,6 +283,13 @@ the model digest and SHA-256 result in each job log. Both canaries and the budge check need repeated passes before considering any required-coverage change. A local Apple Silicon result cannot substitute for Linux CPU and Docker measurements. +Hosted run [`36276039788`](https://github.com/vybestack/llxprt-code/actions/runs/36276039788) +passed shell in both legs but failed replace in all three attempts per leg. +Four replace attempts hit the CLI's default 300,000 ms first-response watchdog; +the remaining host attempt hit the 900,000 ms `TestRig` deadline. Both diagnostic +artifacts uploaded. The pilot-only first-response and replace deadlines above +were increased in response; this does not establish a passing hosted run. + #### Local measurements (September 26, 2026) With Ollama 0.31.1 on an M4 Max, the 2B tag downloaded 2,741,192,820 bytes. diff --git a/integration-tests/replace.test.ts b/integration-tests/replace.test.ts index 9bea4a5c67..a7e99b577e 100644 --- a/integration-tests/replace.test.ts +++ b/integration-tests/replace.test.ts @@ -9,7 +9,7 @@ import { env } from 'node:process'; import { TestRig } from './test-helper.js'; const localModelPilot = env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true'; -const replaceTestOptions = localModelPilot ? { timeout: 1_200_000 } : {}; +const replaceTestOptions = localModelPilot ? { timeout: 1_500_000 } : {}; describe('replace', () => { let rig: TestRig; @@ -40,7 +40,7 @@ describe('replace', () => { await rig.run({ args: `Use the replace tool on '${filePath}' to replace the exact text 'foo content' with 'bar content'. Do not add any whitespace.`, - timeoutMs: localModelPilot ? 900_000 : undefined, + timeoutMs: localModelPilot ? 1_200_000 : undefined, }); const foundToolCall = await rig.waitForToolCall('replace'); diff --git a/packages/test-utils/src/test-rig-setup.ts b/packages/test-utils/src/test-rig-setup.ts index 7331a50d18..1afecdc154 100644 --- a/packages/test-utils/src/test-rig-setup.ts +++ b/packages/test-utils/src/test-rig-setup.ts @@ -224,6 +224,10 @@ function collectEphemeralEntries(): Array<[string, unknown]> { } } + if (env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true') { + entries.push(['stream-first-response-timeout-ms', 600_000]); + } + return entries; } diff --git a/packages/test-utils/src/test-rig.test.ts b/packages/test-utils/src/test-rig.test.ts index fee4c4c59f..3536728dbd 100644 --- a/packages/test-utils/src/test-rig.test.ts +++ b/packages/test-utils/src/test-rig.test.ts @@ -60,6 +60,7 @@ describe('TestRig setup and cleanup behavior', () => { it('writes bounded context and output settings into an opt-in test profile', () => { createRoot(); + setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:2b'); @@ -82,10 +83,31 @@ describe('TestRig setup and cleanup behavior', () => { ephemeralSettings: { 'context-limit': 32768, maxOutputTokens: 8192, + 'stream-first-response-timeout-ms': 600_000, }, }); }); + it('leaves the ordinary test profile first-response deadline unchanged', () => { + createRoot(); + setEnv('LLXPRT_LOCAL_MODEL_PILOT', undefined); + setEnv('LLXPRT_TEST_PROFILE', 'ordinary-profile'); + const rig = new TestRig(); + + rig.setup('ordinary profile'); + + const testDir = requireTestDir(rig.testDir); + const profile = JSON.parse( + readFileSync( + join(testDir, '.llxprt', 'profiles', 'ordinary-profile.json'), + 'utf8', + ), + ); + expect(profile.ephemeralSettings).not.toHaveProperty( + 'stream-first-response-timeout-ms', + ); + }); + it('loads the generated local pilot profile in the real CLI without a global profile', async () => { const root = createRoot(); setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); diff --git a/scripts/bun-test-roots.ts b/scripts/bun-test-roots.ts index 41e928261d..e699bda770 100644 --- a/scripts/bun-test-roots.ts +++ b/scripts/bun-test-roots.ts @@ -153,7 +153,7 @@ export function integrationTestTimeoutOverrides( if (environment['LLXPRT_LOCAL_MODEL_PILOT'] !== 'true') { return []; } - return [{ pattern: /replace\.test\.ts$/, timeout: 1_200_000 }]; + return [{ pattern: /replace\.test\.ts$/, timeout: 1_500_000 }]; } export const BUN_TEST_ROOTS: readonly BunTestRoot[] = [ diff --git a/scripts/tests/bun-test-roots.bun.test.ts b/scripts/tests/bun-test-roots.bun.test.ts index 53129ee8c0..f56f5cb082 100644 --- a/scripts/tests/bun-test-roots.bun.test.ts +++ b/scripts/tests/bun-test-roots.bun.test.ts @@ -101,7 +101,7 @@ describe('BUN_TEST_ROOTS structural guarantees', () => { '/repo/integration-tests/run_shell_command.test.ts', ), ).toBe(false); - expect(overrides[0]?.timeout).toBe(1_200_000); + expect(overrides[0]?.timeout).toBe(1_500_000); }); it('has exactly the expected set of root tokens', () => { From 41b7d9613d926722db6dd29e30bf88e381af46d0 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sat, 26 Sep 2026 21:50:42 -0300 Subject: [PATCH 05/21] Apply pilot watchdog profile and test Gemma 4 QAT (Refs #3764) --- .github/workflows/e2e.yml | 16 +++--- dev-docs/integration-tests.md | 61 +++++++++++++++++++---- packages/test-utils/src/cli-args.test.ts | 24 +++++++++ packages/test-utils/src/cli-args.ts | 11 ++++ packages/test-utils/src/test-rig-setup.ts | 6 ++- packages/test-utils/src/test-rig.test.ts | 48 ++++++++++++++++++ scripts/tests/e2e-local-model.test.ts | 16 +++--- 7 files changed, 155 insertions(+), 27 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5184b51d0d..08e59760f4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -24,7 +24,7 @@ on: default: 'main' type: 'string' pilot_local_model: - description: 'Run local Qwen3.5 canaries instead of credentialed E2E' + description: 'Run local Gemma 4 E2B canaries instead of credentialed E2E' required: false default: false type: 'boolean' @@ -350,7 +350,7 @@ jobs: bun scripts/check-e2e-model-budget.ts --ledger "$LLXPRT_E2E_MODEL_LEDGER" local_model_canaries: - name: 'Local Qwen3.5 pilot - ${{ matrix.sandbox }}' + name: 'Local Gemma 4 E2B pilot - ${{ matrix.sandbox }}' if: github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true permissions: contents: read @@ -411,7 +411,7 @@ jobs: echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" "$RUNNER_TEMP/ollama/bin/ollama" --version - - name: 'Start local Qwen3.5 model' + - name: 'Start local Gemma 4 E2B model' shell: 'bash' env: OLLAMA_HOST: '127.0.0.1:12644' @@ -429,19 +429,19 @@ jobs: sleep 2 done curl --fail --silent "http://$OLLAMA_HOST/api/version" | jq -e '.version == "0.31.1"' - ollama pull qwen3.5:2b + ollama pull gemma4:e2b-it-qat curl --fail --silent "http://$OLLAMA_HOST/api/tags" | - jq -e '.models[] | select(.name == "qwen3.5:2b" and .digest == "324d162be6ca5629ae4517c8710434d0bd2d665bc94dbad46e9af8fbf8a2f0df")' + jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .digest == "07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c")' df -h "$RUNNER_TEMP" - name: 'Run local-model canaries' env: OPENAI_API_KEY: 'ollama-local-only' OPENAI_BASE_URL: 'http://127.0.0.1:12644/v1' - LLXPRT_DEFAULT_MODEL: 'qwen3.5:2b' + LLXPRT_DEFAULT_MODEL: 'gemma4:e2b-it-qat' LLXPRT_DEFAULT_PROVIDER: 'openai' LLXPRT_AUTH_TYPE: 'provider' - LLXPRT_TEST_PROFILE: 'local-qwen35-pilot' + LLXPRT_TEST_PROFILE: 'local-gemma4-pilot' LLXPRT_CONTEXT_LIMIT: '32768' LLXPRT_MAX_OUTPUT_TOKENS: '8192' LLXPRT_LOCAL_MODEL_PILOT: 'true' @@ -494,7 +494,7 @@ jobs: if: always() uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # ratchet:actions/upload-artifact@v7 with: - name: 'local-qwen35-pilot-${{ matrix.artifact_id }}' + name: 'local-gemma4-pilot-${{ matrix.artifact_id }}' if-no-files-found: error include-hidden-files: true path: | diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index 81f8a14322..ecff61c265 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -188,8 +188,8 @@ The credentialed full-platform E2E matrix also remains in `.github/workflows/nightly.yml`. With `pilot_local_model=true`, the pilot starts Ollama 0.31.1 on -`127.0.0.1:12644` and pulls `qwen3.5:2b` (Q8_0, digest -`324d162be6ca5629ae4517c8710434d0bd2d665bc94dbad46e9af8fbf8a2f0df`). +`127.0.0.1:12644` and pulls `gemma4:e2b-it-qat` (Q4_0, digest +`07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c`). It verifies the downloaded Linux runtime archive's SHA-256 digest (`d297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4`) before extraction, then checks the runtime version and model digest before testing. @@ -229,7 +229,7 @@ job has a 90-minute bound. These larger deadlines apply only when To reproduce the two real-model canaries without using an existing Ollama daemon, run these commands from the repository root. Choose a free port if 12644 is in use and change both URLs accordingly. Install Ollama 0.31.1 first and ensure the -`qwen3.5:2b` digest above matches. Store all evidence in the repository's +`gemma4:e2b-it-qat` digest above matches. Store all evidence in the repository's ignored `tmp/` tree: ```bash @@ -239,13 +239,13 @@ OLLAMA_HOST=127.0.0.1:12644 \ OLLAMA_CONTEXT_LENGTH=32768 OLLAMA_NUM_PARALLEL=1 \ ollama serve >tmp/verify3764/ollama-local.log 2>&1 & OLLAMA_HOST=127.0.0.1:12644 \ - OLLAMA_MODELS="$PWD/tmp/verify3764/models" ollama pull qwen3.5:2b -curl -fsS http://127.0.0.1:12644/api/tags | jq '.models[] | select(.name == "qwen3.5:2b") | {digest, size, details}' + OLLAMA_MODELS="$PWD/tmp/verify3764/models" ollama pull gemma4:e2b-it-qat +curl -fsS http://127.0.0.1:12644/api/tags | jq '.models[] | select(.name == "gemma4:e2b-it-qat") | {digest, size, details}' CI=true KEEP_OUTPUT=true VERBOSE=true \ - LLXPRT_DEFAULT_PROVIDER=openai LLXPRT_DEFAULT_MODEL=qwen3.5:2b \ + LLXPRT_DEFAULT_PROVIDER=openai LLXPRT_DEFAULT_MODEL=gemma4:e2b-it-qat \ OPENAI_API_KEY=ollama-local-only \ OPENAI_BASE_URL=http://127.0.0.1:12644/v1 LLXPRT_AUTH_TYPE=provider \ - LLXPRT_TEST_PROFILE=local-qwen35-pilot LLXPRT_CONTEXT_LIMIT=32768 \ + LLXPRT_TEST_PROFILE=local-gemma4-pilot LLXPRT_CONTEXT_LIMIT=32768 \ LLXPRT_MAX_OUTPUT_TOKENS=8192 LLXPRT_LOCAL_MODEL_PILOT=true \ LLXPRT_FORCE_FILE_STORAGE=true \ LLXPRT_E2E_MODEL_LEDGER="$PWD/tmp/verify3764/ledger.jsonl" \ @@ -257,9 +257,8 @@ bun scripts/check-e2e-model-budget.ts --ledger "$PWD/tmp/verify3764/ledger.jsonl curl -fsS http://127.0.0.1:12644/api/ps | jq '.models[] | {name, size, context_length}' ``` -Qwen lists the 2B and 4B releases on March 2, 2026 in its -[release list](https://github.com/QwenLM/Qwen3.5); Ollama publishes the -[quantized tags](https://ollama.com/library/qwen3.5). The +Google [released Gemma 4 E2B on April 2, 2026](https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/); +Ollama publishes the [QAT tag](https://ollama.com/library/gemma4/tags). The [GitHub-hosted runner specifications](https://docs.github.com/en/actions/reference/runners/github-hosted-runners) list 4 CPU, 16 GB RAM and 14 GB SSD for public `ubuntu-latest` jobs. The macOS Apple Silicon measurements below do not establish CPU-only x64 speed, @@ -290,6 +289,31 @@ the remaining host attempt hit the 900,000 ms `TestRig` deadline. Both diagnosti artifacts uploaded. The pilot-only first-response and replace deadlines above were increased in response; this does not establish a passing hosted run. +Hosted [run 36278157447](https://github.com/vybestack/llxprt-code/actions/runs/36278157447) +confirmed why that increase did not work. The test rig generated a profile with +`stream-first-response-timeout-ms: 600000`, but also passed `--provider openai`. +`applyGlobalAndProfileEphemeralSettings` deliberately skips profile ephemerals +when the provider is explicit. The stream guard therefore used its 300000 ms +default. The pilot now relies on its inline profile for provider, model, +credentials and timeout instead of supplying conflicting CLI flags. A real-CLI +test uses a stalled local HTTP server and a shortened profile threshold to +verify that the guard reads the profile setting. This changes neither regular +E2E provider flags nor the default watchdog. + +Both hosted artifacts include `telemetry.log` with the original user prompt: +`Use the replace tool on '' to replace the exact text 'foo +content' with 'bar content'. Do not add any whitespace.` On the Docker leg, +the model first tried `read_file` with a space inserted into the checkout path, +then called `replace` twice after dropping one `llxprt-code` path component. +It subsequently called `glob`, `list_directory`, `direct_web_fetch` and +`run_shell_command`; none repaired the file. On the host leg, a retry read the +correct file but passed tool-result formatting and patch fragments as the +`old_string` to `ast_edit` instead of making the requested replacement. Other +attempts hit the guard or TestRig deadline. The target files existed, observed +prompts fit within 32,768 tokens, and the model chose incorrect tools or paths. +The assertions still require the `replace` call, no unrelated tools and exact +`bar content`. + #### Local measurements (September 26, 2026) With Ollama 0.31.1 on an M4 Max, the 2B tag downloaded 2,741,192,820 bytes. @@ -327,3 +351,20 @@ reported five replace and seven shell invocations including retries. No local model/configuration tested here passed repeat suites consistently, so the pilot is non-required. The three 4B suite logs likewise show only one complete passing suite, with replace exhausting all retries in two. + +#### Gemma 4 pilot candidate (September 26, 2026) + +The official `gemma4:e2b-it-qat` artifact is 4,336,358,185 bytes, with the +digest pinned above. Ollama 0.31.1 reported a 3,784,551,955-byte loaded model +at 32,768 context on the local M4 Max. Three separate full `TestRig` suites +passed both the shell and exact replacement canaries on the first file-level +attempt, six tests in total. The three per-suite budget checks each passed at +four recorded requests across two distinct tests. See +`tmp/verify3764/canaries-gemma4-e2b-{1,2,3}.log` and `gemma-budget.log` for +local evidence. The strict unrelated-tool and exact-content assertions were +unchanged. Qwen3.5:4b Q4_K_M also passed shell but failed replace in a further +complete suite despite retrying three times. Disabling thinking for 4B did not +fix the unrelated-tool failure in its first complete suite; its next run was +stopped to avoid competing with the Gemma measurements. The earlier 2B hosted +failures remain as recorded above. These are local GPU measurements, not proof +that Gemma completes inside the hosted CPU, memory or Docker limits. diff --git a/packages/test-utils/src/cli-args.test.ts b/packages/test-utils/src/cli-args.test.ts index 7e5e55da6c..4902bbc5b7 100644 --- a/packages/test-utils/src/cli-args.test.ts +++ b/packages/test-utils/src/cli-args.test.ts @@ -74,6 +74,30 @@ describe('cli-args helpers', () => { ]); }); + it('uses only the generated profile to configure the local model pilot', () => { + setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-qwen35-pilot'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:4b'); + setEnv('OPENAI_BASE_URL', 'http://127.0.0.1:12644/v1'); + setEnv('OPENAI_API_KEY', 'local-only'); + + expect(buildExtraArgs(undefined, true)).toStrictEqual([ + '--yolo', + '--ide-mode', + 'disable', + ]); + }); + + it('rejects a local pilot without a profile', () => { + setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); + setEnv('LLXPRT_TEST_PROFILE', undefined); + + expect(() => buildExtraArgs(undefined, true)).toThrow( + 'LLXPRT_TEST_PROFILE is required for the local model pilot', + ); + }); + it('builds child env without IDE detection variables and with fake response path', () => { setEnv('TERM_PROGRAM', 'vscode'); setEnv('TERM_PROGRAM_VERSION', '1.0.0'); diff --git a/packages/test-utils/src/cli-args.ts b/packages/test-utils/src/cli-args.ts index 0fced34ca9..0038781cfb 100644 --- a/packages/test-utils/src/cli-args.ts +++ b/packages/test-utils/src/cli-args.ts @@ -53,6 +53,17 @@ export function buildExtraArgs( return extraArgs; } + // The inline pilot profile owns provider, credentials and runtime ephemerals. + // An explicit --provider prevents the CLI from applying those ephemerals. + if (env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true') { + if (getProfileName() === undefined) { + throw new Error( + 'LLXPRT_TEST_PROFILE is required for the local model pilot', + ); + } + return extraArgs; + } + const provider = requireEnvString('LLXPRT_DEFAULT_PROVIDER'); const model = requireEnvString('LLXPRT_DEFAULT_MODEL'); const baseUrl = readEnvString('OPENAI_BASE_URL'); diff --git a/packages/test-utils/src/test-rig-setup.ts b/packages/test-utils/src/test-rig-setup.ts index 1afecdc154..5b716655e0 100644 --- a/packages/test-utils/src/test-rig-setup.ts +++ b/packages/test-utils/src/test-rig-setup.ts @@ -175,7 +175,11 @@ function buildProfile(): Record { version: 1, provider: profileProvider, model: profileModel, - modelParams: {}, + modelParams: + env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true' && + env['LLXPRT_LOCAL_MODEL_THINKING'] === 'none' + ? { reasoning_effort: 'none' } + : {}, ephemeralSettings: Object.fromEntries( ephemeralEntries.filter(([, value]) => value !== undefined), ), diff --git a/packages/test-utils/src/test-rig.test.ts b/packages/test-utils/src/test-rig.test.ts index 3536728dbd..e1d5cb4363 100644 --- a/packages/test-utils/src/test-rig.test.ts +++ b/packages/test-utils/src/test-rig.test.ts @@ -116,6 +116,7 @@ describe('TestRig setup and cleanup behavior', () => { setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:2b'); setEnv('OPENAI_API_KEY', 'local-test-only'); + setEnv('LLXPRT_LOCAL_MODEL_THINKING', 'none'); setEnv('LLXPRT_CONTEXT_LIMIT', '32768'); setEnv('LLXPRT_MAX_OUTPUT_TOKENS', '8192'); @@ -164,9 +165,14 @@ describe('TestRig setup and cleanup behavior', () => { completionRequest === undefined ? undefined : JSON.parse(completionRequest.body).model, + reasoningEffort: + completionRequest === undefined + ? undefined + : JSON.parse(completionRequest.body).reasoning_effort, }).toStrictEqual({ path: '/v1/chat/completions', model: 'qwen3.5:2b', + reasoningEffort: 'none', }); } finally { server.closeAllConnections(); @@ -174,6 +180,48 @@ describe('TestRig setup and cleanup behavior', () => { } }, 30_000); + it('applies the generated profile first-response watchdog in the real CLI', async () => { + const root = createRoot(); + setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); + setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:4b'); + setEnv('OPENAI_API_KEY', 'local-test-only'); + + const server = createServer((_request, _response) => { + // Keep the first response pending so only the CLI watchdog can end it. + }); + server.listen(0, '127.0.0.1'); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Local test server has no TCP port'); + } + setEnv('OPENAI_BASE_URL', `http://127.0.0.1:${address.port}/v1`); + const rig = new TestRig(); + rig.setup('local profile watchdog'); + const profilePath = join( + requireTestDir(rig.testDir), + '.llxprt', + 'profiles', + 'local-model-pilot.json', + ); + const profile = JSON.parse(readFileSync(profilePath, 'utf8')); + profile.ephemeralSettings['stream-first-response-timeout-ms'] = 100; + writeFileSync(profilePath, JSON.stringify(profile)); + + await expect( + rig.run({ args: 'Respond with OK', timeoutMs: 15_000 }), + ).rejects.toThrow( + /threshold 100ms\) from stream-first-response-timeout-ms/, + ); + } finally { + server.closeAllConnections(); + server.close(); + } + }, 20_000); + it('cleans test directories when KEEP_OUTPUT is unset or empty', async () => { createRoot(); const rig = new TestRig(); diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 7e5b56d4af..4622859667 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -80,7 +80,7 @@ describe('optional local-model E2E pilot', () => { 'local_model_canaries', ]); const job = workflowJob(pilot, 'local_model_canaries'); - expect(job.name).toContain('Local Qwen3.5 pilot'); + expect(job.name).toContain('Local Gemma 4 E2B pilot'); expect(job.name).not.toContain('E2E Test (Linux)'); expect(asString(job.if ?? '')).toContain( 'inputs.pilot_local_model == true', @@ -109,7 +109,7 @@ describe('optional local-model E2E pilot', () => { const name = asString( asOptionalRecord(step('Upload local model diagnostics').with)?.name, ); - expect(name).toBe('local-qwen35-pilot-${{ matrix.artifact_id }}'); + expect(name).toBe('local-gemma4-pilot-${{ matrix.artifact_id }}'); const resolvedNames = entries.map((entry) => { const row = asRecord(entry); expect(sandboxes).toContain(row.sandbox); @@ -139,16 +139,16 @@ describe('optional local-model E2E pilot', () => { }); it('checks model version and digest before running either canary', () => { - const server = step('Start local Qwen3.5 model'); + const server = step('Start local Gemma 4 E2B model'); const start = asString(server.run); const env = asOptionalRecord(server.env); expect(env?.OLLAMA_CONTEXT_LENGTH).toBe('32768'); expect(env?.OLLAMA_NUM_PARALLEL).toBe('1'); expect(env?.OLLAMA_HOST).toBe('127.0.0.1:12644'); expect(start).toContain('jq -e \'.version == "0.31.1"\''); - expect(start).toContain('ollama pull qwen3.5:2b'); + expect(start).toContain('ollama pull gemma4:e2b-it-qat'); expect(start).toContain( - '324d162be6ca5629ae4517c8710434d0bd2d665bc94dbad46e9af8fbf8a2f0df', + '07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c', ); expect(steps.indexOf(server)).toBeLessThan( steps.indexOf(step('Run local-model canaries')), @@ -161,14 +161,14 @@ describe('optional local-model E2E pilot', () => { const env = asOptionalRecord(run.env); expect(env?.OPENAI_BASE_URL).toBe('http://127.0.0.1:12644/v1'); expect(env?.LLXPRT_DEFAULT_PROVIDER).toBe('openai'); - expect(env?.LLXPRT_DEFAULT_MODEL).toBe('qwen3.5:2b'); + expect(env?.LLXPRT_DEFAULT_MODEL).toBe('gemma4:e2b-it-qat'); expect(env?.OPENAI_API_KEY).toBe('ollama-local-only'); - expect(env?.LLXPRT_TEST_PROFILE).toBe('local-qwen35-pilot'); + expect(env?.LLXPRT_TEST_PROFILE).toBe('local-gemma4-pilot'); expect(env?.LLXPRT_CONTEXT_LIMIT).toBe('32768'); expect(env?.LLXPRT_MAX_OUTPUT_TOKENS).toBe('8192'); expect(env?.LLXPRT_LOCAL_MODEL_PILOT).toBe('true'); expect(Number(env?.LLXPRT_CONTEXT_LIMIT)).toBe( - Number(step('Start local Qwen3.5 model').env?.OLLAMA_CONTEXT_LENGTH), + Number(step('Start local Gemma 4 E2B model').env?.OLLAMA_CONTEXT_LENGTH), ); expect(Number(env?.LLXPRT_MAX_OUTPUT_TOKENS)).toBeLessThan( Number(env?.LLXPRT_CONTEXT_LIMIT), From 39997cf0c3f379c753b8b468a4e7f962d1f4d76d Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sat, 26 Sep 2026 23:53:05 -0300 Subject: [PATCH 06/21] Extend pilot-only transport and Docker shell deadlines (Refs #3764) --- dev-docs/integration-tests.md | 15 +++- integration-tests/run_shell_command.test.ts | 90 +++++++++++-------- .../src/openai/OpenAIClientFactory.test.ts | 83 +++++++++++++++++ .../src/openai/OpenAIClientFactory.ts | 63 ++++++++++++- .../providers/src/openai/OpenAIProvider.ts | 6 +- packages/test-utils/src/test-rig-setup.ts | 1 + packages/test-utils/src/test-rig.test.ts | 54 +++++++++++ 7 files changed, 270 insertions(+), 42 deletions(-) diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index ecff61c265..1c6122519e 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -221,10 +221,17 @@ its CUDA and Vulkan libraries are excluded during extraction, and the downloaded archive is removed afterward. Ollama is limited to one concurrent context. The manual pilot configures both Ollama and LLxprt with a 32,768-token context and reserves 8,192 tokens for model output. The pilot profile allows 600,000 ms -for the first model response. The real replace invocation has a 1,200,000 ms -`TestRig` deadline, its Bun file has a 1,500,000 ms timeout, and each sandbox -job has a 90-minute bound. These larger deadlines apply only when -`LLXPRT_LOCAL_MODEL_PILOT=true`; normal integration-test deadlines are unchanged. +for the first model response and sets `openai-headers-timeout-ms` to 900,000 ms +on its OpenAI SDK transport. The latter exceeds Undici's ordinary 300,000 ms +response-headers limit without changing the dispatcher for other requests. On +hosted CPU, a 15,200-token replace prompt was still being evaluated at 300 s +when that limit aborted it. The Docker shell model response took about 238 s, +so the pilot shell invocation has a 360,000 ms `TestRig` deadline and a +450,000 ms Bun file timeout to leave room for its tool round trip. The real +replace invocation has a 1,200,000 ms `TestRig` deadline, its Bun file has a +1,500,000 ms timeout, and each sandbox job has a 90-minute bound. These larger +deadlines apply only when `LLXPRT_LOCAL_MODEL_PILOT=true`; normal integration-test +deadlines are unchanged. To reproduce the two real-model canaries without using an existing Ollama daemon, run these commands from the repository root. Choose a free port if 12644 is in diff --git a/integration-tests/run_shell_command.test.ts b/integration-tests/run_shell_command.test.ts index 8685bdc3b6..16149a7116 100644 --- a/integration-tests/run_shell_command.test.ts +++ b/integration-tests/run_shell_command.test.ts @@ -6,10 +6,18 @@ import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; import { join } from 'node:path'; +import { env } from 'node:process'; import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js'; import { getShellConfiguration } from '../packages/core/src/utils/shell-utils.js'; const { shell } = getShellConfiguration(); +const localModelPilot = env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true'; + +function shellCanaryRunTimeoutMs(pilot: boolean): number | undefined { + // Docker's observed 238s model response leaves no room for a tool round trip + // under the original 240s TestRig deadline. + return pilot ? 360_000 : undefined; +} function getLineCountCommand(): { command: string; tool: string } { switch (shell) { @@ -59,6 +67,11 @@ function isExpectedShellTrace( } describe('shell canary command validation', () => { + it('extends only the pilot shell run deadline for the CPU tool round trip', () => { + expect(shellCanaryRunTimeoutMs(true)).toBe(360_000); + expect(shellCanaryRunTimeoutMs(false)).toBeUndefined(); + }); + it('accepts equivalent literal echo commands but not other shell expressions', () => { for (const command of [ 'echo hello-world', @@ -115,47 +128,52 @@ describe('run_shell_command', () => { }); afterEach(async () => await rig.cleanup()); - it('should be able to run a shell command', async () => { - await rig.setup('should be able to run a shell command', { - settings: { tools: { core: ['run_shell_command'] } }, - }); - - const prompt = `Use the run_shell_command tool to execute exactly "echo hello-world". Show me the output.`; - - const result = await rig.run({ - args: ['--allowed-tools=run_shell_command(echo)'], - stdin: prompt, - yolo: false, - }); + it( + 'should be able to run a shell command', + { timeout: localModelPilot ? 450_000 : undefined }, + async () => { + await rig.setup('should be able to run a shell command', { + settings: { tools: { core: ['run_shell_command'] } }, + }); - const foundToolCall = await rig.waitForToolCall('run_shell_command'); + const prompt = `Use the run_shell_command tool to execute exactly "echo hello-world". Show me the output.`; - // Add debugging information - if (!foundToolCall || !result.includes('hello-world')) { - printDebugInfo(rig, result, { - 'Found tool call': foundToolCall, - 'Contains hello-world': result.includes('hello-world'), + const result = await rig.run({ + args: ['--allowed-tools=run_shell_command(echo)'], + stdin: prompt, + yolo: false, + timeoutMs: shellCanaryRunTimeoutMs(localModelPilot), }); - } - expect( - foundToolCall, - 'Expected to find a run_shell_command tool call', - ).toBeTruthy(); - expect( - isExpectedShellTrace(rig.readToolLogs()), - 'Expected only successful echo hello-world shell invocations', - ).toBe(true); - expect(result).toContain('hello-world'); + const foundToolCall = await rig.waitForToolCall('run_shell_command'); - // Validate model output - will throw if no output, warn if missing expected content - // Model often reports exit code instead of showing output - validateModelOutput( - result, - ['hello-world', 'exit code 0'], - 'Shell command test', - ); - }); + // Add debugging information + if (!foundToolCall || !result.includes('hello-world')) { + printDebugInfo(rig, result, { + 'Found tool call': foundToolCall, + 'Contains hello-world': result.includes('hello-world'), + }); + } + + expect( + foundToolCall, + 'Expected to find a run_shell_command tool call', + ).toBeTruthy(); + expect( + isExpectedShellTrace(rig.readToolLogs()), + 'Expected only successful echo hello-world shell invocations', + ).toBe(true); + expect(result).toContain('hello-world'); + + // Validate model output - will throw if no output, warn if missing expected content + // Model often reports exit code instead of showing output + validateModelOutput( + result, + ['hello-world', 'exit code 0'], + 'Shell command test', + ); + }, + ); it('should be able to run a shell command via stdin', async () => { await rig.setup('should be able to run a shell command via stdin', { diff --git a/packages/providers/src/openai/OpenAIClientFactory.test.ts b/packages/providers/src/openai/OpenAIClientFactory.test.ts index e5da269b45..739e1326d0 100644 --- a/packages/providers/src/openai/OpenAIClientFactory.test.ts +++ b/packages/providers/src/openai/OpenAIClientFactory.test.ts @@ -15,8 +15,12 @@ */ import { describe, it, expect } from 'bun:test'; +import { createServer } from 'node:http'; import * as http from 'http'; import * as https from 'https'; +import { createRequire } from 'node:module'; +import type * as Undici from 'undici'; +import type { Agent as UndiciAgent } from 'undici'; import { createHttpAgents, extractModelParamsFromOptions, @@ -26,6 +30,11 @@ import { } from './OpenAIClientFactory.js'; import type { NormalizedGenerateChatOptions } from '../BaseProvider.js'; +const undiciRequire = createRequire( + createRequire(import.meta.url).resolve('undici/package.json'), +); +const { Agent, fetch: undiciFetch } = undiciRequire('./') as typeof Undici; + function getMaxOutputTokens(key: string): number | undefined { return key === 'maxOutputTokens' ? 2000 : undefined; } @@ -324,6 +333,80 @@ describe('OpenAIClientFactory', () => { expect(opts).toBeDefined(); expect(opts.defaultHeaders).toBeUndefined(); }); + + it('keeps the ordinary headers deadline while a scoped extension lets the SDK stream delayed headers', async () => { + const server = createServer((_request, response) => { + setTimeout(() => { + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end( + 'data: {"id":"delayed","choices":[{"index":0,"delta":{"content":"ready"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', + ); + }, 1_500); + }); + const defaultDispatcher = new Agent({ headersTimeout: 100 }); + const testFetch: typeof fetch = async (input, init) => { + const requestInit = init as RequestInit & { dispatcher?: UndiciAgent }; + const response = await undiciFetch(String(input), { + ...requestInit, + dispatcher: requestInit.dispatcher ?? defaultDispatcher, + } as Parameters[1]); + return response as unknown as Response; + }; + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Missing local server port'); + } + const baseURL = `http://127.0.0.1:${address.port}/v1`; + const request = { model: 'local', messages: [], stream: true as const }; + const consume = async ( + client: ReturnType, + ): Promise> => { + const chunks = []; + for await (const chunk of await client.chat.completions.create( + request, + )) { + chunks.push(chunk.choices[0]?.delta.content); + } + return chunks; + }; + const ordinary = instantiateClient( + 'test-token', + baseURL, + undefined, + undefined, + { + fetch: testFetch, + }, + ); + await expect(consume(ordinary)).rejects.toThrow('Request timed out.'); + + const shortPilot = instantiateClient( + 'test-token', + baseURL, + undefined, + undefined, + { headersTimeoutMs: 100 }, + ); + await expect(consume(shortPilot)).rejects.toThrow('Request timed out.'); + + const pilot = instantiateClient( + 'test-token', + baseURL, + undefined, + undefined, + { headersTimeoutMs: 4_000 }, + ); + expect(await consume(pilot)).toStrictEqual(['ready']); + } finally { + server.closeAllConnections(); + server.close(); + await defaultDispatcher.close(); + } + }); }); describe('mergeInvocationHeaders', () => { diff --git a/packages/providers/src/openai/OpenAIClientFactory.ts b/packages/providers/src/openai/OpenAIClientFactory.ts index 5aa4cb038d..d0c820c11c 100644 --- a/packages/providers/src/openai/OpenAIClientFactory.ts +++ b/packages/providers/src/openai/OpenAIClientFactory.ts @@ -26,10 +26,20 @@ import OpenAI from 'openai'; import * as http from 'http'; import * as https from 'https'; import * as net from 'net'; +import { createRequire } from 'node:module'; +import { Readable } from 'node:stream'; +import type * as Undici from 'undici'; import type { NormalizedGenerateChatOptions } from '../BaseProvider.js'; import { OPENAI_TRANSPORT_SELECTOR_KEYS } from './openaiModelPolicy.js'; import { createReaderBasedStreamFetch } from './openaiStreamFetchSafety.js'; +// Bun's 'undici' shim does not apply dispatcher headersTimeout; load the +// installed package's public entry instead of the shim. +const undiciRequire = createRequire( + createRequire(import.meta.url).resolve('undici/package.json'), +); +const { request: undiciRequest } = undiciRequire('./') as typeof Undici; + /** * Create HTTP/HTTPS agents with socket configuration for local AI servers * Returns undefined if no socket settings are configured @@ -163,6 +173,52 @@ export function resolveRuntimeKey( return 'openai.runtime.unscoped'; } +function fetchWithHeadersTimeout(headersTimeoutMs: number): typeof fetch { + if (!Number.isSafeInteger(headersTimeoutMs) || headersTimeoutMs <= 0) { + throw new Error('openai-headers-timeout-ms must be a positive integer'); + } + return async (input, init) => { + if (input instanceof Request) { + throw new Error('Scoped OpenAI transport requires a URL'); + } + const body = init?.body; + if ( + body !== undefined && + body !== null && + typeof body !== 'string' && + !(body instanceof ReadableStream) + ) { + throw new Error('Scoped OpenAI transport requires a JSON body'); + } + const result = await undiciRequest(String(input), { + method: init?.method ?? 'GET', + headers: Object.fromEntries(new Headers(init?.headers)), + body: + body instanceof ReadableStream + ? Readable.fromWeb( + body as unknown as Parameters[0], + ) + : body, + signal: init?.signal ?? undefined, + headersTimeout: headersTimeoutMs, + }); + const headers = new Headers(); + for (const [key, value] of Object.entries(result.headers)) { + if (Array.isArray(value)) { + for (const item of value) headers.append(key, item); + } else if (value !== undefined) { + headers.append(key, String(value)); + } + } + return new Response( + result.statusCode === 204 + ? null + : (Readable.toWeb(result.body) as ReadableStream), + { status: result.statusCode, headers }, + ); + }; +} + /** * @plan:PLAN-20251023-STATELESS-HARDENING.P09 * @requirement:REQ-SP4-002 @@ -173,13 +229,18 @@ export function instantiateClient( baseURL?: string, agents?: { httpAgent: http.Agent; httpsAgent: https.Agent }, headers?: Record, + transport?: { headersTimeoutMs?: number; fetch?: typeof fetch }, ): OpenAI { const clientOptions: Record = { apiKey: authToken || '', maxRetries: 0, }; - clientOptions.fetch = createReaderBasedStreamFetch(); + const scopedFetch = + transport?.headersTimeoutMs === undefined + ? transport?.fetch + : fetchWithHeadersTimeout(transport.headersTimeoutMs); + clientOptions.fetch = createReaderBasedStreamFetch(scopedFetch); if (headers && Object.keys(headers).length > 0) { // Ensure headers like User-Agent are applied even if the SDK call-site diff --git a/packages/providers/src/openai/OpenAIProvider.ts b/packages/providers/src/openai/OpenAIProvider.ts index 877f565947..b4584e0da1 100644 --- a/packages/providers/src/openai/OpenAIProvider.ts +++ b/packages/providers/src/openai/OpenAIProvider.ts @@ -280,7 +280,11 @@ export class OpenAIProvider extends BaseProvider implements IProvider { // based on User-Agent, which must be sent as a real HTTP header. const headers = mergeInvocationHeaders(options); - return instantiateClient(authToken, baseURL, agents, headers); + return instantiateClient(authToken, baseURL, agents, headers, { + headersTimeoutMs: agentSettings['openai-headers-timeout-ms'] as + | number + | undefined, + }); } /** diff --git a/packages/test-utils/src/test-rig-setup.ts b/packages/test-utils/src/test-rig-setup.ts index 5b716655e0..721cc93977 100644 --- a/packages/test-utils/src/test-rig-setup.ts +++ b/packages/test-utils/src/test-rig-setup.ts @@ -230,6 +230,7 @@ function collectEphemeralEntries(): Array<[string, unknown]> { if (env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true') { entries.push(['stream-first-response-timeout-ms', 600_000]); + entries.push(['openai-headers-timeout-ms', 900_000]); } return entries; diff --git a/packages/test-utils/src/test-rig.test.ts b/packages/test-utils/src/test-rig.test.ts index e1d5cb4363..f97f2aac51 100644 --- a/packages/test-utils/src/test-rig.test.ts +++ b/packages/test-utils/src/test-rig.test.ts @@ -84,6 +84,7 @@ describe('TestRig setup and cleanup behavior', () => { 'context-limit': 32768, maxOutputTokens: 8192, 'stream-first-response-timeout-ms': 600_000, + 'openai-headers-timeout-ms': 900_000, }, }); }); @@ -106,6 +107,9 @@ describe('TestRig setup and cleanup behavior', () => { expect(profile.ephemeralSettings).not.toHaveProperty( 'stream-first-response-timeout-ms', ); + expect(profile.ephemeralSettings).not.toHaveProperty( + 'openai-headers-timeout-ms', + ); }); it('loads the generated local pilot profile in the real CLI without a global profile', async () => { @@ -180,6 +184,56 @@ describe('TestRig setup and cleanup behavior', () => { } }, 30_000); + it('applies the pilot transport headers deadline in the real CLI', async () => { + const root = createRoot(); + setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); + setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'local-test-model'); + setEnv('OPENAI_API_KEY', 'local-test-only'); + + const server = createServer((_request, response) => { + setTimeout(() => { + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end( + 'data: {"id":"delayed","choices":[{"index":0,"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', + ); + }, 1_500); + }); + server.listen(0, '127.0.0.1'); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Local test server has no TCP port'); + } + setEnv('OPENAI_BASE_URL', `http://127.0.0.1:${address.port}/v1`); + const rig = new TestRig(); + rig.setup('local profile headers deadline'); + const profilePath = join( + requireTestDir(rig.testDir), + '.llxprt', + 'profiles', + 'local-model-pilot.json', + ); + const profile = JSON.parse(readFileSync(profilePath, 'utf8')); + profile.ephemeralSettings['openai-headers-timeout-ms'] = 100; + writeFileSync(profilePath, JSON.stringify(profile)); + await expect( + rig.run({ args: 'Respond with OK', timeoutMs: 12_000 }), + ).rejects.toThrow(/Request timed out/); + + profile.ephemeralSettings['openai-headers-timeout-ms'] = 4_000; + writeFileSync(profilePath, JSON.stringify(profile)); + expect( + await rig.run({ args: 'Respond with OK', timeoutMs: 12_000 }), + ).toContain('OK'); + } finally { + server.closeAllConnections(); + server.close(); + } + }, 30_000); + it('applies the generated profile first-response watchdog in the real CLI', async () => { const root = createRoot(); setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); From 522090b6b544d7c3c4c003beda24a070c04d283a Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 01:23:12 -0300 Subject: [PATCH 07/21] Allow hosted Docker pilot to finish its first response (Refs #3764) --- dev-docs/integration-tests.md | 2 +- packages/test-utils/src/test-rig-setup.ts | 2 +- packages/test-utils/src/test-rig.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index 1c6122519e..97d0fba21a 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -220,7 +220,7 @@ without discarding file changes. The runtime archive is pinned to 0.31.1; its CUDA and Vulkan libraries are excluded during extraction, and the downloaded archive is removed afterward. Ollama is limited to one concurrent context. The manual pilot configures both Ollama and LLxprt with a 32,768-token context and -reserves 8,192 tokens for model output. The pilot profile allows 600,000 ms +reserves 8,192 tokens for model output. The pilot profile allows 750,000 ms for the first model response and sets `openai-headers-timeout-ms` to 900,000 ms on its OpenAI SDK transport. The latter exceeds Undici's ordinary 300,000 ms response-headers limit without changing the dispatcher for other requests. On diff --git a/packages/test-utils/src/test-rig-setup.ts b/packages/test-utils/src/test-rig-setup.ts index 721cc93977..867945a6af 100644 --- a/packages/test-utils/src/test-rig-setup.ts +++ b/packages/test-utils/src/test-rig-setup.ts @@ -229,7 +229,7 @@ function collectEphemeralEntries(): Array<[string, unknown]> { } if (env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true') { - entries.push(['stream-first-response-timeout-ms', 600_000]); + entries.push(['stream-first-response-timeout-ms', 750_000]); entries.push(['openai-headers-timeout-ms', 900_000]); } diff --git a/packages/test-utils/src/test-rig.test.ts b/packages/test-utils/src/test-rig.test.ts index f97f2aac51..501abaf865 100644 --- a/packages/test-utils/src/test-rig.test.ts +++ b/packages/test-utils/src/test-rig.test.ts @@ -83,7 +83,7 @@ describe('TestRig setup and cleanup behavior', () => { ephemeralSettings: { 'context-limit': 32768, maxOutputTokens: 8192, - 'stream-first-response-timeout-ms': 600_000, + 'stream-first-response-timeout-ms': 750_000, 'openai-headers-timeout-ms': 900_000, }, }); From 0f884b9f9f612d5a3d3bfcfba5665a9970fd0327 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 02:40:31 -0300 Subject: [PATCH 08/21] Exercise full selected E2E suite in manual Gemma pilot (Refs #3764) --- .github/workflows/e2e.yml | 14 +++++--- dev-docs/integration-tests.md | 43 +++++++++++++++-------- scripts/tests/e2e-local-model.test.ts | 50 ++++++++++++++++++++++----- 3 files changed, 81 insertions(+), 26 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 08e59760f4..e753fe30c3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -456,14 +456,20 @@ jobs: set -euo pipefail if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then npm run test:integration:sandbox:docker -- \ + --exclude="**/todo-continuation.e2e.test.ts" \ + --exclude="**/run_shell_command.test.ts" + npm run test:integration:sandbox:docker -- \ + --exclude="**/todo-continuation.e2e.test.ts" \ integration-tests/run_shell_command.test.ts \ - integration-tests/replace.test.ts \ - --testNamePattern="should be able to run a shell command$|should be able to replace content in a file$" + --testNamePattern="should be able to run a shell command|should be able to run a shell command via stdin|should run a platform-specific file listing command" else npm run test:integration:sandbox:none -- \ + --exclude="**/todo-continuation.e2e.test.ts" \ + --exclude="**/run_shell_command.test.ts" + npm run test:integration:sandbox:none -- \ + --exclude="**/todo-continuation.e2e.test.ts" \ integration-tests/run_shell_command.test.ts \ - integration-tests/replace.test.ts \ - --testNamePattern="should be able to run a shell command$|should be able to replace content in a file$" + --testNamePattern="should be able to run a shell command|should be able to run a shell command via stdin|should run a platform-specific file listing command" fi - name: 'Check E2E real-model budget (issue #2278)' diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index 97d0fba21a..088c88cb98 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -194,13 +194,15 @@ It verifies the downloaded Linux runtime archive's SHA-256 digest (`d297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4`) before extraction, then checks the runtime version and model digest before testing. The OpenAI provider uses Ollama's `/v1` compatibility API and a local placeholder -key, with no remote inference service or provider secret. Only the shell and -replace canaries run: both are real `TestRig` tests with tool-choice assertions, -the shell test checks command and tool success, and the replace test checks the -file's exact new contents. The shell step exits on a failed suite. The pilot -retains the four-request ledger budget check for two distinct real-model tests; -the ledger reports retries separately without counting their actual model -requests. +key, with no remote inference service or provider secret. Each sandbox leg now +runs the credentialed Linux job's first full integration invocation, excluding +only `todo-continuation.e2e.test.ts` and `run_shell_command.test.ts` (31 of 33 +selected integration test files), followed by the same three named shell cases +from the credentialed job. The shell step exits on a failed suite. The existing +real-model budget guard remains enabled and the ledger reports retries +separately without counting their actual model requests. This expanded local +pilot does not validate hosted CPU performance or replace required credentialed +E2E coverage. On Linux, the Docker test leg sets `SANDBOX_FLAGS='--network host'` so the sandboxed CLI reaches the host's loopback Ollama endpoint. This networking choice @@ -233,11 +235,11 @@ replace invocation has a 1,200,000 ms `TestRig` deadline, its Bun file has a deadlines apply only when `LLXPRT_LOCAL_MODEL_PILOT=true`; normal integration-test deadlines are unchanged. -To reproduce the two real-model canaries without using an existing Ollama daemon, -run these commands from the repository root. Choose a free port if 12644 is in -use and change both URLs accordingly. Install Ollama 0.31.1 first and ensure the -`gemma4:e2b-it-qat` digest above matches. Store all evidence in the repository's -ignored `tmp/` tree: +To reproduce the expanded real-model test selection without using an existing +Ollama daemon, run these commands from the repository root. Choose a free port +if 12644 is in use and change both URLs accordingly. Install Ollama 0.31.1 first +and ensure the `gemma4:e2b-it-qat` digest above matches. Store all evidence in +the repository's ignored `tmp/` tree: ```bash mkdir -p tmp/verify3764/models @@ -258,8 +260,21 @@ CI=true KEEP_OUTPUT=true VERBOSE=true \ LLXPRT_E2E_MODEL_LEDGER="$PWD/tmp/verify3764/ledger.jsonl" \ GIT_CEILING_DIRECTORIES="$PWD/.integration-tests" \ bun scripts/run_bun_tests.ts --root integration-tests \ - integration-tests/run_shell_command.test.ts integration-tests/replace.test.ts \ - --testNamePattern='should be able to run a shell command$|should be able to replace content in a file' + --exclude='**/todo-continuation.e2e.test.ts' \ + --exclude='**/run_shell_command.test.ts' +CI=true KEEP_OUTPUT=true VERBOSE=true \ + LLXPRT_DEFAULT_PROVIDER=openai LLXPRT_DEFAULT_MODEL=gemma4:e2b-it-qat \ + OPENAI_API_KEY=ollama-local-only \ + OPENAI_BASE_URL=http://127.0.0.1:12644/v1 LLXPRT_AUTH_TYPE=provider \ + LLXPRT_TEST_PROFILE=local-gemma4-pilot LLXPRT_CONTEXT_LIMIT=32768 \ + LLXPRT_MAX_OUTPUT_TOKENS=8192 LLXPRT_LOCAL_MODEL_PILOT=true \ + LLXPRT_FORCE_FILE_STORAGE=true \ + LLXPRT_E2E_MODEL_LEDGER="$PWD/tmp/verify3764/ledger.jsonl" \ + GIT_CEILING_DIRECTORIES="$PWD/.integration-tests" \ + bun scripts/run_bun_tests.ts --root integration-tests \ + --exclude='**/todo-continuation.e2e.test.ts' \ + integration-tests/run_shell_command.test.ts \ + --testNamePattern='should be able to run a shell command|should be able to run a shell command via stdin|should run a platform-specific file listing command' bun scripts/check-e2e-model-budget.ts --ledger "$PWD/tmp/verify3764/ledger.jsonl" curl -fsS http://127.0.0.1:12644/api/ps | jq '.models[] | {name, size, context_length}' ``` diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 4622859667..f5458422c5 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -155,9 +155,43 @@ describe('optional local-model E2E pilot', () => { ); }); - it('runs only real shell and replace TestRig canaries without provider secrets', () => { + it('matches the required Linux E2E test invocations in both sandbox legs without provider secrets', () => { const job = workflowJob(pilot, 'local_model_canaries'); const run = step('Run local-model canaries'); + const command = asString(run.run); + const requiredRun = asString( + (workflowJob(required, 'e2e_linux').steps ?? []).find( + (candidate) => candidate.name === 'Run E2E tests', + )?.run, + ); + const normalize = (script: string): string => + script.replace(/\\\s*/g, ' ').replace(/\s+/g, ' '); + const fullArgs = + '--exclude="**/todo-continuation.e2e.test.ts" --exclude="**/run_shell_command.test.ts"'; + const shellArgs = + 'integration-tests/run_shell_command.test.ts --testNamePattern="should be able to run a shell command|should be able to run a shell command via stdin|should run a platform-specific file listing command"'; + const normalizedRequired = normalize(requiredRun); + const normalizedCommand = normalize(command); + for (const sandbox of ['sandbox:none', 'sandbox:docker']) { + const fullInvocation = `npm run test:integration:${sandbox} -- ${fullArgs}`; + const firstInvocationPattern = new RegExp( + `npm run test:integration:${sandbox} -- (.*?) npm run test:integration:${sandbox} --`, + ); + const requiredFirstInvocation = normalizedRequired.match( + firstInvocationPattern, + ); + const pilotFirstInvocation = normalizedCommand.match( + firstInvocationPattern, + ); + expect(requiredFirstInvocation?.[1]).toBe(fullArgs); + expect(pilotFirstInvocation?.[1]).toBe(fullArgs); + expect(normalizedRequired).toContain(fullInvocation); + expect(normalizedCommand).toContain(fullInvocation); + const shellInvocation = `npm run test:integration:${sandbox} -- --exclude="**/todo-continuation.e2e.test.ts" ${shellArgs}`; + expect(normalizedRequired).toContain(shellInvocation); + expect(normalizedCommand).toContain(shellInvocation); + } + expect(command.match(/--exclude=/g)).toHaveLength(6); const env = asOptionalRecord(run.env); expect(env?.OPENAI_BASE_URL).toBe('http://127.0.0.1:12644/v1'); expect(env?.LLXPRT_DEFAULT_PROVIDER).toBe('openai'); @@ -180,15 +214,15 @@ describe('optional local-model E2E pilot', () => { expect(asString(run.run)).toContain( 'integration-tests/run_shell_command.test.ts', ); - expect(asString(run.run)).toContain('integration-tests/replace.test.ts'); - expect(asString(run.run)).toContain( - 'should be able to replace content in a file', + expect(JSON.stringify(job)).not.toMatch(/\bsecrets(?:\.|\[)/); + const budget = step('Check E2E real-model budget (issue #2278)'); + expect(budget.if).toBe('success()'); + expect(asString(budget.run).trim()).toBe( + 'bun scripts/check-e2e-model-budget.ts --ledger "$LLXPRT_E2E_MODEL_LEDGER"', ); - expect(asString(run.run)).toContain( - 'should be able to run a shell command', + expect(asOptionalRecord(budget.env)?.LLXPRT_E2E_MODEL_LEDGER).toBe( + '${{ runner.temp }}/e2e-model-ledger.jsonl', ); - expect(asString(run.run)).not.toContain('--exclude='); - expect(JSON.stringify(job)).not.toMatch(/\bsecrets(?:\.|\[)/); }); it('uses host networking only for the Docker leg, and retains diagnostics on failure', () => { From f027b8daea2aa3101ea3601cca3f662656afdc05 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 04:10:05 -0300 Subject: [PATCH 09/21] Constrain Intel Docker pilot to AVX2 CPU backend (Refs #3764) --- .github/workflows/e2e.yml | 54 +++++++++++++++++++++++++++ scripts/tests/e2e-local-model.test.ts | 35 +++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e753fe30c3..8ae3aedb02 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -411,6 +411,30 @@ jobs: echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" "$RUNNER_TEMP/ollama/bin/ollama" --version + - name: 'Select pilot CPU backend on Intel Docker' + if: "matrix.sandbox == 'sandbox:docker'" + shell: 'bash' + run: | + set -euo pipefail + if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' /proc/cpuinfo; then + echo 'Docker pilot host is not Intel; leaving Ollama CPU variants untouched' + exit 0 + fi + libdir="$RUNNER_TEMP/ollama/lib/ollama" + test -f "$libdir/libggml-cpu-haswell.so" + shopt -s nullglob + variants=("$libdir"/libggml-cpu-*.so) + (( ${#variants[@]} > 1 )) || { echo 'Expected multiple CPU variants in pinned Ollama runtime' >&2; exit 1; } + mkdir -p "$RUNNER_TEMP/ollama-disabled-cpu-backends" + for library in "${variants[@]}"; do + if [[ "$library" != "$libdir/libggml-cpu-haswell.so" ]]; then + mv "$library" "$RUNNER_TEMP/ollama-disabled-cpu-backends/" + fi + done + echo 'Intel Docker pilot: CPU variants visible to llama-server:' + printf '%s\n' "$libdir"/libggml-cpu-*.so + test "$(find "$libdir" -maxdepth 1 -name 'libggml-cpu-*.so' | wc -l)" -eq 1 + - name: 'Start local Gemma 4 E2B model' shell: 'bash' env: @@ -434,6 +458,36 @@ jobs: jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .digest == "07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c")' df -h "$RUNNER_TEMP" + - name: 'Verify Intel Docker CPU backend warm-up' + if: "matrix.sandbox == 'sandbox:docker'" + shell: 'bash' + env: + OLLAMA_HOST: '127.0.0.1:12644' + run: | + set -euo pipefail + if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' /proc/cpuinfo; then + echo 'Docker pilot host is not Intel; CPU backend A/B not exercised' + exit 0 + fi + curl --fail --show-error --silent --max-time 300 \ + "http://127.0.0.1:12644/api/generate" \ + -H 'Content-Type: application/json' \ + -d '{"model":"gemma4:e2b-it-qat","prompt":"Hello","stream":false,"options":{"num_ctx":32768,"num_predict":1}}' \ + | jq -e '.done == true and .error == null' + info="$(grep 'system_info:.*CPU :' "$RUNNER_TEMP/ollama-server.log" | tail -1)" + echo "$info" + [[ "$info" == *'AVX2 = 1'* && "$info" == *'AVX512 = 0'* && "$info" == *'AMX_INT8 = 0'* ]] || { + echo 'Pinned llama-server did not select the AVX2-only CPU backend' >&2 + exit 1 + } + pid="$(pgrep -f "^$RUNNER_TEMP/ollama/lib/ollama/llama-server --model " | head -1)" + backend="$(grep -Eo '/[^ ]+/libggml-cpu-[^ ]+\.so' "/proc/$pid/maps" | sort -u)" + echo "Loaded CPU backend: $backend" + [[ "$backend" == "$RUNNER_TEMP/ollama/lib/ollama/libggml-cpu-haswell.so" ]] || { + echo 'Unexpected mapped llama-server CPU library' >&2 + exit 1 + } + - name: 'Run local-model canaries' env: OPENAI_API_KEY: 'ollama-local-only' diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index f5458422c5..8689d85c6e 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -138,6 +138,41 @@ describe('optional local-model E2E pilot', () => { expect(install).toContain('rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst"'); }); + it('restricts the CPU-backend A/B to the Intel Docker pilot and proves AVX2 without AVX-512 or AMX from the model-load log', () => { + const select = step('Select pilot CPU backend on Intel Docker'); + expect(select.if).toBe("matrix.sandbox == 'sandbox:docker'"); + const script = asString(select.run); + expect(script).toContain('set -euo pipefail'); + expect(script).toContain('/proc/cpuinfo'); + expect(script).toContain('GenuineIntel'); + expect(script).toContain('libggml-cpu-haswell.so'); + expect(script).toContain('libggml-cpu-*.so'); + expect(script).not.toContain('OLLAMA_LLM_LIBRARY'); + expect(steps.indexOf(select)).toBeGreaterThan( + steps.indexOf(step('Install Ollama CPU runtime')), + ); + expect(steps.indexOf(select)).toBeLessThan( + steps.indexOf(step('Start local Gemma 4 E2B model')), + ); + const verify = step('Verify Intel Docker CPU backend warm-up'); + expect(verify.if).toBe("matrix.sandbox == 'sandbox:docker'"); + expect(asString(verify.run)).toContain( + 'http://127.0.0.1:12644/api/generate', + ); + expect(asString(verify.run)).toContain('ollama-server.log'); + expect(asString(verify.run)).toContain('/proc/$pid/maps'); + expect(asString(verify.run)).toContain('libggml-cpu-haswell.so'); + expect(asString(verify.run)).toContain('AVX2 = 1'); + expect(asString(verify.run)).toContain('AVX512 = 0'); + expect(asString(verify.run)).toContain('AMX_INT8 = 0'); + expect(steps.indexOf(verify)).toBeGreaterThan( + steps.indexOf(step('Start local Gemma 4 E2B model')), + ); + expect(steps.indexOf(verify)).toBeLessThan( + steps.indexOf(step('Run local-model canaries')), + ); + }); + it('checks model version and digest before running either canary', () => { const server = step('Start local Gemma 4 E2B model'); const start = asString(server.run); From 731a778adc5ab2ea3d4e63d4691857aa391b3383 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 05:14:48 -0300 Subject: [PATCH 10/21] Keep pilot SDK alive past first response (Refs #3764) --- .../src/openai/OpenAIClientFactory.ts | 15 +++++- .../providers/src/openai/OpenAIProvider.ts | 3 ++ packages/test-utils/src/test-rig-setup.ts | 1 + packages/test-utils/src/test-rig.test.ts | 54 +++++++++++++++++++ 4 files changed, 72 insertions(+), 1 deletion(-) diff --git a/packages/providers/src/openai/OpenAIClientFactory.ts b/packages/providers/src/openai/OpenAIClientFactory.ts index d0c820c11c..bd665c2a4f 100644 --- a/packages/providers/src/openai/OpenAIClientFactory.ts +++ b/packages/providers/src/openai/OpenAIClientFactory.ts @@ -229,12 +229,25 @@ export function instantiateClient( baseURL?: string, agents?: { httpAgent: http.Agent; httpsAgent: https.Agent }, headers?: Record, - transport?: { headersTimeoutMs?: number; fetch?: typeof fetch }, + transport?: { + headersTimeoutMs?: number; + requestTimeoutMs?: number; + fetch?: typeof fetch; + }, ): OpenAI { const clientOptions: Record = { apiKey: authToken || '', maxRetries: 0, }; + if (transport?.requestTimeoutMs !== undefined) { + if ( + !Number.isSafeInteger(transport.requestTimeoutMs) || + transport.requestTimeoutMs <= 0 + ) { + throw new Error('openai-request-timeout-ms must be a positive integer'); + } + clientOptions.timeout = transport.requestTimeoutMs; + } const scopedFetch = transport?.headersTimeoutMs === undefined diff --git a/packages/providers/src/openai/OpenAIProvider.ts b/packages/providers/src/openai/OpenAIProvider.ts index b4584e0da1..e8ac98cd0a 100644 --- a/packages/providers/src/openai/OpenAIProvider.ts +++ b/packages/providers/src/openai/OpenAIProvider.ts @@ -284,6 +284,9 @@ export class OpenAIProvider extends BaseProvider implements IProvider { headersTimeoutMs: agentSettings['openai-headers-timeout-ms'] as | number | undefined, + requestTimeoutMs: agentSettings['openai-request-timeout-ms'] as + | number + | undefined, }); } diff --git a/packages/test-utils/src/test-rig-setup.ts b/packages/test-utils/src/test-rig-setup.ts index 867945a6af..cd27626bf5 100644 --- a/packages/test-utils/src/test-rig-setup.ts +++ b/packages/test-utils/src/test-rig-setup.ts @@ -230,6 +230,7 @@ function collectEphemeralEntries(): Array<[string, unknown]> { if (env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true') { entries.push(['stream-first-response-timeout-ms', 750_000]); + entries.push(['openai-request-timeout-ms', 850_000]); entries.push(['openai-headers-timeout-ms', 900_000]); } diff --git a/packages/test-utils/src/test-rig.test.ts b/packages/test-utils/src/test-rig.test.ts index 501abaf865..991f95df49 100644 --- a/packages/test-utils/src/test-rig.test.ts +++ b/packages/test-utils/src/test-rig.test.ts @@ -84,6 +84,7 @@ describe('TestRig setup and cleanup behavior', () => { 'context-limit': 32768, maxOutputTokens: 8192, 'stream-first-response-timeout-ms': 750_000, + 'openai-request-timeout-ms': 850_000, 'openai-headers-timeout-ms': 900_000, }, }); @@ -110,6 +111,9 @@ describe('TestRig setup and cleanup behavior', () => { expect(profile.ephemeralSettings).not.toHaveProperty( 'openai-headers-timeout-ms', ); + expect(profile.ephemeralSettings).not.toHaveProperty( + 'openai-request-timeout-ms', + ); }); it('loads the generated local pilot profile in the real CLI without a global profile', async () => { @@ -234,6 +238,56 @@ describe('TestRig setup and cleanup behavior', () => { } }, 30_000); + it('applies the pilot SDK request deadline in the real CLI', async () => { + const root = createRoot(); + setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); + setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'local-test-model'); + setEnv('OPENAI_API_KEY', 'local-test-only'); + + const server = createServer((_request, response) => { + setTimeout(() => { + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end( + 'data: {"id":"delayed","choices":[{"index":0,"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', + ); + }, 1_500); + }); + server.listen(0, '127.0.0.1'); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Local test server has no TCP port'); + } + setEnv('OPENAI_BASE_URL', `http://127.0.0.1:${address.port}/v1`); + const rig = new TestRig(); + rig.setup('local profile SDK deadline'); + const profilePath = join( + requireTestDir(rig.testDir), + '.llxprt', + 'profiles', + 'local-model-pilot.json', + ); + const profile = JSON.parse(readFileSync(profilePath, 'utf8')); + profile.ephemeralSettings['openai-request-timeout-ms'] = 100; + writeFileSync(profilePath, JSON.stringify(profile)); + await expect( + rig.run({ args: 'Respond with OK', timeoutMs: 12_000 }), + ).rejects.toThrow(/Request timed out/); + + profile.ephemeralSettings['openai-request-timeout-ms'] = 4_000; + writeFileSync(profilePath, JSON.stringify(profile)); + expect( + await rig.run({ args: 'Respond with OK', timeoutMs: 12_000 }), + ).toContain('OK'); + } finally { + server.closeAllConnections(); + server.close(); + } + }, 30_000); + it('applies the generated profile first-response watchdog in the real CLI', async () => { const root = createRoot(); setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); From 3965d14c786deb0e47a3aa84584cf88c820ceb39 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 08:05:29 -0300 Subject: [PATCH 11/21] Verify the live Intel CPU backend instead of optional log flags (Refs #3764) --- .github/workflows/e2e.yml | 32 +++++--- scripts/tests/e2e-local-model.test.ts | 110 ++++++++++++++++++++++++-- 2 files changed, 123 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8ae3aedb02..e28a0be983 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -465,7 +465,8 @@ jobs: OLLAMA_HOST: '127.0.0.1:12644' run: | set -euo pipefail - if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' /proc/cpuinfo; then + proc_root="${LLXPRT_PROC_ROOT:-/proc}" + if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' "$proc_root/cpuinfo"; then echo 'Docker pilot host is not Intel; CPU backend A/B not exercised' exit 0 fi @@ -474,17 +475,24 @@ jobs: -H 'Content-Type: application/json' \ -d '{"model":"gemma4:e2b-it-qat","prompt":"Hello","stream":false,"options":{"num_ctx":32768,"num_predict":1}}' \ | jq -e '.done == true and .error == null' - info="$(grep 'system_info:.*CPU :' "$RUNNER_TEMP/ollama-server.log" | tail -1)" - echo "$info" - [[ "$info" == *'AVX2 = 1'* && "$info" == *'AVX512 = 0'* && "$info" == *'AMX_INT8 = 0'* ]] || { - echo 'Pinned llama-server did not select the AVX2-only CPU backend' >&2 - exit 1 - } - pid="$(pgrep -f "^$RUNNER_TEMP/ollama/lib/ollama/llama-server --model " | head -1)" - backend="$(grep -Eo '/[^ ]+/libggml-cpu-[^ ]+\.so' "/proc/$pid/maps" | sort -u)" - echo "Loaded CPU backend: $backend" - [[ "$backend" == "$RUNNER_TEMP/ollama/lib/ollama/libggml-cpu-haswell.so" ]] || { - echo 'Unexpected mapped llama-server CPU library' >&2 + grep 'system_info:.*CPU :' "$RUNNER_TEMP/ollama-server.log" | tail -1 || true + process="$RUNNER_TEMP/ollama/lib/ollama/llama-server" + pid="$(pgrep -f "^$process --model " | head -1)" || true + [[ -n "$pid" ]] || { echo 'Live llama-server process not found' >&2; exit 1; } + maps="$proc_root/$pid/maps" + [[ -r "$maps" && -s "$maps" ]] || { echo 'Live llama-server maps are unavailable or empty' >&2; exit 1; } + expected="$RUNNER_TEMP/ollama/lib/ollama/libggml-cpu-haswell.so" + backends="$(grep -Eo '/[^ ]+/libggml-cpu-[^ ]+\.so( \(deleted\))?' "$maps" | sort -u || true)" + echo "Loaded CPU backends: ${backends:-none}" + [[ -n "$backends" ]] || { echo 'No mapped llama-server CPU backend' >&2; exit 1; } + while IFS= read -r backend; do + [[ "$backend" == "$expected" ]] || { + echo "Unexpected mapped llama-server CPU library: $backend" >&2 + exit 1 + } + done <<< "$backends" + grep -Fxq "$expected" <<< "$backends" || { + echo 'Expected haswell CPU library is not mapped' >&2 exit 1 } diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 8689d85c6e..23c1a3028e 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -5,7 +5,8 @@ */ import { describe, expect, it } from 'bun:test'; -import { readFileSync } from 'node:fs'; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; import { resolve } from 'node:path'; import { asOptionalRecord, @@ -22,6 +23,64 @@ const requiredText = readFileSync( ); const required = parseWorkflowYaml(requiredText); const pilot = required; + +type BackendVerifierResult = { + stdout: string; + stderr: string; + status: number; +}; + +let fixtureCounter = 0; +function runBackendVerifier( + maps: string, + livePid = true, +): BackendVerifierResult { + const fixture = resolve(root, `tmp/verify3764/behavior-${fixtureCounter++}`); + const runnerTemp = resolve(fixture, 'runner-temp'); + const procRoot = resolve(fixture, 'proc'); + const bin = resolve(fixture, 'bin'); + const processDir = resolve(procRoot, '4242'); + const ollamaDir = resolve(runnerTemp, 'ollama/lib/ollama'); + mkdirSync(bin, { recursive: true }); + mkdirSync(processDir, { recursive: true }); + mkdirSync(ollamaDir, { recursive: true }); + writeFileSync(resolve(procRoot, 'cpuinfo'), 'vendor_id : GenuineIntel\n'); + writeFileSync( + resolve(processDir, 'maps'), + maps.replaceAll('/runner-temp', runnerTemp), + ); + writeFileSync( + resolve(runnerTemp, 'ollama-server.log'), + 'cmn common_param: system_info: n_threads = 2 (n_threads_batch = 2) / 4 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | REPACK = 1 | \n', + ); + writeFileSync(resolve(ollamaDir, 'llama-server'), 'fixture process'); + const executable = (name: string, body: string): void => { + const path = resolve(bin, name); + writeFileSync(path, `#!/bin/bash\n${body}\n`); + Bun.spawnSync(['chmod', '+x', path]); + }; + executable('curl', `printf '%s\\n' '{"done":true,"error":null}'`); + executable('jq', 'cat >/dev/null; exit 0'); + executable('pgrep', livePid ? `printf '%s\\n' 4242` : 'exit 1'); + const script = asString(step('Verify Intel Docker CPU backend warm-up').run); + const result = spawnSync('bash', ['-c', script], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + RUNNER_TEMP: runnerTemp, + LLXPRT_PROC_ROOT: procRoot, + OLLAMA_HOST: '127.0.0.1:12644', + }, + }); + rmSync(fixture, { recursive: true, force: true }); + return { + stdout: result.stdout, + stderr: result.stderr, + status: result.status ?? 1, + }; +} + const steps = workflowJob(pilot, 'local_model_canaries').steps ?? []; function step(name: string) { @@ -138,12 +197,12 @@ describe('optional local-model E2E pilot', () => { expect(install).toContain('rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst"'); }); - it('restricts the CPU-backend A/B to the Intel Docker pilot and proves AVX2 without AVX-512 or AMX from the model-load log', () => { + it('restricts the CPU-backend A/B to the Intel Docker pilot and structurally checks live backend verification', () => { const select = step('Select pilot CPU backend on Intel Docker'); expect(select.if).toBe("matrix.sandbox == 'sandbox:docker'"); const script = asString(select.run); expect(script).toContain('set -euo pipefail'); - expect(script).toContain('/proc/cpuinfo'); + expect(script).toContain('grep -qm1'); expect(script).toContain('GenuineIntel'); expect(script).toContain('libggml-cpu-haswell.so'); expect(script).toContain('libggml-cpu-*.so'); @@ -160,11 +219,9 @@ describe('optional local-model E2E pilot', () => { 'http://127.0.0.1:12644/api/generate', ); expect(asString(verify.run)).toContain('ollama-server.log'); - expect(asString(verify.run)).toContain('/proc/$pid/maps'); + expect(asString(verify.run)).toContain('LLXPRT_PROC_ROOT:-/proc'); expect(asString(verify.run)).toContain('libggml-cpu-haswell.so'); - expect(asString(verify.run)).toContain('AVX2 = 1'); - expect(asString(verify.run)).toContain('AVX512 = 0'); - expect(asString(verify.run)).toContain('AMX_INT8 = 0'); + expect(asString(verify.run)).toContain('pgrep -f'); expect(steps.indexOf(verify)).toBeGreaterThan( steps.indexOf(step('Start local Gemma 4 E2B model')), ); @@ -289,3 +346,42 @@ describe('optional local-model E2E pilot', () => { expect(asOptionalRecord(upload.with)?.['include-hidden-files']).toBe(true); }); }); + +describe('workflow backend verification behavior', () => { + const haswell = '/runner-temp/ollama/lib/ollama/libggml-cpu-haswell.so'; + const mapping = (path: string): string => + `7f000000-7f100000 r-xp 00000000 08:01 42 ${path}\n`; + + it('accepts the exact sole haswell mapping when the incident log omits AVX512 and AMX fields', () => { + const result = runBackendVerifier(mapping(haswell)); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('Loaded CPU backends:'); + }); + + it('fails when the live process is absent even though the log reports AVX2', () => { + const result = runBackendVerifier(mapping(haswell), false); + expect(result.status, `${result.stdout}\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain('Live llama-server process not found'); + }); + + it('fails specifically when readable live-process maps contain no CPU backend despite an AVX2 log', () => { + const result = runBackendVerifier(mapping('/usr/lib/libc.so.6')); + expect(result.status, `${result.stdout}\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain('No mapped llama-server CPU backend'); + expect(result.stderr).not.toContain('maps are unavailable or empty'); + }); + + it('rejects additional and alternative mapped CPU library variants', () => { + for (const maps of [ + `${mapping(haswell)}${mapping('/other/libggml-cpu-avx512.so')}`, + mapping('/runner-temp/ollama/lib/ollama/libggml-cpu-skylake.so'), + mapping(`${haswell} (deleted)`), + ]) { + const result = runBackendVerifier(maps); + expect(result.status, `${result.stdout}\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain( + 'Unexpected mapped llama-server CPU library', + ); + } + }); +}); From e705829877da67e932d61ab6bec4d738ebb2b262 Mon Sep 17 00:00:00 2001 From: llxprt <330608044+llxprt@users.noreply.github.com> Date: Sun, 27 Sep 2026 09:49:17 -0300 Subject: [PATCH 12/21] Verify Intel backend selection on both pilot legs (Refs #3764) --- .github/workflows/e2e.yml | 32 ++++++++++++++++++++------- scripts/tests/e2e-local-model.test.ts | 30 ++++++++++++++++++++----- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e28a0be983..1c945b3507 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -411,13 +411,12 @@ jobs: echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" "$RUNNER_TEMP/ollama/bin/ollama" --version - - name: 'Select pilot CPU backend on Intel Docker' - if: "matrix.sandbox == 'sandbox:docker'" + - name: 'Select pilot CPU backend on Intel' shell: 'bash' run: | set -euo pipefail if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' /proc/cpuinfo; then - echo 'Docker pilot host is not Intel; leaving Ollama CPU variants untouched' + echo 'Pilot host is not Intel; leaving Ollama CPU variants untouched' exit 0 fi libdir="$RUNNER_TEMP/ollama/lib/ollama" @@ -458,8 +457,7 @@ jobs: jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .digest == "07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c")' df -h "$RUNNER_TEMP" - - name: 'Verify Intel Docker CPU backend warm-up' - if: "matrix.sandbox == 'sandbox:docker'" + - name: 'Verify Intel CPU backend warm-up' shell: 'bash' env: OLLAMA_HOST: '127.0.0.1:12644' @@ -467,7 +465,7 @@ jobs: set -euo pipefail proc_root="${LLXPRT_PROC_ROOT:-/proc}" if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' "$proc_root/cpuinfo"; then - echo 'Docker pilot host is not Intel; CPU backend A/B not exercised' + echo 'Pilot host is not Intel; CPU backend A/B not exercised' exit 0 fi curl --fail --show-error --silent --max-time 300 \ @@ -480,9 +478,27 @@ jobs: pid="$(pgrep -f "^$process --model " | head -1)" || true [[ -n "$pid" ]] || { echo 'Live llama-server process not found' >&2; exit 1; } maps="$proc_root/$pid/maps" - [[ -r "$maps" && -s "$maps" ]] || { echo 'Live llama-server maps are unavailable or empty' >&2; exit 1; } + [[ -r "$maps" ]] || { echo "Live llama-server maps are unavailable: $maps" >&2; exit 1; } + cmdline="$proc_root/$pid/cmdline" + status="$proc_root/$pid/status" + [[ -r "$cmdline" && -r "$status" ]] || { echo 'Live llama-server process evidence disappeared' >&2; exit 1; } + printf 'llama-server cmdline: ' + tr '\0' ' ' <"$cmdline" + echo + command_line="$(tr '\0' ' ' <"$cmdline")" + [[ "$command_line" == "$process --model "* ]] || { echo 'Matched PID is not the expected llama-server process' >&2; exit 1; } + ppid="$(awk '$1 == "PPid:" { print $2 }' "$status")" + [[ "$ppid" =~ ^[0-9]+$ && "$ppid" != 0 ]] || { echo 'llama-server parent PID is missing' >&2; exit 1; } + parent_cmdline="$proc_root/$ppid/cmdline" + [[ -r "$parent_cmdline" ]] || { echo 'llama-server parent process disappeared' >&2; exit 1; } + printf 'llama-server PPid: %s; parent cmdline: ' "$ppid" + tr '\0' ' ' <"$parent_cmdline" + echo expected="$RUNNER_TEMP/ollama/lib/ollama/libggml-cpu-haswell.so" - backends="$(grep -Eo '/[^ ]+/libggml-cpu-[^ ]+\.so( \(deleted\))?' "$maps" | sort -u || true)" + maps_snapshot="$RUNNER_TEMP/llama-server-$pid.maps" + cat "$maps" >"$maps_snapshot" || { echo 'llama-server maps disappeared while being read' >&2; exit 1; } + cat "$maps_snapshot" + backends="$(grep -Eo '/[^ ]+/libggml-cpu-[^ ]+\.so( \(deleted\))?' "$maps_snapshot" | sort -u || true)" echo "Loaded CPU backends: ${backends:-none}" [[ -n "$backends" ]] || { echo 'No mapped llama-server CPU backend' >&2; exit 1; } while IFS= read -r backend; do diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 23c1a3028e..ef3594ba3b 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -40,11 +40,22 @@ function runBackendVerifier( const procRoot = resolve(fixture, 'proc'); const bin = resolve(fixture, 'bin'); const processDir = resolve(procRoot, '4242'); + const parentDir = resolve(procRoot, '4200'); const ollamaDir = resolve(runnerTemp, 'ollama/lib/ollama'); mkdirSync(bin, { recursive: true }); mkdirSync(processDir, { recursive: true }); + mkdirSync(parentDir, { recursive: true }); mkdirSync(ollamaDir, { recursive: true }); writeFileSync(resolve(procRoot, 'cpuinfo'), 'vendor_id : GenuineIntel\n'); + writeFileSync( + resolve(processDir, 'cmdline'), + `${resolve(ollamaDir, 'llama-server')}\0--model\0gemma\0`, + ); + writeFileSync( + resolve(processDir, 'status'), + 'Name:\tllama-server\nPPid:\t4200\n', + ); + writeFileSync(resolve(parentDir, 'cmdline'), 'ollama\0serve\0'); writeFileSync( resolve(processDir, 'maps'), maps.replaceAll('/runner-temp', runnerTemp), @@ -62,7 +73,7 @@ function runBackendVerifier( executable('curl', `printf '%s\\n' '{"done":true,"error":null}'`); executable('jq', 'cat >/dev/null; exit 0'); executable('pgrep', livePid ? `printf '%s\\n' 4242` : 'exit 1'); - const script = asString(step('Verify Intel Docker CPU backend warm-up').run); + const script = asString(step('Verify Intel CPU backend warm-up').run); const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', env: { @@ -198,8 +209,8 @@ describe('optional local-model E2E pilot', () => { }); it('restricts the CPU-backend A/B to the Intel Docker pilot and structurally checks live backend verification', () => { - const select = step('Select pilot CPU backend on Intel Docker'); - expect(select.if).toBe("matrix.sandbox == 'sandbox:docker'"); + const select = step('Select pilot CPU backend on Intel'); + expect(select.if).toBeUndefined(); const script = asString(select.run); expect(script).toContain('set -euo pipefail'); expect(script).toContain('grep -qm1'); @@ -213,15 +224,20 @@ describe('optional local-model E2E pilot', () => { expect(steps.indexOf(select)).toBeLessThan( steps.indexOf(step('Start local Gemma 4 E2B model')), ); - const verify = step('Verify Intel Docker CPU backend warm-up'); - expect(verify.if).toBe("matrix.sandbox == 'sandbox:docker'"); + const verify = step('Verify Intel CPU backend warm-up'); + expect(verify.if).toBeUndefined(); expect(asString(verify.run)).toContain( 'http://127.0.0.1:12644/api/generate', ); expect(asString(verify.run)).toContain('ollama-server.log'); expect(asString(verify.run)).toContain('LLXPRT_PROC_ROOT:-/proc'); expect(asString(verify.run)).toContain('libggml-cpu-haswell.so'); + expect(verify.if).toBeUndefined(); expect(asString(verify.run)).toContain('pgrep -f'); + expect(asString(verify.run)).toContain('/cmdline'); + expect(asString(verify.run)).toContain('/status'); + expect(asString(verify.run)).toContain('parent cmdline'); + expect(asString(verify.run)).not.toContain('-s \"$maps'); expect(steps.indexOf(verify)).toBeGreaterThan( steps.indexOf(step('Start local Gemma 4 E2B model')), ); @@ -356,6 +372,10 @@ describe('workflow backend verification behavior', () => { const result = runBackendVerifier(mapping(haswell)); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); expect(result.stdout).toContain('Loaded CPU backends:'); + expect(result.stdout).toContain('llama-server cmdline:'); + expect(result.stdout).toContain( + 'llama-server PPid: 4200; parent cmdline: ollama serve', + ); }); it('fails when the live process is absent even though the log reports AVX2', () => { From 68a0346ad0db6e78f8653c155939ada046360ec6 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 10:31:08 -0300 Subject: [PATCH 13/21] test: cover live backend and variant selection evidence --- scripts/tests/e2e-local-model.test.ts | 238 +++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 9 deletions(-) diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index ef3594ba3b..6e3dc349e4 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -5,7 +5,13 @@ */ import { describe, expect, it } from 'bun:test'; -import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + mkdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; import { spawnSync } from 'node:child_process'; import { resolve } from 'node:path'; import { @@ -34,8 +40,14 @@ let fixtureCounter = 0; function runBackendVerifier( maps: string, livePid = true, + options: { + cmdline?: string; + mapsFifo?: boolean; + mapsDisappearAfterPgrep?: boolean; + } = {}, ): BackendVerifierResult { const fixture = resolve(root, `tmp/verify3764/behavior-${fixtureCounter++}`); + rmSync(fixture, { recursive: true, force: true }); const runnerTemp = resolve(fixture, 'runner-temp'); const procRoot = resolve(fixture, 'proc'); const bin = resolve(fixture, 'bin'); @@ -49,17 +61,33 @@ function runBackendVerifier( writeFileSync(resolve(procRoot, 'cpuinfo'), 'vendor_id : GenuineIntel\n'); writeFileSync( resolve(processDir, 'cmdline'), - `${resolve(ollamaDir, 'llama-server')}\0--model\0gemma\0`, + options.cmdline ?? + `${resolve(ollamaDir, 'llama-server')}\0--model\0gemma\0`, ); writeFileSync( resolve(processDir, 'status'), 'Name:\tllama-server\nPPid:\t4200\n', ); writeFileSync(resolve(parentDir, 'cmdline'), 'ollama\0serve\0'); - writeFileSync( - resolve(processDir, 'maps'), - maps.replaceAll('/runner-temp', runnerTemp), - ); + const mapsPath = resolve(processDir, 'maps'); + if (options.mapsFifo) { + const fifo = spawnSync('mkfifo', [mapsPath]); + if (fifo.status !== 0) throw new Error(fifo.stderr); + if (statSync(mapsPath).size !== 0) { + throw new Error('FIFO maps fixture must report zero stat size'); + } + const writer = Bun.spawn([ + 'bash', + '-c', + 'printf %s "$1" > "$2"', + '--', + maps.replaceAll('/runner-temp', runnerTemp), + mapsPath, + ]); + writer.unref(); + } else { + writeFileSync(mapsPath, maps.replaceAll('/runner-temp', runnerTemp)); + } writeFileSync( resolve(runnerTemp, 'ollama-server.log'), 'cmn common_param: system_info: n_threads = 2 (n_threads_batch = 2) / 4 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | REPACK = 1 | \n', @@ -72,7 +100,14 @@ function runBackendVerifier( }; executable('curl', `printf '%s\\n' '{"done":true,"error":null}'`); executable('jq', 'cat >/dev/null; exit 0'); - executable('pgrep', livePid ? `printf '%s\\n' 4242` : 'exit 1'); + executable( + 'pgrep', + options.mapsDisappearAfterPgrep + ? `rm -rf '${processDir}'; printf '%s\\n' 4242` + : livePid + ? `printf '%s\\n' 4242` + : 'exit 1', + ); const script = asString(step('Verify Intel CPU backend warm-up').run); const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', @@ -92,6 +127,90 @@ function runBackendVerifier( }; } +function runLiveLinuxProcVerifier(): BackendVerifierResult { + const fixture = resolve(root, `tmp/verify3764/live-proc-${fixtureCounter++}`); + const runnerTemp = resolve(fixture, 'runner-temp'); + const bin = resolve(fixture, 'bin'); + const ollamaDir = resolve(runnerTemp, 'ollama/lib/ollama'); + mkdirSync(bin, { recursive: true }); + mkdirSync(ollamaDir, { recursive: true }); + const library = resolve(ollamaDir, 'libggml-cpu-haswell.so'); + const executable = resolve(ollamaDir, 'llama-server'); + const compiledLibrary = spawnSync( + 'cc', + ['-shared', '-fPIC', '-x', 'c', '-o', library, '-'], + { input: 'int haswell_fixture(void) { return 1; }', encoding: 'utf8' }, + ); + if (compiledLibrary.status !== 0) { + throw new Error(`${compiledLibrary.stdout}\n${compiledLibrary.stderr}`); + } + const compiledServer = spawnSync( + 'cc', + ['-x', 'c', '-o', executable, '-', '-ldl'], + { + input: + '#include \n#include \nint main(int argc, char **argv) { if (argc < 4 || !dlopen(argv[3], RTLD_NOW)) return 2; for (;;) pause(); }', + encoding: 'utf8', + }, + ); + if (compiledServer.status !== 0) { + throw new Error(`${compiledServer.stdout}\n${compiledServer.stderr}`); + } + const executableScript = (name: string, body: string): void => { + const path = resolve(bin, name); + writeFileSync(path, `#!/bin/bash\n${body}\n`); + Bun.spawnSync(['chmod', '+x', path]); + }; + executableScript('curl', `printf '%s\\n' '{"done":true,"error":null}'`); + executableScript('jq', 'cat >/dev/null; exit 0'); + writeFileSync( + resolve(runnerTemp, 'ollama-server.log'), + 'system_info: CPU : AVX2 = 1\n', + ); + const server = Bun.spawn([executable, '--model', 'gemma', library], { + stdout: 'ignore', + stderr: 'ignore', + }); + try { + const mapsPath = `/proc/${server.pid}/maps`; + let maps = ''; + for (let attempt = 0; attempt < 100; attempt++) { + try { + maps = readFileSync(mapsPath, 'utf8'); + } catch { + maps = ''; + } + if (maps.includes(library)) break; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + if (!maps.includes(library)) + throw new Error('live helper did not map the haswell fixture library'); + if (statSync(mapsPath).size !== 0) + throw new Error('Linux proc maps fixture must report zero stat size'); + const script = asString( + step('Verify Intel CPU backend warm-up').run, + ).replace('"$proc_root/cpuinfo"', `'${resolve(fixture, 'cpuinfo')}'`); + writeFileSync(resolve(fixture, 'cpuinfo'), 'vendor_id : GenuineIntel\\n'); + const result = spawnSync('bash', ['-c', script], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + RUNNER_TEMP: runnerTemp, + OLLAMA_HOST: '127.0.0.1:12644', + }, + }); + return { + stdout: result.stdout, + stderr: result.stderr, + status: result.status ?? 1, + }; + } finally { + server.kill(); + rmSync(fixture, { recursive: true, force: true }); + } +} + const steps = workflowJob(pilot, 'local_model_canaries').steps ?? []; function step(name: string) { @@ -363,11 +482,83 @@ describe('optional local-model E2E pilot', () => { }); }); +describe('workflow CPU variant selection behavior', () => { + const runSelection = (intel: boolean, variants: string[]): string[] => { + const fixture = resolve( + root, + `tmp/verify3764/selection-${fixtureCounter++}`, + ); + const temp = resolve(fixture, 'temp'); + const libdir = resolve(temp, 'ollama/lib/ollama'); + mkdirSync(libdir, { recursive: true }); + mkdirSync(temp, { recursive: true }); + for (const variant of variants) + writeFileSync(resolve(libdir, variant), 'variant'); + const cpuinfo = resolve(fixture, 'cpuinfo'); + writeFileSync( + cpuinfo, + intel ? 'vendor_id : GenuineIntel\n' : 'vendor_id : AuthenticAMD\n', + ); + const script = asString( + step('Select pilot CPU backend on Intel').run, + ).replace('/proc/cpuinfo', cpuinfo); + const result = spawnSync('bash', ['-c', script], { + encoding: 'utf8', + env: { ...process.env, RUNNER_TEMP: temp }, + }); + if (result.status !== 0) + throw new Error(`${result.stdout}\n${result.stderr}`); + const selected = Bun.spawnSync([ + 'find', + libdir, + '-maxdepth', + '1', + '-type', + 'f', + '-name', + 'libggml-cpu-*.so', + ]); + const files = selected.stdout + .toString() + .trim() + .split('\n') + .filter(Boolean) + .map((path) => path.slice(libdir.length + 1)) + .sort(); + rmSync(fixture, { recursive: true, force: true }); + return files; + }; + + it('forces haswell in each Intel matrix leg and leaves AMD variants unchanged', () => { + const variants = [ + 'libggml-cpu-haswell.so', + 'libggml-cpu-avx512.so', + 'libggml-cpu-amx.so', + ]; + for (const _matrixLeg of ['sandbox:none', 'sandbox:docker']) { + expect(runSelection(true, variants)).toEqual(['libggml-cpu-haswell.so']); + } + expect(runSelection(false, variants)).toEqual(variants.sort()); + }); +}); + describe('workflow backend verification behavior', () => { const haswell = '/runner-temp/ollama/lib/ollama/libggml-cpu-haswell.so'; const mapping = (path: string): string => `7f000000-7f100000 r-xp 00000000 08:01 42 ${path}\n`; + it.skipIf(process.platform !== 'linux')( + 'accepts the sole haswell mapping from a live stat-zero Linux proc maps file', + () => { + const result = runLiveLinuxProcVerifier(); + expect(result.status, `${result.stdout}\\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('Loaded CPU backends:'); + expect(result.stdout).toContain('llama-server cmdline:'); + expect(result.stdout).toContain('llama-server PPid:'); + }, + 15_000, + ); + it('accepts the exact sole haswell mapping when the incident log omits AVX512 and AMX fields', () => { const result = runBackendVerifier(mapping(haswell)); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); @@ -380,10 +571,18 @@ describe('workflow backend verification behavior', () => { it('fails when the live process is absent even though the log reports AVX2', () => { const result = runBackendVerifier(mapping(haswell), false); - expect(result.status, `${result.stdout}\n${result.stderr}`).not.toBe(0); + expect(result.status, `${result.stdout}\\n${result.stderr}`).not.toBe(0); expect(result.stderr).toContain('Live llama-server process not found'); }); + it('fails explicitly when the PID vanishes after pgrep returns it', () => { + const result = runBackendVerifier(mapping(haswell), true, { + mapsDisappearAfterPgrep: true, + }); + expect(result.status, `${result.stdout}\\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain('Live llama-server maps are unavailable'); + }); + it('fails specifically when readable live-process maps contain no CPU backend despite an AVX2 log', () => { const result = runBackendVerifier(mapping('/usr/lib/libc.so.6')); expect(result.status, `${result.stdout}\n${result.stderr}`).not.toBe(0); @@ -395,13 +594,34 @@ describe('workflow backend verification behavior', () => { for (const maps of [ `${mapping(haswell)}${mapping('/other/libggml-cpu-avx512.so')}`, mapping('/runner-temp/ollama/lib/ollama/libggml-cpu-skylake.so'), + mapping('/runner-temp/ollama/lib/ollama/libggml-cpu-avx512-vnni.so'), + mapping('/runner-temp/ollama/lib/ollama/libggml-cpu-amx.so'), + mapping('/runner-temp/ollama/lib/ollama/libggml-cpu-sapphirerapids.so'), mapping(`${haswell} (deleted)`), ]) { const result = runBackendVerifier(maps); - expect(result.status, `${result.stdout}\n${result.stderr}`).not.toBe(0); + expect(result.status, `${result.stdout}\\n${result.stderr}`).not.toBe(0); expect(result.stderr).toContain( 'Unexpected mapped llama-server CPU library', ); } + }, 15_000); + + it('reads nonempty maps from a zero-size-reported FIFO', () => { + const result = runBackendVerifier(mapping(haswell), true, { + mapsFifo: true, + }); + expect(result.status, `${result.stdout}\\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('Loaded CPU backends:'); + }, 15_000); + + it('rejects a process whose cmdline does not match the expected server', () => { + const result = runBackendVerifier(mapping(haswell), true, { + cmdline: 'spoofed\\0--model\\0gemma\\0', + }); + expect(result.status, `${result.stdout}\\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain( + 'Matched PID is not the expected llama-server process', + ); }); }); From 50e186fa3cc231cac96c4dec75181e26f99b0bb5 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 10:49:50 -0300 Subject: [PATCH 14/21] Fix live procfs test CPU fixture --- scripts/tests/e2e-local-model.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 6e3dc349e4..c0316622cd 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -190,7 +190,7 @@ function runLiveLinuxProcVerifier(): BackendVerifierResult { const script = asString( step('Verify Intel CPU backend warm-up').run, ).replace('"$proc_root/cpuinfo"', `'${resolve(fixture, 'cpuinfo')}'`); - writeFileSync(resolve(fixture, 'cpuinfo'), 'vendor_id : GenuineIntel\\n'); + writeFileSync(resolve(fixture, 'cpuinfo'), 'vendor_id : GenuineIntel\n'); const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', env: { From fc81a74bde4f7a299b9897107afce1203e905c03 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 11:00:18 -0300 Subject: [PATCH 15/21] Remove workflow test lint violations --- scripts/tests/e2e-local-model.test.ts | 47 ++++++++++++++------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index c0316622cd..09e22f3df8 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -37,6 +37,18 @@ type BackendVerifierResult = { }; let fixtureCounter = 0; + +function writeExecutable(bin: string, name: string, body: string): void { + const path = resolve(bin, name); + writeFileSync( + path, + `#!/bin/bash +${body} +`, + ); + Bun.spawnSync(['chmod', '+x', path]); +} + function runBackendVerifier( maps: string, livePid = true, @@ -93,21 +105,15 @@ function runBackendVerifier( 'cmn common_param: system_info: n_threads = 2 (n_threads_batch = 2) / 4 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | REPACK = 1 | \n', ); writeFileSync(resolve(ollamaDir, 'llama-server'), 'fixture process'); - const executable = (name: string, body: string): void => { - const path = resolve(bin, name); - writeFileSync(path, `#!/bin/bash\n${body}\n`); - Bun.spawnSync(['chmod', '+x', path]); - }; - executable('curl', `printf '%s\\n' '{"done":true,"error":null}'`); - executable('jq', 'cat >/dev/null; exit 0'); - executable( - 'pgrep', - options.mapsDisappearAfterPgrep - ? `rm -rf '${processDir}'; printf '%s\\n' 4242` - : livePid - ? `printf '%s\\n' 4242` - : 'exit 1', - ); + writeExecutable(bin, 'curl', `printf '%s\\n' '{"done":true,"error":null}'`); + writeExecutable(bin, 'jq', 'cat >/dev/null; exit 0'); + let pgrepBody = 'exit 1'; + if (options.mapsDisappearAfterPgrep) { + pgrepBody = `rm -rf '${processDir}'; printf '%s\\n' 4242`; + } else if (livePid) { + pgrepBody = `printf '%s\\n' 4242`; + } + writeExecutable(bin, 'pgrep', pgrepBody); const script = asString(step('Verify Intel CPU backend warm-up').run); const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', @@ -156,13 +162,8 @@ function runLiveLinuxProcVerifier(): BackendVerifierResult { if (compiledServer.status !== 0) { throw new Error(`${compiledServer.stdout}\n${compiledServer.stderr}`); } - const executableScript = (name: string, body: string): void => { - const path = resolve(bin, name); - writeFileSync(path, `#!/bin/bash\n${body}\n`); - Bun.spawnSync(['chmod', '+x', path]); - }; - executableScript('curl', `printf '%s\\n' '{"done":true,"error":null}'`); - executableScript('jq', 'cat >/dev/null; exit 0'); + writeExecutable(bin, 'curl', `printf '%s\\n' '{"done":true,"error":null}'`); + writeExecutable(bin, 'jq', 'cat >/dev/null; exit 0'); writeFileSync( resolve(runnerTemp, 'ollama-server.log'), 'system_info: CPU : AVX2 = 1\n', @@ -356,7 +357,7 @@ describe('optional local-model E2E pilot', () => { expect(asString(verify.run)).toContain('/cmdline'); expect(asString(verify.run)).toContain('/status'); expect(asString(verify.run)).toContain('parent cmdline'); - expect(asString(verify.run)).not.toContain('-s \"$maps'); + expect(asString(verify.run)).not.toContain('-s "$maps'); expect(steps.indexOf(verify)).toBeGreaterThan( steps.indexOf(step('Start local Gemma 4 E2B model')), ); From a086b0bf62138cd5753d1076e0d22b17c9fb0ee5 Mon Sep 17 00:00:00 2001 From: acoliver Date: Sun, 27 Sep 2026 14:49:58 -0300 Subject: [PATCH 16/21] Add ARM runner to optional Ollama pilot --- .github/workflows/e2e.yml | 27 +++++++++++++++++++++------ scripts/tests/e2e-local-model.test.ts | 24 ++++++++++++++++++++---- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1c945b3507..6b0221f127 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -354,7 +354,7 @@ jobs: if: github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true permissions: contents: read - runs-on: 'ubuntu-latest' + runs-on: 'ubuntu-24.04-arm' timeout-minutes: 90 strategy: fail-fast: false @@ -397,17 +397,18 @@ jobs: shell: 'bash' run: | set -euo pipefail + test "$(uname -m)" = 'aarch64' mkdir -p "$RUNNER_TEMP/ollama" curl --fail --show-error --location --retry 3 \ - 'https://ollama.com/download/ollama-linux-amd64.tar.zst?version=0.31.1' \ - -o "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" - printf '%s %s\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check - + 'https://ollama.com/download/ollama-linux-arm64.tar.zst?version=0.31.1' \ + -o "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" + printf '%s %s\n' '47c82a67e59e060a735d1cb50a2acf020126a3a4be3f6847d5b58b7dd59620b6' "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" | sha256sum --check - tar --zstd \ --exclude='lib/ollama/cuda_v12/*' \ --exclude='lib/ollama/cuda_v13/*' \ --exclude='lib/ollama/vulkan/*' \ - -xf "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" -C "$RUNNER_TEMP/ollama" - rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" + -xf "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" -C "$RUNNER_TEMP/ollama" + rm "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" "$RUNNER_TEMP/ollama/bin/ollama" --version @@ -512,6 +513,20 @@ jobs: exit 1 } + - name: 'Verify CPU-only model inference' + shell: 'bash' + env: + OLLAMA_HOST: '127.0.0.1:12644' + run: | + set -euo pipefail + curl --fail --show-error --silent --max-time 300 \ + "http://$OLLAMA_HOST/api/generate" \ + -H 'Content-Type: application/json' \ + -d '{"model":"gemma4:e2b-it-qat","prompt":"Reply with one word.","stream":false,"options":{"num_ctx":32768,"num_predict":1}}' \ + | jq -e '.done == true and .error == null and (.response | length > 0)' + curl --fail --show-error --silent "http://$OLLAMA_HOST/api/ps" | \ + jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .size_vram == 0)' + - name: 'Run local-model canaries' env: OPENAI_API_KEY: 'ollama-local-only' diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 09e22f3df8..8d4c1692f8 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -277,6 +277,7 @@ describe('optional local-model E2E pilot', () => { ); expect(job.permissions).toEqual({ contents: 'read' }); expect(job['timeout-minutes']).toBe(90); + expect(job['runs-on']).toBe('ubuntu-24.04-arm'); expect(asRecord(job.strategy).matrix).toEqual({ sandbox: ['sandbox:none', 'sandbox:docker'], include: [ @@ -315,9 +316,11 @@ describe('optional local-model E2E pilot', () => { it('downloads a pinned Ollama runtime and verifies its archive before extraction', () => { const install = asString(step('Install Ollama CPU runtime').run); - expect(install).toContain('ollama-linux-amd64.tar.zst?version=0.31.1'); + expect(install).toContain('uname -m'); + expect(install).toContain('aarch64'); + expect(install).toContain('ollama-linux-arm64.tar.zst?version=0.31.1'); expect(install).toContain( - `printf '%s %s\\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check -`, + `printf '%s %s\\n' '47c82a67e59e060a735d1cb50a2acf020126a3a4be3f6847d5b58b7dd59620b6' "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" | sha256sum --check -`, ); expect(install.indexOf('sha256sum --check')).toBeLessThan( install.indexOf('tar --zstd'), @@ -325,7 +328,7 @@ describe('optional local-model E2E pilot', () => { expect(install).toContain("--exclude='lib/ollama/cuda_v12/*'"); expect(install).toContain("--exclude='lib/ollama/cuda_v13/*'"); expect(install).toContain("--exclude='lib/ollama/vulkan/*'"); - expect(install).toContain('rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst"'); + expect(install).toContain('rm "$RUNNER_TEMP/ollama-linux-arm64.tar.zst"'); }); it('restricts the CPU-backend A/B to the Intel Docker pilot and structurally checks live backend verification', () => { @@ -378,7 +381,20 @@ describe('optional local-model E2E pilot', () => { expect(start).toContain( '07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c', ); - expect(steps.indexOf(server)).toBeLessThan( + const inference = step('Verify CPU-only model inference'); + const inferenceScript = asString(inference.run); + expect(inferenceScript).toContain('/api/ps'); + expect(inferenceScript).toContain('.size_vram == 0'); + expect(inferenceScript).toContain('/api/generate'); + expect(inferenceScript.indexOf('/api/generate')).toBeLessThan( + inferenceScript.indexOf('/api/ps'), + ); + expect(asString(inference.run)).toContain( + '.done == true and .error == null', + ); + expect(inference.env?.OLLAMA_HOST).toBe('127.0.0.1:12644'); + expect(steps.indexOf(server)).toBeLessThan(steps.indexOf(inference)); + expect(steps.indexOf(inference)).toBeLessThan( steps.indexOf(step('Run local-model canaries')), ); }); From 4782aa8b49915742146a1dbfbb8f6644979899ed Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 15:34:05 -0300 Subject: [PATCH 17/21] Capture ARM pilot CPU inference diagnostics --- .github/workflows/e2e.yml | 9 ++- scripts/tests/e2e-local-model.test.ts | 109 ++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6b0221f127..25a1da6c01 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -522,8 +522,12 @@ jobs: curl --fail --show-error --silent --max-time 300 \ "http://$OLLAMA_HOST/api/generate" \ -H 'Content-Type: application/json' \ - -d '{"model":"gemma4:e2b-it-qat","prompt":"Reply with one word.","stream":false,"options":{"num_ctx":32768,"num_predict":1}}' \ - | jq -e '.done == true and .error == null and (.response | length > 0)' + -d '{"model":"gemma4:e2b-it-qat","prompt":"Reply with one word.","stream":false,"options":{"num_ctx":32768,"num_predict":16}}' \ + -o "$RUNNER_TEMP/ollama-inference.json" + jq -e '.done == true and .error == null and (.response | length > 0)' "$RUNNER_TEMP/ollama-inference.json" || { + cat "$RUNNER_TEMP/ollama-inference.json" >&2 + exit 1 + } curl --fail --show-error --silent "http://$OLLAMA_HOST/api/ps" | \ jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .size_vram == 0)' @@ -598,6 +602,7 @@ jobs: include-hidden-files: true path: | ${{ runner.temp }}/ollama-server.log + ${{ runner.temp }}/ollama-inference.json ${{ runner.temp }}/e2e-model-ledger.jsonl ${{ github.workspace }}/.integration-tests/*/*/telemetry.log ${{ github.workspace }}/.integration-tests/*/*/harness-diagnostics.log diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 8d4c1692f8..71b3c4d4d8 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -642,3 +642,112 @@ describe('workflow backend verification behavior', () => { ); }); }); + +describe('workflow CPU inference warm-up behavior', () => { + function runInference( + generated: { done: boolean; error: string | null; response: string }, + vram: number, + ): { result: BackendVerifierResult; request: unknown; diagnostic: string } { + const fixture = resolve( + root, + `tmp/verify3764/inference-${fixtureCounter++}`, + ); + const bin = resolve(fixture, 'bin'); + mkdirSync(bin, { recursive: true }); + const generation = JSON.stringify(generated); + const ps = JSON.stringify({ + models: [{ name: 'gemma4:e2b-it-qat', size_vram: vram }], + }); + writeExecutable( + bin, + 'curl', + `while (( $# )); do + case "$1" in + */api/generate) endpoint=generate ;; + */api/ps) endpoint=ps ;; + -d) printf '%s' "$2" >"$RUNNER_TEMP/request.json"; shift ;; + -o) output="$2"; shift ;; + esac + shift +done +if [[ "$endpoint" == generate ]]; then + if [[ -n "\${output:-}" ]]; then + printf '%s\\n' '${generation}' >"$output" + else + printf '%s\\n' '${generation}' + fi +else + printf '%s\\n' '${ps}' +fi`, + ); + const script = asString(step('Verify CPU-only model inference').run); + const executed = spawnSync('bash', ['-c', script], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + RUNNER_TEMP: fixture, + OLLAMA_HOST: '127.0.0.1:12644', + }, + }); + const request = JSON.parse( + readFileSync(resolve(fixture, 'request.json'), 'utf8'), + ); + const diagnostic = readFileSync( + resolve(fixture, 'ollama-inference.json'), + 'utf8', + ); + rmSync(fixture, { recursive: true, force: true }); + return { + result: { + status: executed.status ?? 1, + stdout: executed.stdout, + stderr: executed.stderr, + }, + request, + diagnostic, + }; + } + + it('requests sixteen tokens and retains the raw generation response on success', () => { + const output = runInference( + { done: true, error: null, response: 'Hello' }, + 0, + ); + expect(output.result.status, output.result.stderr).toBe(0); + expect(output.request).toMatchObject({ options: { num_predict: 16 } }); + expect(JSON.parse(output.diagnostic)).toEqual({ + done: true, + error: null, + response: 'Hello', + }); + }); + + it('reports raw JSON and rejects empty text or unsuccessful completion', () => { + for (const generated of [ + { done: true, error: null, response: '' }, + { done: false, error: null, response: 'Hello' }, + { done: true, error: 'failure', response: 'Hello' }, + ]) { + const output = runInference(generated, 0); + expect(output.result.status).not.toBe(0); + expect(output.result.stderr).toContain(JSON.stringify(generated)); + expect(output.diagnostic.trim()).toBe(JSON.stringify(generated)); + } + }); + + it('rejects nonzero VRAM despite successful generation', () => { + const output = runInference( + { done: true, error: null, response: 'Hello' }, + 1, + ); + expect(output.result.status).not.toBe(0); + }); + + it('uploads the raw inference JSON alongside the existing diagnostics', () => { + const paths = asString( + asOptionalRecord(step('Upload local model diagnostics').with)?.path, + ).split('\n'); + expect(paths).toContain('${{ runner.temp }}/ollama-inference.json'); + }); +}); From 2bbc746b97f1f2f87e6d5aaf84c42fdf0166e9bf Mon Sep 17 00:00:00 2001 From: acoliver Date: Sun, 27 Sep 2026 15:50:06 -0300 Subject: [PATCH 18/21] Restore x64 pilot with Intel Haswell and AMD native CPU evidence --- .github/workflows/e2e.yml | 39 +++++++++-------- scripts/tests/e2e-local-model.test.ts | 60 +++++++++++++++++++-------- 2 files changed, 65 insertions(+), 34 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 25a1da6c01..5c684e1444 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -354,7 +354,7 @@ jobs: if: github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true permissions: contents: read - runs-on: 'ubuntu-24.04-arm' + runs-on: 'ubuntu-latest' timeout-minutes: 90 strategy: fail-fast: false @@ -397,18 +397,18 @@ jobs: shell: 'bash' run: | set -euo pipefail - test "$(uname -m)" = 'aarch64' + test "$(uname -m)" = 'x86_64' mkdir -p "$RUNNER_TEMP/ollama" curl --fail --show-error --location --retry 3 \ - 'https://ollama.com/download/ollama-linux-arm64.tar.zst?version=0.31.1' \ - -o "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" - printf '%s %s\n' '47c82a67e59e060a735d1cb50a2acf020126a3a4be3f6847d5b58b7dd59620b6' "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" | sha256sum --check - + 'https://ollama.com/download/ollama-linux-amd64.tar.zst?version=0.31.1' \ + -o "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" + printf '%s %s\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check - tar --zstd \ --exclude='lib/ollama/cuda_v12/*' \ --exclude='lib/ollama/cuda_v13/*' \ --exclude='lib/ollama/vulkan/*' \ - -xf "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" -C "$RUNNER_TEMP/ollama" - rm "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" + -xf "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" -C "$RUNNER_TEMP/ollama" + rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" "$RUNNER_TEMP/ollama/bin/ollama" --version @@ -416,6 +416,8 @@ jobs: shell: 'bash' run: | set -euo pipefail + echo "Pilot host architecture: $(uname -m)" + echo "Pilot host CPU vendor: $(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' /proc/cpuinfo)" if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' /proc/cpuinfo; then echo 'Pilot host is not Intel; leaving Ollama CPU variants untouched' exit 0 @@ -431,7 +433,7 @@ jobs: mv "$library" "$RUNNER_TEMP/ollama-disabled-cpu-backends/" fi done - echo 'Intel Docker pilot: CPU variants visible to llama-server:' + echo 'Intel pilot Haswell pin: CPU variants visible to llama-server:' printf '%s\n' "$libdir"/libggml-cpu-*.so test "$(find "$libdir" -maxdepth 1 -name 'libggml-cpu-*.so' | wc -l)" -eq 1 @@ -458,22 +460,21 @@ jobs: jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .digest == "07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c")' df -h "$RUNNER_TEMP" - - name: 'Verify Intel CPU backend warm-up' + - name: 'Verify CPU backend warm-up' shell: 'bash' env: OLLAMA_HOST: '127.0.0.1:12644' run: | set -euo pipefail proc_root="${LLXPRT_PROC_ROOT:-/proc}" - if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' "$proc_root/cpuinfo"; then - echo 'Pilot host is not Intel; CPU backend A/B not exercised' - exit 0 - fi + vendor="$(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' "$proc_root/cpuinfo")" + echo "Pilot host CPU vendor: $vendor" + [[ "$vendor" == GenuineIntel || "$vendor" == AuthenticAMD ]] || { echo "Unsupported CPU vendor: $vendor" >&2; exit 1; } curl --fail --show-error --silent --max-time 300 \ "http://127.0.0.1:12644/api/generate" \ -H 'Content-Type: application/json' \ - -d '{"model":"gemma4:e2b-it-qat","prompt":"Hello","stream":false,"options":{"num_ctx":32768,"num_predict":1}}' \ - | jq -e '.done == true and .error == null' + -d '{"model":"gemma4:e2b-it-qat","prompt":"Hello","stream":false,"think":false,"options":{"num_ctx":32768,"num_predict":16}}' \ + | jq -e '.done == true and .error == null and (.response | length > 0)' grep 'system_info:.*CPU :' "$RUNNER_TEMP/ollama-server.log" | tail -1 || true process="$RUNNER_TEMP/ollama/lib/ollama/llama-server" pid="$(pgrep -f "^$process --model " | head -1)" || true @@ -499,9 +500,13 @@ jobs: maps_snapshot="$RUNNER_TEMP/llama-server-$pid.maps" cat "$maps" >"$maps_snapshot" || { echo 'llama-server maps disappeared while being read' >&2; exit 1; } cat "$maps_snapshot" - backends="$(grep -Eo '/[^ ]+/libggml-cpu-[^ ]+\.so( \(deleted\))?' "$maps_snapshot" | sort -u || true)" + backends="$(grep -Eo '/[^ ]+/libggml-cpu(-[^ ]+)?\.so( \(deleted\))?' "$maps_snapshot" | sort -u || true)" echo "Loaded CPU backends: ${backends:-none}" [[ -n "$backends" ]] || { echo 'No mapped llama-server CPU backend' >&2; exit 1; } + if [[ "$vendor" == AuthenticAMD ]]; then + echo 'AMD pilot: native Ollama CPU backend (variants untouched)' + exit 0 + fi while IFS= read -r backend; do [[ "$backend" == "$expected" ]] || { echo "Unexpected mapped llama-server CPU library: $backend" >&2 @@ -522,7 +527,7 @@ jobs: curl --fail --show-error --silent --max-time 300 \ "http://$OLLAMA_HOST/api/generate" \ -H 'Content-Type: application/json' \ - -d '{"model":"gemma4:e2b-it-qat","prompt":"Reply with one word.","stream":false,"options":{"num_ctx":32768,"num_predict":16}}' \ + -d '{"model":"gemma4:e2b-it-qat","prompt":"Reply with one word.","stream":false,"think":false,"options":{"num_ctx":32768,"num_predict":16}}' \ -o "$RUNNER_TEMP/ollama-inference.json" jq -e '.done == true and .error == null and (.response | length > 0)' "$RUNNER_TEMP/ollama-inference.json" || { cat "$RUNNER_TEMP/ollama-inference.json" >&2 diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 71b3c4d4d8..ca9d9de0de 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -54,6 +54,7 @@ function runBackendVerifier( livePid = true, options: { cmdline?: string; + vendor?: string; mapsFifo?: boolean; mapsDisappearAfterPgrep?: boolean; } = {}, @@ -70,7 +71,10 @@ function runBackendVerifier( mkdirSync(processDir, { recursive: true }); mkdirSync(parentDir, { recursive: true }); mkdirSync(ollamaDir, { recursive: true }); - writeFileSync(resolve(procRoot, 'cpuinfo'), 'vendor_id : GenuineIntel\n'); + writeFileSync( + resolve(procRoot, 'cpuinfo'), + `vendor_id : ${options.vendor ?? 'GenuineIntel'}\n`, + ); writeFileSync( resolve(processDir, 'cmdline'), options.cmdline ?? @@ -105,7 +109,11 @@ function runBackendVerifier( 'cmn common_param: system_info: n_threads = 2 (n_threads_batch = 2) / 4 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | REPACK = 1 | \n', ); writeFileSync(resolve(ollamaDir, 'llama-server'), 'fixture process'); - writeExecutable(bin, 'curl', `printf '%s\\n' '{"done":true,"error":null}'`); + writeExecutable( + bin, + 'curl', + `printf '%s\\n' '{"done":true,"error":null,"response":"Hello"}'`, + ); writeExecutable(bin, 'jq', 'cat >/dev/null; exit 0'); let pgrepBody = 'exit 1'; if (options.mapsDisappearAfterPgrep) { @@ -114,7 +122,7 @@ function runBackendVerifier( pgrepBody = `printf '%s\\n' 4242`; } writeExecutable(bin, 'pgrep', pgrepBody); - const script = asString(step('Verify Intel CPU backend warm-up').run); + const script = asString(step('Verify CPU backend warm-up').run); const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', env: { @@ -188,9 +196,10 @@ function runLiveLinuxProcVerifier(): BackendVerifierResult { throw new Error('live helper did not map the haswell fixture library'); if (statSync(mapsPath).size !== 0) throw new Error('Linux proc maps fixture must report zero stat size'); - const script = asString( - step('Verify Intel CPU backend warm-up').run, - ).replace('"$proc_root/cpuinfo"', `'${resolve(fixture, 'cpuinfo')}'`); + const script = asString(step('Verify CPU backend warm-up').run).replace( + '"$proc_root/cpuinfo"', + `'${resolve(fixture, 'cpuinfo')}'`, + ); writeFileSync(resolve(fixture, 'cpuinfo'), 'vendor_id : GenuineIntel\n'); const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', @@ -277,7 +286,7 @@ describe('optional local-model E2E pilot', () => { ); expect(job.permissions).toEqual({ contents: 'read' }); expect(job['timeout-minutes']).toBe(90); - expect(job['runs-on']).toBe('ubuntu-24.04-arm'); + expect(job['runs-on']).toBe('ubuntu-latest'); expect(asRecord(job.strategy).matrix).toEqual({ sandbox: ['sandbox:none', 'sandbox:docker'], include: [ @@ -317,10 +326,10 @@ describe('optional local-model E2E pilot', () => { it('downloads a pinned Ollama runtime and verifies its archive before extraction', () => { const install = asString(step('Install Ollama CPU runtime').run); expect(install).toContain('uname -m'); - expect(install).toContain('aarch64'); - expect(install).toContain('ollama-linux-arm64.tar.zst?version=0.31.1'); + expect(install).toContain('x86_64'); + expect(install).toContain('ollama-linux-amd64.tar.zst?version=0.31.1'); expect(install).toContain( - `printf '%s %s\\n' '47c82a67e59e060a735d1cb50a2acf020126a3a4be3f6847d5b58b7dd59620b6' "$RUNNER_TEMP/ollama-linux-arm64.tar.zst" | sha256sum --check -`, + `printf '%s %s\\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check -`, ); expect(install.indexOf('sha256sum --check')).toBeLessThan( install.indexOf('tar --zstd'), @@ -328,16 +337,17 @@ describe('optional local-model E2E pilot', () => { expect(install).toContain("--exclude='lib/ollama/cuda_v12/*'"); expect(install).toContain("--exclude='lib/ollama/cuda_v13/*'"); expect(install).toContain("--exclude='lib/ollama/vulkan/*'"); - expect(install).toContain('rm "$RUNNER_TEMP/ollama-linux-arm64.tar.zst"'); + expect(install).toContain('rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst"'); }); - it('restricts the CPU-backend A/B to the Intel Docker pilot and structurally checks live backend verification', () => { + it('pins Intel backend selection before startup and verifies live mapped backends', () => { const select = step('Select pilot CPU backend on Intel'); expect(select.if).toBeUndefined(); const script = asString(select.run); expect(script).toContain('set -euo pipefail'); expect(script).toContain('grep -qm1'); expect(script).toContain('GenuineIntel'); + expect(script).toContain('Pilot host CPU vendor:'); expect(script).toContain('libggml-cpu-haswell.so'); expect(script).toContain('libggml-cpu-*.so'); expect(script).not.toContain('OLLAMA_LLM_LIBRARY'); @@ -347,15 +357,16 @@ describe('optional local-model E2E pilot', () => { expect(steps.indexOf(select)).toBeLessThan( steps.indexOf(step('Start local Gemma 4 E2B model')), ); - const verify = step('Verify Intel CPU backend warm-up'); + const verify = step('Verify CPU backend warm-up'); expect(verify.if).toBeUndefined(); expect(asString(verify.run)).toContain( 'http://127.0.0.1:12644/api/generate', ); expect(asString(verify.run)).toContain('ollama-server.log'); + expect(asString(verify.run)).toContain('"think":false'); + expect(asString(verify.run)).toContain('(.response | length > 0)'); expect(asString(verify.run)).toContain('LLXPRT_PROC_ROOT:-/proc'); expect(asString(verify.run)).toContain('libggml-cpu-haswell.so'); - expect(verify.if).toBeUndefined(); expect(asString(verify.run)).toContain('pgrep -f'); expect(asString(verify.run)).toContain('/cmdline'); expect(asString(verify.run)).toContain('/status'); @@ -518,7 +529,7 @@ describe('workflow CPU variant selection behavior', () => { ); const script = asString( step('Select pilot CPU backend on Intel').run, - ).replace('/proc/cpuinfo', cpuinfo); + ).replaceAll('/proc/cpuinfo', cpuinfo); const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', env: { ...process.env, RUNNER_TEMP: temp }, @@ -560,6 +571,17 @@ describe('workflow CPU variant selection behavior', () => { }); describe('workflow backend verification behavior', () => { + it('reports the mapped native backend on AMD without imposing the Intel Haswell pin', () => { + const result = runBackendVerifier( + '7f000000-7f100000 r-xp 00000000 08:01 42 /runner-temp/ollama/lib/ollama/libggml-cpu-avx2.so\n', + true, + { vendor: 'AuthenticAMD' }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('Pilot host CPU vendor: AuthenticAMD'); + expect(result.stdout).toContain('libggml-cpu-avx2.so'); + }); + const haswell = '/runner-temp/ollama/lib/ollama/libggml-cpu-haswell.so'; const mapping = (path: string): string => `7f000000-7f100000 r-xp 00000000 08:01 42 ${path}\n`; @@ -709,13 +731,17 @@ fi`, }; } - it('requests sixteen tokens and retains the raw generation response on success', () => { + it('disables thinking, requests sixteen tokens and retains the raw generation response on success', () => { const output = runInference( { done: true, error: null, response: 'Hello' }, 0, ); expect(output.result.status, output.result.stderr).toBe(0); - expect(output.request).toMatchObject({ options: { num_predict: 16 } }); + expect(output.request).toMatchObject({ + model: 'gemma4:e2b-it-qat', + think: false, + options: { num_predict: 16 }, + }); expect(JSON.parse(output.diagnostic)).toEqual({ done: true, error: null, From 92e064e05f8cf93d50f4d1321fe36347f54d812a Mon Sep 17 00:00:00 2001 From: acoliver Date: Sun, 27 Sep 2026 17:14:20 -0300 Subject: [PATCH 19/21] Run required Linux E2E with pinned runner-local Gemma (Refs #3764) --- .github/workflows/e2e.yml | 270 +++++++++++++----- dev-docs/integration-tests.md | 121 ++++---- scripts/tests/e2e-local-model.test.ts | 35 ++- .../pr-mergeability-workflow-wiring-b.test.ts | 2 +- .../tests/workflow-quota-selection.test.ts | 160 +++-------- 5 files changed, 339 insertions(+), 249 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5c684e1444..649def2700 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -4,14 +4,14 @@ on: push: branches: - 'main' - # This will run for PRs from the base repository, providing secrets. + # Regular PR E2E uses a local model and does not receive provider secrets. pull_request: branches: - 'main' - 'release/**' - 'dev/**' # This supports explicit labeled reruns for internal PRs. Fork target events - # never reach the secret-bearing E2E jobs. + # never execute their head under the privileged target event. pull_request_target: types: - 'labeled' @@ -24,7 +24,7 @@ on: default: 'main' type: 'string' pilot_local_model: - description: 'Run local Gemma 4 E2B canaries instead of credentialed E2E' + description: 'Run the separate local Gemma 4 E2B pilot instead of regular E2E' required: false default: false type: 'boolean' @@ -164,7 +164,7 @@ jobs: - e2e_doc_change_filter - skip_check - mergeability-gate - timeout-minutes: 60 + timeout-minutes: 90 concurrency: group: >- ${{ github.workflow }}-${{ github.event_name }}-${{ @@ -211,58 +211,12 @@ jobs: - sandbox: ${{ github.event_name == 'push' && 'sandbox:docker' || 'NEVER_MATCH' }} steps: - - name: 'Checkout trusted quota selector' + - name: 'Checkout' uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # ratchet:actions/checkout@v7 + if: "github.event_name != 'pull_request_target' && github.event_name != 'pull_request'" with: - ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.base.sha || github.event.inputs.branch_ref || github.ref }} - - - name: 'Set up Node.js ${{ matrix.node-version }}' - uses: 'actions/setup-node@820762786026740c76f36085b0efc47a31fe5020' # ratchet:actions/setup-node@v7 - with: - node-version: '${{ matrix.node-version }}' - - - name: 'Setup Bun' - # The trusted quota selector runs under Bun before PR code is checked - # out, so Bun must be on PATH before that step. npm run build below is - # also backed by bun scripts/build.ts. - uses: 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' # ratchet:oven-sh/setup-bun@v2 - with: - bun-version-file: '.bun-version' - - - name: 'Check API quota and select optimal key' - id: 'quota' - shell: 'bash' - run: | - set -euo pipefail - quota_selectors=(scripts/ci-quota-check.*) - if (( ${#quota_selectors[@]} != 1 )) || [[ ! -f "${quota_selectors[0]}" ]]; then - echo 'Trusted checkout must contain exactly one quota selector' >&2 - exit 1 - fi - bun "${quota_selectors[0]}" - - # The trusted legacy selector writes the secret to GITHUB_ENV. Remove - # it before checking out PR code; downstream steps resolve one secret. - if [[ -f "$GITHUB_ENV" ]]; then - awk '!/^OPENAI_API_KEY=/' "$GITHUB_ENV" >"${GITHUB_ENV}.quota" - mv "${GITHUB_ENV}.quota" "$GITHUB_ENV" - fi - - if ! grep -Eq '^selected_key=(primary|secondary)$' "$GITHUB_OUTPUT"; then - if grep -q '^selected_key=' "$GITHUB_OUTPUT"; then - echo 'Trusted quota selector emitted an invalid selected_key' >&2 - exit 1 - fi - if [[ "${KEY_VAR_NAME:-}" == *SYNTHETIC* ]]; then - echo 'Trusted Synthetic quota selector did not emit selected_key' >&2 - exit 1 - fi - echo 'selected_key=primary' >>"$GITHUB_OUTPUT" - fi - env: - KEY_VAR_NAME: '${{ vars.KEY_VAR_NAME }}' - OPENAI_API_KEY: '${{ secrets[vars.KEY_VAR_NAME] }}' - OPENAI_API_KEY_2: '${{ secrets[vars.KEY_VAR_NAME_2] }}' + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.branch_ref || github.ref }} + persist-credentials: false - name: 'Checkout PR head (internal target)' uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # ratchet:actions/checkout@v7 @@ -280,6 +234,16 @@ jobs: persist-credentials: false clean: false + - name: 'Set up Node.js ${{ matrix.node-version }}' + uses: 'actions/setup-node@820762786026740c76f36085b0efc47a31fe5020' # ratchet:actions/setup-node@v7 + with: + node-version: '${{ matrix.node-version }}' + + - name: 'Setup Bun' + uses: 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' # ratchet:oven-sh/setup-bun@v2 + with: + bun-version-file: '.bun-version' + - name: 'Install dependencies' run: 'npm ci' @@ -290,39 +254,169 @@ jobs: if: "matrix.sandbox == 'sandbox:docker'" uses: 'docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c' # ratchet:docker/setup-buildx-action@v4 - - name: 'Validate E2E provider environment (Linux)' + - name: 'Install Ollama CPU runtime' + shell: 'bash' + run: | + set -euo pipefail + test "$(uname -m)" = 'x86_64' + mkdir -p "$RUNNER_TEMP/ollama" + curl --fail --show-error --location --retry 3 \ + 'https://ollama.com/download/ollama-linux-amd64.tar.zst?version=0.31.1' \ + -o "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" + printf '%s %s\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check - + tar --zstd \ + --exclude='lib/ollama/cuda_v12/*' \ + --exclude='lib/ollama/cuda_v13/*' \ + --exclude='lib/ollama/vulkan/*' \ + -xf "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" -C "$RUNNER_TEMP/ollama" + rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" + echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" + "$RUNNER_TEMP/ollama/bin/ollama" --version + + - name: 'Select pilot CPU backend on Intel' + shell: 'bash' + run: | + set -euo pipefail + echo "Pilot host architecture: $(uname -m)" + echo "Pilot host CPU vendor: $(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' /proc/cpuinfo)" + if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' /proc/cpuinfo; then + echo 'Pilot host is not Intel; leaving Ollama CPU variants untouched' + exit 0 + fi + libdir="$RUNNER_TEMP/ollama/lib/ollama" + test -f "$libdir/libggml-cpu-haswell.so" + shopt -s nullglob + variants=("$libdir"/libggml-cpu-*.so) + (( ${#variants[@]} > 1 )) || { echo 'Expected multiple CPU variants in pinned Ollama runtime' >&2; exit 1; } + mkdir -p "$RUNNER_TEMP/ollama-disabled-cpu-backends" + for library in "${variants[@]}"; do + if [[ "$library" != "$libdir/libggml-cpu-haswell.so" ]]; then + mv "$library" "$RUNNER_TEMP/ollama-disabled-cpu-backends/" + fi + done + echo 'Intel pilot Haswell pin: CPU variants visible to llama-server:' + printf '%s\n' "$libdir"/libggml-cpu-*.so + test "$(find "$libdir" -maxdepth 1 -name 'libggml-cpu-*.so' | wc -l)" -eq 1 + + - name: 'Start local Gemma 4 E2B model' + shell: 'bash' + env: + OLLAMA_HOST: '127.0.0.1:12644' + OLLAMA_CONTEXT_LENGTH: '32768' + OLLAMA_NUM_PARALLEL: '1' + OLLAMA_MODELS: '${{ runner.temp }}/ollama-models' + run: | + set -euo pipefail + mkdir -p "$OLLAMA_MODELS" + nohup ollama serve >"$RUNNER_TEMP/ollama-server.log" 2>&1 & + for _ in {1..60}; do + if curl --fail --silent "http://$OLLAMA_HOST/api/version"; then + break + fi + sleep 2 + done + curl --fail --silent "http://$OLLAMA_HOST/api/version" | jq -e '.version == "0.31.1"' + ollama pull gemma4:e2b-it-qat + curl --fail --silent "http://$OLLAMA_HOST/api/tags" | + jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .digest == "07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c")' + df -h "$RUNNER_TEMP" + + - name: 'Verify CPU backend warm-up' + shell: 'bash' + env: + OLLAMA_HOST: '127.0.0.1:12644' + run: | + set -euo pipefail + proc_root="${LLXPRT_PROC_ROOT:-/proc}" + vendor="$(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' "$proc_root/cpuinfo")" + echo "Pilot host CPU vendor: $vendor" + [[ "$vendor" == GenuineIntel || "$vendor" == AuthenticAMD ]] || { echo "Unsupported CPU vendor: $vendor" >&2; exit 1; } + curl --fail --show-error --silent --max-time 300 \ + "http://127.0.0.1:12644/api/generate" \ + -H 'Content-Type: application/json' \ + -d '{"model":"gemma4:e2b-it-qat","prompt":"Hello","stream":false,"think":false,"options":{"num_ctx":32768,"num_predict":16}}' \ + | jq -e '.done == true and .error == null and (.response | length > 0)' + grep 'system_info:.*CPU :' "$RUNNER_TEMP/ollama-server.log" | tail -1 || true + process="$RUNNER_TEMP/ollama/lib/ollama/llama-server" + pid="$(pgrep -f "^$process --model " | head -1)" || true + [[ -n "$pid" ]] || { echo 'Live llama-server process not found' >&2; exit 1; } + maps="$proc_root/$pid/maps" + [[ -r "$maps" ]] || { echo "Live llama-server maps are unavailable: $maps" >&2; exit 1; } + cmdline="$proc_root/$pid/cmdline" + status="$proc_root/$pid/status" + [[ -r "$cmdline" && -r "$status" ]] || { echo 'Live llama-server process evidence disappeared' >&2; exit 1; } + printf 'llama-server cmdline: ' + tr '\0' ' ' <"$cmdline" + echo + command_line="$(tr '\0' ' ' <"$cmdline")" + [[ "$command_line" == "$process --model "* ]] || { echo 'Matched PID is not the expected llama-server process' >&2; exit 1; } + ppid="$(awk '$1 == "PPid:" { print $2 }' "$status")" + [[ "$ppid" =~ ^[0-9]+$ && "$ppid" != 0 ]] || { echo 'llama-server parent PID is missing' >&2; exit 1; } + parent_cmdline="$proc_root/$ppid/cmdline" + [[ -r "$parent_cmdline" ]] || { echo 'llama-server parent process disappeared' >&2; exit 1; } + printf 'llama-server PPid: %s; parent cmdline: ' "$ppid" + tr '\0' ' ' <"$parent_cmdline" + echo + expected="$RUNNER_TEMP/ollama/lib/ollama/libggml-cpu-haswell.so" + maps_snapshot="$RUNNER_TEMP/llama-server-$pid.maps" + cat "$maps" >"$maps_snapshot" || { echo 'llama-server maps disappeared while being read' >&2; exit 1; } + cat "$maps_snapshot" + backends="$(grep -Eo '/[^ ]+/libggml-cpu(-[^ ]+)?\.so( \(deleted\))?' "$maps_snapshot" | sort -u || true)" + echo "Loaded CPU backends: ${backends:-none}" + [[ -n "$backends" ]] || { echo 'No mapped llama-server CPU backend' >&2; exit 1; } + if [[ "$vendor" == AuthenticAMD ]]; then + echo 'AMD pilot: native Ollama CPU backend (variants untouched)' + exit 0 + fi + while IFS= read -r backend; do + [[ "$backend" == "$expected" ]] || { + echo "Unexpected mapped llama-server CPU library: $backend" >&2 + exit 1 + } + done <<< "$backends" + grep -Fxq "$expected" <<< "$backends" || { + echo 'Expected haswell CPU library is not mapped' >&2 + exit 1 + } + + - name: 'Verify CPU-only model inference' shell: 'bash' env: - OPENAI_API_KEY: ${{ steps.quota.outputs.selected_key == 'primary' && secrets[vars.KEY_VAR_NAME] || steps.quota.outputs.selected_key == 'secondary' && secrets[vars.KEY_VAR_NAME_2] || '' }} - OPENAI_BASE_URL: '${{ vars.OPENAI_BASE_URL }}' - LLXPRT_DEFAULT_MODEL: '${{ vars.LLXPRT_DEFAULT_MODEL }}' - LLXPRT_DEFAULT_PROVIDER: '${{ vars.LLXPRT_DEFAULT_PROVIDER }}' + OLLAMA_HOST: '127.0.0.1:12644' run: | set -euo pipefail - [[ -n "${OPENAI_API_KEY:-}" ]] || { echo "OPENAI_API_KEY missing after quota selection"; exit 1; } - [[ -n "${OPENAI_BASE_URL:-}" ]] || { echo "OPENAI_BASE_URL is required"; exit 1; } - [[ -n "${LLXPRT_DEFAULT_PROVIDER:-}" ]] || { echo "LLXPRT_DEFAULT_PROVIDER is required"; exit 1; } - [[ -n "${LLXPRT_DEFAULT_MODEL:-}" ]] || { echo "LLXPRT_DEFAULT_MODEL is required"; exit 1; } - echo "E2E env summary: provider=${LLXPRT_DEFAULT_PROVIDER}, model=${LLXPRT_DEFAULT_MODEL}, baseurl=${OPENAI_BASE_URL}" + curl --fail --show-error --silent --max-time 300 \ + "http://$OLLAMA_HOST/api/generate" \ + -H 'Content-Type: application/json' \ + -d '{"model":"gemma4:e2b-it-qat","prompt":"Reply with one word.","stream":false,"think":false,"options":{"num_ctx":32768,"num_predict":16}}' \ + -o "$RUNNER_TEMP/ollama-inference.json" + jq -e '.done == true and .error == null and (.response | length > 0)' "$RUNNER_TEMP/ollama-inference.json" || { + cat "$RUNNER_TEMP/ollama-inference.json" >&2 + exit 1 + } + curl --fail --show-error --silent "http://$OLLAMA_HOST/api/ps" | \ + jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .size_vram == 0)' - name: 'Run E2E tests' env: - # Provider configuration from repository secrets/variables - OPENAI_API_KEY: ${{ steps.quota.outputs.selected_key == 'primary' && secrets[vars.KEY_VAR_NAME] || steps.quota.outputs.selected_key == 'secondary' && secrets[vars.KEY_VAR_NAME_2] || '' }} - KEY_VAR_NAME: '${{ vars.KEY_VAR_NAME }}' - OPENAI_BASE_URL: '${{ vars.OPENAI_BASE_URL }}' - LLXPRT_DEFAULT_MODEL: '${{ vars.LLXPRT_DEFAULT_MODEL }}' - LLXPRT_DEFAULT_PROVIDER: '${{ vars.LLXPRT_DEFAULT_PROVIDER }}' - # Set auth type to provider for API key authentication + OPENAI_API_KEY: 'ollama-local-only' + OPENAI_BASE_URL: 'http://127.0.0.1:12644/v1' + LLXPRT_DEFAULT_MODEL: 'gemma4:e2b-it-qat' + LLXPRT_DEFAULT_PROVIDER: 'openai' LLXPRT_AUTH_TYPE: 'provider' + LLXPRT_TEST_PROFILE: 'local-gemma4-pilot' + LLXPRT_CONTEXT_LIMIT: '32768' + LLXPRT_MAX_OUTPUT_TOKENS: '8192' + LLXPRT_LOCAL_MODEL_PILOT: 'true' + GIT_CEILING_DIRECTORIES: '${{ github.workspace }}/.integration-tests' + SANDBOX_FLAGS: "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}" KEEP_OUTPUT: 'true' VERBOSE: 'true' - # Force file storage to avoid keytar/libsecret dependency in CI LLXPRT_FORCE_FILE_STORAGE: 'true' - # Record real-provider runs for budget enforcement (issue #2278) LLXPRT_E2E_MODEL_LEDGER: '${{ runner.temp }}/e2e-model-ledger.jsonl' shell: 'bash' run: | + set -euo pipefail if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then npm run test:integration:sandbox:docker -- \ --exclude="**/todo-continuation.e2e.test.ts" \ @@ -349,6 +443,36 @@ jobs: run: |- bun scripts/check-e2e-model-budget.ts --ledger "$LLXPRT_E2E_MODEL_LEDGER" + - name: 'Report local model resources' + if: always() + shell: 'bash' + run: | + if curl --fail --silent http://127.0.0.1:12644/api/ps; then + echo + else + echo 'Local Ollama server is unavailable' + fi + free -h + df -h "$RUNNER_TEMP" + du -sh "$RUNNER_TEMP/ollama-models" "$RUNNER_TEMP/ollama" 2>/dev/null || true + if [[ "${{ matrix.sandbox }}" == 'sandbox:docker' ]]; then + docker info --format 'Docker server: {{.ServerVersion}}' + fi + + - name: 'Upload local model diagnostics' + if: always() + uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # ratchet:actions/upload-artifact@v7 + with: + name: "local-gemma4-e2e-${{ matrix.sandbox == 'sandbox:docker' && 'docker' || 'none' }}" + if-no-files-found: error + include-hidden-files: true + path: | + ${{ runner.temp }}/ollama-server.log + ${{ runner.temp }}/ollama-inference.json + ${{ runner.temp }}/e2e-model-ledger.jsonl + ${{ github.workspace }}/.integration-tests/*/*/telemetry.log + ${{ github.workspace }}/.integration-tests/*/*/harness-diagnostics.log + local_model_canaries: name: 'Local Gemma 4 E2B pilot - ${{ matrix.sandbox }}' if: github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index 088c88cb98..4649963d6d 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -173,19 +173,25 @@ The workflow runs the tests in different sandboxing environments to ensure LLxpr - `sandbox:docker`: Runs the tests in a Docker container. - `sandbox:podman`: Runs the tests in a Podman container. -### Manual local-model E2E pilot (issue #3764) - -The existing `.github/workflows/e2e.yml` runs credentialed provider E2E for -internal PRs, main pushes, merge groups and approved labeled reruns. Its required -job names, quota selection, event filtering, full test coverage and budget guard -remain unchanged. The local-model experiment is a separate job in that same -workflow. It runs only on `workflow_dispatch` with the boolean -`pilot_local_model=true`; that dispatch skips the credentialed E2E matrix. -Dispatch without the pilot flag still runs the regular credentialed matrix. -Pilot jobs have distinct, non-required check names and cannot block ordinary -required PR CI. The pilot does not replace any third-party-model E2E coverage. -The credentialed full-platform E2E matrix also remains in -`.github/workflows/nightly.yml`. +### Runner-local model E2E (issue #3764) + +The required `e2e_linux` job uses runner-local Ollama and Gemma 4 E2B on +ordinary internal pull requests, merge groups, main pushes, approved labeled +internal target reruns and manual dispatch without the pilot flag. It sends no +third-party model credentials to the checked-out PR code. The Linux host and +Docker check names, duplicate/doc-only/mergeability gates, main-push host-only +rule, selected integration invocations, assertions, retries and real-model +budget guard remain in place. Fork PR heads do not execute in privileged +`pull_request_target` jobs; fork PRs remain excluded from this E2E job. +The checkout uses the PR merge ref for internal `pull_request`, an approved +internal head SHA for labeled `pull_request_target`, `github.ref` for push and +merge groups, and `branch_ref` for dispatch, always with +`persist-credentials: false`. + +The separate `local_model_canaries` job runs only when explicitly opted in with +`workflow_dispatch` and `pilot_local_model=true`; that dispatch skips the +regular E2E matrix. Its check names remain distinct. Credentialed full-platform +E2E remains in `.github/workflows/nightly.yml` without changes. With `pilot_local_model=true`, the pilot starts Ollama 0.31.1 on `127.0.0.1:12644` and pulls `gemma4:e2b-it-qat` (Q4_0, digest @@ -194,19 +200,19 @@ It verifies the downloaded Linux runtime archive's SHA-256 digest (`d297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4`) before extraction, then checks the runtime version and model digest before testing. The OpenAI provider uses Ollama's `/v1` compatibility API and a local placeholder -key, with no remote inference service or provider secret. Each sandbox leg now -runs the credentialed Linux job's first full integration invocation, excluding +key, with no remote inference service or provider secret. Each sandbox leg +runs the regular Linux job's first full integration invocation, excluding only `todo-continuation.e2e.test.ts` and `run_shell_command.test.ts` (31 of 33 selected integration test files), followed by the same three named shell cases -from the credentialed job. The shell step exits on a failed suite. The existing +from the regular job. The shell step exits on a failed suite. The existing real-model budget guard remains enabled and the ledger reports retries -separately without counting their actual model requests. This expanded local -pilot does not validate hosted CPU performance or replace required credentialed -E2E coverage. +separately without counting their actual model requests. This selection was +validated on hosted x64 runners before its promotion to required Linux E2E; +nightly still covers credentialed full-platform tests. On Linux, the Docker test leg sets `SANDBOX_FLAGS='--network host'` so the sandboxed CLI reaches the host's loopback Ollama endpoint. This networking choice -applies only to the Docker leg. The pilot job supplies no provider credentials. +applies only to the Docker leg. Neither Linux job supplies provider credentials. `GIT_CEILING_DIRECTORIES` prevents Git's implicit parent-repository search from finding the source checkout from a TestRig test directory. The Docker launcher passes the ceiling into the container, where Git tests verify its effect across @@ -220,9 +226,9 @@ path or a generally enabled shell tool. During local evaluation, before this exc ran unrelated Git commands and made an unintended local commit; it was removed without discarding file changes. The runtime archive is pinned to 0.31.1; its CUDA and Vulkan libraries are excluded during extraction, and the downloaded -archive is removed afterward. Ollama is limited to one concurrent context. The -manual pilot configures both Ollama and LLxprt with a 32,768-token context and -reserves 8,192 tokens for model output. The pilot profile allows 750,000 ms +archive is removed afterward. Ollama is limited to one concurrent context. +Both Linux jobs configure Ollama and LLxprt with a 32,768-token context and +reserve 8,192 tokens for model output. The pilot profile allows 750,000 ms for the first model response and sets `openai-headers-timeout-ms` to 900,000 ms on its OpenAI SDK transport. The latter exceeds Undici's ordinary 300,000 ms response-headers limit without changing the dispatcher for other requests. On @@ -232,8 +238,8 @@ so the pilot shell invocation has a 360,000 ms `TestRig` deadline and a 450,000 ms Bun file timeout to leave room for its tool round trip. The real replace invocation has a 1,200,000 ms `TestRig` deadline, its Bun file has a 1,500,000 ms timeout, and each sandbox job has a 90-minute bound. These larger -deadlines apply only when `LLXPRT_LOCAL_MODEL_PILOT=true`; normal integration-test -deadlines are unchanged. +deadlines apply only when `LLXPRT_LOCAL_MODEL_PILOT=true`, set by both Linux +model jobs; local and nightly integration-test deadlines are unchanged. To reproduce the expanded real-model test selection without using an existing Ollama daemon, run these commands from the repository root. Choose a free port @@ -282,27 +288,16 @@ curl -fsS http://127.0.0.1:12644/api/ps | jq '.models[] | {name, size, context_l Google [released Gemma 4 E2B on April 2, 2026](https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/); Ollama publishes the [QAT tag](https://ollama.com/library/gemma4/tags). The [GitHub-hosted runner specifications](https://docs.github.com/en/actions/reference/runners/github-hosted-runners) -list 4 CPU, 16 GB RAM and 14 GB SSD for public `ubuntu-latest` jobs. The -macOS Apple Silicon measurements below do not establish CPU-only x64 speed, -Linux container networking, or the total runtime and disk use of a hosted -Docker E2E leg. Those require an actual Actions run before treating this change -as validated on GitHub's runners. The existing E2E workflow is registered on -the default branch, so after pushing the pilot branch it can be dispatched with -`gh workflow run e2e.yml --ref issue3764 -f branch_ref=issue3764 -f pilot_local_model=true`. -The `--ref` selects the branch version of the workflow and the pilot's checkout; -`branch_ref` identifies the branch for dispatch grouping (and is retained for -normal credentialed reruns). Do not dispatch the new workflow file: GitHub does -not register a new `workflow_dispatch` workflow until it reaches the default -branch. An ordinary `gh workflow run e2e.yml --ref issue3764 -f branch_ref=issue3764` -still runs the credentialed matrix. Local tests validate workflow structure, -but cannot establish hosted-runner performance. Inspect the pilot jobs for both -`sandbox:none` and `sandbox:docker`: job -conclusions, step durations (build, archive download, model pull, E2E), the -`Report local model resources` step (RAM, free disk, model/runtime disk usage, -Docker server), and the uploaded Ollama server log and real-model ledger. Check -the model digest and SHA-256 result in each job log. Both canaries and the budget -check need repeated passes before considering any required-coverage change. -A local Apple Silicon result cannot substitute for Linux CPU and Docker measurements. +list 4 CPU, 16 GB RAM and 14 GB SSD for public `ubuntu-latest` jobs. The macOS +Apple Silicon measurements below alone did not establish CPU-only x64 +performance. Hosted x64 evidence is recorded below. For an opt-in rerun, +use `gh workflow run e2e.yml --ref issue3764 -f branch_ref=issue3764 -f pilot_local_model=true`. +A dispatch without the pilot flag runs the regular local-model job. Do not +start a second dispatch while a run on the same branch is active: the workflow +cancels in-progress jobs on that branch. Inspect both sandbox legs, SHA and +model digest verification, mapped CPU backend, E2E outputs, budget and uploaded +diagnostics. The regular PR checks must pass on the pushed commit before +considering the migration verified. Hosted run [`36276039788`](https://github.com/vybestack/llxprt-code/actions/runs/36276039788) passed shell in both legs but failed replace in all three attempts per leg. @@ -319,8 +314,8 @@ when the provider is explicit. The stream guard therefore used its 300000 ms default. The pilot now relies on its inline profile for provider, model, credentials and timeout instead of supplying conflicting CLI flags. A real-CLI test uses a stalled local HTTP server and a shortened profile threshold to -verify that the guard reads the profile setting. This changes neither regular -E2E provider flags nor the default watchdog. +verify that the guard reads the profile setting. The required Linux job now +uses that same inline profile; the default watchdog is unchanged. Both hosted artifacts include `telemetry.log` with the original user prompt: `Use the replace tool on '' to replace the exact text 'foo @@ -362,9 +357,9 @@ include retries and failed suites (`tmp/verify3764/budget-2b-tuned.log`). A final combined suite with the shell validator accepting only equivalent literal `echo hello-world` commands passed both canaries, with one shell retry (`tmp/verify3764/canaries-2b-final.log`). These are local GPU results. The -failed suite and unmeasured hosted-runner CPU/Docker performance mean this -workflow is not yet validated for PR gating. During remediation, the first two -repeat runs failed because the non-interactive shell canary was initially passed +failed suite and then-unmeasured hosted-runner CPU/Docker performance meant +the pilot was not ready for PR gating at that stage. During remediation, the +first two repeat runs failed because the non-interactive shell canary was initially passed an option array without stdin; that test setup was corrected before the third run. In that third genuine suite, the shell canary passed, but replace failed three of three file-level attempts (`tmp/verify3764/canaries-remediation-2b-3.log`). @@ -388,5 +383,25 @@ unchanged. Qwen3.5:4b Q4_K_M also passed shell but failed replace in a further complete suite despite retrying three times. Disabling thinking for 4B did not fix the unrelated-tool failure in its first complete suite; its next run was stopped to avoid competing with the Gemma measurements. The earlier 2B hosted -failures remain as recorded above. These are local GPU measurements, not proof -that Gemma completes inside the hosted CPU, memory or Docker limits. +failures remain as recorded above. These were local GPU measurements; hosted +Gemma results are recorded below. + +#### Hosted x64 evidence (September 27, 2026) + +Two full hosted Linux pilot runs passed both legs: +[36342130687](https://github.com/vybestack/llxprt-code/actions/runs/36342130687) +used an Intel Docker runner with the Haswell CPU library pinned and a native AMD +host runner; [36344243429](https://github.com/vybestack/llxprt-code/actions/runs/36344243429) +used AMD for both legs. Each leg verified the Ollama 0.31.1 archive SHA-256, +Gemma model digest, live mapped CPU backend and CPU-only inference. Each ran the +same 31 selected integration files plus three named shell cases, produced the +exact `bar content` replacement, and passed the 4/4 real-model budget check. +These successes are specific to hosted x64 CPUs and the pinned model/runtime. +The ARM pilot [36341146657](https://github.com/vybestack/llxprt-code/actions/runs/36341146657) +failed preflight inference in both legs: Ollama returned `done:true` with an +empty `response` and `done_reason:length` after 16 generated tokens. No selected +E2E files or budget guard ran on those ARM legs. The required job checks +`x86_64` and rejects other architectures rather than silently switching backend. +Earlier x64 pilot failures and their observed model/tool issues remain documented +above. The hosted pilots did not themselves +exercise the required PR check names; inspect the ordinary PR run separately. diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index ca9d9de0de..5a24abc4db 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -252,7 +252,7 @@ describe('optional local-model E2E pilot', () => { ); }); - it('leaves required provider E2E events, check names, quota selection and budget unchanged', () => { + it('keeps required E2E events, check names and budget while running without provider credentials', () => { expect(required.on).toHaveProperty('pull_request'); expect(required.on).toHaveProperty('push'); expect(required.on).toHaveProperty('merge_group'); @@ -262,12 +262,43 @@ describe('optional local-model E2E pilot', () => { ); expect(requiredText).not.toContain('Block unqualified local-model gating'); const requiredSteps = workflowJob(required, 'e2e_linux').steps ?? []; - expect(requiredSteps.map((candidate) => candidate.name)).toContain( + expect(requiredSteps.map((candidate) => candidate.name)).not.toContain( 'Check API quota and select optimal key', ); expect(requiredSteps.map((candidate) => candidate.name)).toContain( 'Check E2E real-model budget (issue #2278)', ); + const linuxText = JSON.stringify(workflowJob(required, 'e2e_linux')); + expect(linuxText).not.toContain('secrets['); + expect(linuxText).not.toContain('vars.'); + expect(linuxText).not.toContain('ci-quota-check'); + }); + + it('runs the same pinned local runtime, backend checks, inference and selected suites in required E2E', () => { + const linux = workflowJob(required, 'e2e_linux'); + const linuxSteps = linux.steps ?? []; + for (const name of [ + 'Install Ollama CPU runtime', + 'Select pilot CPU backend on Intel', + 'Start local Gemma 4 E2B model', + 'Verify CPU backend warm-up', + 'Verify CPU-only model inference', + 'Check E2E real-model budget (issue #2278)', + ]) { + const actual = linuxSteps.find((candidate) => candidate.name === name); + expect(actual, name).toEqual(step(name)); + } + const run = linuxSteps.find( + (candidate) => candidate.name === 'Run E2E tests', + ); + expect(run?.env).toEqual(step('Run local-model canaries').env); + expect(run?.run).toBe(step('Run local-model canaries').run); + expect(linux['timeout-minutes']).toBe(90); + expect( + linuxSteps.find( + (candidate) => candidate.name === 'Upload local model diagnostics', + )?.if, + ).toBe('always()'); }); it('only runs on explicit dispatch and uses distinct non-required check names', () => { diff --git a/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts b/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts index 90981445d9..ee06ba2caa 100644 --- a/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts +++ b/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts @@ -366,7 +366,7 @@ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { }); it('bounds the Linux job while preserving concurrency and matrix', () => { - expect(linuxJob?.['timeout-minutes']).toBe(60); + expect(linuxJob?.['timeout-minutes']).toBe(90); const linuxConcurrency = asOptionalRecord(linuxJob?.concurrency); expect(linuxConcurrency?.['cancel-in-progress']).toBe(true); expect(linuxConcurrency?.group).toContain('${{ matrix.sandbox }}'); diff --git a/scripts/tests/workflow-quota-selection.test.ts b/scripts/tests/workflow-quota-selection.test.ts index f3e7be2c28..6b22de7429 100644 --- a/scripts/tests/workflow-quota-selection.test.ts +++ b/scripts/tests/workflow-quota-selection.test.ts @@ -46,78 +46,37 @@ function stepNamed(steps: WorkflowStep[], name: string): WorkflowStep { type WorkflowStep = TypedWorkflowStep; -/** - * Structurally verifies the trusted-ref and PR-code-checkout steps for a - * single E2E job. The trusted checkout ref must cover BOTH pull_request - * and pull_request_target. After quota selection, the PR code must be - * checked out via two mutually-exclusive conditional steps — internal target - * head for pull_request_target and merge ref for pull_request — each with - * persist-credentials:false. Fork target heads must never be checked out. - */ -function assertJobCheckoutSecurity( - steps: WorkflowStep[], - quotaName: string, - quotaId: string, -): void { - const trustedCheckout = stepNamed(steps, 'Checkout trusted quota selector'); - const quota = stepNamed(steps, quotaName); - const targetCheckout = stepNamed(steps, 'Checkout PR head (internal target)'); - const internalCheckout = stepNamed(steps, 'Checkout PR merge ref (internal)'); - - expect(quota.id).toBe(quotaId); - expect(quota.shell).toBe('bash'); - const quotaRun = asString(quota.run); - expect(quotaRun).toContain('quota_selectors=(scripts/ci-quota-check.*)'); - expect(quotaRun).toContain('${#quota_selectors[@]} != 1'); - expect(quotaRun).toContain('[[ ! -f "${quota_selectors[0]}" ]]'); - expect(quotaRun).toContain('bun "${quota_selectors[0]}"'); - expect(quotaRun).not.toContain('ci-quota-check.js'); - expect(quotaRun).toContain('awk \'!/^OPENAI_API_KEY=/\' "$GITHUB_ENV"'); - expect(asString(quota.run)).toContain( - 'grep -Eq \'^selected_key=(primary|secondary)$\' "$GITHUB_OUTPUT"', +/** Verify checkout refs without giving PR code a persistent checkout credential. */ +function assertJobCheckoutSecurity(steps: WorkflowStep[]): void { + const checkout = stepNamed(steps, 'Checkout'); + const target = stepNamed(steps, 'Checkout PR head (internal target)'); + const merge = stepNamed(steps, 'Checkout PR merge ref (internal)'); + expect(checkout.if).toBe( + "github.event_name != 'pull_request_target' && github.event_name != 'pull_request'", ); - expect(asString(quota.run)).toContain( - '[[ "${KEY_VAR_NAME:-}" == *SYNTHETIC* ]]', + const ref = asString(asRecord(checkout.with).ref); + expect(ref).toBe( + "${{ github.event_name == 'workflow_dispatch' && inputs.branch_ref || github.ref }}", ); - expect(asString(quota.run)).toContain( - 'echo \'selected_key=primary\' >>"$GITHUB_OUTPUT"', - ); - - // Trusted checkout ref must use base.sha for BOTH PR event types. - const trustedRef = asString(asRecord(trustedCheckout.with).ref); - expect(trustedRef).toContain("github.event_name == 'pull_request'"); - expect(trustedRef).toContain("github.event_name == 'pull_request_target'"); - expect(trustedRef).toContain('github.event.pull_request.base.sha'); - // Non-PR events must fall back to dispatch input or github.ref. - expect(trustedRef).toContain('github.event.inputs.branch_ref'); - expect(trustedRef).toContain('github.ref'); - - expect(targetCheckout.if).toBe( + expect(asRecord(checkout.with)['persist-credentials']).toBe(false); + expect(target.if).toBe( "github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository", ); - expect(asRecord(targetCheckout.with).ref).toBe( + expect(asRecord(target.with).ref).toBe( '${{ github.event.pull_request.head.sha }}', ); - expect(asRecord(targetCheckout.with).repository).toBe( - '${{ github.repository }}', + expect(asRecord(target.with).repository).toBe('${{ github.repository }}'); + expect(asRecord(target.with)['persist-credentials']).toBe(false); + expect(merge.if).toBe("github.event_name == 'pull_request'"); + expect(asRecord(merge.with).ref).toBe('${{ github.ref }}'); + expect(asRecord(merge.with)['persist-credentials']).toBe(false); + expect(asRecord(merge.with).clean).toBe(false); + expect(steps.indexOf(target)).toBeLessThan( + steps.indexOf(stepNamed(steps, 'Build project')), + ); + expect(steps.indexOf(merge)).toBeLessThan( + steps.indexOf(stepNamed(steps, 'Build project')), ); - expect(asRecord(targetCheckout.with)['persist-credentials']).toBe(false); - - // Internal PR merge-ref checkout: only for pull_request. - expect(internalCheckout.if).toBe("github.event_name == 'pull_request'"); - expect(asRecord(internalCheckout.with).ref).toBe('${{ github.ref }}'); - expect(asRecord(internalCheckout.with)['persist-credentials']).toBe(false); - expect(asRecord(internalCheckout.with).clean).toBe(false); - - // Ordering: trusted → quota → target/internal checkouts. - const idxTrusted = steps.indexOf(trustedCheckout); - const idxQuota = steps.indexOf(quota); - const idxTarget = steps.indexOf(targetCheckout); - const idxInternal = steps.indexOf(internalCheckout); - - expect(idxTrusted).toBeLessThan(idxQuota); - expect(idxQuota).toBeLessThan(idxTarget); - expect(idxQuota).toBeLessThan(idxInternal); } /** Bun's `expect` has no `fail`; throw so the expression stays `never`. */ @@ -126,67 +85,28 @@ function raiseMissing(message: string): never { } describe('quota-selected workflow credentials', () => { - it('maps Linux E2E quota outputs into validation and test steps', () => { + it('runs regular Linux E2E with only a local model and no provider credentials', () => { const workflow = readWorkflow('e2e.yml'); expect(workflow.permissions).toEqual({ actions: 'read', contents: 'read', 'pull-requests': 'read', }); - const jobs = workflow.jobs; - if (!jobs) throw new Error('e2e.yml must define jobs'); - const e2eLinux = jobs.e2e_linux; - if (!e2eLinux) throw new Error('e2e.yml must define e2e_linux'); - expect(hasSecret(e2eLinux.env ?? {})).toBe(false); - const cases: Array<{ - steps: WorkflowStep[]; - quotaName: string; - quotaId: string; - validationName: string; - testName: string; - }> = [ - { - steps: jobSteps(workflowJobOptional(workflow, 'e2e_linux')), - quotaName: 'Check API quota and select optimal key', - quotaId: 'quota', - validationName: 'Validate E2E provider environment (Linux)', - testName: 'Run E2E tests', - }, - ]; - - for (const testCase of cases) { - assertJobCheckoutSecurity( - testCase.steps, - testCase.quotaName, - testCase.quotaId, - ); - - const quota = stepNamed(testCase.steps, testCase.quotaName); - const validation = stepNamed(testCase.steps, testCase.validationName); - const tests = stepNamed(testCase.steps, testCase.testName); - - expect(asRecord(validation.env).OPENAI_API_KEY).toBe( - selectedKeyExpression(testCase.quotaId), - ); - expect(asRecord(tests.env).OPENAI_API_KEY).toBe( - selectedKeyExpression(testCase.quotaId), - ); - expect(asRecord(validation.env).OPENAI_API_KEY_2).toBeUndefined(); - expect(asRecord(tests.env).OPENAI_API_KEY_2).toBeUndefined(); - expect(testCase.steps.filter(hasSecret)).toEqual([ - quota, - validation, - tests, - ]); - - // Quota must precede validation and test steps. - expect(testCase.steps.indexOf(quota)).toBeLessThan( - testCase.steps.indexOf(validation), - ); - expect(testCase.steps.indexOf(quota)).toBeLessThan( - testCase.steps.indexOf(tests), - ); - } + const job = workflowJobOptional(workflow, 'e2e_linux'); + const steps = jobSteps(job); + assertJobCheckoutSecurity(steps); + expect(hasSecret(job)).toBe(false); + expect(JSON.stringify(job)).not.toContain('vars.'); + expect(JSON.stringify(job)).not.toContain('ci-quota-check'); + expect(steps.map((step) => step.name)).not.toContain( + 'Check API quota and select optimal key', + ); + const tests = stepNamed(steps, 'Run E2E tests'); + expect(asRecord(tests.env).OPENAI_API_KEY).toBe('ollama-local-only'); + expect(asRecord(tests.env).OPENAI_BASE_URL).toBe( + 'http://127.0.0.1:12644/v1', + ); + expect(asRecord(tests.env).LLXPRT_DEFAULT_MODEL).toBe('gemma4:e2b-it-qat'); }); it('maps only the selected PR-review key into the walkthrough invocation', () => { From ccba2eae99adbb4cc364fa65f43f27d04c06af63 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Sun, 27 Sep 2026 17:56:18 -0300 Subject: [PATCH 20/21] Keep CPU fixture paths out of workflow shell scripts (Refs #3764) --- scripts/tests/e2e-local-model.test.ts | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index 5a24abc4db..e52b1fac27 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -542,7 +542,11 @@ describe('optional local-model E2E pilot', () => { }); describe('workflow CPU variant selection behavior', () => { - const runSelection = (intel: boolean, variants: string[]): string[] => { + const runSelection = ( + intel: boolean, + variants: string[], + cpuinfoName = 'cpuinfo', + ): string[] => { const fixture = resolve( root, `tmp/verify3764/selection-${fixtureCounter++}`, @@ -553,17 +557,17 @@ describe('workflow CPU variant selection behavior', () => { mkdirSync(temp, { recursive: true }); for (const variant of variants) writeFileSync(resolve(libdir, variant), 'variant'); - const cpuinfo = resolve(fixture, 'cpuinfo'); + const cpuinfo = resolve(fixture, cpuinfoName); writeFileSync( cpuinfo, intel ? 'vendor_id : GenuineIntel\n' : 'vendor_id : AuthenticAMD\n', ); const script = asString( step('Select pilot CPU backend on Intel').run, - ).replaceAll('/proc/cpuinfo', cpuinfo); + ).replaceAll('/proc/cpuinfo', '"$LLXPRT_TEST_CPUINFO"'); const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', - env: { ...process.env, RUNNER_TEMP: temp }, + env: { ...process.env, RUNNER_TEMP: temp, LLXPRT_TEST_CPUINFO: cpuinfo }, }); if (result.status !== 0) throw new Error(`${result.stdout}\n${result.stderr}`); @@ -599,6 +603,18 @@ describe('workflow CPU variant selection behavior', () => { } expect(runSelection(false, variants)).toEqual(variants.sort()); }); + + it('selects Intel and AMD variants with shell syntax in the CPU fixture path', () => { + const variants = ['libggml-cpu-haswell.so', 'libggml-cpu-avx512.so']; + const cpuinfoName = "cpu info '$(printf injected)'"; + expect(runSelection(true, variants, cpuinfoName)).toEqual([ + 'libggml-cpu-haswell.so', + ]); + expect(runSelection(false, variants, cpuinfoName)).toEqual([ + 'libggml-cpu-avx512.so', + 'libggml-cpu-haswell.so', + ]); + }); }); describe('workflow backend verification behavior', () => { From e3a442dbe22e24964068418361ce64a387502087 Mon Sep 17 00:00:00 2001 From: "Andrew C. Oliver" Date: Mon, 28 Sep 2026 01:20:12 -0300 Subject: [PATCH 21/21] Remove obsolete runner-local E2E pilot path (Refs #3764) --- .github/workflows/e2e.yml | 294 +---------------- dev-docs/integration-tests.md | 300 ++++-------------- integration-tests/replace.test.ts | 6 +- integration-tests/run_shell_command.test.ts | 18 +- packages/test-utils/src/cli-args.test.ts | 30 +- packages/test-utils/src/cli-args.ts | 6 +- packages/test-utils/src/test-rig-setup.ts | 4 +- packages/test-utils/src/test-rig.test.ts | 40 +-- packages/test-utils/src/test-rig.ts | 4 +- scripts/bun-test-roots.ts | 2 +- scripts/tests/bun-test-roots.bun.test.ts | 4 +- scripts/tests/e2e-local-model.test.ts | 169 ++++------ .../tests/e2e-platform-cadence.bun.test.ts | 34 +- .../pr-mergeability-workflow-wiring-b.test.ts | 18 -- 14 files changed, 211 insertions(+), 718 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 649def2700..a8f57478a2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -23,11 +23,6 @@ on: required: true default: 'main' type: 'string' - pilot_local_model: - description: 'Run the separate local Gemma 4 E2B pilot instead of regular E2E' - required: false - default: false - type: 'boolean' # Event-type isolation prevents pull_request_target from cancelling pull_request # work and keeps merge_group out of PR-keyed groups. Label isolation prevents an @@ -176,7 +171,6 @@ jobs: needs.skip_check.result == 'success' && !(needs.e2e_doc_change_filter.result == 'success' && needs.e2e_doc_change_filter.outputs.docs_only == 'true') && needs.skip_check.outputs.should_skip != 'true' && - !(github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true) && ( ( ( @@ -201,14 +195,9 @@ jobs: strategy: fail-fast: false matrix: - sandbox: - - 'sandbox:none' - - 'sandbox:docker' + sandbox: ${{ fromJSON(github.event_name == 'push' && '["sandbox:none"]' || '["sandbox:none","sandbox:docker"]') }} node-version: - '24.x' - exclude: - # On push to main, only run sandbox:none (skip docker) - - sandbox: ${{ github.event_name == 'push' && 'sandbox:docker' || 'NEVER_MATCH' }} steps: - name: 'Checkout' @@ -273,14 +262,14 @@ jobs: echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" "$RUNNER_TEMP/ollama/bin/ollama" --version - - name: 'Select pilot CPU backend on Intel' + - name: 'Select CPU backend on Intel' shell: 'bash' run: | set -euo pipefail - echo "Pilot host architecture: $(uname -m)" - echo "Pilot host CPU vendor: $(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' /proc/cpuinfo)" + echo "Host architecture: $(uname -m)" + echo "Host CPU vendor: $(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' /proc/cpuinfo)" if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' /proc/cpuinfo; then - echo 'Pilot host is not Intel; leaving Ollama CPU variants untouched' + echo 'Host is not Intel; leaving Ollama CPU variants untouched' exit 0 fi libdir="$RUNNER_TEMP/ollama/lib/ollama" @@ -294,7 +283,7 @@ jobs: mv "$library" "$RUNNER_TEMP/ollama-disabled-cpu-backends/" fi done - echo 'Intel pilot Haswell pin: CPU variants visible to llama-server:' + echo 'Intel Haswell pin: CPU variants visible to llama-server:' printf '%s\n' "$libdir"/libggml-cpu-*.so test "$(find "$libdir" -maxdepth 1 -name 'libggml-cpu-*.so' | wc -l)" -eq 1 @@ -329,7 +318,7 @@ jobs: set -euo pipefail proc_root="${LLXPRT_PROC_ROOT:-/proc}" vendor="$(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' "$proc_root/cpuinfo")" - echo "Pilot host CPU vendor: $vendor" + echo "Host CPU vendor: $vendor" [[ "$vendor" == GenuineIntel || "$vendor" == AuthenticAMD ]] || { echo "Unsupported CPU vendor: $vendor" >&2; exit 1; } curl --fail --show-error --silent --max-time 300 \ "http://127.0.0.1:12644/api/generate" \ @@ -365,7 +354,7 @@ jobs: echo "Loaded CPU backends: ${backends:-none}" [[ -n "$backends" ]] || { echo 'No mapped llama-server CPU backend' >&2; exit 1; } if [[ "$vendor" == AuthenticAMD ]]; then - echo 'AMD pilot: native Ollama CPU backend (variants untouched)' + echo 'AMD: native Ollama CPU backend (variants untouched)' exit 0 fi while IFS= read -r backend; do @@ -404,10 +393,10 @@ jobs: LLXPRT_DEFAULT_MODEL: 'gemma4:e2b-it-qat' LLXPRT_DEFAULT_PROVIDER: 'openai' LLXPRT_AUTH_TYPE: 'provider' - LLXPRT_TEST_PROFILE: 'local-gemma4-pilot' + LLXPRT_TEST_PROFILE: 'local-gemma4-e2e' LLXPRT_CONTEXT_LIMIT: '32768' LLXPRT_MAX_OUTPUT_TOKENS: '8192' - LLXPRT_LOCAL_MODEL_PILOT: 'true' + LLXPRT_LOCAL_MODEL_E2E: 'true' GIT_CEILING_DIRECTORIES: '${{ github.workspace }}/.integration-tests' SANDBOX_FLAGS: "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}" KEEP_OUTPUT: 'true' @@ -472,266 +461,3 @@ jobs: ${{ runner.temp }}/e2e-model-ledger.jsonl ${{ github.workspace }}/.integration-tests/*/*/telemetry.log ${{ github.workspace }}/.integration-tests/*/*/harness-diagnostics.log - - local_model_canaries: - name: 'Local Gemma 4 E2B pilot - ${{ matrix.sandbox }}' - if: github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true - permissions: - contents: read - runs-on: 'ubuntu-latest' - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - sandbox: - - 'sandbox:none' - - 'sandbox:docker' - include: - - sandbox: 'sandbox:none' - artifact_id: 'none' - - sandbox: 'sandbox:docker' - artifact_id: 'docker' - steps: - - name: 'Checkout' - uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # ratchet:actions/checkout@v7 - with: - persist-credentials: false - - - name: 'Set up Node.js' - uses: 'actions/setup-node@820762786026740c76f36085b0efc47a31fe5020' # ratchet:actions/setup-node@v7 - with: - node-version: '24.x' - - - name: 'Setup Bun' - uses: 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' # ratchet:oven-sh/setup-bun@v2 - with: - bun-version-file: '.bun-version' - - - name: 'Install dependencies' - run: 'npm ci' - - - name: 'Build project' - run: 'npm run build' - - - name: 'Set up Docker' - if: "matrix.sandbox == 'sandbox:docker'" - uses: 'docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c' # ratchet:docker/setup-buildx-action@v4 - - - name: 'Install Ollama CPU runtime' - shell: 'bash' - run: | - set -euo pipefail - test "$(uname -m)" = 'x86_64' - mkdir -p "$RUNNER_TEMP/ollama" - curl --fail --show-error --location --retry 3 \ - 'https://ollama.com/download/ollama-linux-amd64.tar.zst?version=0.31.1' \ - -o "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" - printf '%s %s\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check - - tar --zstd \ - --exclude='lib/ollama/cuda_v12/*' \ - --exclude='lib/ollama/cuda_v13/*' \ - --exclude='lib/ollama/vulkan/*' \ - -xf "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" -C "$RUNNER_TEMP/ollama" - rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" - echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" - "$RUNNER_TEMP/ollama/bin/ollama" --version - - - name: 'Select pilot CPU backend on Intel' - shell: 'bash' - run: | - set -euo pipefail - echo "Pilot host architecture: $(uname -m)" - echo "Pilot host CPU vendor: $(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' /proc/cpuinfo)" - if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' /proc/cpuinfo; then - echo 'Pilot host is not Intel; leaving Ollama CPU variants untouched' - exit 0 - fi - libdir="$RUNNER_TEMP/ollama/lib/ollama" - test -f "$libdir/libggml-cpu-haswell.so" - shopt -s nullglob - variants=("$libdir"/libggml-cpu-*.so) - (( ${#variants[@]} > 1 )) || { echo 'Expected multiple CPU variants in pinned Ollama runtime' >&2; exit 1; } - mkdir -p "$RUNNER_TEMP/ollama-disabled-cpu-backends" - for library in "${variants[@]}"; do - if [[ "$library" != "$libdir/libggml-cpu-haswell.so" ]]; then - mv "$library" "$RUNNER_TEMP/ollama-disabled-cpu-backends/" - fi - done - echo 'Intel pilot Haswell pin: CPU variants visible to llama-server:' - printf '%s\n' "$libdir"/libggml-cpu-*.so - test "$(find "$libdir" -maxdepth 1 -name 'libggml-cpu-*.so' | wc -l)" -eq 1 - - - name: 'Start local Gemma 4 E2B model' - shell: 'bash' - env: - OLLAMA_HOST: '127.0.0.1:12644' - OLLAMA_CONTEXT_LENGTH: '32768' - OLLAMA_NUM_PARALLEL: '1' - OLLAMA_MODELS: '${{ runner.temp }}/ollama-models' - run: | - set -euo pipefail - mkdir -p "$OLLAMA_MODELS" - nohup ollama serve >"$RUNNER_TEMP/ollama-server.log" 2>&1 & - for _ in {1..60}; do - if curl --fail --silent "http://$OLLAMA_HOST/api/version"; then - break - fi - sleep 2 - done - curl --fail --silent "http://$OLLAMA_HOST/api/version" | jq -e '.version == "0.31.1"' - ollama pull gemma4:e2b-it-qat - curl --fail --silent "http://$OLLAMA_HOST/api/tags" | - jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .digest == "07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c")' - df -h "$RUNNER_TEMP" - - - name: 'Verify CPU backend warm-up' - shell: 'bash' - env: - OLLAMA_HOST: '127.0.0.1:12644' - run: | - set -euo pipefail - proc_root="${LLXPRT_PROC_ROOT:-/proc}" - vendor="$(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' "$proc_root/cpuinfo")" - echo "Pilot host CPU vendor: $vendor" - [[ "$vendor" == GenuineIntel || "$vendor" == AuthenticAMD ]] || { echo "Unsupported CPU vendor: $vendor" >&2; exit 1; } - curl --fail --show-error --silent --max-time 300 \ - "http://127.0.0.1:12644/api/generate" \ - -H 'Content-Type: application/json' \ - -d '{"model":"gemma4:e2b-it-qat","prompt":"Hello","stream":false,"think":false,"options":{"num_ctx":32768,"num_predict":16}}' \ - | jq -e '.done == true and .error == null and (.response | length > 0)' - grep 'system_info:.*CPU :' "$RUNNER_TEMP/ollama-server.log" | tail -1 || true - process="$RUNNER_TEMP/ollama/lib/ollama/llama-server" - pid="$(pgrep -f "^$process --model " | head -1)" || true - [[ -n "$pid" ]] || { echo 'Live llama-server process not found' >&2; exit 1; } - maps="$proc_root/$pid/maps" - [[ -r "$maps" ]] || { echo "Live llama-server maps are unavailable: $maps" >&2; exit 1; } - cmdline="$proc_root/$pid/cmdline" - status="$proc_root/$pid/status" - [[ -r "$cmdline" && -r "$status" ]] || { echo 'Live llama-server process evidence disappeared' >&2; exit 1; } - printf 'llama-server cmdline: ' - tr '\0' ' ' <"$cmdline" - echo - command_line="$(tr '\0' ' ' <"$cmdline")" - [[ "$command_line" == "$process --model "* ]] || { echo 'Matched PID is not the expected llama-server process' >&2; exit 1; } - ppid="$(awk '$1 == "PPid:" { print $2 }' "$status")" - [[ "$ppid" =~ ^[0-9]+$ && "$ppid" != 0 ]] || { echo 'llama-server parent PID is missing' >&2; exit 1; } - parent_cmdline="$proc_root/$ppid/cmdline" - [[ -r "$parent_cmdline" ]] || { echo 'llama-server parent process disappeared' >&2; exit 1; } - printf 'llama-server PPid: %s; parent cmdline: ' "$ppid" - tr '\0' ' ' <"$parent_cmdline" - echo - expected="$RUNNER_TEMP/ollama/lib/ollama/libggml-cpu-haswell.so" - maps_snapshot="$RUNNER_TEMP/llama-server-$pid.maps" - cat "$maps" >"$maps_snapshot" || { echo 'llama-server maps disappeared while being read' >&2; exit 1; } - cat "$maps_snapshot" - backends="$(grep -Eo '/[^ ]+/libggml-cpu(-[^ ]+)?\.so( \(deleted\))?' "$maps_snapshot" | sort -u || true)" - echo "Loaded CPU backends: ${backends:-none}" - [[ -n "$backends" ]] || { echo 'No mapped llama-server CPU backend' >&2; exit 1; } - if [[ "$vendor" == AuthenticAMD ]]; then - echo 'AMD pilot: native Ollama CPU backend (variants untouched)' - exit 0 - fi - while IFS= read -r backend; do - [[ "$backend" == "$expected" ]] || { - echo "Unexpected mapped llama-server CPU library: $backend" >&2 - exit 1 - } - done <<< "$backends" - grep -Fxq "$expected" <<< "$backends" || { - echo 'Expected haswell CPU library is not mapped' >&2 - exit 1 - } - - - name: 'Verify CPU-only model inference' - shell: 'bash' - env: - OLLAMA_HOST: '127.0.0.1:12644' - run: | - set -euo pipefail - curl --fail --show-error --silent --max-time 300 \ - "http://$OLLAMA_HOST/api/generate" \ - -H 'Content-Type: application/json' \ - -d '{"model":"gemma4:e2b-it-qat","prompt":"Reply with one word.","stream":false,"think":false,"options":{"num_ctx":32768,"num_predict":16}}' \ - -o "$RUNNER_TEMP/ollama-inference.json" - jq -e '.done == true and .error == null and (.response | length > 0)' "$RUNNER_TEMP/ollama-inference.json" || { - cat "$RUNNER_TEMP/ollama-inference.json" >&2 - exit 1 - } - curl --fail --show-error --silent "http://$OLLAMA_HOST/api/ps" | \ - jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .size_vram == 0)' - - - name: 'Run local-model canaries' - env: - OPENAI_API_KEY: 'ollama-local-only' - OPENAI_BASE_URL: 'http://127.0.0.1:12644/v1' - LLXPRT_DEFAULT_MODEL: 'gemma4:e2b-it-qat' - LLXPRT_DEFAULT_PROVIDER: 'openai' - LLXPRT_AUTH_TYPE: 'provider' - LLXPRT_TEST_PROFILE: 'local-gemma4-pilot' - LLXPRT_CONTEXT_LIMIT: '32768' - LLXPRT_MAX_OUTPUT_TOKENS: '8192' - LLXPRT_LOCAL_MODEL_PILOT: 'true' - GIT_CEILING_DIRECTORIES: '${{ github.workspace }}/.integration-tests' - SANDBOX_FLAGS: "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}" - KEEP_OUTPUT: 'true' - VERBOSE: 'true' - LLXPRT_FORCE_FILE_STORAGE: 'true' - LLXPRT_E2E_MODEL_LEDGER: '${{ runner.temp }}/e2e-model-ledger.jsonl' - shell: 'bash' - run: | - set -euo pipefail - if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then - npm run test:integration:sandbox:docker -- \ - --exclude="**/todo-continuation.e2e.test.ts" \ - --exclude="**/run_shell_command.test.ts" - npm run test:integration:sandbox:docker -- \ - --exclude="**/todo-continuation.e2e.test.ts" \ - integration-tests/run_shell_command.test.ts \ - --testNamePattern="should be able to run a shell command|should be able to run a shell command via stdin|should run a platform-specific file listing command" - else - npm run test:integration:sandbox:none -- \ - --exclude="**/todo-continuation.e2e.test.ts" \ - --exclude="**/run_shell_command.test.ts" - npm run test:integration:sandbox:none -- \ - --exclude="**/todo-continuation.e2e.test.ts" \ - integration-tests/run_shell_command.test.ts \ - --testNamePattern="should be able to run a shell command|should be able to run a shell command via stdin|should run a platform-specific file listing command" - fi - - - name: 'Check E2E real-model budget (issue #2278)' - if: success() - env: - LLXPRT_E2E_MODEL_LEDGER: '${{ runner.temp }}/e2e-model-ledger.jsonl' - shell: 'bash' - run: |- - bun scripts/check-e2e-model-budget.ts --ledger "$LLXPRT_E2E_MODEL_LEDGER" - - - name: 'Report local model resources' - if: always() - shell: 'bash' - run: | - if curl --fail --silent http://127.0.0.1:12644/api/ps; then - echo - else - echo 'Local Ollama server is unavailable' - fi - free -h - df -h "$RUNNER_TEMP" - du -sh "$RUNNER_TEMP/ollama-models" "$RUNNER_TEMP/ollama" 2>/dev/null || true - if [[ "${{ matrix.sandbox }}" == 'sandbox:docker' ]]; then - docker info --format 'Docker server: {{.ServerVersion}}' - fi - - - name: 'Upload local model diagnostics' - if: always() - uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # ratchet:actions/upload-artifact@v7 - with: - name: 'local-gemma4-pilot-${{ matrix.artifact_id }}' - if-no-files-found: error - include-hidden-files: true - path: | - ${{ runner.temp }}/ollama-server.log - ${{ runner.temp }}/ollama-inference.json - ${{ runner.temp }}/e2e-model-ledger.jsonl - ${{ github.workspace }}/.integration-tests/*/*/telemetry.log - ${{ github.workspace }}/.integration-tests/*/*/harness-diagnostics.log diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index 4649963d6d..b39b86eb1c 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -165,243 +165,81 @@ This structure makes it easy to locate the artifacts for a specific test run, fi ## Continuous integration -To ensure the integration tests are always run, a GitHub Actions workflow is defined in `.github/workflows/e2e.yml`. This workflow automatically runs the integrations tests for pull requests against the `main` branch, or when a pull request is added to a merge queue. +`.github/workflows/e2e.yml` runs integration tests for internal pull requests against `main`, `release/**` and `dev/**`, and for merge groups. The workflow runs the tests in different sandboxing environments to ensure LLxprt Code is tested across each: - `sandbox:none`: Runs the tests without any sandboxing. - `sandbox:docker`: Runs the tests in a Docker container. -- `sandbox:podman`: Runs the tests in a Podman container. ### Runner-local model E2E (issue #3764) -The required `e2e_linux` job uses runner-local Ollama and Gemma 4 E2B on -ordinary internal pull requests, merge groups, main pushes, approved labeled -internal target reruns and manual dispatch without the pilot flag. It sends no -third-party model credentials to the checked-out PR code. The Linux host and -Docker check names, duplicate/doc-only/mergeability gates, main-push host-only -rule, selected integration invocations, assertions, retries and real-model -budget guard remain in place. Fork PR heads do not execute in privileged -`pull_request_target` jobs; fork PRs remain excluded from this E2E job. -The checkout uses the PR merge ref for internal `pull_request`, an approved -internal head SHA for labeled `pull_request_target`, `github.ref` for push and -merge groups, and `branch_ref` for dispatch, always with -`persist-credentials: false`. - -The separate `local_model_canaries` job runs only when explicitly opted in with -`workflow_dispatch` and `pilot_local_model=true`; that dispatch skips the -regular E2E matrix. Its check names remain distinct. Credentialed full-platform -E2E remains in `.github/workflows/nightly.yml` without changes. - -With `pilot_local_model=true`, the pilot starts Ollama 0.31.1 on -`127.0.0.1:12644` and pulls `gemma4:e2b-it-qat` (Q4_0, digest -`07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c`). -It verifies the downloaded Linux runtime archive's SHA-256 digest -(`d297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4`) -before extraction, then checks the runtime version and model digest before testing. -The OpenAI provider uses Ollama's `/v1` compatibility API and a local placeholder -key, with no remote inference service or provider secret. Each sandbox leg -runs the regular Linux job's first full integration invocation, excluding -only `todo-continuation.e2e.test.ts` and `run_shell_command.test.ts` (31 of 33 -selected integration test files), followed by the same three named shell cases -from the regular job. The shell step exits on a failed suite. The existing -real-model budget guard remains enabled and the ledger reports retries -separately without counting their actual model requests. This selection was -validated on hosted x64 runners before its promotion to required Linux E2E; -nightly still covers credentialed full-platform tests. - -On Linux, the Docker test leg sets `SANDBOX_FLAGS='--network host'` so the -sandboxed CLI reaches the host's loopback Ollama endpoint. This networking choice -applies only to the Docker leg. Neither Linux job supplies provider credentials. -`GIT_CEILING_DIRECTORIES` prevents Git's implicit parent-repository search from -finding the source checkout from a TestRig test directory. The Docker launcher -passes the ceiling into the container, where Git tests verify its effect across -a real bind mount; the host Git test checks the same boundary without Docker. -The shell canary permits only a literal echo shell command in non-interactive -mode. The replace canary excludes `run_shell_command` and rejects unintended -tool logs. These controls prevent the -observed incidental Git discovery and shell calls in these canaries, but they -are not a security boundary against a model supplied an explicit repository -path or a generally enabled shell tool. During local evaluation, before this exclusion, the 2B model -ran unrelated Git commands and made an unintended local commit; it was removed -without discarding file changes. The runtime archive is pinned to 0.31.1; -its CUDA and Vulkan libraries are excluded during extraction, and the downloaded -archive is removed afterward. Ollama is limited to one concurrent context. -Both Linux jobs configure Ollama and LLxprt with a 32,768-token context and -reserve 8,192 tokens for model output. The pilot profile allows 750,000 ms -for the first model response and sets `openai-headers-timeout-ms` to 900,000 ms -on its OpenAI SDK transport. The latter exceeds Undici's ordinary 300,000 ms -response-headers limit without changing the dispatcher for other requests. On -hosted CPU, a 15,200-token replace prompt was still being evaluated at 300 s -when that limit aborted it. The Docker shell model response took about 238 s, -so the pilot shell invocation has a 360,000 ms `TestRig` deadline and a -450,000 ms Bun file timeout to leave room for its tool round trip. The real -replace invocation has a 1,200,000 ms `TestRig` deadline, its Bun file has a -1,500,000 ms timeout, and each sandbox job has a 90-minute bound. These larger -deadlines apply only when `LLXPRT_LOCAL_MODEL_PILOT=true`, set by both Linux -model jobs; local and nightly integration-test deadlines are unchanged. - -To reproduce the expanded real-model test selection without using an existing -Ollama daemon, run these commands from the repository root. Choose a free port -if 12644 is in use and change both URLs accordingly. Install Ollama 0.31.1 first -and ensure the `gemma4:e2b-it-qat` digest above matches. Store all evidence in -the repository's ignored `tmp/` tree: - -```bash -mkdir -p tmp/verify3764/models -OLLAMA_HOST=127.0.0.1:12644 \ - OLLAMA_MODELS="$PWD/tmp/verify3764/models" \ - OLLAMA_CONTEXT_LENGTH=32768 OLLAMA_NUM_PARALLEL=1 \ - ollama serve >tmp/verify3764/ollama-local.log 2>&1 & -OLLAMA_HOST=127.0.0.1:12644 \ - OLLAMA_MODELS="$PWD/tmp/verify3764/models" ollama pull gemma4:e2b-it-qat -curl -fsS http://127.0.0.1:12644/api/tags | jq '.models[] | select(.name == "gemma4:e2b-it-qat") | {digest, size, details}' -CI=true KEEP_OUTPUT=true VERBOSE=true \ - LLXPRT_DEFAULT_PROVIDER=openai LLXPRT_DEFAULT_MODEL=gemma4:e2b-it-qat \ - OPENAI_API_KEY=ollama-local-only \ - OPENAI_BASE_URL=http://127.0.0.1:12644/v1 LLXPRT_AUTH_TYPE=provider \ - LLXPRT_TEST_PROFILE=local-gemma4-pilot LLXPRT_CONTEXT_LIMIT=32768 \ - LLXPRT_MAX_OUTPUT_TOKENS=8192 LLXPRT_LOCAL_MODEL_PILOT=true \ - LLXPRT_FORCE_FILE_STORAGE=true \ - LLXPRT_E2E_MODEL_LEDGER="$PWD/tmp/verify3764/ledger.jsonl" \ - GIT_CEILING_DIRECTORIES="$PWD/.integration-tests" \ - bun scripts/run_bun_tests.ts --root integration-tests \ - --exclude='**/todo-continuation.e2e.test.ts' \ - --exclude='**/run_shell_command.test.ts' -CI=true KEEP_OUTPUT=true VERBOSE=true \ - LLXPRT_DEFAULT_PROVIDER=openai LLXPRT_DEFAULT_MODEL=gemma4:e2b-it-qat \ - OPENAI_API_KEY=ollama-local-only \ - OPENAI_BASE_URL=http://127.0.0.1:12644/v1 LLXPRT_AUTH_TYPE=provider \ - LLXPRT_TEST_PROFILE=local-gemma4-pilot LLXPRT_CONTEXT_LIMIT=32768 \ - LLXPRT_MAX_OUTPUT_TOKENS=8192 LLXPRT_LOCAL_MODEL_PILOT=true \ - LLXPRT_FORCE_FILE_STORAGE=true \ - LLXPRT_E2E_MODEL_LEDGER="$PWD/tmp/verify3764/ledger.jsonl" \ - GIT_CEILING_DIRECTORIES="$PWD/.integration-tests" \ - bun scripts/run_bun_tests.ts --root integration-tests \ - --exclude='**/todo-continuation.e2e.test.ts' \ - integration-tests/run_shell_command.test.ts \ - --testNamePattern='should be able to run a shell command|should be able to run a shell command via stdin|should run a platform-specific file listing command' -bun scripts/check-e2e-model-budget.ts --ledger "$PWD/tmp/verify3764/ledger.jsonl" -curl -fsS http://127.0.0.1:12644/api/ps | jq '.models[] | {name, size, context_length}' -``` - -Google [released Gemma 4 E2B on April 2, 2026](https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/); -Ollama publishes the [QAT tag](https://ollama.com/library/gemma4/tags). The -[GitHub-hosted runner specifications](https://docs.github.com/en/actions/reference/runners/github-hosted-runners) -list 4 CPU, 16 GB RAM and 14 GB SSD for public `ubuntu-latest` jobs. The macOS -Apple Silicon measurements below alone did not establish CPU-only x64 -performance. Hosted x64 evidence is recorded below. For an opt-in rerun, -use `gh workflow run e2e.yml --ref issue3764 -f branch_ref=issue3764 -f pilot_local_model=true`. -A dispatch without the pilot flag runs the regular local-model job. Do not -start a second dispatch while a run on the same branch is active: the workflow -cancels in-progress jobs on that branch. Inspect both sandbox legs, SHA and -model digest verification, mapped CPU backend, E2E outputs, budget and uploaded -diagnostics. The regular PR checks must pass on the pushed commit before -considering the migration verified. - -Hosted run [`36276039788`](https://github.com/vybestack/llxprt-code/actions/runs/36276039788) -passed shell in both legs but failed replace in all three attempts per leg. -Four replace attempts hit the CLI's default 300,000 ms first-response watchdog; -the remaining host attempt hit the 900,000 ms `TestRig` deadline. Both diagnostic -artifacts uploaded. The pilot-only first-response and replace deadlines above -were increased in response; this does not establish a passing hosted run. - -Hosted [run 36278157447](https://github.com/vybestack/llxprt-code/actions/runs/36278157447) -confirmed why that increase did not work. The test rig generated a profile with -`stream-first-response-timeout-ms: 600000`, but also passed `--provider openai`. -`applyGlobalAndProfileEphemeralSettings` deliberately skips profile ephemerals -when the provider is explicit. The stream guard therefore used its 300000 ms -default. The pilot now relies on its inline profile for provider, model, -credentials and timeout instead of supplying conflicting CLI flags. A real-CLI -test uses a stalled local HTTP server and a shortened profile threshold to -verify that the guard reads the profile setting. The required Linux job now -uses that same inline profile; the default watchdog is unchanged. - -Both hosted artifacts include `telemetry.log` with the original user prompt: -`Use the replace tool on '' to replace the exact text 'foo -content' with 'bar content'. Do not add any whitespace.` On the Docker leg, -the model first tried `read_file` with a space inserted into the checkout path, -then called `replace` twice after dropping one `llxprt-code` path component. -It subsequently called `glob`, `list_directory`, `direct_web_fetch` and -`run_shell_command`; none repaired the file. On the host leg, a retry read the -correct file but passed tool-result formatting and patch fragments as the -`old_string` to `ast_edit` instead of making the requested replacement. Other -attempts hit the guard or TestRig deadline. The target files existed, observed -prompts fit within 32,768 tokens, and the model chose incorrect tools or paths. -The assertions still require the `replace` call, no unrelated tools and exact -`bar content`. - -#### Local measurements (September 26, 2026) - -With Ollama 0.31.1 on an M4 Max, the 2B tag downloaded 2,741,192,820 bytes. -The pinned Linux x64 runtime archive downloaded 1,408,625,102 bytes and a -CPU-only extraction occupied 64 MiB locally. The first three real `TestRig` -suites with the original shell assertion passed 3/3 shell and 3/3 replace after -two replace retries (one timeout, one incorrect tool path). Those runs used the -model's 262,144-token context. At a 32,768-token limit, the first CI-style -replace attempt failed after 205 seconds with malformed file paths; the repeat -experiment was stopped rather than counted as a passing suite. The 4B model -(digest `2a654d98e6fba55d452b7043684e9b57a947e393bbffa62485a7aac05ee4eefd`) -passed only 1/3 complete suites at 32,768 tokens, so it was not selected. -With shell-command validation and exact file-content assertions, two initial -2B suites passed after retries; a third suite repeatedly searched outside the -TestRig workspace and was stopped during its second attempt. After giving the -model the absolute test-file path and naming the required tools, two of three -further suites passed (`tmp/verify3764/canaries-2b-tuned-{1,2,3}.log`). The -remaining suite failed all three replace attempts: the model wrote incorrect -content, including conversational text. The shell canary passed after retries. -At the 262,144-token context, `/api/ps` reported 6,816,498,972 bytes loaded for -2B on the M4 Max (`tmp/verify3764/ps-2b-tuned.json`). The budget guard passed: -four declared requests across two distinct tests; six recorded runs of each -include retries and failed suites (`tmp/verify3764/budget-2b-tuned.log`). A -final combined suite with the shell validator accepting only equivalent literal -`echo hello-world` commands passed both canaries, with one shell retry -(`tmp/verify3764/canaries-2b-final.log`). These are local GPU results. The -failed suite and then-unmeasured hosted-runner CPU/Docker performance meant -the pilot was not ready for PR gating at that stage. During remediation, the -first two repeat runs failed because the non-interactive shell canary was initially passed -an option array without stdin; that test setup was corrected before the third -run. In that third genuine suite, the shell canary passed, but replace failed -three of three file-level attempts (`tmp/verify3764/canaries-remediation-2b-3.log`). -The budget guard still passed with four distinct-test requests and separately -reported five replace and seven shell invocations including retries. No local -model/configuration tested here passed repeat suites consistently, so the pilot -is non-required. The three 4B suite logs likewise show only one complete passing -suite, with replace exhausting all retries in two. - -#### Gemma 4 pilot candidate (September 26, 2026) - -The official `gemma4:e2b-it-qat` artifact is 4,336,358,185 bytes, with the -digest pinned above. Ollama 0.31.1 reported a 3,784,551,955-byte loaded model -at 32,768 context on the local M4 Max. Three separate full `TestRig` suites -passed both the shell and exact replacement canaries on the first file-level -attempt, six tests in total. The three per-suite budget checks each passed at -four recorded requests across two distinct tests. See -`tmp/verify3764/canaries-gemma4-e2b-{1,2,3}.log` and `gemma-budget.log` for -local evidence. The strict unrelated-tool and exact-content assertions were -unchanged. Qwen3.5:4b Q4_K_M also passed shell but failed replace in a further -complete suite despite retrying three times. Disabling thinking for 4B did not -fix the unrelated-tool failure in its first complete suite; its next run was -stopped to avoid competing with the Gemma measurements. The earlier 2B hosted -failures remain as recorded above. These were local GPU measurements; hosted -Gemma results are recorded below. - -#### Hosted x64 evidence (September 27, 2026) - -Two full hosted Linux pilot runs passed both legs: +`.github/workflows/e2e.yml` runs one required Linux x64 job with runner-local +Ollama 0.31.1 and `gemma4:e2b-it-qat`. Internal PRs, merge groups and manual +`workflow_dispatch` run both host (`sandbox:none`) and Docker +(`sandbox:docker`) legs; pushes to main run only the host leg. Dispatch accepts +`branch_ref` and runs the same job. Pull requests check out their merge ref, +approved labeled internal `pull_request_target` runs check out the head SHA, +and manual runs check out `branch_ref`; checkout never persists credentials. +Fork PR heads do not run under the privileged target event or receive this E2E +job. Duplicate and doc-only filters and the internal-target mergeability gate +remain in effect. Credentialed full-platform E2E remains in `nightly.yml`. + +The workflow verifies the x64 Ollama archive SHA-256 +`d297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4`, +Ollama version 0.31.1 and Gemma digest +`07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c`. +On Intel it exposes only the Haswell CPU backend before startup and verifies +that library in the live llama-server process mapping. On AMD it retains the +native Ollama backend and verifies a mapped CPU library. A warm-up and a second +CPU-only inference check require a nonempty response and zero VRAM usage. +The Docker leg uses host networking to reach the Ollama loopback endpoint. +No third-party model credentials are available: the OpenAI-compatible `/v1` +endpoint uses a local placeholder key. + +Each leg runs the selected integration suite excluding +`todo-continuation.e2e.test.ts` and `run_shell_command.test.ts`, followed by +exactly three named shell cases. The shell assertion permits only successful +literal `echo hello-world` commands; replace requires the `replace` tool, +exact `bar content`, and no unrelated tools. File-level retries stay enabled. +`GIT_CEILING_DIRECTORIES` prevents incidental parent repository discovery in +TestRig workspaces, but does not prevent a model given an explicit path from +accessing it. The real-model ledger enforces a four-request ceiling +across two distinct tests; retries are reported separately. Logs, +telemetry, inference JSON and ledger are uploaded even on failure. + +The local-model E2E profile sets a 32,768-token context and reserves 8,192 +output tokens. Only when `LLXPRT_LOCAL_MODEL_E2E=true`, the generated inline +profile extends first-response, SDK request and transport headers deadlines +to 750,000, 850,000 and 900,000 ms respectively. The shell case has a +360,000 ms TestRig deadline and 450,000 ms Bun timeout; replace has a +1,200,000 ms TestRig deadline and 1,500,000 ms Bun timeout. Each job has a +90-minute bound. Other providers, local runs and nightly E2E keep their +ordinary deadlines. The inline profile controls provider selection so explicit +`--provider` flags cannot bypass its timeout settings. + +To reproduce a run on a branch, use +`gh workflow run e2e.yml --ref issue3764 -f branch_ref=issue3764`. +Do not dispatch while another run on that branch is active: branch concurrency +cancels an earlier run. Verify both sandbox check names, runtime and model +pins, CPU mapping, selected suites, budget and diagnostics. To reproduce the +test selection locally, set `OPENAI_BASE_URL` to a private Ollama 0.31.1 +instance, `OPENAI_API_KEY=ollama-local-only`, +`LLXPRT_DEFAULT_PROVIDER=openai`, +`LLXPRT_DEFAULT_MODEL=gemma4:e2b-it-qat`, +`LLXPRT_TEST_PROFILE=local-gemma4-e2e`, `LLXPRT_CONTEXT_LIMIT=32768`, +`LLXPRT_MAX_OUTPUT_TOKENS=8192`, and `LLXPRT_LOCAL_MODEL_E2E=true`. +Run `npm run test:integration:sandbox:none -- +--exclude="**/todo-continuation.e2e.test.ts" +--exclude="**/run_shell_command.test.ts"` and then the three named shell +cases with the same invocations as the workflow. Keep local evidence under the +repository's ignored `tmp/` directory. + +Earlier local and hosted trials with smaller models did not consistently +produce the required exact replacement. An ARM trial returned an empty +response after its first 16 generated tokens. Two hosted x64 Gemma runs [36342130687](https://github.com/vybestack/llxprt-code/actions/runs/36342130687) -used an Intel Docker runner with the Haswell CPU library pinned and a native AMD -host runner; [36344243429](https://github.com/vybestack/llxprt-code/actions/runs/36344243429) -used AMD for both legs. Each leg verified the Ollama 0.31.1 archive SHA-256, -Gemma model digest, live mapped CPU backend and CPU-only inference. Each ran the -same 31 selected integration files plus three named shell cases, produced the -exact `bar content` replacement, and passed the 4/4 real-model budget check. -These successes are specific to hosted x64 CPUs and the pinned model/runtime. -The ARM pilot [36341146657](https://github.com/vybestack/llxprt-code/actions/runs/36341146657) -failed preflight inference in both legs: Ollama returned `done:true` with an -empty `response` and `done_reason:length` after 16 generated tokens. No selected -E2E files or budget guard ran on those ARM legs. The required job checks -`x86_64` and rejects other architectures rather than silently switching backend. -Earlier x64 pilot failures and their observed model/tool issues remain documented -above. The hosted pilots did not themselves -exercise the required PR check names; inspect the ordinary PR run separately. +and [36344243429](https://github.com/vybestack/llxprt-code/actions/runs/36344243429) +passed both legs with the selected files, shell cases, CPU verification and +budget. These results are specific to the pinned x64 runtime and model; they +do not establish equivalence with the credentialed nightly full-platform suite. diff --git a/integration-tests/replace.test.ts b/integration-tests/replace.test.ts index a7e99b577e..7ec60f9e0f 100644 --- a/integration-tests/replace.test.ts +++ b/integration-tests/replace.test.ts @@ -8,8 +8,8 @@ import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; import { env } from 'node:process'; import { TestRig } from './test-helper.js'; -const localModelPilot = env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true'; -const replaceTestOptions = localModelPilot ? { timeout: 1_500_000 } : {}; +const localModelE2E = env['LLXPRT_LOCAL_MODEL_E2E'] === 'true'; +const replaceTestOptions = localModelE2E ? { timeout: 1_500_000 } : {}; describe('replace', () => { let rig: TestRig; @@ -40,7 +40,7 @@ describe('replace', () => { await rig.run({ args: `Use the replace tool on '${filePath}' to replace the exact text 'foo content' with 'bar content'. Do not add any whitespace.`, - timeoutMs: localModelPilot ? 1_200_000 : undefined, + timeoutMs: localModelE2E ? 1_200_000 : undefined, }); const foundToolCall = await rig.waitForToolCall('replace'); diff --git a/integration-tests/run_shell_command.test.ts b/integration-tests/run_shell_command.test.ts index 16149a7116..dc2cca2988 100644 --- a/integration-tests/run_shell_command.test.ts +++ b/integration-tests/run_shell_command.test.ts @@ -11,12 +11,12 @@ import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js'; import { getShellConfiguration } from '../packages/core/src/utils/shell-utils.js'; const { shell } = getShellConfiguration(); -const localModelPilot = env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true'; +const localModelE2E = env['LLXPRT_LOCAL_MODEL_E2E'] === 'true'; -function shellCanaryRunTimeoutMs(pilot: boolean): number | undefined { +function shellE2ERunTimeoutMs(localModelE2E: boolean): number | undefined { // Docker's observed 238s model response leaves no room for a tool round trip // under the original 240s TestRig deadline. - return pilot ? 360_000 : undefined; + return localModelE2E ? 360_000 : undefined; } function getLineCountCommand(): { command: string; tool: string } { @@ -66,10 +66,10 @@ function isExpectedShellTrace( ); } -describe('shell canary command validation', () => { - it('extends only the pilot shell run deadline for the CPU tool round trip', () => { - expect(shellCanaryRunTimeoutMs(true)).toBe(360_000); - expect(shellCanaryRunTimeoutMs(false)).toBeUndefined(); +describe('local-model E2E shell command validation', () => { + it('extends only the local model E2E shell run deadline for the CPU tool round trip', () => { + expect(shellE2ERunTimeoutMs(true)).toBe(360_000); + expect(shellE2ERunTimeoutMs(false)).toBeUndefined(); }); it('accepts equivalent literal echo commands but not other shell expressions', () => { @@ -130,7 +130,7 @@ describe('run_shell_command', () => { afterEach(async () => await rig.cleanup()); it( 'should be able to run a shell command', - { timeout: localModelPilot ? 450_000 : undefined }, + { timeout: localModelE2E ? 450_000 : undefined }, async () => { await rig.setup('should be able to run a shell command', { settings: { tools: { core: ['run_shell_command'] } }, @@ -142,7 +142,7 @@ describe('run_shell_command', () => { args: ['--allowed-tools=run_shell_command(echo)'], stdin: prompt, yolo: false, - timeoutMs: shellCanaryRunTimeoutMs(localModelPilot), + timeoutMs: shellE2ERunTimeoutMs(localModelE2E), }); const foundToolCall = await rig.waitForToolCall('run_shell_command'); diff --git a/packages/test-utils/src/cli-args.test.ts b/packages/test-utils/src/cli-args.test.ts index 4902bbc5b7..650b19611b 100644 --- a/packages/test-utils/src/cli-args.test.ts +++ b/packages/test-utils/src/cli-args.test.ts @@ -35,6 +35,24 @@ describe('cli-args helpers', () => { ]); }); + it('uses fake responses without provider credentials even with local E2E selected', () => { + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', undefined); + setEnv('LLXPRT_DEFAULT_PROVIDER', undefined); + setEnv('OPENAI_API_KEY', undefined); + const fakePath = path.join(os.tmpdir(), 'fake-responses.json'); + + expect(() => assertProviderConfig(fakePath)).not.toThrow(); + expect(buildExtraArgs(fakePath, false)).toStrictEqual([ + '--ide-mode', + 'disable', + '--provider', + 'fake', + '--model', + 'fake-model', + ]); + }); + it('requires non-empty provider configuration without fake responses', () => { setEnv('LLXPRT_DEFAULT_PROVIDER', ''); setEnv('LLXPRT_DEFAULT_MODEL', 'model'); @@ -74,9 +92,9 @@ describe('cli-args helpers', () => { ]); }); - it('uses only the generated profile to configure the local model pilot', () => { - setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); - setEnv('LLXPRT_TEST_PROFILE', 'local-qwen35-pilot'); + it('uses only the generated profile to configure the local model E2E', () => { + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-qwen35-e2e'); setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:4b'); setEnv('OPENAI_BASE_URL', 'http://127.0.0.1:12644/v1'); @@ -89,12 +107,12 @@ describe('cli-args helpers', () => { ]); }); - it('rejects a local pilot without a profile', () => { - setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); + it('rejects a local model E2E without a profile', () => { + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); setEnv('LLXPRT_TEST_PROFILE', undefined); expect(() => buildExtraArgs(undefined, true)).toThrow( - 'LLXPRT_TEST_PROFILE is required for the local model pilot', + 'LLXPRT_TEST_PROFILE is required for the local model E2E', ); }); diff --git a/packages/test-utils/src/cli-args.ts b/packages/test-utils/src/cli-args.ts index 0038781cfb..348fe6f46e 100644 --- a/packages/test-utils/src/cli-args.ts +++ b/packages/test-utils/src/cli-args.ts @@ -53,12 +53,12 @@ export function buildExtraArgs( return extraArgs; } - // The inline pilot profile owns provider, credentials and runtime ephemerals. + // The inline local model E2E profile owns provider, credentials and runtime ephemerals. // An explicit --provider prevents the CLI from applying those ephemerals. - if (env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true') { + if (env['LLXPRT_LOCAL_MODEL_E2E'] === 'true') { if (getProfileName() === undefined) { throw new Error( - 'LLXPRT_TEST_PROFILE is required for the local model pilot', + 'LLXPRT_TEST_PROFILE is required for the local model E2E', ); } return extraArgs; diff --git a/packages/test-utils/src/test-rig-setup.ts b/packages/test-utils/src/test-rig-setup.ts index cd27626bf5..52336cabf6 100644 --- a/packages/test-utils/src/test-rig-setup.ts +++ b/packages/test-utils/src/test-rig-setup.ts @@ -176,7 +176,7 @@ function buildProfile(): Record { provider: profileProvider, model: profileModel, modelParams: - env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true' && + env['LLXPRT_LOCAL_MODEL_E2E'] === 'true' && env['LLXPRT_LOCAL_MODEL_THINKING'] === 'none' ? { reasoning_effort: 'none' } : {}, @@ -228,7 +228,7 @@ function collectEphemeralEntries(): Array<[string, unknown]> { } } - if (env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true') { + if (env['LLXPRT_LOCAL_MODEL_E2E'] === 'true') { entries.push(['stream-first-response-timeout-ms', 750_000]); entries.push(['openai-request-timeout-ms', 850_000]); entries.push(['openai-headers-timeout-ms', 900_000]); diff --git a/packages/test-utils/src/test-rig.test.ts b/packages/test-utils/src/test-rig.test.ts index 991f95df49..4170b9ca81 100644 --- a/packages/test-utils/src/test-rig.test.ts +++ b/packages/test-utils/src/test-rig.test.ts @@ -60,20 +60,20 @@ describe('TestRig setup and cleanup behavior', () => { it('writes bounded context and output settings into an opt-in test profile', () => { createRoot(); - setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); - setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:2b'); setEnv('LLXPRT_CONTEXT_LIMIT', '32768'); setEnv('LLXPRT_MAX_OUTPUT_TOKENS', '8192'); const rig = new TestRig(); - rig.setup('local model pilot profile'); + rig.setup('local model E2E profile'); const testDir = requireTestDir(rig.testDir); const profile = JSON.parse( readFileSync( - join(testDir, '.llxprt', 'profiles', 'local-model-pilot.json'), + join(testDir, '.llxprt', 'profiles', 'local-model-e2e.json'), 'utf8', ), ); @@ -92,7 +92,7 @@ describe('TestRig setup and cleanup behavior', () => { it('leaves the ordinary test profile first-response deadline unchanged', () => { createRoot(); - setEnv('LLXPRT_LOCAL_MODEL_PILOT', undefined); + setEnv('LLXPRT_LOCAL_MODEL_E2E', undefined); setEnv('LLXPRT_TEST_PROFILE', 'ordinary-profile'); const rig = new TestRig(); @@ -116,11 +116,11 @@ describe('TestRig setup and cleanup behavior', () => { ); }); - it('loads the generated local pilot profile in the real CLI without a global profile', async () => { + it('loads the generated local model E2E profile in the real CLI without a global profile', async () => { const root = createRoot(); setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); - setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); - setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:2b'); setEnv('OPENAI_API_KEY', 'local-test-only'); @@ -160,7 +160,7 @@ describe('TestRig setup and cleanup behavior', () => { } setEnv('OPENAI_BASE_URL', `http://127.0.0.1:${address.port}/v1`); const rig = new TestRig(); - rig.setup('real CLI local pilot profile'); + rig.setup('real CLI local model E2E profile'); await rig.run({ args: 'Respond with OK', timeoutMs: 20_000 }); @@ -188,11 +188,11 @@ describe('TestRig setup and cleanup behavior', () => { } }, 30_000); - it('applies the pilot transport headers deadline in the real CLI', async () => { + it('applies the local model E2E transport headers deadline in the real CLI', async () => { const root = createRoot(); setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); - setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); - setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); setEnv('LLXPRT_DEFAULT_MODEL', 'local-test-model'); setEnv('OPENAI_API_KEY', 'local-test-only'); @@ -218,7 +218,7 @@ describe('TestRig setup and cleanup behavior', () => { requireTestDir(rig.testDir), '.llxprt', 'profiles', - 'local-model-pilot.json', + 'local-model-e2e.json', ); const profile = JSON.parse(readFileSync(profilePath, 'utf8')); profile.ephemeralSettings['openai-headers-timeout-ms'] = 100; @@ -238,11 +238,11 @@ describe('TestRig setup and cleanup behavior', () => { } }, 30_000); - it('applies the pilot SDK request deadline in the real CLI', async () => { + it('applies the local model E2E SDK request deadline in the real CLI', async () => { const root = createRoot(); setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); - setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); - setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); setEnv('LLXPRT_DEFAULT_MODEL', 'local-test-model'); setEnv('OPENAI_API_KEY', 'local-test-only'); @@ -268,7 +268,7 @@ describe('TestRig setup and cleanup behavior', () => { requireTestDir(rig.testDir), '.llxprt', 'profiles', - 'local-model-pilot.json', + 'local-model-e2e.json', ); const profile = JSON.parse(readFileSync(profilePath, 'utf8')); profile.ephemeralSettings['openai-request-timeout-ms'] = 100; @@ -291,8 +291,8 @@ describe('TestRig setup and cleanup behavior', () => { it('applies the generated profile first-response watchdog in the real CLI', async () => { const root = createRoot(); setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); - setEnv('LLXPRT_LOCAL_MODEL_PILOT', 'true'); - setEnv('LLXPRT_TEST_PROFILE', 'local-model-pilot'); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:4b'); setEnv('OPENAI_API_KEY', 'local-test-only'); @@ -313,7 +313,7 @@ describe('TestRig setup and cleanup behavior', () => { requireTestDir(rig.testDir), '.llxprt', 'profiles', - 'local-model-pilot.json', + 'local-model-e2e.json', ); const profile = JSON.parse(readFileSync(profilePath, 'utf8')); profile.ephemeralSettings['stream-first-response-timeout-ms'] = 100; diff --git a/packages/test-utils/src/test-rig.ts b/packages/test-utils/src/test-rig.ts index c400a18d6d..0996ef6257 100644 --- a/packages/test-utils/src/test-rig.ts +++ b/packages/test-utils/src/test-rig.ts @@ -599,7 +599,7 @@ function appendUserArgs( } /** - * Append the configured test profile, using inline JSON for the local pilot. + * Append the configured test profile, using inline JSON for the local model E2E. */ function appendProfileFlag(commandArgs: string[], testDir: string): void { const profileName = getProfileName(); @@ -608,7 +608,7 @@ function appendProfileFlag(commandArgs: string[], testDir: string): void { } const ideFlagIndex = commandArgs.indexOf('--ide-mode'); const insertionIndex = ideFlagIndex >= 0 ? ideFlagIndex : commandArgs.length; - if (env['LLXPRT_LOCAL_MODEL_PILOT'] === 'true') { + if (env['LLXPRT_LOCAL_MODEL_E2E'] === 'true') { const profileJson = readFileSync( join(testDir, '.llxprt', 'profiles', `${profileName}.json`), 'utf8', diff --git a/scripts/bun-test-roots.ts b/scripts/bun-test-roots.ts index e699bda770..58e1b3f340 100644 --- a/scripts/bun-test-roots.ts +++ b/scripts/bun-test-roots.ts @@ -150,7 +150,7 @@ export function getErrorCode(error: unknown): string | undefined { export function integrationTestTimeoutOverrides( environment: Readonly, ): readonly BunTestTimeoutOverride[] { - if (environment['LLXPRT_LOCAL_MODEL_PILOT'] !== 'true') { + if (environment['LLXPRT_LOCAL_MODEL_E2E'] !== 'true') { return []; } return [{ pattern: /replace\.test\.ts$/, timeout: 1_500_000 }]; diff --git a/scripts/tests/bun-test-roots.bun.test.ts b/scripts/tests/bun-test-roots.bun.test.ts index f56f5cb082..2b4eff3c81 100644 --- a/scripts/tests/bun-test-roots.bun.test.ts +++ b/scripts/tests/bun-test-roots.bun.test.ts @@ -85,11 +85,11 @@ describe('BUN_TEST_ROOTS structural guarantees', () => { } }); - it('extends only the replace file timeout for the opt-in local-model pilot', () => { + it('extends only the replace file timeout for local-model E2E', () => { expect(integrationTestTimeoutOverrides({})).toEqual([]); const overrides = integrationTestTimeoutOverrides({ - LLXPRT_LOCAL_MODEL_PILOT: 'true', + LLXPRT_LOCAL_MODEL_E2E: 'true', }); expect(overrides).toHaveLength(1); diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts index e52b1fac27..784eb4fcb8 100644 --- a/scripts/tests/e2e-local-model.test.ts +++ b/scripts/tests/e2e-local-model.test.ts @@ -28,7 +28,6 @@ const requiredText = readFileSync( 'utf8', ); const required = parseWorkflowYaml(requiredText); -const pilot = required; type BackendVerifierResult = { stdout: string; @@ -221,34 +220,34 @@ function runLiveLinuxProcVerifier(): BackendVerifierResult { } } -const steps = workflowJob(pilot, 'local_model_canaries').steps ?? []; +const steps = workflowJob(required, 'e2e_linux').steps ?? []; function step(name: string) { const found = steps.find((candidate) => candidate.name === name); - if (!found) throw new Error(`Missing pilot step: ${name}`); + if (!found) throw new Error(`Missing local model E2E step: ${name}`); return found; } -describe('optional local-model E2E pilot', () => { - it('dispatches the default-branch workflow against a selected ref with explicit opt-in', () => { +describe('ordinary local-model E2E', () => { + it('dispatches the selected branch through the ordinary E2E job with no extra input', () => { const dispatch = asRecord(required.on?.workflow_dispatch); - const inputs = asRecord(dispatch.inputs); - expect(asRecord(inputs.branch_ref)).toEqual({ - description: 'Branch to run on', - required: true, - default: 'main', - type: 'string', - }); - expect(asRecord(inputs.pilot_local_model)).toMatchObject({ - required: false, - default: false, - type: 'boolean', + expect(asRecord(dispatch.inputs)).toEqual({ + branch_ref: { + description: 'Branch to run on', + required: true, + default: 'main', + type: 'string', + }, }); - expect(asString(workflowJob(pilot, 'local_model_canaries').if)).toContain( - "github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true", - ); - expect(asString(workflowJob(required, 'e2e_linux').if)).toContain( - "!(github.event_name == 'workflow_dispatch' && inputs.pilot_local_model == true)", + expect(Object.keys(required.jobs ?? {})).toEqual([ + 'skip_check', + 'mergeability-gate', + 'e2e_doc_change_filter', + 'e2e_linux', + ]); + const checkout = step('Checkout'); + expect(asOptionalRecord(checkout.with)?.ref).toBe( + "${{ github.event_name == 'workflow_dispatch' && inputs.branch_ref || github.ref }}", ); }); @@ -274,84 +273,37 @@ describe('optional local-model E2E pilot', () => { expect(linuxText).not.toContain('ci-quota-check'); }); - it('runs the same pinned local runtime, backend checks, inference and selected suites in required E2E', () => { + it('runs the pinned local runtime and both E2E legs in one required job', () => { const linux = workflowJob(required, 'e2e_linux'); - const linuxSteps = linux.steps ?? []; + expect(linux['timeout-minutes']).toBe(90); + expect(asRecord(asRecord(linux.strategy).matrix).sandbox).toBe( + '${{ fromJSON(github.event_name == \'push\' && \'["sandbox:none"]\' || \'["sandbox:none","sandbox:docker"]\') }}', + ); for (const name of [ 'Install Ollama CPU runtime', - 'Select pilot CPU backend on Intel', + 'Select CPU backend on Intel', 'Start local Gemma 4 E2B model', 'Verify CPU backend warm-up', 'Verify CPU-only model inference', + 'Run E2E tests', 'Check E2E real-model budget (issue #2278)', + 'Upload local model diagnostics', ]) { - const actual = linuxSteps.find((candidate) => candidate.name === name); - expect(actual, name).toEqual(step(name)); + expect(step(name).name).toBe(name); } - const run = linuxSteps.find( - (candidate) => candidate.name === 'Run E2E tests', - ); - expect(run?.env).toEqual(step('Run local-model canaries').env); - expect(run?.run).toBe(step('Run local-model canaries').run); - expect(linux['timeout-minutes']).toBe(90); - expect( - linuxSteps.find( - (candidate) => candidate.name === 'Upload local model diagnostics', - )?.if, - ).toBe('always()'); - }); - - it('only runs on explicit dispatch and uses distinct non-required check names', () => { - expect(Object.keys(pilot.jobs ?? {})).toEqual([ - 'skip_check', - 'mergeability-gate', - 'e2e_doc_change_filter', - 'e2e_linux', - 'local_model_canaries', - ]); - const job = workflowJob(pilot, 'local_model_canaries'); - expect(job.name).toContain('Local Gemma 4 E2B pilot'); - expect(job.name).not.toContain('E2E Test (Linux)'); - expect(asString(job.if ?? '')).toContain( - 'inputs.pilot_local_model == true', - ); - expect(job.permissions).toEqual({ contents: 'read' }); - expect(job['timeout-minutes']).toBe(90); - expect(job['runs-on']).toBe('ubuntu-latest'); - expect(asRecord(job.strategy).matrix).toEqual({ - sandbox: ['sandbox:none', 'sandbox:docker'], - include: [ - { sandbox: 'sandbox:none', artifact_id: 'none' }, - { sandbox: 'sandbox:docker', artifact_id: 'docker' }, - ], - }); + expect(step('Upload local model diagnostics').if).toBe('always()'); }); - it('gives every sandbox a distinct artifact name without forbidden characters', () => { - const matrix = asRecord( - asRecord(workflowJob(pilot, 'local_model_canaries').strategy).matrix, - ); - const sandboxes = matrix.sandbox; - expect(sandboxes).toEqual(['sandbox:none', 'sandbox:docker']); - const entries = matrix.include; - expect(Array.isArray(entries)).toBe(true); - if (!Array.isArray(entries)) - throw new Error('Pilot matrix include is required'); + it('names distinct safe diagnostic artifacts for host and Docker', () => { const name = asString( asOptionalRecord(step('Upload local model diagnostics').with)?.name, ); - expect(name).toBe('local-gemma4-pilot-${{ matrix.artifact_id }}'); - const resolvedNames = entries.map((entry) => { - const row = asRecord(entry); - expect(sandboxes).toContain(row.sandbox); - const id = asString(row.artifact_id); - expect(id).toMatch(/^[a-z0-9-]+$/); - return name.replace('${{ matrix.artifact_id }}', id); - }); - expect(new Set(resolvedNames).size).toBe(sandboxes.length); - expect( - resolvedNames.every((resolved) => !/[\\/:*?"<>|\r\n]/.test(resolved)), - ).toBe(true); + expect(name).toBe( + "local-gemma4-e2e-${{ matrix.sandbox == 'sandbox:docker' && 'docker' || 'none' }}", + ); + for (const sandbox of ['none', 'docker']) { + expect(`local-gemma4-e2e-${sandbox}`).not.toMatch(/[\\/:*?"<>|\r\n]/); + } }); it('downloads a pinned Ollama runtime and verifies its archive before extraction', () => { @@ -372,13 +324,13 @@ describe('optional local-model E2E pilot', () => { }); it('pins Intel backend selection before startup and verifies live mapped backends', () => { - const select = step('Select pilot CPU backend on Intel'); + const select = step('Select CPU backend on Intel'); expect(select.if).toBeUndefined(); const script = asString(select.run); expect(script).toContain('set -euo pipefail'); expect(script).toContain('grep -qm1'); expect(script).toContain('GenuineIntel'); - expect(script).toContain('Pilot host CPU vendor:'); + expect(script).toContain('Host CPU vendor:'); expect(script).toContain('libggml-cpu-haswell.so'); expect(script).toContain('libggml-cpu-*.so'); expect(script).not.toContain('OLLAMA_LLM_LIBRARY'); @@ -407,11 +359,11 @@ describe('optional local-model E2E pilot', () => { steps.indexOf(step('Start local Gemma 4 E2B model')), ); expect(steps.indexOf(verify)).toBeLessThan( - steps.indexOf(step('Run local-model canaries')), + steps.indexOf(step('Run E2E tests')), ); }); - it('checks model version and digest before running either canary', () => { + it('checks model version and digest before running integration tests', () => { const server = step('Start local Gemma 4 E2B model'); const start = asString(server.run); const env = asOptionalRecord(server.env); @@ -437,44 +389,30 @@ describe('optional local-model E2E pilot', () => { expect(inference.env?.OLLAMA_HOST).toBe('127.0.0.1:12644'); expect(steps.indexOf(server)).toBeLessThan(steps.indexOf(inference)); expect(steps.indexOf(inference)).toBeLessThan( - steps.indexOf(step('Run local-model canaries')), + steps.indexOf(step('Run E2E tests')), ); }); it('matches the required Linux E2E test invocations in both sandbox legs without provider secrets', () => { - const job = workflowJob(pilot, 'local_model_canaries'); - const run = step('Run local-model canaries'); + const job = workflowJob(required, 'e2e_linux'); + const run = step('Run E2E tests'); const command = asString(run.run); - const requiredRun = asString( - (workflowJob(required, 'e2e_linux').steps ?? []).find( - (candidate) => candidate.name === 'Run E2E tests', - )?.run, - ); const normalize = (script: string): string => script.replace(/\\\s*/g, ' ').replace(/\s+/g, ' '); const fullArgs = '--exclude="**/todo-continuation.e2e.test.ts" --exclude="**/run_shell_command.test.ts"'; const shellArgs = 'integration-tests/run_shell_command.test.ts --testNamePattern="should be able to run a shell command|should be able to run a shell command via stdin|should run a platform-specific file listing command"'; - const normalizedRequired = normalize(requiredRun); const normalizedCommand = normalize(command); for (const sandbox of ['sandbox:none', 'sandbox:docker']) { const fullInvocation = `npm run test:integration:${sandbox} -- ${fullArgs}`; const firstInvocationPattern = new RegExp( `npm run test:integration:${sandbox} -- (.*?) npm run test:integration:${sandbox} --`, ); - const requiredFirstInvocation = normalizedRequired.match( - firstInvocationPattern, - ); - const pilotFirstInvocation = normalizedCommand.match( - firstInvocationPattern, - ); - expect(requiredFirstInvocation?.[1]).toBe(fullArgs); - expect(pilotFirstInvocation?.[1]).toBe(fullArgs); - expect(normalizedRequired).toContain(fullInvocation); + const firstInvocation = normalizedCommand.match(firstInvocationPattern); + expect(firstInvocation?.[1]).toBe(fullArgs); expect(normalizedCommand).toContain(fullInvocation); const shellInvocation = `npm run test:integration:${sandbox} -- --exclude="**/todo-continuation.e2e.test.ts" ${shellArgs}`; - expect(normalizedRequired).toContain(shellInvocation); expect(normalizedCommand).toContain(shellInvocation); } expect(command.match(/--exclude=/g)).toHaveLength(6); @@ -483,10 +421,10 @@ describe('optional local-model E2E pilot', () => { expect(env?.LLXPRT_DEFAULT_PROVIDER).toBe('openai'); expect(env?.LLXPRT_DEFAULT_MODEL).toBe('gemma4:e2b-it-qat'); expect(env?.OPENAI_API_KEY).toBe('ollama-local-only'); - expect(env?.LLXPRT_TEST_PROFILE).toBe('local-gemma4-pilot'); + expect(env?.LLXPRT_TEST_PROFILE).toBe('local-gemma4-e2e'); expect(env?.LLXPRT_CONTEXT_LIMIT).toBe('32768'); expect(env?.LLXPRT_MAX_OUTPUT_TOKENS).toBe('8192'); - expect(env?.LLXPRT_LOCAL_MODEL_PILOT).toBe('true'); + expect(env?.LLXPRT_LOCAL_MODEL_E2E).toBe('true'); expect(Number(env?.LLXPRT_CONTEXT_LIMIT)).toBe( Number(step('Start local Gemma 4 E2B model').env?.OLLAMA_CONTEXT_LENGTH), ); @@ -512,7 +450,7 @@ describe('optional local-model E2E pilot', () => { }); it('uses host networking only for the Docker leg, and retains diagnostics on failure', () => { - const env = asOptionalRecord(step('Run local-model canaries').env); + const env = asOptionalRecord(step('Run E2E tests').env); expect(env?.SANDBOX_FLAGS).toBe( "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}", ); @@ -562,9 +500,10 @@ describe('workflow CPU variant selection behavior', () => { cpuinfo, intel ? 'vendor_id : GenuineIntel\n' : 'vendor_id : AuthenticAMD\n', ); - const script = asString( - step('Select pilot CPU backend on Intel').run, - ).replaceAll('/proc/cpuinfo', '"$LLXPRT_TEST_CPUINFO"'); + const script = asString(step('Select CPU backend on Intel').run).replaceAll( + '/proc/cpuinfo', + '"$LLXPRT_TEST_CPUINFO"', + ); const result = spawnSync('bash', ['-c', script], { encoding: 'utf8', env: { ...process.env, RUNNER_TEMP: temp, LLXPRT_TEST_CPUINFO: cpuinfo }, @@ -625,7 +564,7 @@ describe('workflow backend verification behavior', () => { { vendor: 'AuthenticAMD' }, ); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); - expect(result.stdout).toContain('Pilot host CPU vendor: AuthenticAMD'); + expect(result.stdout).toContain('Host CPU vendor: AuthenticAMD'); expect(result.stdout).toContain('libggml-cpu-avx2.so'); }); diff --git a/scripts/tests/e2e-platform-cadence.bun.test.ts b/scripts/tests/e2e-platform-cadence.bun.test.ts index 0bc2da58c3..06e60ab1f0 100644 --- a/scripts/tests/e2e-platform-cadence.bun.test.ts +++ b/scripts/tests/e2e-platform-cadence.bun.test.ts @@ -18,9 +18,8 @@ * YAML helpers and assert the approved split: * * - e2e.yml defines no e2e_mac job and allocates no E2E job to a macOS or - * Windows runner. The Linux PR matrix retains exactly sandbox:none and - * sandbox:docker, with a single exclusion that drops sandbox:docker only on - * push to main. + * Windows runner. The Linux PR matrix selects sandbox:none and sandbox:docker; + * push to main selects sandbox:none only. * - nightly.yml retains the e2e_full rows for Ubuntu none, Ubuntu Docker, * macOS none, and Windows none; both schedule and workflow_dispatch * triggers exercise the matrix; the matrix is non-fail-fast; E2E failures @@ -102,27 +101,18 @@ describe('e2e.yml: macOS E2E is off the PR feedback path (issue #3189)', () => { expect(jobRunsOn(linux)).toBe('ubuntu-latest'); const strategy = asOptionalRecord(linux.strategy); const matrix = asOptionalRecord(strategy?.matrix); - const sandbox = asStringArray(matrix?.sandbox); - expect(sandbox).toEqual(['sandbox:none', 'sandbox:docker']); + expect(asString(matrix?.sandbox)).toContain('sandbox:none'); + expect(asString(matrix?.sandbox)).toContain('sandbox:docker'); }); - it('excludes only sandbox:docker on push, leaving both legs for all other events', () => { - const linux = workflowJob(workflow, 'e2e_linux'); - const strategy = asOptionalRecord(linux.strategy); - const matrix = asOptionalRecord(strategy?.matrix); - const exclude = asArray(matrix?.exclude).map((row) => { - const rec = asRecord(row); - return { sandbox: asString(rec.sandbox) }; - }); - // Exactly one exclusion: drop sandbox:docker only when the event is push. - // pull_request, approved pull_request_target, merge_group, and - // workflow_dispatch all receive both Linux sandbox legs. - expect(exclude).toEqual([ - { - sandbox: - "${{ github.event_name == 'push' && 'sandbox:docker' || 'NEVER_MATCH' }}", - }, - ]); + it('selects only the host on push and both legs on other events without a placeholder row', () => { + const matrix = asOptionalRecord( + asOptionalRecord(workflowJob(workflow, 'e2e_linux').strategy)?.matrix, + ); + expect(matrix?.exclude).toBeUndefined(); + expect(matrix?.sandbox).toBe( + '${{ fromJSON(github.event_name == \'push\' && \'["sandbox:none"]\' || \'["sandbox:none","sandbox:docker"]\') }}', + ); }); it('wires the Linux E2E real-model budget ledger and enforcement step', () => { diff --git a/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts b/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts index ee06ba2caa..f05a2c8ef2 100644 --- a/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts +++ b/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts @@ -69,7 +69,6 @@ interface E2eContextParams { gateResult?: string; shouldRun?: string; cancelled?: boolean; - pilotLocalModel?: boolean; } function e2eContext({ @@ -85,11 +84,9 @@ function e2eContext({ gateResult = 'skipped', shouldRun, cancelled = false, - pilotLocalModel = false, }: E2eContextParams) { return { cancelled, - inputs: { pilot_local_model: pilotLocalModel }, github: { event_name: eventName, event: { @@ -116,7 +113,6 @@ function e2eContext({ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { let parsed: Record; let linuxJob: Record | undefined; - let pilotJob: Record | undefined; let gateJob: Record | undefined; let docFilterJob: Record | undefined; @@ -125,7 +121,6 @@ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { parsed = wf.parsed; const jobs = wf.jobs; linuxJob = asRecord(jobs?.e2e_linux ?? undefined); - pilotJob = asRecord(jobs?.local_model_canaries ?? undefined); gateJob = asRecord(jobs?.['mergeability-gate'] ?? undefined); docFilterJob = asRecord(jobs?.e2e_doc_change_filter ?? undefined); }); @@ -180,12 +175,6 @@ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { eventName: 'workflow_dispatch', linux: true, }, - { - name: 'explicitly opted-in pilot dispatch skips credentialed E2E', - eventName: 'workflow_dispatch', - pilotLocalModel: true, - linux: false, - }, { name: 'internal pull request with intentionally skipped gate', eventName: 'pull_request', @@ -308,13 +297,6 @@ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { expect( Boolean(evaluateE2ECondition(asString(linuxJob?.if), context)), ).toBe(scenario.linux); - expect( - Boolean(evaluateE2ECondition(asString(pilotJob?.if), context)), - ).toBe( - scenario.eventName === 'workflow_dispatch' && - 'pilotLocalModel' in scenario && - scenario.pilotLocalModel === true, - ); }); }