diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4924faf13d..a8f57478a2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -4,14 +4,14 @@ on: push: branches: - 'main' - # This will run for PRs from the base repository, providing secrets. + # Regular PR E2E uses a local model and does not receive provider secrets. pull_request: branches: - 'main' - 'release/**' - 'dev/**' # This supports explicit labeled reruns for internal PRs. Fork target events - # never reach the secret-bearing E2E jobs. + # never execute their head under the privileged target event. pull_request_target: types: - 'labeled' @@ -159,7 +159,7 @@ jobs: - e2e_doc_change_filter - skip_check - mergeability-gate - timeout-minutes: 60 + timeout-minutes: 90 concurrency: group: >- ${{ github.workflow }}-${{ github.event_name }}-${{ @@ -195,68 +195,17 @@ jobs: strategy: fail-fast: false matrix: - sandbox: - - 'sandbox:none' - - 'sandbox:docker' + sandbox: ${{ fromJSON(github.event_name == 'push' && '["sandbox:none"]' || '["sandbox:none","sandbox:docker"]') }} node-version: - '24.x' - exclude: - # On push to main, only run sandbox:none (skip docker) - - sandbox: ${{ github.event_name == 'push' && 'sandbox:docker' || 'NEVER_MATCH' }} steps: - - name: 'Checkout trusted quota selector' + - name: 'Checkout' uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # ratchet:actions/checkout@v7 + if: "github.event_name != 'pull_request_target' && github.event_name != 'pull_request'" with: - ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.base.sha || github.event.inputs.branch_ref || github.ref }} - - - name: 'Set up Node.js ${{ matrix.node-version }}' - uses: 'actions/setup-node@820762786026740c76f36085b0efc47a31fe5020' # ratchet:actions/setup-node@v7 - with: - node-version: '${{ matrix.node-version }}' - - - name: 'Setup Bun' - # The trusted quota selector runs under Bun before PR code is checked - # out, so Bun must be on PATH before that step. npm run build below is - # also backed by bun scripts/build.ts. - uses: 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' # ratchet:oven-sh/setup-bun@v2 - with: - bun-version-file: '.bun-version' - - - name: 'Check API quota and select optimal key' - id: 'quota' - shell: 'bash' - run: | - set -euo pipefail - quota_selectors=(scripts/ci-quota-check.*) - if (( ${#quota_selectors[@]} != 1 )) || [[ ! -f "${quota_selectors[0]}" ]]; then - echo 'Trusted checkout must contain exactly one quota selector' >&2 - exit 1 - fi - bun "${quota_selectors[0]}" - - # The trusted legacy selector writes the secret to GITHUB_ENV. Remove - # it before checking out PR code; downstream steps resolve one secret. - if [[ -f "$GITHUB_ENV" ]]; then - awk '!/^OPENAI_API_KEY=/' "$GITHUB_ENV" >"${GITHUB_ENV}.quota" - mv "${GITHUB_ENV}.quota" "$GITHUB_ENV" - fi - - if ! grep -Eq '^selected_key=(primary|secondary)$' "$GITHUB_OUTPUT"; then - if grep -q '^selected_key=' "$GITHUB_OUTPUT"; then - echo 'Trusted quota selector emitted an invalid selected_key' >&2 - exit 1 - fi - if [[ "${KEY_VAR_NAME:-}" == *SYNTHETIC* ]]; then - echo 'Trusted Synthetic quota selector did not emit selected_key' >&2 - exit 1 - fi - echo 'selected_key=primary' >>"$GITHUB_OUTPUT" - fi - env: - KEY_VAR_NAME: '${{ vars.KEY_VAR_NAME }}' - OPENAI_API_KEY: '${{ secrets[vars.KEY_VAR_NAME] }}' - OPENAI_API_KEY_2: '${{ secrets[vars.KEY_VAR_NAME_2] }}' + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.branch_ref || github.ref }} + persist-credentials: false - name: 'Checkout PR head (internal target)' uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # ratchet:actions/checkout@v7 @@ -274,6 +223,16 @@ jobs: persist-credentials: false clean: false + - name: 'Set up Node.js ${{ matrix.node-version }}' + uses: 'actions/setup-node@820762786026740c76f36085b0efc47a31fe5020' # ratchet:actions/setup-node@v7 + with: + node-version: '${{ matrix.node-version }}' + + - name: 'Setup Bun' + uses: 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' # ratchet:oven-sh/setup-bun@v2 + with: + bun-version-file: '.bun-version' + - name: 'Install dependencies' run: 'npm ci' @@ -284,39 +243,169 @@ jobs: if: "matrix.sandbox == 'sandbox:docker'" uses: 'docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c' # ratchet:docker/setup-buildx-action@v4 - - name: 'Validate E2E provider environment (Linux)' + - name: 'Install Ollama CPU runtime' + shell: 'bash' + run: | + set -euo pipefail + test "$(uname -m)" = 'x86_64' + mkdir -p "$RUNNER_TEMP/ollama" + curl --fail --show-error --location --retry 3 \ + 'https://ollama.com/download/ollama-linux-amd64.tar.zst?version=0.31.1' \ + -o "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" + printf '%s %s\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check - + tar --zstd \ + --exclude='lib/ollama/cuda_v12/*' \ + --exclude='lib/ollama/cuda_v13/*' \ + --exclude='lib/ollama/vulkan/*' \ + -xf "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" -C "$RUNNER_TEMP/ollama" + rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" + echo "$RUNNER_TEMP/ollama/bin" >>"$GITHUB_PATH" + "$RUNNER_TEMP/ollama/bin/ollama" --version + + - name: 'Select CPU backend on Intel' + shell: 'bash' + run: | + set -euo pipefail + echo "Host architecture: $(uname -m)" + echo "Host CPU vendor: $(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' /proc/cpuinfo)" + if ! grep -qm1 '^vendor_id[[:space:]]*: GenuineIntel$' /proc/cpuinfo; then + echo 'Host is not Intel; leaving Ollama CPU variants untouched' + exit 0 + fi + libdir="$RUNNER_TEMP/ollama/lib/ollama" + test -f "$libdir/libggml-cpu-haswell.so" + shopt -s nullglob + variants=("$libdir"/libggml-cpu-*.so) + (( ${#variants[@]} > 1 )) || { echo 'Expected multiple CPU variants in pinned Ollama runtime' >&2; exit 1; } + mkdir -p "$RUNNER_TEMP/ollama-disabled-cpu-backends" + for library in "${variants[@]}"; do + if [[ "$library" != "$libdir/libggml-cpu-haswell.so" ]]; then + mv "$library" "$RUNNER_TEMP/ollama-disabled-cpu-backends/" + fi + done + echo 'Intel Haswell pin: CPU variants visible to llama-server:' + printf '%s\n' "$libdir"/libggml-cpu-*.so + test "$(find "$libdir" -maxdepth 1 -name 'libggml-cpu-*.so' | wc -l)" -eq 1 + + - name: 'Start local Gemma 4 E2B model' + shell: 'bash' + env: + OLLAMA_HOST: '127.0.0.1:12644' + OLLAMA_CONTEXT_LENGTH: '32768' + OLLAMA_NUM_PARALLEL: '1' + OLLAMA_MODELS: '${{ runner.temp }}/ollama-models' + run: | + set -euo pipefail + mkdir -p "$OLLAMA_MODELS" + nohup ollama serve >"$RUNNER_TEMP/ollama-server.log" 2>&1 & + for _ in {1..60}; do + if curl --fail --silent "http://$OLLAMA_HOST/api/version"; then + break + fi + sleep 2 + done + curl --fail --silent "http://$OLLAMA_HOST/api/version" | jq -e '.version == "0.31.1"' + ollama pull gemma4:e2b-it-qat + curl --fail --silent "http://$OLLAMA_HOST/api/tags" | + jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .digest == "07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c")' + df -h "$RUNNER_TEMP" + + - name: 'Verify CPU backend warm-up' + shell: 'bash' + env: + OLLAMA_HOST: '127.0.0.1:12644' + run: | + set -euo pipefail + proc_root="${LLXPRT_PROC_ROOT:-/proc}" + vendor="$(awk -F: '/^vendor_id[[:space:]]*:/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' "$proc_root/cpuinfo")" + echo "Host CPU vendor: $vendor" + [[ "$vendor" == GenuineIntel || "$vendor" == AuthenticAMD ]] || { echo "Unsupported CPU vendor: $vendor" >&2; exit 1; } + curl --fail --show-error --silent --max-time 300 \ + "http://127.0.0.1:12644/api/generate" \ + -H 'Content-Type: application/json' \ + -d '{"model":"gemma4:e2b-it-qat","prompt":"Hello","stream":false,"think":false,"options":{"num_ctx":32768,"num_predict":16}}' \ + | jq -e '.done == true and .error == null and (.response | length > 0)' + grep 'system_info:.*CPU :' "$RUNNER_TEMP/ollama-server.log" | tail -1 || true + process="$RUNNER_TEMP/ollama/lib/ollama/llama-server" + pid="$(pgrep -f "^$process --model " | head -1)" || true + [[ -n "$pid" ]] || { echo 'Live llama-server process not found' >&2; exit 1; } + maps="$proc_root/$pid/maps" + [[ -r "$maps" ]] || { echo "Live llama-server maps are unavailable: $maps" >&2; exit 1; } + cmdline="$proc_root/$pid/cmdline" + status="$proc_root/$pid/status" + [[ -r "$cmdline" && -r "$status" ]] || { echo 'Live llama-server process evidence disappeared' >&2; exit 1; } + printf 'llama-server cmdline: ' + tr '\0' ' ' <"$cmdline" + echo + command_line="$(tr '\0' ' ' <"$cmdline")" + [[ "$command_line" == "$process --model "* ]] || { echo 'Matched PID is not the expected llama-server process' >&2; exit 1; } + ppid="$(awk '$1 == "PPid:" { print $2 }' "$status")" + [[ "$ppid" =~ ^[0-9]+$ && "$ppid" != 0 ]] || { echo 'llama-server parent PID is missing' >&2; exit 1; } + parent_cmdline="$proc_root/$ppid/cmdline" + [[ -r "$parent_cmdline" ]] || { echo 'llama-server parent process disappeared' >&2; exit 1; } + printf 'llama-server PPid: %s; parent cmdline: ' "$ppid" + tr '\0' ' ' <"$parent_cmdline" + echo + expected="$RUNNER_TEMP/ollama/lib/ollama/libggml-cpu-haswell.so" + maps_snapshot="$RUNNER_TEMP/llama-server-$pid.maps" + cat "$maps" >"$maps_snapshot" || { echo 'llama-server maps disappeared while being read' >&2; exit 1; } + cat "$maps_snapshot" + backends="$(grep -Eo '/[^ ]+/libggml-cpu(-[^ ]+)?\.so( \(deleted\))?' "$maps_snapshot" | sort -u || true)" + echo "Loaded CPU backends: ${backends:-none}" + [[ -n "$backends" ]] || { echo 'No mapped llama-server CPU backend' >&2; exit 1; } + if [[ "$vendor" == AuthenticAMD ]]; then + echo 'AMD: native Ollama CPU backend (variants untouched)' + exit 0 + fi + while IFS= read -r backend; do + [[ "$backend" == "$expected" ]] || { + echo "Unexpected mapped llama-server CPU library: $backend" >&2 + exit 1 + } + done <<< "$backends" + grep -Fxq "$expected" <<< "$backends" || { + echo 'Expected haswell CPU library is not mapped' >&2 + exit 1 + } + + - name: 'Verify CPU-only model inference' shell: 'bash' env: - OPENAI_API_KEY: ${{ steps.quota.outputs.selected_key == 'primary' && secrets[vars.KEY_VAR_NAME] || steps.quota.outputs.selected_key == 'secondary' && secrets[vars.KEY_VAR_NAME_2] || '' }} - OPENAI_BASE_URL: '${{ vars.OPENAI_BASE_URL }}' - LLXPRT_DEFAULT_MODEL: '${{ vars.LLXPRT_DEFAULT_MODEL }}' - LLXPRT_DEFAULT_PROVIDER: '${{ vars.LLXPRT_DEFAULT_PROVIDER }}' + OLLAMA_HOST: '127.0.0.1:12644' run: | set -euo pipefail - [[ -n "${OPENAI_API_KEY:-}" ]] || { echo "OPENAI_API_KEY missing after quota selection"; exit 1; } - [[ -n "${OPENAI_BASE_URL:-}" ]] || { echo "OPENAI_BASE_URL is required"; exit 1; } - [[ -n "${LLXPRT_DEFAULT_PROVIDER:-}" ]] || { echo "LLXPRT_DEFAULT_PROVIDER is required"; exit 1; } - [[ -n "${LLXPRT_DEFAULT_MODEL:-}" ]] || { echo "LLXPRT_DEFAULT_MODEL is required"; exit 1; } - echo "E2E env summary: provider=${LLXPRT_DEFAULT_PROVIDER}, model=${LLXPRT_DEFAULT_MODEL}, baseurl=${OPENAI_BASE_URL}" + curl --fail --show-error --silent --max-time 300 \ + "http://$OLLAMA_HOST/api/generate" \ + -H 'Content-Type: application/json' \ + -d '{"model":"gemma4:e2b-it-qat","prompt":"Reply with one word.","stream":false,"think":false,"options":{"num_ctx":32768,"num_predict":16}}' \ + -o "$RUNNER_TEMP/ollama-inference.json" + jq -e '.done == true and .error == null and (.response | length > 0)' "$RUNNER_TEMP/ollama-inference.json" || { + cat "$RUNNER_TEMP/ollama-inference.json" >&2 + exit 1 + } + curl --fail --show-error --silent "http://$OLLAMA_HOST/api/ps" | \ + jq -e '.models[] | select(.name == "gemma4:e2b-it-qat" and .size_vram == 0)' - name: 'Run E2E tests' env: - # Provider configuration from repository secrets/variables - OPENAI_API_KEY: ${{ steps.quota.outputs.selected_key == 'primary' && secrets[vars.KEY_VAR_NAME] || steps.quota.outputs.selected_key == 'secondary' && secrets[vars.KEY_VAR_NAME_2] || '' }} - KEY_VAR_NAME: '${{ vars.KEY_VAR_NAME }}' - OPENAI_BASE_URL: '${{ vars.OPENAI_BASE_URL }}' - LLXPRT_DEFAULT_MODEL: '${{ vars.LLXPRT_DEFAULT_MODEL }}' - LLXPRT_DEFAULT_PROVIDER: '${{ vars.LLXPRT_DEFAULT_PROVIDER }}' - # Set auth type to provider for API key authentication + OPENAI_API_KEY: 'ollama-local-only' + OPENAI_BASE_URL: 'http://127.0.0.1:12644/v1' + LLXPRT_DEFAULT_MODEL: 'gemma4:e2b-it-qat' + LLXPRT_DEFAULT_PROVIDER: 'openai' LLXPRT_AUTH_TYPE: 'provider' + LLXPRT_TEST_PROFILE: 'local-gemma4-e2e' + LLXPRT_CONTEXT_LIMIT: '32768' + LLXPRT_MAX_OUTPUT_TOKENS: '8192' + LLXPRT_LOCAL_MODEL_E2E: 'true' + GIT_CEILING_DIRECTORIES: '${{ github.workspace }}/.integration-tests' + SANDBOX_FLAGS: "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}" KEEP_OUTPUT: 'true' VERBOSE: 'true' - # Force file storage to avoid keytar/libsecret dependency in CI LLXPRT_FORCE_FILE_STORAGE: 'true' - # Record real-provider runs for budget enforcement (issue #2278) LLXPRT_E2E_MODEL_LEDGER: '${{ runner.temp }}/e2e-model-ledger.jsonl' shell: 'bash' run: | + set -euo pipefail if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then npm run test:integration:sandbox:docker -- \ --exclude="**/todo-continuation.e2e.test.ts" \ @@ -342,3 +431,33 @@ jobs: shell: 'bash' run: |- bun scripts/check-e2e-model-budget.ts --ledger "$LLXPRT_E2E_MODEL_LEDGER" + + - name: 'Report local model resources' + if: always() + shell: 'bash' + run: | + if curl --fail --silent http://127.0.0.1:12644/api/ps; then + echo + else + echo 'Local Ollama server is unavailable' + fi + free -h + df -h "$RUNNER_TEMP" + du -sh "$RUNNER_TEMP/ollama-models" "$RUNNER_TEMP/ollama" 2>/dev/null || true + if [[ "${{ matrix.sandbox }}" == 'sandbox:docker' ]]; then + docker info --format 'Docker server: {{.ServerVersion}}' + fi + + - name: 'Upload local model diagnostics' + if: always() + uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # ratchet:actions/upload-artifact@v7 + with: + name: "local-gemma4-e2e-${{ matrix.sandbox == 'sandbox:docker' && 'docker' || 'none' }}" + if-no-files-found: error + include-hidden-files: true + path: | + ${{ runner.temp }}/ollama-server.log + ${{ runner.temp }}/ollama-inference.json + ${{ runner.temp }}/e2e-model-ledger.jsonl + ${{ github.workspace }}/.integration-tests/*/*/telemetry.log + ${{ github.workspace }}/.integration-tests/*/*/harness-diagnostics.log diff --git a/dev-docs/integration-tests.md b/dev-docs/integration-tests.md index 70c4c84f2f..b39b86eb1c 100644 --- a/dev-docs/integration-tests.md +++ b/dev-docs/integration-tests.md @@ -165,10 +165,81 @@ This structure makes it easy to locate the artifacts for a specific test run, fi ## Continuous integration -To ensure the integration tests are always run, a GitHub Actions workflow is defined in `.github/workflows/e2e.yml`. This workflow automatically runs the integrations tests for pull requests against the `main` branch, or when a pull request is added to a merge queue. +`.github/workflows/e2e.yml` runs integration tests for internal pull requests against `main`, `release/**` and `dev/**`, and for merge groups. The workflow runs the tests in different sandboxing environments to ensure LLxprt Code is tested across each: - `sandbox:none`: Runs the tests without any sandboxing. - `sandbox:docker`: Runs the tests in a Docker container. -- `sandbox:podman`: Runs the tests in a Podman container. + +### Runner-local model E2E (issue #3764) + +`.github/workflows/e2e.yml` runs one required Linux x64 job with runner-local +Ollama 0.31.1 and `gemma4:e2b-it-qat`. Internal PRs, merge groups and manual +`workflow_dispatch` run both host (`sandbox:none`) and Docker +(`sandbox:docker`) legs; pushes to main run only the host leg. Dispatch accepts +`branch_ref` and runs the same job. Pull requests check out their merge ref, +approved labeled internal `pull_request_target` runs check out the head SHA, +and manual runs check out `branch_ref`; checkout never persists credentials. +Fork PR heads do not run under the privileged target event or receive this E2E +job. Duplicate and doc-only filters and the internal-target mergeability gate +remain in effect. Credentialed full-platform E2E remains in `nightly.yml`. + +The workflow verifies the x64 Ollama archive SHA-256 +`d297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4`, +Ollama version 0.31.1 and Gemma digest +`07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c`. +On Intel it exposes only the Haswell CPU backend before startup and verifies +that library in the live llama-server process mapping. On AMD it retains the +native Ollama backend and verifies a mapped CPU library. A warm-up and a second +CPU-only inference check require a nonempty response and zero VRAM usage. +The Docker leg uses host networking to reach the Ollama loopback endpoint. +No third-party model credentials are available: the OpenAI-compatible `/v1` +endpoint uses a local placeholder key. + +Each leg runs the selected integration suite excluding +`todo-continuation.e2e.test.ts` and `run_shell_command.test.ts`, followed by +exactly three named shell cases. The shell assertion permits only successful +literal `echo hello-world` commands; replace requires the `replace` tool, +exact `bar content`, and no unrelated tools. File-level retries stay enabled. +`GIT_CEILING_DIRECTORIES` prevents incidental parent repository discovery in +TestRig workspaces, but does not prevent a model given an explicit path from +accessing it. The real-model ledger enforces a four-request ceiling +across two distinct tests; retries are reported separately. Logs, +telemetry, inference JSON and ledger are uploaded even on failure. + +The local-model E2E profile sets a 32,768-token context and reserves 8,192 +output tokens. Only when `LLXPRT_LOCAL_MODEL_E2E=true`, the generated inline +profile extends first-response, SDK request and transport headers deadlines +to 750,000, 850,000 and 900,000 ms respectively. The shell case has a +360,000 ms TestRig deadline and 450,000 ms Bun timeout; replace has a +1,200,000 ms TestRig deadline and 1,500,000 ms Bun timeout. Each job has a +90-minute bound. Other providers, local runs and nightly E2E keep their +ordinary deadlines. The inline profile controls provider selection so explicit +`--provider` flags cannot bypass its timeout settings. + +To reproduce a run on a branch, use +`gh workflow run e2e.yml --ref issue3764 -f branch_ref=issue3764`. +Do not dispatch while another run on that branch is active: branch concurrency +cancels an earlier run. Verify both sandbox check names, runtime and model +pins, CPU mapping, selected suites, budget and diagnostics. To reproduce the +test selection locally, set `OPENAI_BASE_URL` to a private Ollama 0.31.1 +instance, `OPENAI_API_KEY=ollama-local-only`, +`LLXPRT_DEFAULT_PROVIDER=openai`, +`LLXPRT_DEFAULT_MODEL=gemma4:e2b-it-qat`, +`LLXPRT_TEST_PROFILE=local-gemma4-e2e`, `LLXPRT_CONTEXT_LIMIT=32768`, +`LLXPRT_MAX_OUTPUT_TOKENS=8192`, and `LLXPRT_LOCAL_MODEL_E2E=true`. +Run `npm run test:integration:sandbox:none -- +--exclude="**/todo-continuation.e2e.test.ts" +--exclude="**/run_shell_command.test.ts"` and then the three named shell +cases with the same invocations as the workflow. Keep local evidence under the +repository's ignored `tmp/` directory. + +Earlier local and hosted trials with smaller models did not consistently +produce the required exact replacement. An ARM trial returned an empty +response after its first 16 generated tokens. Two hosted x64 Gemma runs +[36342130687](https://github.com/vybestack/llxprt-code/actions/runs/36342130687) +and [36344243429](https://github.com/vybestack/llxprt-code/actions/runs/36344243429) +passed both legs with the selected files, shell cases, CPU verification and +budget. These results are specific to the pinned x64 runtime and model; they +do not establish equivalence with the credentialed nightly full-platform suite. diff --git a/integration-tests/replace.test.ts b/integration-tests/replace.test.ts index 12234b9d7a..7ec60f9e0f 100644 --- a/integration-tests/replace.test.ts +++ b/integration-tests/replace.test.ts @@ -5,8 +5,12 @@ */ import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; +import { env } from 'node:process'; import { TestRig } from './test-helper.js'; +const localModelE2E = env['LLXPRT_LOCAL_MODEL_E2E'] === 'true'; +const replaceTestOptions = localModelE2E ? { timeout: 1_500_000 } : {}; + describe('replace', () => { let rig: TestRig; @@ -15,24 +19,45 @@ describe('replace', () => { }); afterEach(async () => await rig.cleanup()); - it('should be able to replace content in a file', async () => { - await rig.setup('should be able to replace content in a file', { - settings: { tools: { core: ['replace', 'read_file'] } }, - }); - - const fileName = 'file_to_replace.txt'; - const originalContent = 'foo content'; - const expectedContent = 'bar content'; - - rig.createFile(fileName, originalContent); - - await rig.run({ - args: `Replace 'foo' with 'bar' in the file 'file_to_replace.txt'`, - }); - - const foundToolCall = await rig.waitForToolCall('replace'); - expect(foundToolCall, 'Expected to find a replace tool call').toBeTruthy(); - - expect(rig.readFile(fileName)).toBe(expectedContent); - }); + it( + 'should be able to replace content in a file', + replaceTestOptions, + async () => { + await rig.setup('should be able to replace content in a file', { + settings: { + tools: { + core: ['replace', 'read_file'], + exclude: ['run_shell_command'], + }, + }, + }); + + const fileName = 'file_to_replace.txt'; + const originalContent = 'foo content'; + const expectedContent = 'bar content'; + + const filePath = rig.createFile(fileName, originalContent); + + await rig.run({ + args: `Use the replace tool on '${filePath}' to replace the exact text 'foo content' with 'bar content'. Do not add any whitespace.`, + timeoutMs: localModelE2E ? 1_200_000 : undefined, + }); + + const foundToolCall = await rig.waitForToolCall('replace'); + expect( + foundToolCall, + 'Expected to find a replace tool call', + ).toBeTruthy(); + expect( + rig + .readToolLogs() + .filter( + (log) => !['replace', 'read_file'].includes(log.toolRequest.name), + ), + 'Text replacement must not invoke unrelated tools', + ).toHaveLength(0); + + expect(rig.readFile(fileName)).toBe(expectedContent); + }, + ); }); diff --git a/integration-tests/run_shell_command.test.ts b/integration-tests/run_shell_command.test.ts index 87dff22db9..dc2cca2988 100644 --- a/integration-tests/run_shell_command.test.ts +++ b/integration-tests/run_shell_command.test.ts @@ -6,10 +6,18 @@ import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; import { join } from 'node:path'; +import { env } from 'node:process'; import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js'; import { getShellConfiguration } from '../packages/core/src/utils/shell-utils.js'; const { shell } = getShellConfiguration(); +const localModelE2E = env['LLXPRT_LOCAL_MODEL_E2E'] === 'true'; + +function shellE2ERunTimeoutMs(localModelE2E: boolean): number | undefined { + // Docker's observed 238s model response leaves no room for a tool round trip + // under the original 240s TestRig deadline. + return localModelE2E ? 360_000 : undefined; +} function getLineCountCommand(): { command: string; tool: string } { switch (shell) { @@ -22,6 +30,96 @@ function getLineCountCommand(): { command: string; tool: string } { } } +function isExpectedEchoCommand(args: string): boolean { + const request: unknown = JSON.parse(args); + if ( + typeof request !== 'object' || + request === null || + !('command' in request) + ) { + return false; + } + const command = request.command; + return ( + typeof command === 'string' && + new Set([ + 'echo hello-world', + 'echo "hello-world"', + "echo 'hello-world'", + ]).has(command) + ); +} + +function isExpectedShellTrace( + logs: ReadonlyArray<{ + toolRequest: { name: string; args: string; success?: boolean }; + }>, +): boolean { + return ( + logs.length > 0 && + logs.every( + ({ toolRequest }) => + toolRequest.name === 'run_shell_command' && + isExpectedEchoCommand(toolRequest.args) && + toolRequest.success === true, + ) + ); +} + +describe('local-model E2E shell command validation', () => { + it('extends only the local model E2E shell run deadline for the CPU tool round trip', () => { + expect(shellE2ERunTimeoutMs(true)).toBe(360_000); + expect(shellE2ERunTimeoutMs(false)).toBeUndefined(); + }); + + it('accepts equivalent literal echo commands but not other shell expressions', () => { + for (const command of [ + 'echo hello-world', + 'echo "hello-world"', + "echo 'hello-world'", + ]) { + expect(isExpectedEchoCommand(JSON.stringify({ command }))).toBe(true); + } + expect( + isExpectedEchoCommand(JSON.stringify({ command: 'echo goodbye' })), + ).toBe(false); + expect( + isExpectedEchoCommand( + JSON.stringify({ command: 'echo hello-world; git commit -am oops' }), + ), + ).toBe(false); + }); + + it('rejects wrong or additional tools even when the echo succeeded', () => { + const echo = { + toolRequest: { + name: 'run_shell_command', + args: JSON.stringify({ command: 'echo hello-world' }), + success: true, + }, + }; + expect(isExpectedShellTrace([echo])).toBe(true); + expect( + isExpectedShellTrace([ + echo, + { toolRequest: { name: 'read_file', args: '{}', success: true } }, + ]), + ).toBe(false); + expect( + isExpectedShellTrace([ + echo, + { + toolRequest: { + name: 'run_shell_command', + args: JSON.stringify({ command: 'git status' }), + success: true, + }, + }, + ]), + ).toBe(false); + }); +}); + describe('run_shell_command', () => { let rig: TestRig; @@ -30,38 +128,52 @@ describe('run_shell_command', () => { }); afterEach(async () => await rig.cleanup()); - it('should be able to run a shell command', async () => { - await rig.setup('should be able to run a shell command', { - settings: { tools: { core: ['run_shell_command'] } }, - }); - - const prompt = `Please run the command "echo hello-world" and show me the output`; - - const result = await rig.run({ args: prompt }); + it( + 'should be able to run a shell command', + { timeout: localModelE2E ? 450_000 : undefined }, + async () => { + await rig.setup('should be able to run a shell command', { + settings: { tools: { core: ['run_shell_command'] } }, + }); - const foundToolCall = await rig.waitForToolCall('run_shell_command'); + const prompt = `Use the run_shell_command tool to execute exactly "echo hello-world". Show me the output.`; - // Add debugging information - if (!foundToolCall || !result.includes('hello-world')) { - printDebugInfo(rig, result, { - 'Found tool call': foundToolCall, - 'Contains hello-world': result.includes('hello-world'), + const result = await rig.run({ + args: ['--allowed-tools=run_shell_command(echo)'], + stdin: prompt, + yolo: false, + timeoutMs: shellE2ERunTimeoutMs(localModelE2E), }); - } - expect( - foundToolCall, - 'Expected to find a run_shell_command tool call', - ).toBeTruthy(); + const foundToolCall = await rig.waitForToolCall('run_shell_command'); - // Validate model output - will throw if no output, warn if missing expected content - // Model often reports exit code instead of showing output - validateModelOutput( - result, - ['hello-world', 'exit code 0'], - 'Shell command test', - ); - }); + // Add debugging information + if (!foundToolCall || !result.includes('hello-world')) { + printDebugInfo(rig, result, { + 'Found tool call': foundToolCall, + 'Contains hello-world': result.includes('hello-world'), + }); + } + + expect( + foundToolCall, + 'Expected to find a run_shell_command tool call', + ).toBeTruthy(); + expect( + isExpectedShellTrace(rig.readToolLogs()), + 'Expected only successful echo hello-world shell invocations', + ).toBe(true); + expect(result).toContain('hello-world'); + + // Validate model output - will throw if no output, warn if missing expected content + // Model often reports exit code instead of showing output + validateModelOutput( + result, + ['hello-world', 'exit code 0'], + 'Shell command test', + ); + }, + ); it('should be able to run a shell command via stdin', async () => { await rig.setup('should be able to run a shell command via stdin', { diff --git a/integration-tests/sandbox-git-ceiling.real.test.ts b/integration-tests/sandbox-git-ceiling.real.test.ts new file mode 100644 index 0000000000..466fd20c1d --- /dev/null +++ b/integration-tests/sandbox-git-ceiling.real.test.ts @@ -0,0 +1,136 @@ +/** + * @license + * Copyright 2026 Vybestack LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'bun:test'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, realpathSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + addContainerEnvVars, + buildContainerRunArgs, +} from '../packages/cli/src/utils/sandbox-containers.js'; + +const image = 'ghcr.io/vybestack/llxprt-code/sandbox:0.11.0'; +const runtime = process.env.LLXPRT_SANDBOX; + +function dockerReady(): boolean { + if (runtime !== 'docker') return false; + try { + execFileSync('docker', ['info'], { stdio: 'ignore', timeout: 20_000 }); + execFileSync('docker', ['image', 'inspect', image], { + stdio: 'ignore', + timeout: 20_000, + }); + return true; + } catch { + return false; + } +} + +function gitRoot(dir: string, env: NodeJS.ProcessEnv): string { + return execFileSync('git', ['-C', dir, 'rev-parse', '--show-toplevel'], { + env, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); +} + +describe('TestRig Git discovery boundary', () => { + it('blocks discovery of the host checkout from a nested test directory', () => { + const root = realpathSync( + mkdtempSync(join(tmpdir(), 'llxprt-git-ceiling-')), + ); + const ceiling = join(root, '.integration-tests'); + const testDir = join(ceiling, 'run', 'test'); + try { + mkdirSync(testDir, { recursive: true }); + execFileSync('git', ['init', '-q', root]); + const env = { ...process.env }; + delete env.GIT_CEILING_DIRECTORIES; + expect(gitRoot(testDir, env)).toBe(root); + expect(() => + gitRoot(testDir, { ...env, GIT_CEILING_DIRECTORIES: ceiling }), + ).toThrow(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it.skipIf(!dockerReady())( + 'blocks discovery of a bind-mounted parent repo inside the real Docker sandbox', + () => { + const root = realpathSync( + mkdtempSync(join(tmpdir(), 'llxprt-docker-git-ceiling-')), + ); + const ceiling = join(root, '.integration-tests'); + const testDir = join(ceiling, 'run', 'test'); + const sessionDir = join(root, 'session'); + const originalCeiling = process.env.GIT_CEILING_DIRECTORIES; + try { + mkdirSync(testDir, { recursive: true }); + mkdirSync(sessionDir); + execFileSync('git', ['init', '-q', root]); + const config = { command: 'docker', image } as const; + const containerArgs = buildContainerRunArgs( + config, + image, + root, + root, + sessionDir, + ).filter((arg) => arg !== '-t'); + containerArgs[containerArgs.indexOf('--workdir') + 1] = testDir; + delete process.env.GIT_CEILING_DIRECTORIES; + const baselineArgs = [...containerArgs]; + addContainerEnvVars(baselineArgs, config, 'git-ceiling-test', [], root); + expect( + execFileSync( + 'docker', + [ + ...baselineArgs, + '--entrypoint', + 'sh', + image, + '-c', + 'git rev-parse --show-toplevel', + ], + { encoding: 'utf8', timeout: 30_000 }, + ).trim(), + ).toBe(root); + process.env.GIT_CEILING_DIRECTORIES = ceiling; + addContainerEnvVars( + containerArgs, + config, + 'git-ceiling-test', + [], + root, + ); + const args = [ + ...containerArgs, + '--entrypoint', + 'sh', + image, + '-c', + 'git rev-parse --show-toplevel', + ]; + expect(() => + execFileSync('docker', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 30_000, + }), + ).toThrow(); + } finally { + if (originalCeiling === undefined) { + delete process.env.GIT_CEILING_DIRECTORIES; + } else { + process.env.GIT_CEILING_DIRECTORIES = originalCeiling; + } + rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/packages/cli/src/utils/sandbox-containers.ts b/packages/cli/src/utils/sandbox-containers.ts index 02802ece9c..fe015dc0b4 100644 --- a/packages/cli/src/utils/sandbox-containers.ts +++ b/packages/cli/src/utils/sandbox-containers.ts @@ -272,6 +272,7 @@ const RESERVED_SANDBOX_ENV_KEYS = new Set([ // marker gate) at attacker-chosen paths inside the container. 'LLXPRT_SANDBOX_PROJECT_KEY', 'LLXPRT_SANDBOX_CHECKPOINT_STORE', + 'GIT_CEILING_DIRECTORIES', ]); function parseSandboxEnvVars(): string[] { @@ -340,6 +341,11 @@ export function addContainerEnvVars( GEMINI_MODEL: process.env.GEMINI_MODEL, TERM: process.env.TERM, COLORTERM: process.env.COLORTERM, + GIT_CEILING_DIRECTORIES: process.env.GIT_CEILING_DIRECTORIES?.split( + path.delimiter, + ) + .map(getContainerPath) + .join(':'), }; for (const [key, val] of Object.entries(envMap)) { if (val !== undefined) { diff --git a/packages/cli/src/utils/sandbox-git-ceiling.test.ts b/packages/cli/src/utils/sandbox-git-ceiling.test.ts new file mode 100644 index 0000000000..e284641388 --- /dev/null +++ b/packages/cli/src/utils/sandbox-git-ceiling.test.ts @@ -0,0 +1,46 @@ +/** + * @license + * Copyright 2026 Vybestack LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, describe, expect, it } from 'bun:test'; +import { FatalSandboxError } from '@vybestack/llxprt-code-core'; +import { addContainerEnvVars } from './sandbox-containers.js'; + +const config = { command: 'docker', image: 'test' } as const; +const originalCeiling = process.env.GIT_CEILING_DIRECTORIES; +const originalSandboxEnv = process.env.SANDBOX_ENV; + +describe('container Git discovery ceiling', () => { + afterEach(() => { + if (originalCeiling === undefined) { + delete process.env.GIT_CEILING_DIRECTORIES; + } else { + process.env.GIT_CEILING_DIRECTORIES = originalCeiling; + } + if (originalSandboxEnv === undefined) { + delete process.env.SANDBOX_ENV; + } else { + process.env.SANDBOX_ENV = originalSandboxEnv; + } + }); + + it('passes the host ceiling into the container', () => { + process.env.GIT_CEILING_DIRECTORIES = '/workspace/.integration-tests'; + delete process.env.SANDBOX_ENV; + const args: string[] = []; + addContainerEnvVars(args, config, 'git-ceiling-test', [], '/workspace'); + expect(args).toContain( + 'GIT_CEILING_DIRECTORIES=/workspace/.integration-tests', + ); + }); + + it('rejects an override that could detach the pinned Git ceiling', () => { + process.env.GIT_CEILING_DIRECTORIES = '/workspace/.integration-tests'; + process.env.SANDBOX_ENV = 'GIT_CEILING_DIRECTORIES=/'; + expect(() => + addContainerEnvVars([], config, 'git-ceiling-test', [], '/workspace'), + ).toThrow(FatalSandboxError); + }); +}); diff --git a/packages/providers/src/openai/OpenAIClientFactory.test.ts b/packages/providers/src/openai/OpenAIClientFactory.test.ts index e5da269b45..739e1326d0 100644 --- a/packages/providers/src/openai/OpenAIClientFactory.test.ts +++ b/packages/providers/src/openai/OpenAIClientFactory.test.ts @@ -15,8 +15,12 @@ */ import { describe, it, expect } from 'bun:test'; +import { createServer } from 'node:http'; import * as http from 'http'; import * as https from 'https'; +import { createRequire } from 'node:module'; +import type * as Undici from 'undici'; +import type { Agent as UndiciAgent } from 'undici'; import { createHttpAgents, extractModelParamsFromOptions, @@ -26,6 +30,11 @@ import { } from './OpenAIClientFactory.js'; import type { NormalizedGenerateChatOptions } from '../BaseProvider.js'; +const undiciRequire = createRequire( + createRequire(import.meta.url).resolve('undici/package.json'), +); +const { Agent, fetch: undiciFetch } = undiciRequire('./') as typeof Undici; + function getMaxOutputTokens(key: string): number | undefined { return key === 'maxOutputTokens' ? 2000 : undefined; } @@ -324,6 +333,80 @@ describe('OpenAIClientFactory', () => { expect(opts).toBeDefined(); expect(opts.defaultHeaders).toBeUndefined(); }); + + it('keeps the ordinary headers deadline while a scoped extension lets the SDK stream delayed headers', async () => { + const server = createServer((_request, response) => { + setTimeout(() => { + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end( + 'data: {"id":"delayed","choices":[{"index":0,"delta":{"content":"ready"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', + ); + }, 1_500); + }); + const defaultDispatcher = new Agent({ headersTimeout: 100 }); + const testFetch: typeof fetch = async (input, init) => { + const requestInit = init as RequestInit & { dispatcher?: UndiciAgent }; + const response = await undiciFetch(String(input), { + ...requestInit, + dispatcher: requestInit.dispatcher ?? defaultDispatcher, + } as Parameters[1]); + return response as unknown as Response; + }; + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Missing local server port'); + } + const baseURL = `http://127.0.0.1:${address.port}/v1`; + const request = { model: 'local', messages: [], stream: true as const }; + const consume = async ( + client: ReturnType, + ): Promise> => { + const chunks = []; + for await (const chunk of await client.chat.completions.create( + request, + )) { + chunks.push(chunk.choices[0]?.delta.content); + } + return chunks; + }; + const ordinary = instantiateClient( + 'test-token', + baseURL, + undefined, + undefined, + { + fetch: testFetch, + }, + ); + await expect(consume(ordinary)).rejects.toThrow('Request timed out.'); + + const shortPilot = instantiateClient( + 'test-token', + baseURL, + undefined, + undefined, + { headersTimeoutMs: 100 }, + ); + await expect(consume(shortPilot)).rejects.toThrow('Request timed out.'); + + const pilot = instantiateClient( + 'test-token', + baseURL, + undefined, + undefined, + { headersTimeoutMs: 4_000 }, + ); + expect(await consume(pilot)).toStrictEqual(['ready']); + } finally { + server.closeAllConnections(); + server.close(); + await defaultDispatcher.close(); + } + }); }); describe('mergeInvocationHeaders', () => { diff --git a/packages/providers/src/openai/OpenAIClientFactory.ts b/packages/providers/src/openai/OpenAIClientFactory.ts index 5aa4cb038d..bd665c2a4f 100644 --- a/packages/providers/src/openai/OpenAIClientFactory.ts +++ b/packages/providers/src/openai/OpenAIClientFactory.ts @@ -26,10 +26,20 @@ import OpenAI from 'openai'; import * as http from 'http'; import * as https from 'https'; import * as net from 'net'; +import { createRequire } from 'node:module'; +import { Readable } from 'node:stream'; +import type * as Undici from 'undici'; import type { NormalizedGenerateChatOptions } from '../BaseProvider.js'; import { OPENAI_TRANSPORT_SELECTOR_KEYS } from './openaiModelPolicy.js'; import { createReaderBasedStreamFetch } from './openaiStreamFetchSafety.js'; +// Bun's 'undici' shim does not apply dispatcher headersTimeout; load the +// installed package's public entry instead of the shim. +const undiciRequire = createRequire( + createRequire(import.meta.url).resolve('undici/package.json'), +); +const { request: undiciRequest } = undiciRequire('./') as typeof Undici; + /** * Create HTTP/HTTPS agents with socket configuration for local AI servers * Returns undefined if no socket settings are configured @@ -163,6 +173,52 @@ export function resolveRuntimeKey( return 'openai.runtime.unscoped'; } +function fetchWithHeadersTimeout(headersTimeoutMs: number): typeof fetch { + if (!Number.isSafeInteger(headersTimeoutMs) || headersTimeoutMs <= 0) { + throw new Error('openai-headers-timeout-ms must be a positive integer'); + } + return async (input, init) => { + if (input instanceof Request) { + throw new Error('Scoped OpenAI transport requires a URL'); + } + const body = init?.body; + if ( + body !== undefined && + body !== null && + typeof body !== 'string' && + !(body instanceof ReadableStream) + ) { + throw new Error('Scoped OpenAI transport requires a JSON body'); + } + const result = await undiciRequest(String(input), { + method: init?.method ?? 'GET', + headers: Object.fromEntries(new Headers(init?.headers)), + body: + body instanceof ReadableStream + ? Readable.fromWeb( + body as unknown as Parameters[0], + ) + : body, + signal: init?.signal ?? undefined, + headersTimeout: headersTimeoutMs, + }); + const headers = new Headers(); + for (const [key, value] of Object.entries(result.headers)) { + if (Array.isArray(value)) { + for (const item of value) headers.append(key, item); + } else if (value !== undefined) { + headers.append(key, String(value)); + } + } + return new Response( + result.statusCode === 204 + ? null + : (Readable.toWeb(result.body) as ReadableStream), + { status: result.statusCode, headers }, + ); + }; +} + /** * @plan:PLAN-20251023-STATELESS-HARDENING.P09 * @requirement:REQ-SP4-002 @@ -173,13 +229,31 @@ export function instantiateClient( baseURL?: string, agents?: { httpAgent: http.Agent; httpsAgent: https.Agent }, headers?: Record, + transport?: { + headersTimeoutMs?: number; + requestTimeoutMs?: number; + fetch?: typeof fetch; + }, ): OpenAI { const clientOptions: Record = { apiKey: authToken || '', maxRetries: 0, }; + if (transport?.requestTimeoutMs !== undefined) { + if ( + !Number.isSafeInteger(transport.requestTimeoutMs) || + transport.requestTimeoutMs <= 0 + ) { + throw new Error('openai-request-timeout-ms must be a positive integer'); + } + clientOptions.timeout = transport.requestTimeoutMs; + } - clientOptions.fetch = createReaderBasedStreamFetch(); + const scopedFetch = + transport?.headersTimeoutMs === undefined + ? transport?.fetch + : fetchWithHeadersTimeout(transport.headersTimeoutMs); + clientOptions.fetch = createReaderBasedStreamFetch(scopedFetch); if (headers && Object.keys(headers).length > 0) { // Ensure headers like User-Agent are applied even if the SDK call-site diff --git a/packages/providers/src/openai/OpenAIProvider.ts b/packages/providers/src/openai/OpenAIProvider.ts index 877f565947..e8ac98cd0a 100644 --- a/packages/providers/src/openai/OpenAIProvider.ts +++ b/packages/providers/src/openai/OpenAIProvider.ts @@ -280,7 +280,14 @@ export class OpenAIProvider extends BaseProvider implements IProvider { // based on User-Agent, which must be sent as a real HTTP header. const headers = mergeInvocationHeaders(options); - return instantiateClient(authToken, baseURL, agents, headers); + return instantiateClient(authToken, baseURL, agents, headers, { + headersTimeoutMs: agentSettings['openai-headers-timeout-ms'] as + | number + | undefined, + requestTimeoutMs: agentSettings['openai-request-timeout-ms'] as + | number + | undefined, + }); } /** diff --git a/packages/test-utils/src/cli-args.test.ts b/packages/test-utils/src/cli-args.test.ts index 7e5e55da6c..650b19611b 100644 --- a/packages/test-utils/src/cli-args.test.ts +++ b/packages/test-utils/src/cli-args.test.ts @@ -35,6 +35,24 @@ describe('cli-args helpers', () => { ]); }); + it('uses fake responses without provider credentials even with local E2E selected', () => { + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', undefined); + setEnv('LLXPRT_DEFAULT_PROVIDER', undefined); + setEnv('OPENAI_API_KEY', undefined); + const fakePath = path.join(os.tmpdir(), 'fake-responses.json'); + + expect(() => assertProviderConfig(fakePath)).not.toThrow(); + expect(buildExtraArgs(fakePath, false)).toStrictEqual([ + '--ide-mode', + 'disable', + '--provider', + 'fake', + '--model', + 'fake-model', + ]); + }); + it('requires non-empty provider configuration without fake responses', () => { setEnv('LLXPRT_DEFAULT_PROVIDER', ''); setEnv('LLXPRT_DEFAULT_MODEL', 'model'); @@ -74,6 +92,30 @@ describe('cli-args helpers', () => { ]); }); + it('uses only the generated profile to configure the local model E2E', () => { + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-qwen35-e2e'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:4b'); + setEnv('OPENAI_BASE_URL', 'http://127.0.0.1:12644/v1'); + setEnv('OPENAI_API_KEY', 'local-only'); + + expect(buildExtraArgs(undefined, true)).toStrictEqual([ + '--yolo', + '--ide-mode', + 'disable', + ]); + }); + + it('rejects a local model E2E without a profile', () => { + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', undefined); + + expect(() => buildExtraArgs(undefined, true)).toThrow( + 'LLXPRT_TEST_PROFILE is required for the local model E2E', + ); + }); + it('builds child env without IDE detection variables and with fake response path', () => { setEnv('TERM_PROGRAM', 'vscode'); setEnv('TERM_PROGRAM_VERSION', '1.0.0'); diff --git a/packages/test-utils/src/cli-args.ts b/packages/test-utils/src/cli-args.ts index 0fced34ca9..348fe6f46e 100644 --- a/packages/test-utils/src/cli-args.ts +++ b/packages/test-utils/src/cli-args.ts @@ -53,6 +53,17 @@ export function buildExtraArgs( return extraArgs; } + // The inline local model E2E profile owns provider, credentials and runtime ephemerals. + // An explicit --provider prevents the CLI from applying those ephemerals. + if (env['LLXPRT_LOCAL_MODEL_E2E'] === 'true') { + if (getProfileName() === undefined) { + throw new Error( + 'LLXPRT_TEST_PROFILE is required for the local model E2E', + ); + } + return extraArgs; + } + const provider = requireEnvString('LLXPRT_DEFAULT_PROVIDER'); const model = requireEnvString('LLXPRT_DEFAULT_MODEL'); const baseUrl = readEnvString('OPENAI_BASE_URL'); diff --git a/packages/test-utils/src/test-rig-setup.ts b/packages/test-utils/src/test-rig-setup.ts index 0e519bb6db..52336cabf6 100644 --- a/packages/test-utils/src/test-rig-setup.ts +++ b/packages/test-utils/src/test-rig-setup.ts @@ -175,7 +175,11 @@ function buildProfile(): Record { version: 1, provider: profileProvider, model: profileModel, - modelParams: {}, + modelParams: + env['LLXPRT_LOCAL_MODEL_E2E'] === 'true' && + env['LLXPRT_LOCAL_MODEL_THINKING'] === 'none' + ? { reasoning_effort: 'none' } + : {}, ephemeralSettings: Object.fromEntries( ephemeralEntries.filter(([, value]) => value !== undefined), ), @@ -216,6 +220,20 @@ function collectEphemeralEntries(): Array<[string, unknown]> { } } + const maxOutputTokens = env['LLXPRT_MAX_OUTPUT_TOKENS']; + if (maxOutputTokens !== undefined) { + const parsedMaxOutputTokens = Number(maxOutputTokens); + if (Number.isFinite(parsedMaxOutputTokens) && parsedMaxOutputTokens > 0) { + entries.push(['maxOutputTokens', parsedMaxOutputTokens]); + } + } + + if (env['LLXPRT_LOCAL_MODEL_E2E'] === 'true') { + entries.push(['stream-first-response-timeout-ms', 750_000]); + entries.push(['openai-request-timeout-ms', 850_000]); + entries.push(['openai-headers-timeout-ms', 900_000]); + } + return entries; } diff --git a/packages/test-utils/src/test-rig.test.ts b/packages/test-utils/src/test-rig.test.ts index 51f7689611..4170b9ca81 100644 --- a/packages/test-utils/src/test-rig.test.ts +++ b/packages/test-utils/src/test-rig.test.ts @@ -4,8 +4,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; +import { createServer } from 'node:http'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'bun:test'; import { restoreEnv, setEnv } from './env-test-helpers.js'; @@ -51,6 +58,278 @@ describe('TestRig setup and cleanup behavior', () => { expect(rig.originalFakeResponsesPath).toBe(fakeResponsesPath); }); + it('writes bounded context and output settings into an opt-in test profile', () => { + createRoot(); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:2b'); + setEnv('LLXPRT_CONTEXT_LIMIT', '32768'); + setEnv('LLXPRT_MAX_OUTPUT_TOKENS', '8192'); + const rig = new TestRig(); + + rig.setup('local model E2E profile'); + + const testDir = requireTestDir(rig.testDir); + const profile = JSON.parse( + readFileSync( + join(testDir, '.llxprt', 'profiles', 'local-model-e2e.json'), + 'utf8', + ), + ); + expect(profile).toMatchObject({ + provider: 'openai', + model: 'qwen3.5:2b', + ephemeralSettings: { + 'context-limit': 32768, + maxOutputTokens: 8192, + 'stream-first-response-timeout-ms': 750_000, + 'openai-request-timeout-ms': 850_000, + 'openai-headers-timeout-ms': 900_000, + }, + }); + }); + + it('leaves the ordinary test profile first-response deadline unchanged', () => { + createRoot(); + setEnv('LLXPRT_LOCAL_MODEL_E2E', undefined); + setEnv('LLXPRT_TEST_PROFILE', 'ordinary-profile'); + const rig = new TestRig(); + + rig.setup('ordinary profile'); + + const testDir = requireTestDir(rig.testDir); + const profile = JSON.parse( + readFileSync( + join(testDir, '.llxprt', 'profiles', 'ordinary-profile.json'), + 'utf8', + ), + ); + expect(profile.ephemeralSettings).not.toHaveProperty( + 'stream-first-response-timeout-ms', + ); + expect(profile.ephemeralSettings).not.toHaveProperty( + 'openai-headers-timeout-ms', + ); + expect(profile.ephemeralSettings).not.toHaveProperty( + 'openai-request-timeout-ms', + ); + }); + + it('loads the generated local model E2E profile in the real CLI without a global profile', async () => { + const root = createRoot(); + setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:2b'); + setEnv('OPENAI_API_KEY', 'local-test-only'); + setEnv('LLXPRT_LOCAL_MODEL_THINKING', 'none'); + setEnv('LLXPRT_CONTEXT_LIMIT', '32768'); + setEnv('LLXPRT_MAX_OUTPUT_TOKENS', '8192'); + + const requests: Array<{ path: string | undefined; body: string }> = []; + const server = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on('data', (chunk: Buffer) => chunks.push(chunk)); + request.on('end', () => { + requests.push({ + path: request.url, + body: Buffer.concat(chunks).toString(), + }); + const chunk = JSON.stringify({ + id: 'chatcmpl-local-profile-test', + object: 'chat.completion.chunk', + choices: [ + { + delta: { content: 'OK' }, + index: 0, + finish_reason: 'stop', + }, + ], + }); + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end(`data: ${chunk}\n\ndata: [DONE]\n\n`); + }); + }); + server.listen(0, '127.0.0.1'); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Local test server has no TCP port'); + } + setEnv('OPENAI_BASE_URL', `http://127.0.0.1:${address.port}/v1`); + const rig = new TestRig(); + rig.setup('real CLI local model E2E profile'); + + await rig.run({ args: 'Respond with OK', timeoutMs: 20_000 }); + + const completionRequest = requests.find( + (request) => request.path === '/v1/chat/completions', + ); + expect({ + path: completionRequest?.path, + model: + completionRequest === undefined + ? undefined + : JSON.parse(completionRequest.body).model, + reasoningEffort: + completionRequest === undefined + ? undefined + : JSON.parse(completionRequest.body).reasoning_effort, + }).toStrictEqual({ + path: '/v1/chat/completions', + model: 'qwen3.5:2b', + reasoningEffort: 'none', + }); + } finally { + server.closeAllConnections(); + server.close(); + } + }, 30_000); + + it('applies the local model E2E transport headers deadline in the real CLI', async () => { + const root = createRoot(); + setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'local-test-model'); + setEnv('OPENAI_API_KEY', 'local-test-only'); + + const server = createServer((_request, response) => { + setTimeout(() => { + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end( + 'data: {"id":"delayed","choices":[{"index":0,"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', + ); + }, 1_500); + }); + server.listen(0, '127.0.0.1'); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Local test server has no TCP port'); + } + setEnv('OPENAI_BASE_URL', `http://127.0.0.1:${address.port}/v1`); + const rig = new TestRig(); + rig.setup('local profile headers deadline'); + const profilePath = join( + requireTestDir(rig.testDir), + '.llxprt', + 'profiles', + 'local-model-e2e.json', + ); + const profile = JSON.parse(readFileSync(profilePath, 'utf8')); + profile.ephemeralSettings['openai-headers-timeout-ms'] = 100; + writeFileSync(profilePath, JSON.stringify(profile)); + await expect( + rig.run({ args: 'Respond with OK', timeoutMs: 12_000 }), + ).rejects.toThrow(/Request timed out/); + + profile.ephemeralSettings['openai-headers-timeout-ms'] = 4_000; + writeFileSync(profilePath, JSON.stringify(profile)); + expect( + await rig.run({ args: 'Respond with OK', timeoutMs: 12_000 }), + ).toContain('OK'); + } finally { + server.closeAllConnections(); + server.close(); + } + }, 30_000); + + it('applies the local model E2E SDK request deadline in the real CLI', async () => { + const root = createRoot(); + setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'local-test-model'); + setEnv('OPENAI_API_KEY', 'local-test-only'); + + const server = createServer((_request, response) => { + setTimeout(() => { + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end( + 'data: {"id":"delayed","choices":[{"index":0,"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', + ); + }, 1_500); + }); + server.listen(0, '127.0.0.1'); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Local test server has no TCP port'); + } + setEnv('OPENAI_BASE_URL', `http://127.0.0.1:${address.port}/v1`); + const rig = new TestRig(); + rig.setup('local profile SDK deadline'); + const profilePath = join( + requireTestDir(rig.testDir), + '.llxprt', + 'profiles', + 'local-model-e2e.json', + ); + const profile = JSON.parse(readFileSync(profilePath, 'utf8')); + profile.ephemeralSettings['openai-request-timeout-ms'] = 100; + writeFileSync(profilePath, JSON.stringify(profile)); + await expect( + rig.run({ args: 'Respond with OK', timeoutMs: 12_000 }), + ).rejects.toThrow(/Request timed out/); + + profile.ephemeralSettings['openai-request-timeout-ms'] = 4_000; + writeFileSync(profilePath, JSON.stringify(profile)); + expect( + await rig.run({ args: 'Respond with OK', timeoutMs: 12_000 }), + ).toContain('OK'); + } finally { + server.closeAllConnections(); + server.close(); + } + }, 30_000); + + it('applies the generated profile first-response watchdog in the real CLI', async () => { + const root = createRoot(); + setEnv('LLXPRT_CONFIG_HOME', join(root, 'global-config')); + setEnv('LLXPRT_LOCAL_MODEL_E2E', 'true'); + setEnv('LLXPRT_TEST_PROFILE', 'local-model-e2e'); + setEnv('LLXPRT_DEFAULT_PROVIDER', 'openai'); + setEnv('LLXPRT_DEFAULT_MODEL', 'qwen3.5:4b'); + setEnv('OPENAI_API_KEY', 'local-test-only'); + + const server = createServer((_request, _response) => { + // Keep the first response pending so only the CLI watchdog can end it. + }); + server.listen(0, '127.0.0.1'); + try { + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Local test server has no TCP port'); + } + setEnv('OPENAI_BASE_URL', `http://127.0.0.1:${address.port}/v1`); + const rig = new TestRig(); + rig.setup('local profile watchdog'); + const profilePath = join( + requireTestDir(rig.testDir), + '.llxprt', + 'profiles', + 'local-model-e2e.json', + ); + const profile = JSON.parse(readFileSync(profilePath, 'utf8')); + profile.ephemeralSettings['stream-first-response-timeout-ms'] = 100; + writeFileSync(profilePath, JSON.stringify(profile)); + + await expect( + rig.run({ args: 'Respond with OK', timeoutMs: 15_000 }), + ).rejects.toThrow( + /threshold 100ms\) from stream-first-response-timeout-ms/, + ); + } finally { + server.closeAllConnections(); + server.close(); + } + }, 20_000); + it('cleans test directories when KEEP_OUTPUT is unset or empty', async () => { createRoot(); const rig = new TestRig(); @@ -171,4 +450,32 @@ describe('TestRig setup and cleanup behavior', () => { expect(existsSync(ledgerPath)).toBe(false); }); + + it('uses a per-run deadline without changing the default TestRig timeout', async () => { + const root = createRoot(); + const fixturePath = join(root, 'fake.jsonl'); + const fixture = JSON.stringify({ + chunks: [ + { + speaker: 'ai', + blocks: [{ type: 'text', text: 'OK' }], + metadata: { + usage: { + promptTokens: 1, + completionTokens: 1, + totalTokens: 2, + }, + }, + }, + ], + }); + writeFileSync(fixturePath, `${fixture}\n`); + setEnv('LLXPRT_TEST_PROFILE', undefined); + const rig = new TestRig(); + rig.setup('scoped run timeout', { fakeResponsesPath: fixturePath }); + + await expect( + rig.run({ args: 'test prompt', timeoutMs: 1 }), + ).rejects.toThrow('TestRig.run() timed out after 1ms'); + }); }); diff --git a/packages/test-utils/src/test-rig.ts b/packages/test-utils/src/test-rig.ts index 8766d58b47..0996ef6257 100644 --- a/packages/test-utils/src/test-rig.ts +++ b/packages/test-utils/src/test-rig.ts @@ -87,6 +87,7 @@ interface RunMethodOptions { stdin?: string; stdinDoesNotEnd?: boolean; yolo?: boolean; + timeoutMs?: number; } /** @@ -226,7 +227,7 @@ export class TestRig { const commandArgs = [...initialArgs]; appendUserArgs(commandArgs, options.args); - appendProfileFlag(commandArgs); + appendProfileFlag(commandArgs, this.testDir as string); const childEnv = buildChildEnv( this.testDir as string, @@ -254,7 +255,7 @@ export class TestRig { options, isJsonOutput, transform, - getDefaultTimeout() * 4, + options.timeoutMs ?? getDefaultTimeout() * 4, (capture) => { this._lastRunCapture = capture; this._lastRunStdout = capture.stdout; @@ -508,7 +509,7 @@ export class TestRig { const commandArgs = [...initialArgs]; appendInteractiveArgs(commandArgs, options?.args); - appendProfileFlag(commandArgs); + appendProfileFlag(commandArgs, this.testDir as string); const childEnv = buildChildEnv( this.testDir as string, @@ -598,16 +599,24 @@ function appendUserArgs( } /** - * Append the `--profile-load` flag when a test profile is configured. + * Append the configured test profile, using inline JSON for the local model E2E. */ -function appendProfileFlag(commandArgs: string[]): void { +function appendProfileFlag(commandArgs: string[], testDir: string): void { const profileName = getProfileName(); if (profileName === undefined) { return; } const ideFlagIndex = commandArgs.indexOf('--ide-mode'); const insertionIndex = ideFlagIndex >= 0 ? ideFlagIndex : commandArgs.length; - commandArgs.splice(insertionIndex, 0, '--profile-load', profileName); + if (env['LLXPRT_LOCAL_MODEL_E2E'] === 'true') { + const profileJson = readFileSync( + join(testDir, '.llxprt', 'profiles', `${profileName}.json`), + 'utf8', + ); + commandArgs.splice(insertionIndex, 0, '--profile', profileJson); + } else { + commandArgs.splice(insertionIndex, 0, '--profile-load', profileName); + } } /** diff --git a/scripts/bun-test-roots.ts b/scripts/bun-test-roots.ts index 1ec052c627..58e1b3f340 100644 --- a/scripts/bun-test-roots.ts +++ b/scripts/bun-test-roots.ts @@ -147,6 +147,15 @@ export function getErrorCode(error: unknown): string | undefined { // Root table // --------------------------------------------------------------------------- +export function integrationTestTimeoutOverrides( + environment: Readonly, +): readonly BunTestTimeoutOverride[] { + if (environment['LLXPRT_LOCAL_MODEL_E2E'] !== 'true') { + return []; + } + return [{ pattern: /replace\.test\.ts$/, timeout: 1_500_000 }]; +} + export const BUN_TEST_ROOTS: readonly BunTestRoot[] = [ { root: 'a2a-server', @@ -258,6 +267,7 @@ export const BUN_TEST_ROOTS: readonly BunTestRoot[] = [ timeout: 300_000, retries: 2, credentialed: true, + timeoutOverrides: integrationTestTimeoutOverrides(process.env), }, ]; diff --git a/scripts/tests/bun-test-roots.bun.test.ts b/scripts/tests/bun-test-roots.bun.test.ts index 8fb9c8d0a0..2b4eff3c81 100644 --- a/scripts/tests/bun-test-roots.bun.test.ts +++ b/scripts/tests/bun-test-roots.bun.test.ts @@ -26,6 +26,7 @@ import { discoverTestFilesInDirectory, getErrorCode, isTestFileName, + integrationTestTimeoutOverrides, resolveBunTestFiles, resolveRoot, resolveRootCwd, @@ -84,6 +85,25 @@ describe('BUN_TEST_ROOTS structural guarantees', () => { } }); + it('extends only the replace file timeout for local-model E2E', () => { + expect(integrationTestTimeoutOverrides({})).toEqual([]); + + const overrides = integrationTestTimeoutOverrides({ + LLXPRT_LOCAL_MODEL_E2E: 'true', + }); + + expect(overrides).toHaveLength(1); + expect( + overrides[0]?.pattern.test('/repo/integration-tests/replace.test.ts'), + ).toBe(true); + expect( + overrides[0]?.pattern.test( + '/repo/integration-tests/run_shell_command.test.ts', + ), + ).toBe(false); + expect(overrides[0]?.timeout).toBe(1_500_000); + }); + it('has exactly the expected set of root tokens', () => { const tokens = BUN_TEST_ROOTS.map((r) => r.root); expect(tokens).toEqual([ diff --git a/scripts/tests/e2e-local-model.test.ts b/scripts/tests/e2e-local-model.test.ts new file mode 100644 index 0000000000..784eb4fcb8 --- /dev/null +++ b/scripts/tests/e2e-local-model.test.ts @@ -0,0 +1,765 @@ +/** + * @license + * Copyright 2026 Vybestack LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'bun:test'; +import { + mkdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { resolve } from 'node:path'; +import { + asOptionalRecord, + asRecord, + asString, + parseWorkflowYaml, + workflowJob, +} from './typed-test-helpers.ts'; + +const root = resolve(import.meta.dirname, '../..'); +const requiredText = readFileSync( + resolve(root, '.github/workflows/e2e.yml'), + 'utf8', +); +const required = parseWorkflowYaml(requiredText); + +type BackendVerifierResult = { + stdout: string; + stderr: string; + status: number; +}; + +let fixtureCounter = 0; + +function writeExecutable(bin: string, name: string, body: string): void { + const path = resolve(bin, name); + writeFileSync( + path, + `#!/bin/bash +${body} +`, + ); + Bun.spawnSync(['chmod', '+x', path]); +} + +function runBackendVerifier( + maps: string, + livePid = true, + options: { + cmdline?: string; + vendor?: string; + mapsFifo?: boolean; + mapsDisappearAfterPgrep?: boolean; + } = {}, +): BackendVerifierResult { + const fixture = resolve(root, `tmp/verify3764/behavior-${fixtureCounter++}`); + rmSync(fixture, { recursive: true, force: true }); + const runnerTemp = resolve(fixture, 'runner-temp'); + const procRoot = resolve(fixture, 'proc'); + const bin = resolve(fixture, 'bin'); + const processDir = resolve(procRoot, '4242'); + const parentDir = resolve(procRoot, '4200'); + const ollamaDir = resolve(runnerTemp, 'ollama/lib/ollama'); + mkdirSync(bin, { recursive: true }); + mkdirSync(processDir, { recursive: true }); + mkdirSync(parentDir, { recursive: true }); + mkdirSync(ollamaDir, { recursive: true }); + writeFileSync( + resolve(procRoot, 'cpuinfo'), + `vendor_id : ${options.vendor ?? 'GenuineIntel'}\n`, + ); + writeFileSync( + resolve(processDir, 'cmdline'), + options.cmdline ?? + `${resolve(ollamaDir, 'llama-server')}\0--model\0gemma\0`, + ); + writeFileSync( + resolve(processDir, 'status'), + 'Name:\tllama-server\nPPid:\t4200\n', + ); + writeFileSync(resolve(parentDir, 'cmdline'), 'ollama\0serve\0'); + const mapsPath = resolve(processDir, 'maps'); + if (options.mapsFifo) { + const fifo = spawnSync('mkfifo', [mapsPath]); + if (fifo.status !== 0) throw new Error(fifo.stderr); + if (statSync(mapsPath).size !== 0) { + throw new Error('FIFO maps fixture must report zero stat size'); + } + const writer = Bun.spawn([ + 'bash', + '-c', + 'printf %s "$1" > "$2"', + '--', + maps.replaceAll('/runner-temp', runnerTemp), + mapsPath, + ]); + writer.unref(); + } else { + writeFileSync(mapsPath, maps.replaceAll('/runner-temp', runnerTemp)); + } + writeFileSync( + resolve(runnerTemp, 'ollama-server.log'), + 'cmn common_param: system_info: n_threads = 2 (n_threads_batch = 2) / 4 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | LLAMAFILE = 1 | REPACK = 1 | \n', + ); + writeFileSync(resolve(ollamaDir, 'llama-server'), 'fixture process'); + writeExecutable( + bin, + 'curl', + `printf '%s\\n' '{"done":true,"error":null,"response":"Hello"}'`, + ); + writeExecutable(bin, 'jq', 'cat >/dev/null; exit 0'); + let pgrepBody = 'exit 1'; + if (options.mapsDisappearAfterPgrep) { + pgrepBody = `rm -rf '${processDir}'; printf '%s\\n' 4242`; + } else if (livePid) { + pgrepBody = `printf '%s\\n' 4242`; + } + writeExecutable(bin, 'pgrep', pgrepBody); + const script = asString(step('Verify CPU backend warm-up').run); + const result = spawnSync('bash', ['-c', script], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + RUNNER_TEMP: runnerTemp, + LLXPRT_PROC_ROOT: procRoot, + OLLAMA_HOST: '127.0.0.1:12644', + }, + }); + rmSync(fixture, { recursive: true, force: true }); + return { + stdout: result.stdout, + stderr: result.stderr, + status: result.status ?? 1, + }; +} + +function runLiveLinuxProcVerifier(): BackendVerifierResult { + const fixture = resolve(root, `tmp/verify3764/live-proc-${fixtureCounter++}`); + const runnerTemp = resolve(fixture, 'runner-temp'); + const bin = resolve(fixture, 'bin'); + const ollamaDir = resolve(runnerTemp, 'ollama/lib/ollama'); + mkdirSync(bin, { recursive: true }); + mkdirSync(ollamaDir, { recursive: true }); + const library = resolve(ollamaDir, 'libggml-cpu-haswell.so'); + const executable = resolve(ollamaDir, 'llama-server'); + const compiledLibrary = spawnSync( + 'cc', + ['-shared', '-fPIC', '-x', 'c', '-o', library, '-'], + { input: 'int haswell_fixture(void) { return 1; }', encoding: 'utf8' }, + ); + if (compiledLibrary.status !== 0) { + throw new Error(`${compiledLibrary.stdout}\n${compiledLibrary.stderr}`); + } + const compiledServer = spawnSync( + 'cc', + ['-x', 'c', '-o', executable, '-', '-ldl'], + { + input: + '#include \n#include \nint main(int argc, char **argv) { if (argc < 4 || !dlopen(argv[3], RTLD_NOW)) return 2; for (;;) pause(); }', + encoding: 'utf8', + }, + ); + if (compiledServer.status !== 0) { + throw new Error(`${compiledServer.stdout}\n${compiledServer.stderr}`); + } + writeExecutable(bin, 'curl', `printf '%s\\n' '{"done":true,"error":null}'`); + writeExecutable(bin, 'jq', 'cat >/dev/null; exit 0'); + writeFileSync( + resolve(runnerTemp, 'ollama-server.log'), + 'system_info: CPU : AVX2 = 1\n', + ); + const server = Bun.spawn([executable, '--model', 'gemma', library], { + stdout: 'ignore', + stderr: 'ignore', + }); + try { + const mapsPath = `/proc/${server.pid}/maps`; + let maps = ''; + for (let attempt = 0; attempt < 100; attempt++) { + try { + maps = readFileSync(mapsPath, 'utf8'); + } catch { + maps = ''; + } + if (maps.includes(library)) break; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + if (!maps.includes(library)) + throw new Error('live helper did not map the haswell fixture library'); + if (statSync(mapsPath).size !== 0) + throw new Error('Linux proc maps fixture must report zero stat size'); + const script = asString(step('Verify CPU backend warm-up').run).replace( + '"$proc_root/cpuinfo"', + `'${resolve(fixture, 'cpuinfo')}'`, + ); + writeFileSync(resolve(fixture, 'cpuinfo'), 'vendor_id : GenuineIntel\n'); + const result = spawnSync('bash', ['-c', script], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + RUNNER_TEMP: runnerTemp, + OLLAMA_HOST: '127.0.0.1:12644', + }, + }); + return { + stdout: result.stdout, + stderr: result.stderr, + status: result.status ?? 1, + }; + } finally { + server.kill(); + rmSync(fixture, { recursive: true, force: true }); + } +} + +const steps = workflowJob(required, 'e2e_linux').steps ?? []; + +function step(name: string) { + const found = steps.find((candidate) => candidate.name === name); + if (!found) throw new Error(`Missing local model E2E step: ${name}`); + return found; +} + +describe('ordinary local-model E2E', () => { + it('dispatches the selected branch through the ordinary E2E job with no extra input', () => { + const dispatch = asRecord(required.on?.workflow_dispatch); + expect(asRecord(dispatch.inputs)).toEqual({ + branch_ref: { + description: 'Branch to run on', + required: true, + default: 'main', + type: 'string', + }, + }); + expect(Object.keys(required.jobs ?? {})).toEqual([ + 'skip_check', + 'mergeability-gate', + 'e2e_doc_change_filter', + 'e2e_linux', + ]); + const checkout = step('Checkout'); + expect(asOptionalRecord(checkout.with)?.ref).toBe( + "${{ github.event_name == 'workflow_dispatch' && inputs.branch_ref || github.ref }}", + ); + }); + + it('keeps required E2E events, check names and budget while running without provider credentials', () => { + expect(required.on).toHaveProperty('pull_request'); + expect(required.on).toHaveProperty('push'); + expect(required.on).toHaveProperty('merge_group'); + expect(required.on).toHaveProperty('pull_request_target'); + expect(workflowJob(required, 'e2e_linux').name).toBe( + 'E2E Test (Linux) - ${{ matrix.sandbox }}', + ); + expect(requiredText).not.toContain('Block unqualified local-model gating'); + const requiredSteps = workflowJob(required, 'e2e_linux').steps ?? []; + expect(requiredSteps.map((candidate) => candidate.name)).not.toContain( + 'Check API quota and select optimal key', + ); + expect(requiredSteps.map((candidate) => candidate.name)).toContain( + 'Check E2E real-model budget (issue #2278)', + ); + const linuxText = JSON.stringify(workflowJob(required, 'e2e_linux')); + expect(linuxText).not.toContain('secrets['); + expect(linuxText).not.toContain('vars.'); + expect(linuxText).not.toContain('ci-quota-check'); + }); + + it('runs the pinned local runtime and both E2E legs in one required job', () => { + const linux = workflowJob(required, 'e2e_linux'); + expect(linux['timeout-minutes']).toBe(90); + expect(asRecord(asRecord(linux.strategy).matrix).sandbox).toBe( + '${{ fromJSON(github.event_name == \'push\' && \'["sandbox:none"]\' || \'["sandbox:none","sandbox:docker"]\') }}', + ); + for (const name of [ + 'Install Ollama CPU runtime', + 'Select CPU backend on Intel', + 'Start local Gemma 4 E2B model', + 'Verify CPU backend warm-up', + 'Verify CPU-only model inference', + 'Run E2E tests', + 'Check E2E real-model budget (issue #2278)', + 'Upload local model diagnostics', + ]) { + expect(step(name).name).toBe(name); + } + expect(step('Upload local model diagnostics').if).toBe('always()'); + }); + + it('names distinct safe diagnostic artifacts for host and Docker', () => { + const name = asString( + asOptionalRecord(step('Upload local model diagnostics').with)?.name, + ); + expect(name).toBe( + "local-gemma4-e2e-${{ matrix.sandbox == 'sandbox:docker' && 'docker' || 'none' }}", + ); + for (const sandbox of ['none', 'docker']) { + expect(`local-gemma4-e2e-${sandbox}`).not.toMatch(/[\\/:*?"<>|\r\n]/); + } + }); + + it('downloads a pinned Ollama runtime and verifies its archive before extraction', () => { + const install = asString(step('Install Ollama CPU runtime').run); + expect(install).toContain('uname -m'); + expect(install).toContain('x86_64'); + expect(install).toContain('ollama-linux-amd64.tar.zst?version=0.31.1'); + expect(install).toContain( + `printf '%s %s\\n' 'd297381efc136451f6fabb9dd644a67f70fe51c16815a0c4a95ff0e327a3afb4' "$RUNNER_TEMP/ollama-linux-amd64.tar.zst" | sha256sum --check -`, + ); + expect(install.indexOf('sha256sum --check')).toBeLessThan( + install.indexOf('tar --zstd'), + ); + expect(install).toContain("--exclude='lib/ollama/cuda_v12/*'"); + expect(install).toContain("--exclude='lib/ollama/cuda_v13/*'"); + expect(install).toContain("--exclude='lib/ollama/vulkan/*'"); + expect(install).toContain('rm "$RUNNER_TEMP/ollama-linux-amd64.tar.zst"'); + }); + + it('pins Intel backend selection before startup and verifies live mapped backends', () => { + const select = step('Select CPU backend on Intel'); + expect(select.if).toBeUndefined(); + const script = asString(select.run); + expect(script).toContain('set -euo pipefail'); + expect(script).toContain('grep -qm1'); + expect(script).toContain('GenuineIntel'); + expect(script).toContain('Host CPU vendor:'); + expect(script).toContain('libggml-cpu-haswell.so'); + expect(script).toContain('libggml-cpu-*.so'); + expect(script).not.toContain('OLLAMA_LLM_LIBRARY'); + expect(steps.indexOf(select)).toBeGreaterThan( + steps.indexOf(step('Install Ollama CPU runtime')), + ); + expect(steps.indexOf(select)).toBeLessThan( + steps.indexOf(step('Start local Gemma 4 E2B model')), + ); + const verify = step('Verify CPU backend warm-up'); + expect(verify.if).toBeUndefined(); + expect(asString(verify.run)).toContain( + 'http://127.0.0.1:12644/api/generate', + ); + expect(asString(verify.run)).toContain('ollama-server.log'); + expect(asString(verify.run)).toContain('"think":false'); + expect(asString(verify.run)).toContain('(.response | length > 0)'); + expect(asString(verify.run)).toContain('LLXPRT_PROC_ROOT:-/proc'); + expect(asString(verify.run)).toContain('libggml-cpu-haswell.so'); + expect(asString(verify.run)).toContain('pgrep -f'); + expect(asString(verify.run)).toContain('/cmdline'); + expect(asString(verify.run)).toContain('/status'); + expect(asString(verify.run)).toContain('parent cmdline'); + expect(asString(verify.run)).not.toContain('-s "$maps'); + expect(steps.indexOf(verify)).toBeGreaterThan( + steps.indexOf(step('Start local Gemma 4 E2B model')), + ); + expect(steps.indexOf(verify)).toBeLessThan( + steps.indexOf(step('Run E2E tests')), + ); + }); + + it('checks model version and digest before running integration tests', () => { + const server = step('Start local Gemma 4 E2B model'); + const start = asString(server.run); + const env = asOptionalRecord(server.env); + expect(env?.OLLAMA_CONTEXT_LENGTH).toBe('32768'); + expect(env?.OLLAMA_NUM_PARALLEL).toBe('1'); + expect(env?.OLLAMA_HOST).toBe('127.0.0.1:12644'); + expect(start).toContain('jq -e \'.version == "0.31.1"\''); + expect(start).toContain('ollama pull gemma4:e2b-it-qat'); + expect(start).toContain( + '07ea59a474013479c8b6b802bef095c40e964a1d776ba02f264c0e30e1aede0c', + ); + const inference = step('Verify CPU-only model inference'); + const inferenceScript = asString(inference.run); + expect(inferenceScript).toContain('/api/ps'); + expect(inferenceScript).toContain('.size_vram == 0'); + expect(inferenceScript).toContain('/api/generate'); + expect(inferenceScript.indexOf('/api/generate')).toBeLessThan( + inferenceScript.indexOf('/api/ps'), + ); + expect(asString(inference.run)).toContain( + '.done == true and .error == null', + ); + expect(inference.env?.OLLAMA_HOST).toBe('127.0.0.1:12644'); + expect(steps.indexOf(server)).toBeLessThan(steps.indexOf(inference)); + expect(steps.indexOf(inference)).toBeLessThan( + steps.indexOf(step('Run E2E tests')), + ); + }); + + it('matches the required Linux E2E test invocations in both sandbox legs without provider secrets', () => { + const job = workflowJob(required, 'e2e_linux'); + const run = step('Run E2E tests'); + const command = asString(run.run); + const normalize = (script: string): string => + script.replace(/\\\s*/g, ' ').replace(/\s+/g, ' '); + const fullArgs = + '--exclude="**/todo-continuation.e2e.test.ts" --exclude="**/run_shell_command.test.ts"'; + const shellArgs = + 'integration-tests/run_shell_command.test.ts --testNamePattern="should be able to run a shell command|should be able to run a shell command via stdin|should run a platform-specific file listing command"'; + const normalizedCommand = normalize(command); + for (const sandbox of ['sandbox:none', 'sandbox:docker']) { + const fullInvocation = `npm run test:integration:${sandbox} -- ${fullArgs}`; + const firstInvocationPattern = new RegExp( + `npm run test:integration:${sandbox} -- (.*?) npm run test:integration:${sandbox} --`, + ); + const firstInvocation = normalizedCommand.match(firstInvocationPattern); + expect(firstInvocation?.[1]).toBe(fullArgs); + expect(normalizedCommand).toContain(fullInvocation); + const shellInvocation = `npm run test:integration:${sandbox} -- --exclude="**/todo-continuation.e2e.test.ts" ${shellArgs}`; + expect(normalizedCommand).toContain(shellInvocation); + } + expect(command.match(/--exclude=/g)).toHaveLength(6); + const env = asOptionalRecord(run.env); + expect(env?.OPENAI_BASE_URL).toBe('http://127.0.0.1:12644/v1'); + expect(env?.LLXPRT_DEFAULT_PROVIDER).toBe('openai'); + expect(env?.LLXPRT_DEFAULT_MODEL).toBe('gemma4:e2b-it-qat'); + expect(env?.OPENAI_API_KEY).toBe('ollama-local-only'); + expect(env?.LLXPRT_TEST_PROFILE).toBe('local-gemma4-e2e'); + expect(env?.LLXPRT_CONTEXT_LIMIT).toBe('32768'); + expect(env?.LLXPRT_MAX_OUTPUT_TOKENS).toBe('8192'); + expect(env?.LLXPRT_LOCAL_MODEL_E2E).toBe('true'); + expect(Number(env?.LLXPRT_CONTEXT_LIMIT)).toBe( + Number(step('Start local Gemma 4 E2B model').env?.OLLAMA_CONTEXT_LENGTH), + ); + expect(Number(env?.LLXPRT_MAX_OUTPUT_TOKENS)).toBeLessThan( + Number(env?.LLXPRT_CONTEXT_LIMIT), + ); + expect(env?.GIT_CEILING_DIRECTORIES).toBe( + '${{ github.workspace }}/.integration-tests', + ); + expect(asString(run.run).startsWith('set -euo pipefail')).toBe(true); + expect(asString(run.run)).toContain( + 'integration-tests/run_shell_command.test.ts', + ); + expect(JSON.stringify(job)).not.toMatch(/\bsecrets(?:\.|\[)/); + const budget = step('Check E2E real-model budget (issue #2278)'); + expect(budget.if).toBe('success()'); + expect(asString(budget.run).trim()).toBe( + 'bun scripts/check-e2e-model-budget.ts --ledger "$LLXPRT_E2E_MODEL_LEDGER"', + ); + expect(asOptionalRecord(budget.env)?.LLXPRT_E2E_MODEL_LEDGER).toBe( + '${{ runner.temp }}/e2e-model-ledger.jsonl', + ); + }); + + it('uses host networking only for the Docker leg, and retains diagnostics on failure', () => { + const env = asOptionalRecord(step('Run E2E tests').env); + expect(env?.SANDBOX_FLAGS).toBe( + "${{ matrix.sandbox == 'sandbox:docker' && '--network host' || '' }}", + ); + expect(env?.KEEP_OUTPUT).toBe('true'); + expect(env?.LLXPRT_E2E_MODEL_LEDGER).toBe( + '${{ runner.temp }}/e2e-model-ledger.jsonl', + ); + const report = step('Report local model resources'); + const upload = step('Upload local model diagnostics'); + expect(report.if).toBe('always()'); + expect(asString(report.run)).toContain('/api/ps'); + expect(asString(report.run)).toContain('free -h'); + expect(asString(report.run)).toContain('df -h'); + expect(upload.if).toBe('always()'); + const paths = asString(asOptionalRecord(upload.with)?.path).split('\n'); + expect(paths).toContain('${{ runner.temp }}/ollama-server.log'); + expect(paths).toContain('${{ runner.temp }}/e2e-model-ledger.jsonl'); + expect(paths).toContain( + '${{ github.workspace }}/.integration-tests/*/*/telemetry.log', + ); + expect(paths).toContain( + '${{ github.workspace }}/.integration-tests/*/*/harness-diagnostics.log', + ); + expect(asOptionalRecord(upload.with)?.['if-no-files-found']).toBe('error'); + expect(asOptionalRecord(upload.with)?.['include-hidden-files']).toBe(true); + }); +}); + +describe('workflow CPU variant selection behavior', () => { + const runSelection = ( + intel: boolean, + variants: string[], + cpuinfoName = 'cpuinfo', + ): string[] => { + const fixture = resolve( + root, + `tmp/verify3764/selection-${fixtureCounter++}`, + ); + const temp = resolve(fixture, 'temp'); + const libdir = resolve(temp, 'ollama/lib/ollama'); + mkdirSync(libdir, { recursive: true }); + mkdirSync(temp, { recursive: true }); + for (const variant of variants) + writeFileSync(resolve(libdir, variant), 'variant'); + const cpuinfo = resolve(fixture, cpuinfoName); + writeFileSync( + cpuinfo, + intel ? 'vendor_id : GenuineIntel\n' : 'vendor_id : AuthenticAMD\n', + ); + const script = asString(step('Select CPU backend on Intel').run).replaceAll( + '/proc/cpuinfo', + '"$LLXPRT_TEST_CPUINFO"', + ); + const result = spawnSync('bash', ['-c', script], { + encoding: 'utf8', + env: { ...process.env, RUNNER_TEMP: temp, LLXPRT_TEST_CPUINFO: cpuinfo }, + }); + if (result.status !== 0) + throw new Error(`${result.stdout}\n${result.stderr}`); + const selected = Bun.spawnSync([ + 'find', + libdir, + '-maxdepth', + '1', + '-type', + 'f', + '-name', + 'libggml-cpu-*.so', + ]); + const files = selected.stdout + .toString() + .trim() + .split('\n') + .filter(Boolean) + .map((path) => path.slice(libdir.length + 1)) + .sort(); + rmSync(fixture, { recursive: true, force: true }); + return files; + }; + + it('forces haswell in each Intel matrix leg and leaves AMD variants unchanged', () => { + const variants = [ + 'libggml-cpu-haswell.so', + 'libggml-cpu-avx512.so', + 'libggml-cpu-amx.so', + ]; + for (const _matrixLeg of ['sandbox:none', 'sandbox:docker']) { + expect(runSelection(true, variants)).toEqual(['libggml-cpu-haswell.so']); + } + expect(runSelection(false, variants)).toEqual(variants.sort()); + }); + + it('selects Intel and AMD variants with shell syntax in the CPU fixture path', () => { + const variants = ['libggml-cpu-haswell.so', 'libggml-cpu-avx512.so']; + const cpuinfoName = "cpu info '$(printf injected)'"; + expect(runSelection(true, variants, cpuinfoName)).toEqual([ + 'libggml-cpu-haswell.so', + ]); + expect(runSelection(false, variants, cpuinfoName)).toEqual([ + 'libggml-cpu-avx512.so', + 'libggml-cpu-haswell.so', + ]); + }); +}); + +describe('workflow backend verification behavior', () => { + it('reports the mapped native backend on AMD without imposing the Intel Haswell pin', () => { + const result = runBackendVerifier( + '7f000000-7f100000 r-xp 00000000 08:01 42 /runner-temp/ollama/lib/ollama/libggml-cpu-avx2.so\n', + true, + { vendor: 'AuthenticAMD' }, + ); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('Host CPU vendor: AuthenticAMD'); + expect(result.stdout).toContain('libggml-cpu-avx2.so'); + }); + + const haswell = '/runner-temp/ollama/lib/ollama/libggml-cpu-haswell.so'; + const mapping = (path: string): string => + `7f000000-7f100000 r-xp 00000000 08:01 42 ${path}\n`; + + it.skipIf(process.platform !== 'linux')( + 'accepts the sole haswell mapping from a live stat-zero Linux proc maps file', + () => { + const result = runLiveLinuxProcVerifier(); + expect(result.status, `${result.stdout}\\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('Loaded CPU backends:'); + expect(result.stdout).toContain('llama-server cmdline:'); + expect(result.stdout).toContain('llama-server PPid:'); + }, + 15_000, + ); + + it('accepts the exact sole haswell mapping when the incident log omits AVX512 and AMX fields', () => { + const result = runBackendVerifier(mapping(haswell)); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('Loaded CPU backends:'); + expect(result.stdout).toContain('llama-server cmdline:'); + expect(result.stdout).toContain( + 'llama-server PPid: 4200; parent cmdline: ollama serve', + ); + }); + + it('fails when the live process is absent even though the log reports AVX2', () => { + const result = runBackendVerifier(mapping(haswell), false); + expect(result.status, `${result.stdout}\\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain('Live llama-server process not found'); + }); + + it('fails explicitly when the PID vanishes after pgrep returns it', () => { + const result = runBackendVerifier(mapping(haswell), true, { + mapsDisappearAfterPgrep: true, + }); + expect(result.status, `${result.stdout}\\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain('Live llama-server maps are unavailable'); + }); + + it('fails specifically when readable live-process maps contain no CPU backend despite an AVX2 log', () => { + const result = runBackendVerifier(mapping('/usr/lib/libc.so.6')); + expect(result.status, `${result.stdout}\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain('No mapped llama-server CPU backend'); + expect(result.stderr).not.toContain('maps are unavailable or empty'); + }); + + it('rejects additional and alternative mapped CPU library variants', () => { + for (const maps of [ + `${mapping(haswell)}${mapping('/other/libggml-cpu-avx512.so')}`, + mapping('/runner-temp/ollama/lib/ollama/libggml-cpu-skylake.so'), + mapping('/runner-temp/ollama/lib/ollama/libggml-cpu-avx512-vnni.so'), + mapping('/runner-temp/ollama/lib/ollama/libggml-cpu-amx.so'), + mapping('/runner-temp/ollama/lib/ollama/libggml-cpu-sapphirerapids.so'), + mapping(`${haswell} (deleted)`), + ]) { + const result = runBackendVerifier(maps); + expect(result.status, `${result.stdout}\\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain( + 'Unexpected mapped llama-server CPU library', + ); + } + }, 15_000); + + it('reads nonempty maps from a zero-size-reported FIFO', () => { + const result = runBackendVerifier(mapping(haswell), true, { + mapsFifo: true, + }); + expect(result.status, `${result.stdout}\\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('Loaded CPU backends:'); + }, 15_000); + + it('rejects a process whose cmdline does not match the expected server', () => { + const result = runBackendVerifier(mapping(haswell), true, { + cmdline: 'spoofed\\0--model\\0gemma\\0', + }); + expect(result.status, `${result.stdout}\\n${result.stderr}`).not.toBe(0); + expect(result.stderr).toContain( + 'Matched PID is not the expected llama-server process', + ); + }); +}); + +describe('workflow CPU inference warm-up behavior', () => { + function runInference( + generated: { done: boolean; error: string | null; response: string }, + vram: number, + ): { result: BackendVerifierResult; request: unknown; diagnostic: string } { + const fixture = resolve( + root, + `tmp/verify3764/inference-${fixtureCounter++}`, + ); + const bin = resolve(fixture, 'bin'); + mkdirSync(bin, { recursive: true }); + const generation = JSON.stringify(generated); + const ps = JSON.stringify({ + models: [{ name: 'gemma4:e2b-it-qat', size_vram: vram }], + }); + writeExecutable( + bin, + 'curl', + `while (( $# )); do + case "$1" in + */api/generate) endpoint=generate ;; + */api/ps) endpoint=ps ;; + -d) printf '%s' "$2" >"$RUNNER_TEMP/request.json"; shift ;; + -o) output="$2"; shift ;; + esac + shift +done +if [[ "$endpoint" == generate ]]; then + if [[ -n "\${output:-}" ]]; then + printf '%s\\n' '${generation}' >"$output" + else + printf '%s\\n' '${generation}' + fi +else + printf '%s\\n' '${ps}' +fi`, + ); + const script = asString(step('Verify CPU-only model inference').run); + const executed = spawnSync('bash', ['-c', script], { + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH ?? ''}`, + RUNNER_TEMP: fixture, + OLLAMA_HOST: '127.0.0.1:12644', + }, + }); + const request = JSON.parse( + readFileSync(resolve(fixture, 'request.json'), 'utf8'), + ); + const diagnostic = readFileSync( + resolve(fixture, 'ollama-inference.json'), + 'utf8', + ); + rmSync(fixture, { recursive: true, force: true }); + return { + result: { + status: executed.status ?? 1, + stdout: executed.stdout, + stderr: executed.stderr, + }, + request, + diagnostic, + }; + } + + it('disables thinking, requests sixteen tokens and retains the raw generation response on success', () => { + const output = runInference( + { done: true, error: null, response: 'Hello' }, + 0, + ); + expect(output.result.status, output.result.stderr).toBe(0); + expect(output.request).toMatchObject({ + model: 'gemma4:e2b-it-qat', + think: false, + options: { num_predict: 16 }, + }); + expect(JSON.parse(output.diagnostic)).toEqual({ + done: true, + error: null, + response: 'Hello', + }); + }); + + it('reports raw JSON and rejects empty text or unsuccessful completion', () => { + for (const generated of [ + { done: true, error: null, response: '' }, + { done: false, error: null, response: 'Hello' }, + { done: true, error: 'failure', response: 'Hello' }, + ]) { + const output = runInference(generated, 0); + expect(output.result.status).not.toBe(0); + expect(output.result.stderr).toContain(JSON.stringify(generated)); + expect(output.diagnostic.trim()).toBe(JSON.stringify(generated)); + } + }); + + it('rejects nonzero VRAM despite successful generation', () => { + const output = runInference( + { done: true, error: null, response: 'Hello' }, + 1, + ); + expect(output.result.status).not.toBe(0); + }); + + it('uploads the raw inference JSON alongside the existing diagnostics', () => { + const paths = asString( + asOptionalRecord(step('Upload local model diagnostics').with)?.path, + ).split('\n'); + expect(paths).toContain('${{ runner.temp }}/ollama-inference.json'); + }); +}); diff --git a/scripts/tests/e2e-platform-cadence.bun.test.ts b/scripts/tests/e2e-platform-cadence.bun.test.ts index 0bc2da58c3..06e60ab1f0 100644 --- a/scripts/tests/e2e-platform-cadence.bun.test.ts +++ b/scripts/tests/e2e-platform-cadence.bun.test.ts @@ -18,9 +18,8 @@ * YAML helpers and assert the approved split: * * - e2e.yml defines no e2e_mac job and allocates no E2E job to a macOS or - * Windows runner. The Linux PR matrix retains exactly sandbox:none and - * sandbox:docker, with a single exclusion that drops sandbox:docker only on - * push to main. + * Windows runner. The Linux PR matrix selects sandbox:none and sandbox:docker; + * push to main selects sandbox:none only. * - nightly.yml retains the e2e_full rows for Ubuntu none, Ubuntu Docker, * macOS none, and Windows none; both schedule and workflow_dispatch * triggers exercise the matrix; the matrix is non-fail-fast; E2E failures @@ -102,27 +101,18 @@ describe('e2e.yml: macOS E2E is off the PR feedback path (issue #3189)', () => { expect(jobRunsOn(linux)).toBe('ubuntu-latest'); const strategy = asOptionalRecord(linux.strategy); const matrix = asOptionalRecord(strategy?.matrix); - const sandbox = asStringArray(matrix?.sandbox); - expect(sandbox).toEqual(['sandbox:none', 'sandbox:docker']); + expect(asString(matrix?.sandbox)).toContain('sandbox:none'); + expect(asString(matrix?.sandbox)).toContain('sandbox:docker'); }); - it('excludes only sandbox:docker on push, leaving both legs for all other events', () => { - const linux = workflowJob(workflow, 'e2e_linux'); - const strategy = asOptionalRecord(linux.strategy); - const matrix = asOptionalRecord(strategy?.matrix); - const exclude = asArray(matrix?.exclude).map((row) => { - const rec = asRecord(row); - return { sandbox: asString(rec.sandbox) }; - }); - // Exactly one exclusion: drop sandbox:docker only when the event is push. - // pull_request, approved pull_request_target, merge_group, and - // workflow_dispatch all receive both Linux sandbox legs. - expect(exclude).toEqual([ - { - sandbox: - "${{ github.event_name == 'push' && 'sandbox:docker' || 'NEVER_MATCH' }}", - }, - ]); + it('selects only the host on push and both legs on other events without a placeholder row', () => { + const matrix = asOptionalRecord( + asOptionalRecord(workflowJob(workflow, 'e2e_linux').strategy)?.matrix, + ); + expect(matrix?.exclude).toBeUndefined(); + expect(matrix?.sandbox).toBe( + '${{ fromJSON(github.event_name == \'push\' && \'["sandbox:none"]\' || \'["sandbox:none","sandbox:docker"]\') }}', + ); }); it('wires the Linux E2E real-model budget ledger and enforcement step', () => { diff --git a/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts b/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts index 68b5ba3fd3..f05a2c8ef2 100644 --- a/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts +++ b/scripts/tests/pr-mergeability-workflow-wiring-b.test.ts @@ -348,7 +348,7 @@ describe('E2E mergeability gate wiring (.github/workflows/e2e.yml)', () => { }); it('bounds the Linux job while preserving concurrency and matrix', () => { - expect(linuxJob?.['timeout-minutes']).toBe(60); + expect(linuxJob?.['timeout-minutes']).toBe(90); const linuxConcurrency = asOptionalRecord(linuxJob?.concurrency); expect(linuxConcurrency?.['cancel-in-progress']).toBe(true); expect(linuxConcurrency?.group).toContain('${{ matrix.sandbox }}'); diff --git a/scripts/tests/workflow-quota-selection.test.ts b/scripts/tests/workflow-quota-selection.test.ts index f3e7be2c28..6b22de7429 100644 --- a/scripts/tests/workflow-quota-selection.test.ts +++ b/scripts/tests/workflow-quota-selection.test.ts @@ -46,78 +46,37 @@ function stepNamed(steps: WorkflowStep[], name: string): WorkflowStep { type WorkflowStep = TypedWorkflowStep; -/** - * Structurally verifies the trusted-ref and PR-code-checkout steps for a - * single E2E job. The trusted checkout ref must cover BOTH pull_request - * and pull_request_target. After quota selection, the PR code must be - * checked out via two mutually-exclusive conditional steps — internal target - * head for pull_request_target and merge ref for pull_request — each with - * persist-credentials:false. Fork target heads must never be checked out. - */ -function assertJobCheckoutSecurity( - steps: WorkflowStep[], - quotaName: string, - quotaId: string, -): void { - const trustedCheckout = stepNamed(steps, 'Checkout trusted quota selector'); - const quota = stepNamed(steps, quotaName); - const targetCheckout = stepNamed(steps, 'Checkout PR head (internal target)'); - const internalCheckout = stepNamed(steps, 'Checkout PR merge ref (internal)'); - - expect(quota.id).toBe(quotaId); - expect(quota.shell).toBe('bash'); - const quotaRun = asString(quota.run); - expect(quotaRun).toContain('quota_selectors=(scripts/ci-quota-check.*)'); - expect(quotaRun).toContain('${#quota_selectors[@]} != 1'); - expect(quotaRun).toContain('[[ ! -f "${quota_selectors[0]}" ]]'); - expect(quotaRun).toContain('bun "${quota_selectors[0]}"'); - expect(quotaRun).not.toContain('ci-quota-check.js'); - expect(quotaRun).toContain('awk \'!/^OPENAI_API_KEY=/\' "$GITHUB_ENV"'); - expect(asString(quota.run)).toContain( - 'grep -Eq \'^selected_key=(primary|secondary)$\' "$GITHUB_OUTPUT"', +/** Verify checkout refs without giving PR code a persistent checkout credential. */ +function assertJobCheckoutSecurity(steps: WorkflowStep[]): void { + const checkout = stepNamed(steps, 'Checkout'); + const target = stepNamed(steps, 'Checkout PR head (internal target)'); + const merge = stepNamed(steps, 'Checkout PR merge ref (internal)'); + expect(checkout.if).toBe( + "github.event_name != 'pull_request_target' && github.event_name != 'pull_request'", ); - expect(asString(quota.run)).toContain( - '[[ "${KEY_VAR_NAME:-}" == *SYNTHETIC* ]]', + const ref = asString(asRecord(checkout.with).ref); + expect(ref).toBe( + "${{ github.event_name == 'workflow_dispatch' && inputs.branch_ref || github.ref }}", ); - expect(asString(quota.run)).toContain( - 'echo \'selected_key=primary\' >>"$GITHUB_OUTPUT"', - ); - - // Trusted checkout ref must use base.sha for BOTH PR event types. - const trustedRef = asString(asRecord(trustedCheckout.with).ref); - expect(trustedRef).toContain("github.event_name == 'pull_request'"); - expect(trustedRef).toContain("github.event_name == 'pull_request_target'"); - expect(trustedRef).toContain('github.event.pull_request.base.sha'); - // Non-PR events must fall back to dispatch input or github.ref. - expect(trustedRef).toContain('github.event.inputs.branch_ref'); - expect(trustedRef).toContain('github.ref'); - - expect(targetCheckout.if).toBe( + expect(asRecord(checkout.with)['persist-credentials']).toBe(false); + expect(target.if).toBe( "github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository", ); - expect(asRecord(targetCheckout.with).ref).toBe( + expect(asRecord(target.with).ref).toBe( '${{ github.event.pull_request.head.sha }}', ); - expect(asRecord(targetCheckout.with).repository).toBe( - '${{ github.repository }}', + expect(asRecord(target.with).repository).toBe('${{ github.repository }}'); + expect(asRecord(target.with)['persist-credentials']).toBe(false); + expect(merge.if).toBe("github.event_name == 'pull_request'"); + expect(asRecord(merge.with).ref).toBe('${{ github.ref }}'); + expect(asRecord(merge.with)['persist-credentials']).toBe(false); + expect(asRecord(merge.with).clean).toBe(false); + expect(steps.indexOf(target)).toBeLessThan( + steps.indexOf(stepNamed(steps, 'Build project')), + ); + expect(steps.indexOf(merge)).toBeLessThan( + steps.indexOf(stepNamed(steps, 'Build project')), ); - expect(asRecord(targetCheckout.with)['persist-credentials']).toBe(false); - - // Internal PR merge-ref checkout: only for pull_request. - expect(internalCheckout.if).toBe("github.event_name == 'pull_request'"); - expect(asRecord(internalCheckout.with).ref).toBe('${{ github.ref }}'); - expect(asRecord(internalCheckout.with)['persist-credentials']).toBe(false); - expect(asRecord(internalCheckout.with).clean).toBe(false); - - // Ordering: trusted → quota → target/internal checkouts. - const idxTrusted = steps.indexOf(trustedCheckout); - const idxQuota = steps.indexOf(quota); - const idxTarget = steps.indexOf(targetCheckout); - const idxInternal = steps.indexOf(internalCheckout); - - expect(idxTrusted).toBeLessThan(idxQuota); - expect(idxQuota).toBeLessThan(idxTarget); - expect(idxQuota).toBeLessThan(idxInternal); } /** Bun's `expect` has no `fail`; throw so the expression stays `never`. */ @@ -126,67 +85,28 @@ function raiseMissing(message: string): never { } describe('quota-selected workflow credentials', () => { - it('maps Linux E2E quota outputs into validation and test steps', () => { + it('runs regular Linux E2E with only a local model and no provider credentials', () => { const workflow = readWorkflow('e2e.yml'); expect(workflow.permissions).toEqual({ actions: 'read', contents: 'read', 'pull-requests': 'read', }); - const jobs = workflow.jobs; - if (!jobs) throw new Error('e2e.yml must define jobs'); - const e2eLinux = jobs.e2e_linux; - if (!e2eLinux) throw new Error('e2e.yml must define e2e_linux'); - expect(hasSecret(e2eLinux.env ?? {})).toBe(false); - const cases: Array<{ - steps: WorkflowStep[]; - quotaName: string; - quotaId: string; - validationName: string; - testName: string; - }> = [ - { - steps: jobSteps(workflowJobOptional(workflow, 'e2e_linux')), - quotaName: 'Check API quota and select optimal key', - quotaId: 'quota', - validationName: 'Validate E2E provider environment (Linux)', - testName: 'Run E2E tests', - }, - ]; - - for (const testCase of cases) { - assertJobCheckoutSecurity( - testCase.steps, - testCase.quotaName, - testCase.quotaId, - ); - - const quota = stepNamed(testCase.steps, testCase.quotaName); - const validation = stepNamed(testCase.steps, testCase.validationName); - const tests = stepNamed(testCase.steps, testCase.testName); - - expect(asRecord(validation.env).OPENAI_API_KEY).toBe( - selectedKeyExpression(testCase.quotaId), - ); - expect(asRecord(tests.env).OPENAI_API_KEY).toBe( - selectedKeyExpression(testCase.quotaId), - ); - expect(asRecord(validation.env).OPENAI_API_KEY_2).toBeUndefined(); - expect(asRecord(tests.env).OPENAI_API_KEY_2).toBeUndefined(); - expect(testCase.steps.filter(hasSecret)).toEqual([ - quota, - validation, - tests, - ]); - - // Quota must precede validation and test steps. - expect(testCase.steps.indexOf(quota)).toBeLessThan( - testCase.steps.indexOf(validation), - ); - expect(testCase.steps.indexOf(quota)).toBeLessThan( - testCase.steps.indexOf(tests), - ); - } + const job = workflowJobOptional(workflow, 'e2e_linux'); + const steps = jobSteps(job); + assertJobCheckoutSecurity(steps); + expect(hasSecret(job)).toBe(false); + expect(JSON.stringify(job)).not.toContain('vars.'); + expect(JSON.stringify(job)).not.toContain('ci-quota-check'); + expect(steps.map((step) => step.name)).not.toContain( + 'Check API quota and select optimal key', + ); + const tests = stepNamed(steps, 'Run E2E tests'); + expect(asRecord(tests.env).OPENAI_API_KEY).toBe('ollama-local-only'); + expect(asRecord(tests.env).OPENAI_BASE_URL).toBe( + 'http://127.0.0.1:12644/v1', + ); + expect(asRecord(tests.env).LLXPRT_DEFAULT_MODEL).toBe('gemma4:e2b-it-qat'); }); it('maps only the selected PR-review key into the walkthrough invocation', () => {