From d10a333c579997383f281184d9fecd0cc0e85ca3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 06:38:21 -0700 Subject: [PATCH 1/3] test(noema): require free-first auto fallback route --- tests/test_noema_orchestrator_workflow_contract.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index dfa9aa2c8f..c40a30b5c4 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -1,4 +1,4 @@ -"""Noema review now uses the vendored orchestrator sidecar, not NVIDIA NIM.""" +"""Noema review uses the vendored orchestrator auto pool with free-first fallback.""" from __future__ import annotations @@ -11,8 +11,8 @@ from tests.test_required_workflow_queue_contract import workflow_step, workflow_text -def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: - """Require reviewer credentials and the sidecar; the public NIM hardcode is gone.""" +def test_noema_review_credentials_and_llm_use_orchestrator_auto() -> None: + """Require reviewer credentials and the auto sidecar; direct NIM stays absent.""" workflow = workflow_text("noema-review.yml") assert "fail_unavailable()" in workflow @@ -32,6 +32,7 @@ def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: assert "Resolve Noema target repository visibility" in workflow assert "target_visibility.outputs.require_zdr" in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow + assert "CONTEXTUAL_ORCHESTRATOR_POOL: auto" in workflow assert ( "NOEMA_LLM_API_KEY: ${{ secrets.NOEMA_LLM_API_KEY || secrets.OPENAI_API_KEY || '' }}" not in workflow @@ -42,7 +43,8 @@ def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: assert "NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }}" in workflow assert "OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}" in workflow assert "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}" in workflow - assert 'export NOEMA_LLM_MODEL="orchestrator/free"' in workflow + assert 'export NOEMA_LLM_MODEL="orchestrator/auto"' in workflow + assert 'export NOEMA_LLM_MODEL="orchestrator/free"' not in workflow assert ( "contextual-orchestrator review sidecar must be provisioned before Noema LLM review." in workflow From 0c4f0992f6f4130ce277d8017061246177ff6f0c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 06:43:29 -0700 Subject: [PATCH 2/3] ci(noema): run gateway contracts on changed paths --- .../noema-orchestrator-quality-ci.yml | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/noema-orchestrator-quality-ci.yml diff --git a/.github/workflows/noema-orchestrator-quality-ci.yml b/.github/workflows/noema-orchestrator-quality-ci.yml new file mode 100644 index 0000000000..09b806cbb9 --- /dev/null +++ b/.github/workflows/noema-orchestrator-quality-ci.yml @@ -0,0 +1,74 @@ +name: Noema Orchestrator Quality CI + +on: + pull_request: + branches: [main] + paths: + - ".github/workflows/noema-orchestrator-quality-ci.yml" + - ".github/workflows/noema-review.yml" + - "CHANGELOG.md" + - "docs/doctoring/noema-orchestrator-auto-fallback.md" + - "scripts/ci/contextual_orchestrator_review_launcher.py" + - "scripts/ci/contextual_orchestrator_review_policy.py" + - "scripts/ci/contextual_orchestrator_review_sidecar.sh" + - "scripts/ci/load_contextual_orchestrator_token.sh" + - "scripts/ci/zdr_policy.py" + - "tests/test_contextual_orchestrator_review_sidecar_contract.py" + - "tests/test_noema_orchestrator_workflow_contract.py" + +permissions: + contents: read + +concurrency: + group: noema-orchestrator-quality-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + exact-head-contract: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout exact source revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install exact hash-verified test runner dependencies + env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PIP_NO_INPUT: "1" + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + cat >"${RUNNER_TEMP}/noema-quality-requirements.txt" <<'EOF' + coverage==7.15.2 --hash=sha256:b9a6367e4aff723e8ee8190836836124284e8fcd4265e307c844010cfa074f3f + iniconfig==2.1.0 --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 + packaging==26.2 --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e + pluggy==1.6.0 --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + pygments==2.20.0 --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + pytest==9.1.1 --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + EOF + python -m pip install \ + --only-binary=:all: \ + --require-hashes \ + -r "${RUNNER_TEMP}/noema-quality-requirements.txt" + + - name: Verify exact-head Noema gateway contracts + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}" + python -m coverage run -m pytest \ + tests/test_noema_orchestrator_workflow_contract.py \ + tests/test_contextual_orchestrator_review_sidecar_contract.py \ + -q + python -m compileall -q \ + tests/test_noema_orchestrator_workflow_contract.py \ + tests/test_contextual_orchestrator_review_sidecar_contract.py \ + scripts/ci/contextual_orchestrator_review_launcher.py + bash -n scripts/ci/contextual_orchestrator_review_sidecar.sh + git diff --exit-code From 4741d8e0450b8537beeb8ec4cfe2ea9213a829e5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 06:58:36 -0700 Subject: [PATCH 3/3] fix(noema): use free-first auto provider fallback --- .github/workflows/noema-review.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 5c60782adb..9affc48bef 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -297,6 +297,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ steps.target_visibility.outputs.require_zdr }} + CONTEXTUAL_ORCHESTRATOR_POOL: auto run: | set -euo pipefail bash "$GITHUB_WORKSPACE/scripts/ci/contextual_orchestrator_review_sidecar.sh" @@ -322,7 +323,7 @@ jobs: fi source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh" export NOEMA_LLM_API_URL="${CONTEXTUAL_ORCHESTRATOR_BASE_URL%/}/v1/chat/completions" - export NOEMA_LLM_MODEL="orchestrator/free" + export NOEMA_LLM_MODEL="orchestrator/auto" export NOEMA_LLM_API_KEY="${CONTEXTUAL_ORCHESTRATOR_TOKEN}" export NOEMA_LLM_VIA_ORCHESTRATOR=1 python3 scripts/ci/noema_review_gate.py \