From 5b2bf6b3268a7f08536a38a6a001dfcdff553abe Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 11:31:50 +0000 Subject: [PATCH] chore(ci): remove orphaned direct-NVIDIA-NIM model resolver Audited this repository for CI consumers that talk to a provider directly instead of through the vendored contextual-orchestrator gateway (ADR-0003). scripts/ci/select_nvidia_nim_model.py opened a direct HTTPSConnection to integrate.api.nvidia.com with a raw provider API key to resolve a live NVIDIA NIM model id for the scheduled autofix worker -- a real direct-provider bypass, exactly the pattern ADR-0003 migrated pr-review-autofix.yml away from. Confirmed by repository-wide search that it was wired into nothing: pr-review-autofix.yml (its only plausible caller) has no reference to it, and no live NVIDIA/mistral model id, base URL, or provider name -- test_scheduled_autofix_routes_through_contextual_orchestrator already pins "https://integrate.api.nvidia.com/v1" as a forbidden string in that workflow. The only reference anywhere in the repository was the script's own dedicated test. Orphaned dead code predating the ADR-0003 gateway migration; removed along with tests/test_select_nvidia_nim_model.py. Not the same thing as contextual-orchestrator's own legitimate internal support for NVIDIA NIM as one of its five backend providers, which is unmodified and out of this repository's scope. Also corrects four stale pre-ADR-0003 documentation references that still described the scheduled autofix worker's model credential as a single hardcoded NVIDIA_NIM_API_KEY used directly, rather than the current five-secret contextual-orchestrator gateway routing: docs/doctoring/hourly-nvidia-nim-autofix.md (dated addendum, extensive historical detail kept as history), docs/doctoring/originweave-hourly-review-caller.md, docs/doctoring/nonnest2-hourly-review-caller.md (one stale sentence each), and docs/automation/hourly-review-repair.md (summary bullet and the Clearfolio credential paragraph, to match language its own already-accurate Orgmetra section used). Split out of ContextualWisdomLab/.github#1437 per its exact-head governance review's fifth acceptance criterion: this cleanup is orthogonal to that PR's Strix pool-gating behavior repair (a per-family cap does not create a second outage-domain family) and belongs in its own, independently reviewable and mergeable change, not bundled into a broader policy transition. Test/coverage results: - coverage run -m pytest tests -q: 1864 passed, 1 skipped, 21 subtests - coverage report: 99% total, single pre-existing miss in scripts/ci/pingora_edge_policy.py:274 (untouched by this PR, tracked separately as #1398) - interrogate: 100.0% Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01BV96rXhqoR3tYZ9AeAVur4 --- CHANGELOG.md | 23 + docs/automation/hourly-review-repair.md | 26 +- docs/doctoring/hourly-nvidia-nim-autofix.md | 37 ++ .../nonnest2-hourly-review-caller.md | 13 +- .../originweave-hourly-review-caller.md | 13 +- scripts/ci/select_nvidia_nim_model.py | 209 --------- tests/test_select_nvidia_nim_model.py | 398 ------------------ 7 files changed, 96 insertions(+), 623 deletions(-) delete mode 100644 scripts/ci/select_nvidia_nim_model.py delete mode 100644 tests/test_select_nvidia_nim_model.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fc84661ed6..3ee9ee56f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,29 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Remove `scripts/ci/select_nvidia_nim_model.py`: an orphaned helper that + opened a direct `HTTPSConnection` to `integrate.api.nvidia.com` with a raw + provider API key to resolve a live NVIDIA NIM model id, predating the + ADR-0003 gateway migration and wired into nothing (no workflow or script + referenced it — only its own dedicated test, `test_scheduled_autofix_routes_through_contextual_orchestrator` + already pins `https://integrate.api.nvidia.com/v1` as a **forbidden** + string in `pr-review-autofix.yml`). Removed with its test + (`tests/test_select_nvidia_nim_model.py`). Not the same thing as + `contextual-orchestrator`'s own legitimate internal support for NVIDIA NIM + as one of its five backend providers, which is unmodified and out of this + repository's scope. Corrected four stale pre-ADR-0003 "single hardcoded + `NVIDIA_NIM_API_KEY`" descriptions of the scheduled autofix worker's model + credential in `docs/doctoring/hourly-nvidia-nim-autofix.md` (dated + addendum, extensive historical detail kept as history), + `docs/doctoring/originweave-hourly-review-caller.md`, + `docs/doctoring/nonnest2-hourly-review-caller.md` (one stale sentence + each), and `docs/automation/hourly-review-repair.md` (summary bullet and + the Clearfolio credential paragraph) to describe the current five-secret + `contextual-orchestrator` gateway routing instead. Split out of + `ContextualWisdomLab/.github#1437` per its exact-head governance review's + fifth acceptance criterion: this cleanup is orthogonal to that PR's + Strix pool-gating behavior repair and belongs in its own, + independently-reviewable change. - Raise `contextual_orchestrator_review_sidecar.sh`'s `ORCHESTRATOR_CATALOG_FAMILY_CAP` default from 4 to 8: root-caused the live "no provider route passed the Strix plain-chat preflight" outage diff --git a/docs/automation/hourly-review-repair.md b/docs/automation/hourly-review-repair.md index 7227249584..d95786095e 100644 --- a/docs/automation/hourly-review-repair.md +++ b/docs/automation/hourly-review-repair.md @@ -12,13 +12,15 @@ engine**. contextual-orchestrator, Inkspan, or another CWL service with an explicit repository and base branch. - `pr-review-autofix.yml` is the bounded write-capable worker. It uses OpenCode - with NVIDIA NIM and does not approve or merge pull requests. + routed through the vendored contextual-orchestrator gateway and does not + approve or merge pull requests. -Orgmetra's caller remains provider-neutral. The intended model boundary is the -contextual-orchestrator gateway: provider keys stay in its KV registry and -automatic model discovery selects upstream models. A caller schedule is not -evidence that gateway credentials, discovery, or a live OpenCode tool loop are -available; those facts require exact worker-run evidence. +Every caller, including Clearfolio's, is provider-neutral by design. The model +boundary is the contextual-orchestrator gateway: provider keys stay in its KV +registry and automatic model discovery selects upstream models. A caller +schedule is not evidence that gateway credentials, discovery, or a live +OpenCode tool loop are available; those facts require exact worker-run +evidence. Merge eligibility remains owned by the separate merge scheduler, branch protection, required checks, independent review, and unresolved-thread policy. @@ -44,8 +46,10 @@ not overlap its successor. At most one repair dispatch is created per run. The caller passes only the established `PR_REVIEW_MERGE_TOKEN` and `OPENCODE_APPROVE_TOKEN` scheduler credentials. It does not receive or forward -`NVIDIA_NIM_API_KEY`; the model credential is scoped exclusively to the two -OpenCode execution steps in the separately reviewed autofix worker. +any of the five gateway provider secrets (`BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, +`NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`); those stay +scoped exclusively to the contextual-orchestrator sidecar provisioning step in +the separately reviewed autofix worker. ## Orgmetra execution contract @@ -199,7 +203,8 @@ organization-level queue inspection and bounded repair dispatch. When a scheduled run fails, classify the result before rerunning: - no actionable file-scoped feedback: expected no-op; -- missing `NVIDIA_NIM_API_KEY`: central secret configuration failure; +- missing gateway provider secrets or an unprovisioned contextual-orchestrator + sidecar: central secret/configuration failure; - head changed: safe optimistic-concurrency refusal; inspect the new head rather than retrying predecessor evidence; - out-of-scope or ignored-path change: treat as a security failure and preserve @@ -225,7 +230,8 @@ Permanent tests prove: - the dispatch budget and same-head retry floor remain one; - caller and reusable-workflow secrets are explicit and never use `secrets: inherit`; -- immutable source, NVIDIA-only model authentication, child-process credential +- immutable source, gateway-only model authentication (no direct-provider + credential in the model child process), child-process credential stripping, live-head guards, and independent reviewer identity remain intact; - ordinary and conflict repair share the complete ignored-inclusive snapshot and NUL-delimited allowlist boundary; diff --git a/docs/doctoring/hourly-nvidia-nim-autofix.md b/docs/doctoring/hourly-nvidia-nim-autofix.md index 6b05c6bd60..62ad714914 100644 --- a/docs/doctoring/hourly-nvidia-nim-autofix.md +++ b/docs/doctoring/hourly-nvidia-nim-autofix.md @@ -1,5 +1,42 @@ # Hourly NVIDIA NIM Review-Autofix Boundary +## Update (2026-08-30): model execution migrated to the contextual-orchestrator gateway + +This record's "Decision", "Provider contract", and "Credential boundary" +sections below describe the worker's design **as it stood before** +[ADR-0003](../adr/0003-contextual-orchestrator-vendored-free-zdr.md) +(2026-08-27): a single direct NVIDIA NIM provider +(`https://integrate.api.nvidia.com/v1`), the `mistralai/mistral-small-4-119b-2603` +writer model, and a lone `NVIDIA_NIM_API_KEY` credential bound to the two +OpenCode execution steps. That description is now superseded, not deleted — +kept below as the historical record of the write-scope/security work this +file documents, most of which (the allowlist, snapshot/verification, hook +suppression, and explicit push-destination controls) is provider-independent +and unchanged. + +What actually changed: `pr-review-autofix.yml`'s OpenCode execution now +routes through the vendored `contextual-orchestrator` gateway sidecar +(`scripts/ci/contextual_orchestrator_review_sidecar.sh`), provisioned with +all five provider secrets (`BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, +`NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`) and the +fail-closed zero-cost virtual model `contextual-orchestrator/orchestrator/free` +for both the primary and small model. There is no direct +`https://integrate.api.nvidia.com/v1` base URL, no hardcoded +`mistralai/mistral-small-4-119b-2603` or `nvidia/nemotron-3-nano-30b-a3b` +model id, and no bare `NVIDIA_API_KEY`/`NVIDIA_NIM_API_KEY` environment +variable read anywhere in the workflow's execution steps — enforced by +`tests/test_pr_review_autofix_nvidia_nim_contract.py::test_scheduled_autofix_routes_through_contextual_orchestrator`, +which asserts the gateway strings are present and the direct-NVIDIA base URL +and legacy model ids are absent. Separately, +`scripts/ci/select_nvidia_nim_model.py` — a standalone helper that resolved a +live NVIDIA NIM model id by calling `integrate.api.nvidia.com` directly — +was found wired into nothing (no workflow or script referenced it, only its +own test) and removed as orphaned dead code predating this migration. + +This does not change contextual-orchestrator's own internal use of NVIDIA NIM +as one of its five auto-discovered backend providers; that remains entirely +the gateway's concern, not this worker's. + ## Decision Materialize accepts only exact SHA-256 pins or a bounded relative `-r` include; a lone `--require-hashes` line is not lock evidence. diff --git a/docs/doctoring/nonnest2-hourly-review-caller.md b/docs/doctoring/nonnest2-hourly-review-caller.md index eba36c7878..942fa2e789 100644 --- a/docs/doctoring/nonnest2-hourly-review-caller.md +++ b/docs/doctoring/nonnest2-hourly-review-caller.md @@ -74,9 +74,16 @@ OpenCode GitHub App token from GitHub OIDC when the mapped PAT is absent forbids executing the caller with write or model privileges it does not need (MITRE, 2026). -Model execution remains inside the central worker. The model credential -is the GitHub Secret `NVIDIA_NIM_API_KEY`; the caller does not receive or -forward it. +Model execution remains inside the central worker, which routes through the +vendored `contextual-orchestrator` gateway sidecar +(`scripts/ci/contextual_orchestrator_review_sidecar.sh`) using all five +provider secrets (`BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, +`NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`) and the +fail-closed zero-cost `orchestrator/free` pool (see +[ADR-0003](../adr/0003-contextual-orchestrator-vendored-free-zdr.md)); the +caller receives none of them. (Updated 2026-08-30: this paragraph originally +described a single hardcoded `NVIDIA_NIM_API_KEY` model credential, which +predates the ADR-0003 gateway migration.) Before protected-master activation, the repository variable `OPENCODE_REPOSITORY_DISPATCH_TARGETS` must contain the exact diff --git a/docs/doctoring/originweave-hourly-review-caller.md b/docs/doctoring/originweave-hourly-review-caller.md index 8ed460abbe..ebd518e0c5 100644 --- a/docs/doctoring/originweave-hourly-review-caller.md +++ b/docs/doctoring/originweave-hourly-review-caller.md @@ -75,9 +75,16 @@ OpenCode GitHub App token from GitHub OIDC when the mapped PAT is absent forbids executing the caller with write or model privileges it does not need (MITRE, 2026). -Model execution remains inside the central worker. The model credential -is the GitHub Secret `NVIDIA_NIM_API_KEY`; the caller does not receive or -forward it. +Model execution remains inside the central worker, which routes through the +vendored `contextual-orchestrator` gateway sidecar +(`scripts/ci/contextual_orchestrator_review_sidecar.sh`) using all five +provider secrets (`BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, +`NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`) and the +fail-closed zero-cost `orchestrator/free` pool (see +[ADR-0003](../adr/0003-contextual-orchestrator-vendored-free-zdr.md)); the +caller receives none of them. (Updated 2026-08-30: this paragraph originally +described a single hardcoded `NVIDIA_NIM_API_KEY` model credential, which +predates the ADR-0003 gateway migration.) Before protected-main activation, the repository variable `OPENCODE_REPOSITORY_DISPATCH_TARGETS` must contain the exact diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py deleted file mode 100644 index 3a501f8374..0000000000 --- a/scripts/ci/select_nvidia_nim_model.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python3 -"""Resolve the first live NVIDIA NIM model from an ordered candidate pool. - -Why this exists ---------------- -The scheduled autofix worker used to hard-code one NVIDIA NIM model id. NVIDIA -retires hosted models on published end-of-life dates, and the endpoint then -answers every request with HTTP 410 ``Gone``, e.g. - - The model 'mistralai/mistral-small-4-119b-2603' has reached its end of life - on 2026-07-27T00:00:00Z and is no longer available. - -A single hard-coded id therefore turns a normal provider lifecycle event into a -total outage of the repair loop. This helper asks the provider which models are -actually served right now (``GET /v1/models``, the OpenAI-compatible catalog -route NVIDIA NIM implements) and returns the first entry of an ordered, -operator-controlled preference list that the provider still serves. - -The helper is deliberately fail-closed: an unreachable catalog, an unparsable -catalog, or a pool with no served candidate is an error, never a silent -fallback to an arbitrary model. - -References: - NVIDIA. (2025). *NVIDIA NIM for large language models: OpenAI-compatible - API reference*. https://docs.nvidia.com/nim/large-language-models/latest/api-reference.html - OpenAI. (2025). *API reference: List models*. - https://platform.openai.com/docs/api-reference/models/list -""" - -from __future__ import annotations - -import argparse -import http.client -import json -import os -import ssl -import sys -from urllib.parse import urlsplit - -DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1" -ALLOWED_CATALOG_HOSTS = frozenset({"integrate.api.nvidia.com"}) -DEFAULT_TIMEOUT_SECONDS = 30.0 -EX_TEMPFAIL = 75 - - -class ModelResolutionUnavailable(RuntimeError): - """The reviewed model pool cannot be resolved due to provider availability.""" - - -def parse_candidates(raw_candidates: str) -> list[str]: - """Split a whitespace-separated candidate pool into ordered model ids. - - Duplicate ids are removed while the operator's preference order is kept, so - a pool may be assembled from several sources without changing behavior. - """ - ordered: list[str] = [] - for candidate in raw_candidates.split(): - if candidate not in ordered: - ordered.append(candidate) - return ordered - - -def validate_catalog_base_url(base_url: str) -> str: - """Return the catalog base URL after refusing untrusted endpoints. - - Only HTTPS URLs on the known NVIDIA NIM integration host are accepted, so a - tampered variable cannot redirect the API key to another host. - """ - parts = urlsplit(base_url) - if parts.scheme != "https": - raise ValueError(f"NVIDIA NIM base URL must use https; got {parts.scheme or ''}") - if parts.hostname not in ALLOWED_CATALOG_HOSTS: - raise ValueError(f"NVIDIA NIM base URL host is not allowed: {parts.hostname or ''}") - if parts.port not in (None, 443): - raise ValueError(f"NVIDIA NIM base URL must use the default HTTPS port; got {parts.port}") - if parts.username or parts.password: - raise ValueError("NVIDIA NIM base URL must not embed credentials") - if parts.query or parts.fragment: - raise ValueError("NVIDIA NIM base URL must not include a query or fragment") - return base_url.rstrip("/") - - -def fetch_served_model_ids( - base_url: str, - api_key: str, - *, - timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, -) -> set[str]: - """Return the model ids the provider currently serves. - - Any transport or payload problem raises, because guessing a model id would - hide a provider outage behind a confusing downstream model error. - """ - normalized_base_url = validate_catalog_base_url(base_url) - parts = urlsplit(normalized_base_url) - request_path = f"{parts.path.rstrip('/')}/models" - try: - connection = http.client.HTTPSConnection( # nosemgrep: python.lang.security.audit.httpsconnection-detected.httpsconnection-detected - parts.hostname, - parts.port or 443, - timeout=timeout_seconds, - context=ssl.create_default_context(), - ) - try: - connection.request( - "GET", - request_path, - headers={ - "Authorization": f"Bearer {api_key}", - "Accept": "application/json", - }, - ) - response = connection.getresponse() - if response.status >= 400: - error = RuntimeError( - f"NVIDIA NIM model catalog request failed with HTTP {response.status}" - ) - if response.status == 429 or response.status >= 500: - raise ModelResolutionUnavailable(str(error)) - raise error - payload = json.loads(response.read().decode("utf-8")) - finally: - connection.close() - except RuntimeError: - raise - except (OSError, http.client.HTTPException) as error: - raise ModelResolutionUnavailable("NVIDIA NIM model catalog is unreachable") from error - except (UnicodeDecodeError, json.JSONDecodeError) as error: - raise ModelResolutionUnavailable("NVIDIA NIM model catalog returned a non-JSON body") from error - entries = payload.get("data") if isinstance(payload, dict) else None - if not isinstance(entries, list): - raise ModelResolutionUnavailable("NVIDIA NIM model catalog payload has no model list") - served = { - str(entry["id"]) - for entry in entries - if isinstance(entry, dict) and isinstance(entry.get("id"), str) and entry["id"] - } - if not served: - raise ModelResolutionUnavailable("NVIDIA NIM model catalog listed no usable model id") - return served - - -def select_model(candidates: list[str], served_model_ids: set[str], *, role: str) -> str: - """Return the first candidate the provider still serves for this role.""" - if not candidates: - raise ValueError(f"no {role} NVIDIA NIM model candidates were configured") - for candidate in candidates: - if candidate in served_model_ids: - return candidate - raise ModelResolutionUnavailable( - f"no configured {role} NVIDIA NIM model candidate is currently served: {' '.join(candidates)}. " - "Add a live model id to the candidate pool variable so the repair worker can run." - ) - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse the command line for the model resolver.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--candidates", required=True, help="whitespace-separated ordered model ids") - parser.add_argument( - "--exclude", - default="", - help="whitespace-separated model ids that cannot be selected", - ) - parser.add_argument("--role", default="primary", help="candidate pool role used in error messages") - parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="NVIDIA NIM OpenAI-compatible base URL") - parser.add_argument( - "--timeout-seconds", - type=float, - default=DEFAULT_TIMEOUT_SECONDS, - help="model catalog request timeout", - ) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - """Print the resolved model id, or report an actionable failure.""" - args = parse_args(argv) - api_key = os.environ.get("NVIDIA_API_KEY") or os.environ.get("NVIDIA_NIM_API_KEY") or "" - if not api_key: - print( - "::error::NVIDIA_API_KEY is required to resolve a live NVIDIA NIM model.", - file=sys.stderr, - ) - return 1 - try: - served = fetch_served_model_ids(args.base_url, api_key, timeout_seconds=args.timeout_seconds) - excluded = set(parse_candidates(args.exclude)) - configured_candidates = parse_candidates(args.candidates) - candidates = [candidate for candidate in configured_candidates if candidate not in excluded] - if configured_candidates and not candidates: - raise ModelResolutionUnavailable( - f"no distinct {args.role} NVIDIA NIM model candidate remains after exclusions" - ) - print(select_model(candidates, served, role=args.role)) - except ValueError as error: - print(f"::error::{error}", file=sys.stderr) - return 1 - except ModelResolutionUnavailable as error: - print(f"::error::{error}", file=sys.stderr) - return EX_TEMPFAIL - except RuntimeError as error: - print(f"::error::{error}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": # pragma: no cover - raise SystemExit(main()) diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py deleted file mode 100644 index 2f8d72c5f2..0000000000 --- a/tests/test_select_nvidia_nim_model.py +++ /dev/null @@ -1,398 +0,0 @@ -"""Tests for resolving a live NVIDIA NIM model from an ordered candidate pool.""" - -from __future__ import annotations - -import io -import http.client -import json -from pathlib import Path -import ssl -from typing import Any - -import pytest - -from scripts.ci import select_nvidia_nim_model as resolver - - -class _FakeResponse(io.BytesIO): - """Minimal context-managed HTTP response body for catalog stubs.""" - - status = 200 - - def __enter__(self) -> "_FakeResponse": - """Return the response itself, matching urlopen's context manager.""" - return self - - def __exit__(self, *_exc_info: object) -> bool: - """Close the buffer and never suppress an exception.""" - self.close() - return False - - -class _FakeConnection: - """Minimal non-context-managed HTTPS connection stub for catalog requests.""" - - def __init__( - self, - host: str, - port: int, - *, - timeout: float, - context: ssl.SSLContext, - response: _FakeResponse, - requests: list[Any], - ) -> None: - """Record the validated destination and canned response.""" - self.host = host - self.port = port - self.timeout = timeout - self.context = context - self.response = response - self.requests = requests - self.closed = False - - def close(self) -> None: - """Record explicit cleanup, matching ``HTTPSConnection.close``.""" - self.closed = True - - def request(self, method: str, path: str, *, headers: dict[str, str]) -> None: - """Record one outbound request without opening a network socket.""" - self.requests.append((self, method, path, headers)) - - def getresponse(self) -> _FakeResponse: - """Return the canned provider response.""" - return self.response - - -def _catalog(*model_ids: str) -> bytes: - """Render an OpenAI-compatible model catalog payload for the given ids.""" - return json.dumps({"object": "list", "data": [{"id": model_id} for model_id in model_ids]}).encode("utf-8") - - -def _stub_catalog(monkeypatch: pytest.MonkeyPatch, payload: bytes) -> list[Any]: - """Serve one canned catalog payload and record the issued requests.""" - requests: list[Any] = [] - - def fake_connection( - host: str, port: int, *, timeout: float, context: ssl.SSLContext - ) -> _FakeConnection: - """Return a canned HTTPS connection and record its destination.""" - return _FakeConnection( - host, - port, - timeout=timeout, - context=context, - response=_FakeResponse(payload), - requests=requests, - ) - - monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) - return requests - - -def test_catalog_sink_has_one_scoped_semgrep_exception_and_explicit_tls() -> None: - """Keep the reviewed HTTPS sink suppressed only for its known false positive.""" - source_text = Path(resolver.__file__).read_text(encoding="utf-8") - rule = "python.lang.security.audit.httpsconnection-detected.httpsconnection-detected" - sink_lines = [ - line for line in source_text.splitlines() if "http.client.HTTPSConnection(" in line - ] - - assert len(sink_lines) == 1 - assert f"# nosemgrep: {rule}" in sink_lines[0] - assert source_text.count(f"# nosemgrep: {rule}") == 1 - assert "context=ssl.create_default_context()" in source_text - - -def test_parse_candidates_keeps_preference_order_without_duplicates() -> None: - """Operators may concatenate pools; order wins and repeats are dropped.""" - assert resolver.parse_candidates(" a/one\n b/two a/one ") == ["a/one", "b/two"] - assert resolver.parse_candidates(" ") == [] - - -@pytest.mark.parametrize( - ("base_url", "message"), - [ - ("http://integrate.api.nvidia.com/v1", "must use https"), - ("https://models.example.invalid/v1", "host is not allowed"), - ("https://integrate.api.nvidia.com:8443/v1", "default HTTPS port"), - ("https://user:pass@integrate.api.nvidia.com/v1", "must not embed credentials"), - ("https://integrate.api.nvidia.com/v1?mode=models", "query or fragment"), - ("https://integrate.api.nvidia.com/v1#models", "query or fragment"), - ], -) -def test_validate_catalog_base_url_refuses_untrusted_endpoints(base_url: str, message: str) -> None: - """A tampered base URL must never receive the provider API key.""" - with pytest.raises(ValueError, match=message): - resolver.validate_catalog_base_url(base_url) - - -def test_validate_catalog_base_url_normalizes_the_trusted_endpoint() -> None: - """The trusted endpoint is accepted with any trailing slash removed.""" - assert resolver.validate_catalog_base_url(f"{resolver.DEFAULT_BASE_URL}/") == resolver.DEFAULT_BASE_URL - - -def test_fetch_served_model_ids_returns_the_live_catalog(monkeypatch: pytest.MonkeyPatch) -> None: - """The resolver reads ids from the provider's OpenAI-compatible catalog.""" - requests = _stub_catalog(monkeypatch, _catalog("a/one", "b/two")) - - served = resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key", timeout_seconds=7.0) - - assert served == {"a/one", "b/two"} - connection, method, path, headers = requests[0] - assert connection.host == "integrate.api.nvidia.com" - assert connection.port == 443 - assert connection.timeout == 7.0 - assert connection.context.verify_mode == ssl.CERT_REQUIRED - assert connection.context.check_hostname is True - assert connection.closed is True - assert method == "GET" - assert path == "/v1/models" - assert headers["Authorization"] == "Bearer secret-key" - - -def test_fetch_served_model_ids_ignores_malformed_entries(monkeypatch: pytest.MonkeyPatch) -> None: - """Entries without a usable string id cannot become selectable models.""" - payload = json.dumps({"data": [{"id": ""}, {"id": 7}, "not-an-object", {"id": "a/one"}]}).encode("utf-8") - _stub_catalog(monkeypatch, payload) - - assert resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") == {"a/one"} - - -@pytest.mark.parametrize( - ("error", "message"), - [ - (http.client.RemoteDisconnected("closed"), "unreachable"), - (OSError("dns"), "unreachable"), - ], -) -def test_fetch_served_model_ids_fails_closed_on_transport_errors( - monkeypatch: pytest.MonkeyPatch, error: Exception, message: str -) -> None: - """A catalog outage is reported, never masked by guessing a model id.""" - - def fake_connection( - _host: str, _port: int, *, timeout: float, context: ssl.SSLContext - ) -> _FakeConnection: - """Raise the configured provider failure from the HTTP boundary.""" - del timeout - del context - raise error - - monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) - - with pytest.raises(RuntimeError, match=message): - resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") - - -@pytest.mark.parametrize( - ("status", "error_type"), - [ - (401, RuntimeError), - (429, resolver.ModelResolutionUnavailable), - (503, resolver.ModelResolutionUnavailable), - ], -) -def test_fetch_served_model_ids_reports_http_status( - monkeypatch: pytest.MonkeyPatch, - status: int, - error_type: type[RuntimeError], -) -> None: - """Provider HTTP failures identify the status without exposing credentials.""" - response = _FakeResponse(b"{}") - response.status = status - - def fake_connection( - _host: str, _port: int, *, timeout: float, context: ssl.SSLContext - ) -> _FakeConnection: - """Return an unauthorized provider response.""" - return _FakeConnection( - "integrate.api.nvidia.com", - 443, - timeout=timeout, - context=context, - response=response, - requests=[], - ) - - monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) - - with pytest.raises(error_type, match=f"HTTP {status}"): - resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") - - -@pytest.mark.parametrize( - ("payload", "message"), - [ - (b"maintenance", "non-JSON body"), - (b"\x80", "non-JSON body"), - (b'{"object": "list"}', "no model list"), - (b'{"data": []}', "no usable model id"), - ], -) -def test_fetch_served_model_ids_fails_closed_on_unusable_payloads( - monkeypatch: pytest.MonkeyPatch, payload: bytes, message: str -) -> None: - """Unparsable or empty catalogs are errors rather than silent fallbacks.""" - _stub_catalog(monkeypatch, payload) - - with pytest.raises(RuntimeError, match=message): - resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") - - -def test_select_model_prefers_the_first_served_candidate() -> None: - """A retired first choice transparently falls through to the next live one.""" - candidates = ["retired/model", "live/model", "other/model"] - - assert resolver.select_model(candidates, {"live/model", "other/model"}, role="primary") == "live/model" - - -def test_select_model_requires_a_configured_pool() -> None: - """An empty pool is a configuration error with the role named.""" - with pytest.raises(ValueError, match="no small NVIDIA NIM model candidates"): - resolver.select_model([], {"live/model"}, role="small") - - -def test_select_model_reports_a_fully_retired_pool() -> None: - """When no candidate is served, the message tells the operator what to do.""" - with pytest.raises(RuntimeError, match="Add a live model id to the candidate pool"): - resolver.select_model(["retired/model"], {"live/model"}, role="primary") - - -def test_main_prints_the_resolved_model_id( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """The successful path prints exactly the resolved id for shell capture.""" - monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") - _stub_catalog(monkeypatch, _catalog("live/model")) - - exit_code = resolver.main(["--role", "primary", "--candidates", "retired/model live/model"]) - - assert exit_code == 0 - assert capsys.readouterr().out == "live/model\n" - - -def test_main_excludes_the_resolved_primary_from_fallback_selection( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """Fallback resolution selects a distinct live model from an overlapping pool.""" - monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") - _stub_catalog(monkeypatch, _catalog("primary/model", "fallback/model")) - - exit_code = resolver.main( - [ - "--role", - "fallback", - "--candidates", - "primary/model fallback/model", - "--exclude", - "primary/model", - ] - ) - - assert exit_code == 0 - assert capsys.readouterr().out == "fallback/model\n" - - -def test_main_treats_exclusion_only_empty_pool_as_temporarily_unavailable( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """A valid pool exhausted by exclusion keeps cross-provider failover available.""" - monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") - _stub_catalog(monkeypatch, _catalog("primary/model")) - - exit_code = resolver.main( - [ - "--role", - "fallback", - "--candidates", - "primary/model", - "--exclude", - "primary/model", - ] - ) - - assert exit_code == resolver.EX_TEMPFAIL - assert "no distinct fallback" in capsys.readouterr().err - - -def test_main_accepts_the_workflow_secret_name( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """Either credential variable name works, so callers need no shim.""" - monkeypatch.delenv("NVIDIA_API_KEY", raising=False) - monkeypatch.setenv("NVIDIA_NIM_API_KEY", "secret-key") - _stub_catalog(monkeypatch, _catalog("live/model")) - - assert resolver.main(["--candidates", "live/model"]) == 0 - assert capsys.readouterr().out == "live/model\n" - - -def test_main_requires_a_provider_credential( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """Without a credential the resolver fails closed with a CI annotation.""" - monkeypatch.delenv("NVIDIA_API_KEY", raising=False) - monkeypatch.delenv("NVIDIA_NIM_API_KEY", raising=False) - - assert resolver.main(["--candidates", "live/model"]) == 1 - assert "NVIDIA_API_KEY is required" in capsys.readouterr().err - - -def test_main_annotates_a_resolution_failure( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """Resolution failures surface as GitHub error annotations, not tracebacks.""" - monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") - _stub_catalog(monkeypatch, _catalog("live/model")) - - assert resolver.main(["--candidates", "retired/model"]) == resolver.EX_TEMPFAIL - assert "::error::no configured primary NVIDIA NIM model candidate" in capsys.readouterr().err - - -def test_main_treats_invalid_catalog_utf8_as_temporary( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """Malformed provider bytes preserve the workflow's fallback exit code.""" - monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") - _stub_catalog(monkeypatch, b"\x80") - - assert resolver.main(["--candidates", "live/model"]) == resolver.EX_TEMPFAIL - assert "non-JSON body" in capsys.readouterr().err - - -def test_main_keeps_invalid_operator_configuration_nonrecoverable( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """An empty operator pool is invalid rather than provider unavailability.""" - monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") - _stub_catalog(monkeypatch, _catalog("live/model")) - - assert resolver.main(["--candidates", ""]) == 1 - assert "no primary NVIDIA NIM model candidates" in capsys.readouterr().err - - -def test_main_keeps_catalog_authentication_errors_nonrecoverable( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """An invalid provider credential must not silently switch providers.""" - monkeypatch.setenv("NVIDIA_API_KEY", "invalid-key") - response = _FakeResponse(b"{}") - response.status = 401 - - def fake_connection( - _host: str, _port: int, *, timeout: float, context: ssl.SSLContext - ) -> _FakeConnection: - return _FakeConnection( - "integrate.api.nvidia.com", - 443, - timeout=timeout, - context=context, - response=response, - requests=[], - ) - - monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) - - assert resolver.main(["--candidates", "live/model"]) == 1 - assert "HTTP 401" in capsys.readouterr().err