diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 4e79a3025..428cbfaf1 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -8,7 +8,7 @@ on: push: branches: [main] pull_request: - branches: [main] + types: [opened, synchronize, reopened, ready_for_review, converted_to_draft, closed] schedule: - cron: "17 3 * * 1" - cron: "41 4 * * 2" @@ -18,13 +18,15 @@ permissions: contents: read concurrency: - group: local-quality-${{ github.repository }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.event.schedule || github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || github.event.schedule || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: tests: name: Tests and package quality - if: github.event_name != 'schedule' + if: >- + github.event_name != 'schedule' && + (github.event_name != 'pull_request' || (github.event.action != 'closed' && github.event.pull_request.draft == false)) runs-on: ubuntu-latest steps: - name: Checkout repository @@ -81,7 +83,9 @@ jobs: fuzz: name: Property and coverage-guided fuzzing - if: github.event_name != 'schedule' || github.event.schedule == '41 4 * * 2' + if: >- + (github.event_name != 'schedule' || github.event.schedule == '41 4 * * 2') && + (github.event_name != 'pull_request' || (github.event.action != 'closed' && github.event.pull_request.draft == false)) runs-on: ubuntu-latest steps: - name: Checkout repository @@ -134,7 +138,10 @@ jobs: name: CodeQL, supply chain, and SBOM if: >- (github.event_name != 'schedule' || github.event.schedule == '17 3 * * 1') && - (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + (github.event_name != 'pull_request' || + (github.event.action != 'closed' && + github.event.pull_request.draft == false && + github.event.pull_request.head.repo.full_name == github.repository)) runs-on: ubuntu-latest permissions: actions: read diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0b268c9..83a3be438 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- Refresh the time-bounded NVIDIA hosted NIM access evidence against the + official Run NIM Anywhere terms, restoring fail-closed live benchmark + execution through 2026-10-05 without treating prototype access as production + pricing or licensing evidence. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. diff --git a/contextual_orchestrator/nim_benchmark.py b/contextual_orchestrator/nim_benchmark.py index 3db7aca5e..286f39496 100644 --- a/contextual_orchestrator/nim_benchmark.py +++ b/contextual_orchestrator/nim_benchmark.py @@ -112,8 +112,8 @@ def estimate_tokens(text: str) -> int: ACTUAL_COST_EVIDENCE: dict[str, Any] = { "evidence_schema_version": "1.0.0", - "source_title": "NVIDIA NIM General FAQ", - "source_url": "https://docs.api.nvidia.com/nim/docs/product", + "source_title": "Run NIM Anywhere", + "source_url": "https://docs.api.nvidia.com/nim/docs/run-anywhere", "reviewed_at_date": "2026-09-05", "valid_until_date": "2026-10-05", "access_program": "NVIDIA Developer Program API Catalog hosted endpoints", @@ -2535,9 +2535,9 @@ def _validate_actual_cost_evidence(report: dict[str, Any]) -> None: raise BenchmarkContractError( "actual cost evidence must preserve the reviewed zero-cost value" ) - if evidence["source_url"] != "https://docs.api.nvidia.com/nim/docs/product": + if evidence["source_url"] != "https://docs.api.nvidia.com/nim/docs/run-anywhere": raise BenchmarkContractError( - "actual cost evidence must cite the reviewed NVIDIA NIM General FAQ" + "actual cost evidence must cite the reviewed NVIDIA NIM access terms" ) reviewed_at = _parse_evidence_date(evidence["reviewed_at_date"], "reviewed_at_date") valid_until = _parse_evidence_date(evidence["valid_until_date"], "valid_until_date") diff --git a/docs/doctoring/nim-benchmark-evidence-grade.md b/docs/doctoring/nim-benchmark-evidence-grade.md index add6a2b1b..7bca8d728 100644 --- a/docs/doctoring/nim-benchmark-evidence-grade.md +++ b/docs/doctoring/nim-benchmark-evidence-grade.md @@ -127,7 +127,7 @@ issuing five calls. Actual access cost and hypothetical production cost are separate fields and separate evidence classes. -As reviewed on 2026-09-05, NVIDIA's NIM General FAQ states that NVIDIA Developer +As reviewed on 2026-09-05, NVIDIA's Run NIM Anywhere page states that NVIDIA Developer Program members have free access to hosted NIM API endpoints for prototyping. The same source distinguishes development, testing, research, and evaluation from production and states that production requires NVIDIA AI Enterprise. The @@ -237,8 +237,9 @@ Cosgrove, C., Manning, C. D., Ré, C., Acosta-Navas, D., Hudson, D. A., … Kore Y. (2023). Holistic evaluation of language models. *Transactions on Machine Learning Research*. https://doi.org/10.48550/arXiv.2211.09110 -NVIDIA Corporation. (n.d.). *General FAQ*. NVIDIA NIM Documentation. Retrieved -August 5, 2026, from https://docs.api.nvidia.com/nim/docs/product +NVIDIA Corporation. (n.d.). *Run NIM Anywhere*. NVIDIA NIM Documentation. +Retrieved September 5, 2026, from +https://docs.api.nvidia.com/nim/docs/run-anywhere NVIDIA Corporation. (2026, June 4). *NIM offerings*. NVIDIA NIM for Large Language Models. https://docs.nvidia.com/nim/large-language-models/2.0.5/about-nim-llm/nim-offerings.html diff --git a/docs/nim_benchmark.md b/docs/nim_benchmark.md index 4e6a81c7b..2a5ecea04 100644 --- a/docs/nim_benchmark.md +++ b/docs/nim_benchmark.md @@ -150,7 +150,7 @@ actual and hypothetical cost fields, and a response SHA-256. Actual endpoint access and hypothetical paid cost remain separate evidence classes. -As reviewed on 2026-09-05, NVIDIA's current General FAQ states that NVIDIA +As reviewed on 2026-09-05, NVIDIA's Run NIM Anywhere page states that NVIDIA Developer Program members have free access to hosted NIM API endpoints for prototyping. The report records that exact source, review date, validity horizon, program context, production distinction, and uncertainty. A live run fails diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d145a0b1d..8198808fa 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2185,7 +2185,7 @@ live work item. | P0 | Operational failure paths are not yet one buyer-verifiable contract. | [#771](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/771) and [#772](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/772) are open. | Exact-head full suite, focused edge tests, security scans, and a buyer-facing failure/rollback trace pass. | | P1 | PII can remain usable without blanket masking, but authorization/encryption is unfinished. | [ADR 0010](planning/adrs/0010-pii-audit-not-mask.md) records the no-blanket-masking policy and explicitly leaves authorization/encryption as follow-up. The actual design is proposed [ADR 0011 at #762's exact head](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/8f87bcaeddff0866e26900e41deeafe208d8f9e4/docs/planning/adrs/0011-pii-purpose-authorization-and-field-encryption.md); both design [#762](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/762) and implementation [#803](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/803) remain open and are not protected-main evidence. | Protected main has purpose-scoped caller/role authorization, field-level encryption at rest, credential-only redaction, and audit tests proving raw PII is returned only to an authorized purpose. | | P1 | Deep-workflow compute policy lacks provider-neutral measured ablation. | PR [#785](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/785) supplies opt-in profiles, snapshot replay, and synthetic/estimated RMSE; the production gate remains closed pending buyer-held-out measurement. | Equal-budget shallow/deep/role-effort/access-list replay with reproducible quality, verifier, cost, and trace metrics. | -| P1 | Model discovery lacks live NVIDIA NIM evidence. | Issue [#86](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/86) remains open; active PR [#906](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/906) now provides the bounded benchmark, but it is not protected-main evidence while OpenCode/Strix and independent approval remain incomplete. | KV-backed NIM discovery benchmark records model-level declared capability, price provenance, failure class, and quality result without secret leakage; protected main then activates only capability-qualified deployments. | +| P1 | Model discovery lacks live NVIDIA NIM evidence. | Issue [#86](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/86) remains open; active PR [#906](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/906) provides the bounded benchmark, but it is not protected-main evidence while OpenCode/Strix and independent approval remain incomplete. PR [#1063](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1063) source commit `9928e4ab` refreshes the expired prototype-access evidence against NVIDIA's official Run NIM Anywhere terms; its full local tree is `3394 passed, 2 skipped`, while protected exact-head checks and independent review remain required. | KV-backed NIM discovery benchmark records model-level declared capability, price provenance, failure class, and quality result without secret leakage; protected main then activates only capability-qualified deployments. | | P1 | Release gate and hourly loop need exact operational proof. | Central scheduler workflows own the loop; PR [#784](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/784) adds the exact-head authority evaluator/collector, but protected approval and release evidence remain open. | One scheduler owner, no duplicate workflow, exact-head release gate, version/changelog update, and normal protected release evidence. | | P2 | LineageWeave has no protected-main consumer acceptance gate. | [#801](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/801) added explicit CLI `argv` only to a non-main stack. Main-target [#823](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/823) has the explicit contract at `6bb3fe2c54cda9f574cd239922bc91ece5ea2585`, but remains `REVIEW_REQUIRED`/blocked despite terminal hosted checks; documented protected main still exposes `contextual_orchestrator.__main__.main()` without an `argv` argument. LineageWeave `main@ef6f5a5f` still assigns `sys.argv` in `docker/contextual-orchestrator/start.py`, and its bootstrap test observes that mutation; open LineageWeave [#468](https://github.com/ContextualWisdomLab/LineageWeave/pull/468) retains it. Its opt-in real-provider test bypasses that bootstrap, so neither it nor #823's mocked-server unit test is authenticated consumer proof. | PR #823 explicit CLI invocation contract is merged to protected main and update LineageWeave at that exact upstream pin to invoke the server with explicit arguments rather than mutating process arguments. Then run a LineageWeave-owned authenticated `/v1/chat/completions` end-to-end test that proves process `sys.argv` is unchanged; retain authorization and chat-completion evidence against the exact protected main SHA. | | P2 | Ecosystem boundaries need consumer proof. | `naruon`, `.github`, and sibling components are named consumers, but this repo remains one deployable product. | test_naruon_ecosystem_connector.py proves the exact JSON schema and endpoint consumption without speculatively extracting the codebase. | diff --git a/tests/test_nim_benchmark_release_acceptance.py b/tests/test_nim_benchmark_release_acceptance.py index 59ecde9ff..c4344405d 100644 --- a/tests/test_nim_benchmark_release_acceptance.py +++ b/tests/test_nim_benchmark_release_acceptance.py @@ -659,7 +659,7 @@ def test_actual_cost_evidence_validation_and_expiry_paths( wrong_source = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} wrong_source["actual_cost_evidence"]["source_url"] = "https://example.test" - with pytest.raises(nb.BenchmarkContractError, match="General FAQ"): + with pytest.raises(nb.BenchmarkContractError, match="NVIDIA NIM access terms"): nb._validate_actual_cost_evidence(wrong_source) invalid_dates = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} diff --git a/tests/test_repository_security_metadata.py b/tests/test_repository_security_metadata.py index 1105543dd..d7c7c4523 100644 --- a/tests/test_repository_security_metadata.py +++ b/tests/test_repository_security_metadata.py @@ -68,9 +68,14 @@ def test_security_workflow_covers_core_repository_security_process(): for duplicate_scanner in removed_duplicate_scanners: assert duplicate_scanner not in workflow_text - assert "local-quality-${{ github.repository }}-${{ github.event_name }}-${{" in workflow_text - assert "github.event.pull_request.number || github.event.schedule || github.ref" in workflow_text - assert "cancel-in-progress: true" in workflow_text + assert "${{ github.workflow }}-${{ github.repository }}-${{" in workflow_text + assert "github.event.pull_request.number || github.event.schedule || github.run_id" in workflow_text + assert "cancel-in-progress: ${{ github.event_name == 'pull_request' }}" in workflow_text + assert ( + "types: [opened, synchronize, reopened, ready_for_review, converted_to_draft, closed]" + in workflow_text + ) + assert workflow_text.count("github.event.pull_request.draft == false") == 3 assert not (ROOT_DIR / ".github/workflows/ci.yml").exists() assert not (ROOT_DIR / ".github/workflows/fuzz.yml").exists() @@ -81,6 +86,15 @@ def test_security_workflow_covers_core_repository_security_process(): assert all(re.search(r"@[0-9a-f]{40}(?:\s+#|$)", line) for line in uses_lines) +def test_security_workflow_supports_stacked_pull_requests(): + workflow_text = read_text(".github/workflows/security.yml") + pull_request_trigger = workflow_text.split(" pull_request:\n", 1)[1].split( + " schedule:\n", 1 + )[0] + + assert "branches:" not in pull_request_trigger + + def test_dependabot_tracks_actions_and_python_dependencies(): dependabot_text = read_text(".github/dependabot.yml")