From b0bd71dfa5d6ca065052bc951ad9c9209b88413b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 10:16:08 +0900 Subject: [PATCH] fix(nim): refresh hosted access evidence before CI stack --- CHANGELOG.md | 4 ++++ contextual_orchestrator/nim_benchmark.py | 12 ++++++------ docs/doctoring/nim-benchmark-evidence-grade.md | 9 +++++---- docs/nim_benchmark.md | 4 ++-- docs/product-technical-gap-baseline.md | 2 +- tests/test_nim_benchmark_release_acceptance.py | 4 ++-- 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0b268c9..63e2c4b70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- Refresh the time-bounded NVIDIA hosted NIM access evidence against the + official Run NIM Anywhere terms, restoring fail-closed live benchmark + execution through 2026-10-04 without treating prototype access as production + pricing or licensing evidence. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. diff --git a/contextual_orchestrator/nim_benchmark.py b/contextual_orchestrator/nim_benchmark.py index 1823b14cc..c7a247bf2 100644 --- a/contextual_orchestrator/nim_benchmark.py +++ b/contextual_orchestrator/nim_benchmark.py @@ -112,10 +112,10 @@ def estimate_tokens(text: str) -> int: ACTUAL_COST_EVIDENCE: dict[str, Any] = { "evidence_schema_version": "1.0.0", - "source_title": "NVIDIA NIM General FAQ", - "source_url": "https://docs.api.nvidia.com/nim/docs/product", - "reviewed_at_date": "2026-08-05", - "valid_until_date": "2026-09-04", + "source_title": "Run NIM Anywhere", + "source_url": "https://docs.api.nvidia.com/nim/docs/run-anywhere", + "reviewed_at_date": "2026-09-05", + "valid_until_date": "2026-10-04", "access_program": "NVIDIA Developer Program API Catalog hosted endpoints", "access_scope": "free API endpoint access for prototyping", "production_access_note": ( @@ -2535,9 +2535,9 @@ def _validate_actual_cost_evidence(report: dict[str, Any]) -> None: raise BenchmarkContractError( "actual cost evidence must preserve the reviewed zero-cost value" ) - if evidence["source_url"] != "https://docs.api.nvidia.com/nim/docs/product": + if evidence["source_url"] != "https://docs.api.nvidia.com/nim/docs/run-anywhere": raise BenchmarkContractError( - "actual cost evidence must cite the reviewed NVIDIA NIM General FAQ" + "actual cost evidence must cite the reviewed NVIDIA NIM access terms" ) reviewed_at = _parse_evidence_date(evidence["reviewed_at_date"], "reviewed_at_date") valid_until = _parse_evidence_date(evidence["valid_until_date"], "valid_until_date") diff --git a/docs/doctoring/nim-benchmark-evidence-grade.md b/docs/doctoring/nim-benchmark-evidence-grade.md index 14a197491..48609a427 100644 --- a/docs/doctoring/nim-benchmark-evidence-grade.md +++ b/docs/doctoring/nim-benchmark-evidence-grade.md @@ -127,14 +127,14 @@ issuing five calls. Actual access cost and hypothetical production cost are separate fields and separate evidence classes. -As reviewed on 2026-08-05, NVIDIA's NIM General FAQ states that NVIDIA Developer +As reviewed on 2026-09-05, NVIDIA's Run NIM Anywhere page states that NVIDIA Developer Program members have free access to hosted NIM API endpoints for prototyping. The same source distinguishes development, testing, research, and evaluation from production and states that production requires NVIDIA AI Enterprise. The report therefore records `actual_cost_usd = 0.0` only for the reviewed hosted endpoint access context, includes the exact source, review date, validity horizon, program scope, production distinction, and uncertainty, and refuses a -live run after 2026-09-04 until the source is reviewed again. +live run after 2026-10-04 until the source is reviewed again. No NVIDIA model price is embedded or inferred. A live hypothetical pricing scenario is optional; absence means `unknown`. If supplied, it must be marked @@ -237,8 +237,9 @@ Cosgrove, C., Manning, C. D., Ré, C., Acosta-Navas, D., Hudson, D. A., … Kore Y. (2023). Holistic evaluation of language models. *Transactions on Machine Learning Research*. https://doi.org/10.48550/arXiv.2211.09110 -NVIDIA Corporation. (n.d.). *General FAQ*. NVIDIA NIM Documentation. Retrieved -August 5, 2026, from https://docs.api.nvidia.com/nim/docs/product +NVIDIA Corporation. (n.d.). *Run NIM Anywhere*. NVIDIA NIM Documentation. +Retrieved September 5, 2026, from +https://docs.api.nvidia.com/nim/docs/run-anywhere NVIDIA Corporation. (2026, June 4). *NIM offerings*. NVIDIA NIM for Large Language Models. https://docs.nvidia.com/nim/large-language-models/2.0.5/about-nim-llm/nim-offerings.html diff --git a/docs/nim_benchmark.md b/docs/nim_benchmark.md index 479afdcb2..abf64bdfb 100644 --- a/docs/nim_benchmark.md +++ b/docs/nim_benchmark.md @@ -150,11 +150,11 @@ actual and hypothetical cost fields, and a response SHA-256. Actual endpoint access and hypothetical paid cost remain separate evidence classes. -As reviewed on 2026-08-05, NVIDIA's current General FAQ states that NVIDIA +As reviewed on 2026-09-05, NVIDIA's Run NIM Anywhere page states that NVIDIA Developer Program members have free access to hosted NIM API endpoints for prototyping. The report records that exact source, review date, validity horizon, program context, production distinction, and uncertainty. A live run fails -closed after 2026-09-04 until the official source is reviewed again. Production +closed after 2026-10-04 until the official source is reviewed again. Production support and licensing are not inferred from prototype access and require NVIDIA AI Enterprise under the reviewed documentation. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d145a0b1d..8198808fa 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2185,7 +2185,7 @@ live work item. | P0 | Operational failure paths are not yet one buyer-verifiable contract. | [#771](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/771) and [#772](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/772) are open. | Exact-head full suite, focused edge tests, security scans, and a buyer-facing failure/rollback trace pass. | | P1 | PII can remain usable without blanket masking, but authorization/encryption is unfinished. | [ADR 0010](planning/adrs/0010-pii-audit-not-mask.md) records the no-blanket-masking policy and explicitly leaves authorization/encryption as follow-up. The actual design is proposed [ADR 0011 at #762's exact head](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/8f87bcaeddff0866e26900e41deeafe208d8f9e4/docs/planning/adrs/0011-pii-purpose-authorization-and-field-encryption.md); both design [#762](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/762) and implementation [#803](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/803) remain open and are not protected-main evidence. | Protected main has purpose-scoped caller/role authorization, field-level encryption at rest, credential-only redaction, and audit tests proving raw PII is returned only to an authorized purpose. | | P1 | Deep-workflow compute policy lacks provider-neutral measured ablation. | PR [#785](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/785) supplies opt-in profiles, snapshot replay, and synthetic/estimated RMSE; the production gate remains closed pending buyer-held-out measurement. | Equal-budget shallow/deep/role-effort/access-list replay with reproducible quality, verifier, cost, and trace metrics. | -| P1 | Model discovery lacks live NVIDIA NIM evidence. | Issue [#86](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/86) remains open; active PR [#906](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/906) now provides the bounded benchmark, but it is not protected-main evidence while OpenCode/Strix and independent approval remain incomplete. | KV-backed NIM discovery benchmark records model-level declared capability, price provenance, failure class, and quality result without secret leakage; protected main then activates only capability-qualified deployments. | +| P1 | Model discovery lacks live NVIDIA NIM evidence. | Issue [#86](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/86) remains open; active PR [#906](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/906) provides the bounded benchmark, but it is not protected-main evidence while OpenCode/Strix and independent approval remain incomplete. PR [#1063](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1063) source commit `9928e4ab` refreshes the expired prototype-access evidence against NVIDIA's official Run NIM Anywhere terms; its full local tree is `3394 passed, 2 skipped`, while protected exact-head checks and independent review remain required. | KV-backed NIM discovery benchmark records model-level declared capability, price provenance, failure class, and quality result without secret leakage; protected main then activates only capability-qualified deployments. | | P1 | Release gate and hourly loop need exact operational proof. | Central scheduler workflows own the loop; PR [#784](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/784) adds the exact-head authority evaluator/collector, but protected approval and release evidence remain open. | One scheduler owner, no duplicate workflow, exact-head release gate, version/changelog update, and normal protected release evidence. | | P2 | LineageWeave has no protected-main consumer acceptance gate. | [#801](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/801) added explicit CLI `argv` only to a non-main stack. Main-target [#823](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/823) has the explicit contract at `6bb3fe2c54cda9f574cd239922bc91ece5ea2585`, but remains `REVIEW_REQUIRED`/blocked despite terminal hosted checks; documented protected main still exposes `contextual_orchestrator.__main__.main()` without an `argv` argument. LineageWeave `main@ef6f5a5f` still assigns `sys.argv` in `docker/contextual-orchestrator/start.py`, and its bootstrap test observes that mutation; open LineageWeave [#468](https://github.com/ContextualWisdomLab/LineageWeave/pull/468) retains it. Its opt-in real-provider test bypasses that bootstrap, so neither it nor #823's mocked-server unit test is authenticated consumer proof. | PR #823 explicit CLI invocation contract is merged to protected main and update LineageWeave at that exact upstream pin to invoke the server with explicit arguments rather than mutating process arguments. Then run a LineageWeave-owned authenticated `/v1/chat/completions` end-to-end test that proves process `sys.argv` is unchanged; retain authorization and chat-completion evidence against the exact protected main SHA. | | P2 | Ecosystem boundaries need consumer proof. | `naruon`, `.github`, and sibling components are named consumers, but this repo remains one deployable product. | test_naruon_ecosystem_connector.py proves the exact JSON schema and endpoint consumption without speculatively extracting the codebase. | diff --git a/tests/test_nim_benchmark_release_acceptance.py b/tests/test_nim_benchmark_release_acceptance.py index 3171841e6..3cacf2273 100644 --- a/tests/test_nim_benchmark_release_acceptance.py +++ b/tests/test_nim_benchmark_release_acceptance.py @@ -659,11 +659,11 @@ def test_actual_cost_evidence_validation_and_expiry_paths( wrong_source = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} wrong_source["actual_cost_evidence"]["source_url"] = "https://example.test" - with pytest.raises(nb.BenchmarkContractError, match="General FAQ"): + with pytest.raises(nb.BenchmarkContractError, match="NVIDIA NIM access terms"): nb._validate_actual_cost_evidence(wrong_source) invalid_dates = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} - invalid_dates["actual_cost_evidence"]["reviewed_at_date"] = "2026-09-05" + invalid_dates["actual_cost_evidence"]["reviewed_at_date"] = "2026-10-05" with pytest.raises(nb.BenchmarkContractError, match="validity precedes"): nb._validate_actual_cost_evidence(invalid_dates)