diff --git a/.github/workflows/baseline.yml b/.github/workflows/baseline.yml index 17148b089..620d9945e 100644 --- a/.github/workflows/baseline.yml +++ b/.github/workflows/baseline.yml @@ -17,6 +17,8 @@ on: - vendor.lock.json - qualification/** - Makefile + - scripts/ci-install-apt-deps.sh + - tests/ci/test_apt_installer.py - .github/workflows/baseline.yml pull_request: paths: @@ -31,6 +33,8 @@ on: - vendor.lock.json - qualification/** - Makefile + - scripts/ci-install-apt-deps.sh + - tests/ci/test_apt_installer.py - .github/workflows/baseline.yml permissions: @@ -178,14 +182,15 @@ jobs: key: engine-nnue-${{ runner.os }}-${{ hashFiles('vendor.lock.json') }} - name: Fetch and verify pinned engine NNUE artifacts + timeout-minutes: 5 run: | ./scripts/fetch-stockfish-network.sh >/dev/null ./scripts/fetch-reckless-network.sh >/dev/null - name: Install LC0 validation dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev + bash scripts/ci-install-apt-deps.sh meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev - name: Toolchain versions run: | diff --git a/.github/workflows/engine-optimization.yml b/.github/workflows/engine-optimization.yml index 700c07d1b..34cf596b7 100644 --- a/.github/workflows/engine-optimization.yml +++ b/.github/workflows/engine-optimization.yml @@ -10,6 +10,9 @@ on: - 'qualification/online-hybrid-v1-resource-observations.json' - 'qualification/replay-history-sources.json' - 'qualification/resource-profile-catalog-v1.json' + - 'qualification/resource-profile-catalog-v1-j7-baseline.json' + - 'qualification/online-engine-opt-v2-j7-baseline.json' + - 'config/allfather.online-hybrid-v2.validation-j7-baseline.json' - 'qualification/adaptive-clock-v1.json' - 'qualification/work-grant-scheduler-v1.json' - 'qualification/adaptive-resource-allocation-v1.json' @@ -17,6 +20,7 @@ on: - 'qualification/online-engine-opt-v2.json' - 'qualification/online-hybrid-v2.json' - 'qualification/online-hybrid-v2-candidate.json' + - 'qualification/online-hybrid-v2-b4.json' - 'qualification/online-hybrid-v2-candidate-witnesses.json' - 'qualification/g3-candidate-discovery-source.json' - 'qualification/local-full-game-v2.json' @@ -38,6 +42,8 @@ on: - 'scripts/engine-opt-*.py' - 'scripts/build-online-engine-opt-v2.sh' - 'scripts/qualify-engine-opt-v2.py' + - 'scripts/qualify-engine-opt-evidence.py' + - 'scripts/qualify-engine-opt-v2-promotion-gate.py' - 'scripts/qualify-adaptive-time.py' - 'scripts/adaptive-time-contract.py' - 'scripts/qualify-work-grant-scheduler.py' @@ -50,6 +56,7 @@ on: - 'scripts/build-j10-allocation-calibration.py' - 'scripts/qualify-online-hybrid-v2.py' - 'scripts/qualify-online-hybrid-v2-candidate.py' + - 'scripts/qualify-online-hybrid-v2-b4.py' - 'scripts/qualify-engine-opt-v2-candidate.py' - 'tests/engine_opt/**' - 'tests/controller/test_engine_opt_runtime.py' @@ -71,6 +78,8 @@ on: - 'tests/harness/uci_session.py' - 'Makefile' - 'Makefile.local-game' + - 'scripts/ci-install-apt-deps.sh' + - 'tests/ci/test_apt_installer.py' - '.github/workflows/engine-optimization.yml' permissions: @@ -139,10 +148,14 @@ jobs: python3 tests/engine_opt/test_execution_domain.py python3 tests/engine_opt/test_matrix_qualification.py python3 tests/engine_opt/test_matrix_scope.py + python3 tests/engine_opt/test_aggregate_paths.py + python3 tests/ci/test_apt_installer.py python3 tests/engine_opt/test_g3_witness_extractor.py + python3 tests/engine_opt/test_g3_b4.py python3 tests/engine_opt/test_constituent_hash.py python3 tests/engine_opt/test_legacy_calibration_reconciliation.py python3 tests/engine_opt/test_candidate_gate.py + python3 tests/engine_opt/test_promotion_gate.py python3 tests/controller/test_partition_versions.py python3 -m json.tool qualification/constituent-benchmark-v2.json >/dev/null python3 -m json.tool qualification/online-hybrid-v1-resource-calibration.json >/dev/null @@ -172,10 +185,11 @@ jobs: python3 tests/controller/test_runtime_profiles.py python3 tests/adapters/test_linux_affinity.py python3 tests/controller/test_resource_control.py - python3 -m py_compile scripts/qualify-engine-opt-evidence.py scripts/qualify-engine-opt-v2-candidate.py scripts/qualify-engine-opt-v2-candidate-gate.py scripts/qualify-engine-opt-matrix.py scripts/qualify-online-hybrid-v2-candidate.py scripts/extract-g3-candidate-witnesses.py tools/engine_opt/matrix_qualification.py tools/engine_opt/candidate_gate.py + python3 -m py_compile scripts/qualify-engine-opt-evidence.py scripts/qualify-engine-opt-v2-candidate.py scripts/qualify-engine-opt-v2-candidate-gate.py scripts/qualify-engine-opt-v2-promotion-gate.py scripts/qualify-engine-opt-matrix.py scripts/qualify-online-hybrid-v2-candidate.py scripts/extract-g3-candidate-witnesses.py tools/engine_opt/matrix_qualification.py tools/engine_opt/candidate_gate.py tools/engine_opt/promotion_gate.py python3 scripts/qualify-online-hybrid-v2.py --static python3 scripts/qualify-online-hybrid-v2-candidate.py --static - for f in qualification/resource-profile-catalog-v1.json qualification/adaptive-clock-v1.json qualification/work-grant-scheduler-v1.json qualification/adaptive-resource-allocation-v1.json qualification/j10-calibration-source-v1.json qualification/j10-calibration-corpus-v1.json qualification/j10-staged-decision-model-v1.json qualification/j10-regime-support-model-v1.json qualification/engine-opt-v2.json qualification/engine-opt-v2-selection.json qualification/engine-opt-v2-evidence.json qualification/engine-opt-v2-host-binding.json qualification/engine-opt-v2-candidate-selection.json qualification/engine-opt-v2-candidate-evidence.json qualification/engine-opt-v2-candidate-native-work-policy.json qualification/engine-derived-lock.json qualification/online-engine-opt-v2.json qualification/online-hybrid-v2.json qualification/online-hybrid-v2-candidate.json qualification/online-hybrid-v2-candidate-witnesses.json qualification/g3-candidate-discovery-source.json qualification/local-full-game-v2.json qualification/local-full-game-v2-candidate.json qualification/local-full-game-orchestrated-v1.json config/allfather.online-engine-opt-v2.json config/allfather.online-engine-opt-v2.candidate.json config/allfather.m14-j-j8.validation.json config/allfather.m14-j-j9.validation.json config/allfather.m14-j-j10.validation.json config/allfather.orchestrated-v1.validation.json config/allfather.online-hybrid-v2.validation.json config/allfather.online-hybrid-v2.candidate.validation.json; do python3 -m json.tool "$f" >/dev/null; done + python3 scripts/qualify-online-hybrid-v2-b4.py --static + for f in qualification/resource-profile-catalog-v1.json qualification/resource-profile-catalog-v1-j7-baseline.json qualification/online-engine-opt-v2-j7-baseline.json qualification/adaptive-clock-v1.json qualification/work-grant-scheduler-v1.json qualification/adaptive-resource-allocation-v1.json qualification/j10-calibration-source-v1.json qualification/j10-calibration-corpus-v1.json qualification/j10-staged-decision-model-v1.json qualification/j10-regime-support-model-v1.json qualification/engine-opt-v2.json qualification/engine-opt-v2-selection.json qualification/engine-opt-v2-evidence.json qualification/engine-opt-v2-host-binding.json qualification/engine-opt-v2-candidate-selection.json qualification/engine-opt-v2-candidate-evidence.json qualification/engine-opt-v2-candidate-native-work-policy.json qualification/engine-opt-v2-native-work-policy.json qualification/engine-derived-lock.json qualification/online-engine-opt-v2.json qualification/online-hybrid-v2.json qualification/online-hybrid-v2-candidate.json qualification/online-hybrid-v2-candidate-witnesses.json qualification/g3-candidate-discovery-source.json qualification/local-full-game-v2.json qualification/local-full-game-v2-candidate.json qualification/local-full-game-orchestrated-v1.json config/allfather.online-engine-opt-v2.json config/allfather.online-engine-opt-v2.candidate.json config/allfather.m14-j-j8.validation.json config/allfather.m14-j-j9.validation.json config/allfather.m14-j-j10.validation.json config/allfather.orchestrated-v1.validation.json config/allfather.online-hybrid-v2.validation.json config/allfather.online-hybrid-v2.validation-j7-baseline.json config/allfather.online-hybrid-v2.candidate.validation.json; do python3 -m json.tool "$f" >/dev/null; done bash -n scripts/build-online-engine-opt-v2.sh build-candidate: @@ -188,11 +202,12 @@ jobs: with: ref: ${{ env.ALLFATHER_SOURCE_SHA }} - name: Verify exact source checkout + timeout-minutes: 5 run: test "$(git rev-parse HEAD)" = "$ALLFATHER_SOURCE_SHA" - name: Install build dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y build-essential meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev + bash scripts/ci-install-apt-deps.sh build-essential meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev - name: Cache networks uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: @@ -223,17 +238,19 @@ jobs: with: ref: ${{ env.ALLFATHER_SOURCE_SHA }} - name: Verify exact source checkout + timeout-minutes: 5 run: test "$(git rev-parse HEAD)" = "$ALLFATHER_SOURCE_SHA" - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: engine-opt-v2-candidate path: build/candidate-download - name: Restore candidate executable bits + timeout-minutes: 5 run: chmod +x build/candidate-download/online-engine-opt-v2/bin/* - name: Install constituent build dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y build-essential meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev + bash scripts/ci-install-apt-deps.sh build-essential meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev - name: Checkout pristine LC0 control uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: @@ -361,18 +378,16 @@ jobs: name: engine-opt-v2-fastchess-source-test path: build/fastchess-source-test - name: Restore exact v2 candidate bundle + timeout-minutes: 5 run: | test -d build/candidate-download/online-engine-opt-v2 rm -rf build/online-engine-opt-v2 mv build/candidate-download/online-engine-opt-v2 build/online-engine-opt-v2 chmod +x build/online-engine-opt-v2/bin/* - name: Install one shared qualification substrate + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y \ - build-essential git meson ninja-build pkg-config \ - libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev \ - python3-pip python3-setuptools python3-wheel + bash scripts/ci-install-apt-deps.sh build-essential git meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev python3-pip python3-setuptools python3-wheel python3 -m pip install --break-system-packages --no-build-isolation --no-deps \ --require-hashes -r qualification/local-game-requirements.txt - name: Capture one canonical execution domain @@ -438,32 +453,12 @@ jobs: --policy qualification/local-full-game-orchestrated-v1.json \ --execution-domain build/test-results/engine-opt-v2-profile-domain/execution-domain.json \ --output build/test-results/engine-opt-v2-profile-domain/j12-local1/campaign - - name: Bind candidate overlay to the exact-head engine bundle + - name: Bind promoted canonical profile to the exact-head engine bundle run: | - python3 scripts/qualify-engine-opt-v2-candidate.py - mkdir -p build/test-results/engine-opt-v2-profile-domain/candidate - cp build/test-results/engine-opt-v2-candidate/report.json \ - build/test-results/engine-opt-v2-profile-domain/candidate/report.json - - name: Run b4 candidate LC0 matrix on the bound domain - run: | - python3 scripts/engine-opt-matrix.py \ - --binary build/online-engine-opt-v2/bin/lc0 \ - --weights build/online-engine-opt-v2/networks/791556.pb.gz \ - --selection qualification/engine-opt-v2-candidate-selection.json \ - --qualification-only \ - --execution-domain build/test-results/engine-opt-v2-profile-domain/execution-domain.json \ - --output build/test-results/engine-opt-v2-profile-domain/matrix/candidate-lc0.json - - name: Build pinned Fastchess against source-test attestation - run: make -f Makefile.local-game build-fastchess - - name: Run canonical LOCAL-1 regressions unchanged - run: make -f Makefile.local-game test - - name: Qualify b4 candidate G3 and LOCAL-1 on the same bound worker - run: | - python3 -m tools.local_game.runner \ - --mode required \ - --policy qualification/local-full-game-v2-candidate.json \ - --execution-domain build/test-results/engine-opt-v2-profile-domain/execution-domain.json \ - --output build/test-results/engine-opt-v2-profile-domain/local1/campaign + python3 scripts/qualify-engine-opt-v2.py + mkdir -p build/test-results/engine-opt-v2-profile-domain/canonical + cp build/test-results/engine-opt-v2/report.json \ + build/test-results/engine-opt-v2-profile-domain/canonical/report.json - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 if: always() with: @@ -484,11 +479,21 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: ref: ${{ env.ALLFATHER_SOURCE_SHA }} + fetch-depth: 2 - name: Verify exact source checkout run: test "$(git rev-parse HEAD)" = "$ALLFATHER_SOURCE_SHA" - name: Fetch PR base for canonical surface guard if: ${{ env.ALLFATHER_BASE_SHA != '' }} run: git fetch --no-tags --depth=1 origin "$ALLFATHER_BASE_SHA" + - name: Observe promotion ancestry; gate seals any missing-object failure + continue-on-error: true + run: | + set -euo pipefail + git rev-parse --verify 'HEAD^{commit}' + git rev-parse --verify 'HEAD^' + if [ -n "$ALLFATHER_BASE_SHA" ]; then + test "$(git rev-parse --verify "$ALLFATHER_BASE_SHA^{commit}")" = "$ALLFATHER_BASE_SHA" + fi - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: pattern: engine-opt-v2-* @@ -499,6 +504,7 @@ jobs: python3 scripts/qualify-engine-opt-matrix.py \ --root build/engine-opt-artifacts \ --selection qualification/engine-opt-v2-selection.json \ + --candidate-report-pattern 'engine-opt-v2-profile-domain/**/canonical/report.json' \ --output build/test-results/engine-opt-v2-canonical-matrix/report.json - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 if: always() @@ -507,34 +513,37 @@ jobs: path: build/test-results/engine-opt-v2-canonical-matrix if-no-files-found: error retention-days: 14 - - name: Aggregate exact-head b4 candidate evidence + - name: Aggregate exact-head canonical b4 evidence run: | python3 scripts/qualify-engine-opt-evidence.py \ --root build/engine-opt-artifacts \ - --selection qualification/engine-opt-v2-candidate-selection.json \ - --candidate-report-pattern 'engine-opt-v2-profile-domain/**/candidate/report.json' \ - --candidate-mode \ - --output build/test-results/engine-opt-v2-candidate-aggregate/report.json - - name: Summarize b4 candidate disposition + --selection qualification/engine-opt-v2-selection.json \ + --candidate-report-pattern 'engine-opt-v2-profile-domain/**/canonical/report.json' \ + --output build/test-results/engine-opt-v2-aggregate/report.json + - name: Summarize canonical b4 disposition if: always() run: | python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json from pathlib import Path - path=Path("build/test-results/engine-opt-v2-candidate-aggregate/report.json") - print("## ENGINE-OPT-V2 b4 candidate aggregate") + path=Path("build/test-results/engine-opt-v2-aggregate/report.json") + print("## ENGINE-OPT-V2 canonical b4 aggregate") if not path.is_file(): - print("\nCandidate aggregate report was not produced.") + print("\nCanonical aggregate report was not produced.") else: report=json.loads(path.read_text(encoding="utf-8")) for label,key in ( ("Evidence valid","evidence_valid"), - ("Candidate qualified","candidate_qualified"), - ("Promotion ready","promotion_ready"), - ("Canonical profile changed","canonical_profile_changed"), + ("Profile qualified","profile_qualified"), + ("Passed","passed"), ("Disposition","qualification_disposition"), ): print(f"- **{label}:** `{report.get(key)}`") + invalid=report.get("invalid_evidence") or [] + if invalid: + print("\n### Invalid evidence") + for issue in invalid: + print(f"- {issue}") failures=report.get("qualification_failures") or [] if failures: print("\n### Qualification failures") @@ -544,42 +553,42 @@ jobs: - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 if: always() with: - name: engine-opt-v2-candidate-aggregate - path: build/test-results/engine-opt-v2-candidate-aggregate + name: engine-opt-v2-aggregate + path: build/test-results/engine-opt-v2-aggregate if-no-files-found: error retention-days: 14 - - name: Evaluate host-aware canonical/candidate gate + - name: Evaluate canonical b4 promotion gate + id: promotion_gate run: | - python3 scripts/qualify-engine-opt-v2-candidate-gate.py \ - --canonical build/test-results/engine-opt-v2-canonical-matrix/report.json \ - --candidate build/test-results/engine-opt-v2-candidate-aggregate/report.json \ + python3 scripts/qualify-engine-opt-v2-promotion-gate.py \ + --aggregate build/test-results/engine-opt-v2-aggregate/report.json \ --host-binding qualification/engine-opt-v2-host-binding.json \ + --catalog qualification/resource-profile-catalog-v1.json \ --base-sha "$ALLFATHER_BASE_SHA" \ - --output build/test-results/engine-opt-v2-final-gate/report.json - - name: Summarize final host-aware gate + --output build/test-results/engine-opt-v2-promotion-gate/report.json + - name: Summarize canonical b4 promotion gate if: always() run: | python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json from pathlib import Path - path=Path("build/test-results/engine-opt-v2-final-gate/report.json") - print("\n## ENGINE-OPT-V2 final gate") + path=Path("build/test-results/engine-opt-v2-promotion-gate/report.json") + print("\n## ENGINE-OPT-V2 canonical b4 promotion gate") if not path.is_file(): - print("\nFinal gate report was not produced.") + print("\nPromotion gate report was not produced.") else: report=json.loads(path.read_text(encoding="utf-8")) print(f"- **Passed:** `{report.get('passed')}`") print(f"- **Disposition:** `{report.get('disposition')}`") - canonical=report.get('canonical') or {} - print(f"- **Canonical status:** `{canonical.get('status')}`") - print(f"- **Canonical authority on current host:** `{canonical.get('authority_on_current_host')}`") - candidate=report.get('candidate') or {} - print(f"- **Candidate promotion ready:** `{candidate.get('promotion_ready')}`") + failure=report.get('failure') or {} + if failure: + print(f"- **Blocking reason:** `{failure.get('message')}`") PY - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - if: always() + if: ${{ always() && (steps.promotion_gate.outcome == 'success' || steps.promotion_gate.outcome == 'failure') }} with: - name: engine-opt-v2-final-gate - path: build/test-results/engine-opt-v2-final-gate + name: engine-opt-v2-promotion-gate + path: build/test-results/engine-opt-v2-promotion-gate if-no-files-found: error retention-days: 14 + diff --git a/.github/workflows/full-game-qualification.yml b/.github/workflows/full-game-qualification.yml index 5da847632..dc5d0661f 100644 --- a/.github/workflows/full-game-qualification.yml +++ b/.github/workflows/full-game-qualification.yml @@ -31,6 +31,8 @@ on: - 'tests/fixtures/fake_uci_engine.py' - 'vendor.lock.json' - 'Makefile*' + - 'scripts/ci-install-apt-deps.sh' + - 'tests/ci/test_apt_installer.py' - '.github/workflows/full-game-qualification.yml' push: branches: [main] @@ -51,6 +53,8 @@ on: - 'tests/fixtures/fake_uci_engine.py' - 'vendor.lock.json' - 'Makefile*' + - 'scripts/ci-install-apt-deps.sh' + - 'tests/ci/test_apt_installer.py' - '.github/workflows/full-game-qualification.yml' permissions: @@ -146,16 +150,18 @@ jobs: with: ref: ${{ env.ALLFATHER_SOURCE_SHA }} - name: Verify checkout identity + timeout-minutes: 5 run: test "$(git rev-parse HEAD)" = "$ALLFATHER_SOURCE_SHA" - name: Download source-test attestation + timeout-minutes: 5 uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: local1-fastchess-source-test path: build/fastchess-source-test - name: Install build dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y build-essential git meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev + bash scripts/ci-install-apt-deps.sh build-essential git meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev - name: Build exact qualification binaries run: | make -f Makefile.local-game build-fastchess @@ -193,16 +199,17 @@ jobs: test "$(git rev-parse HEAD)" = "$ALLFATHER_SOURCE_SHA" echo "ALLFATHER_RUNNER_ENVIRONMENT=${{ runner.environment }}" >> "$GITHUB_ENV" - name: Download exact qualification build bundle + timeout-minutes: 5 uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: local1-qualification-build path: . - name: Restore executable bits and install runtime/test dependencies + timeout-minutes: 5 run: | chmod +x build/tools/fastchess/bin/fastchess engines/lc0/build/release/lc0 find build -path '*/bin/*' -type f -exec chmod +x {} + - sudo apt-get update - sudo apt-get install -y meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev python3-pip python3-setuptools python3-wheel + bash scripts/ci-install-apt-deps.sh meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev python3-pip python3-setuptools python3-wheel python3 -m pip install --break-system-packages --no-build-isolation --no-deps --require-hashes -r qualification/local-game-requirements.txt - name: Validate legacy G3 resource calibration run: | @@ -247,18 +254,20 @@ jobs: with: ref: ${{ env.ALLFATHER_SOURCE_SHA }} - name: Verify checkout identity + timeout-minutes: 5 run: test "$(git rev-parse HEAD)" = "$ALLFATHER_SOURCE_SHA" - name: Download exact qualification build bundle + timeout-minutes: 5 uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: local1-qualification-build path: build - name: Restore executable bits and install dependencies + timeout-minutes: 5 run: | chmod +x build/tools/fastchess/bin/fastchess engines/lc0/build/release/lc0 find build -path '*/bin/*' -type f -exec chmod +x {} + - sudo apt-get update - sudo apt-get install -y meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev python3-pip python3-setuptools python3-wheel + bash scripts/ci-install-apt-deps.sh meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev python3-pip python3-setuptools python3-wheel python3 -m pip install --break-system-packages --no-build-isolation --no-deps --require-hashes -r qualification/local-game-requirements.txt - name: Validate legacy G3 resource calibration run: | @@ -293,16 +302,18 @@ jobs: with: ref: ${{ env.ALLFATHER_SOURCE_SHA }} - name: Verify checkout identity + timeout-minutes: 5 run: test "$(git rev-parse HEAD)" = "$ALLFATHER_SOURCE_SHA" - name: Download exact qualification build bundle + timeout-minutes: 5 uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: local1-qualification-build path: build - name: Install independent validator dependency + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y python3-pip python3-setuptools python3-wheel + bash scripts/ci-install-apt-deps.sh python3-pip python3-setuptools python3-wheel python3 -m pip install --break-system-packages --no-build-isolation --no-deps --require-hashes -r qualification/local-game-requirements.txt - name: Download all soak shard artifacts uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 diff --git a/.github/workflows/g3-canonical-b4-triage.yml b/.github/workflows/g3-canonical-b4-triage.yml new file mode 100644 index 000000000..8093ceee7 --- /dev/null +++ b/.github/workflows/g3-canonical-b4-triage.yml @@ -0,0 +1,100 @@ +name: G3 canonical-b4 23-case diagnostic (non-promoting) + +on: + workflow_dispatch: + pull_request: + paths: + - 'scripts/diagnose-g3-canonical-b4.py' + - 'tests/engine_opt/test_g3_canonical_b4_triage.py' + - 'config/allfather.online-hybrid-v2.g3-triage.json' + - '.github/workflows/g3-canonical-b4-triage.yml' + +permissions: + contents: read + +env: + TRIAGE_HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + ALLFATHER_RUNNER_ENVIRONMENT: github-hosted + +concurrency: + group: g3-b4-triage-${{ github.ref }} + cancel-in-progress: false + +jobs: + static-contracts: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ env.TRIAGE_HEAD_SHA }} + - name: Verify exact HEAD and frozen policy/config contrasts + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$TRIAGE_HEAD_SHA" + python3 -m py_compile scripts/diagnose-g3-canonical-b4.py + python3 tests/engine_opt/test_g3_canonical_b4_triage.py + python3 scripts/diagnose-g3-canonical-b4.py --static + + canonical-b4-triage: + needs: static-contracts + runs-on: ubuntu-24.04 + timeout-minutes: 105 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ env.TRIAGE_HEAD_SHA }} + - name: Verify exact source checkout + run: test "$(git rev-parse HEAD)" = "$TRIAGE_HEAD_SHA" + - name: Install pinned CPU inference build dependencies + timeout-minutes: 5 + run: | + bash scripts/ci-install-apt-deps.sh build-essential git meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev + - name: Build actual canonical-b4 engine bundle + timeout-minutes: 70 + run: make build-online-engine-opt-v2 + - name: Validate exact-head engine bundle + run: python3 scripts/qualify-engine-opt-v2.py + - name: Capture exact execution domain on this worker + run: | + set -euo pipefail + mkdir -p build/test-results/g3-canonical-b4-triage + python3 scripts/lc0-hardware-probe.py > build/test-results/g3-canonical-b4-triage/host.json + python3 -m tools.engine_opt.domain capture \ + --probe build/test-results/g3-canonical-b4-triage/host.json \ + --output build/test-results/g3-canonical-b4-triage/execution-domain.json + python3 -m tools.engine_opt.domain validate \ + --input build/test-results/g3-canonical-b4-triage/execution-domain.json \ + --expected-source-commit "$TRIAGE_HEAD_SHA" + - name: Diagnose 23 frozen G3 cases with canonical b4 (no authority promotion) + timeout-minutes: 30 + run: | + python3 scripts/diagnose-g3-canonical-b4.py \ + --expected-head "$TRIAGE_HEAD_SHA" \ + --execution-domain build/test-results/g3-canonical-b4-triage/execution-domain.json + - name: Summarize diagnostic disposition + if: always() + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + from pathlib import Path + p=Path("build/test-results/g3-canonical-b4-triage/report.json") + print("## G3 23-case b4 triage: diagnostic only") + if p.exists(): + r=json.loads(p.read_text()) + for key in ("diagnostic_complete","diagnostic_positive_witness","cases_exercised", + "qualification_disposition","promotion_evidence"): + print(f"- {key}: {r.get(key)}") + else: + print("- No diagnostic disposition was generated.") + PY + - name: Upload complete source-bound diagnostic and replays + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: g3-canonical-b4-triage-nonpromoting + path: | + build/test-results/g3-canonical-b4-triage/** + build/replays-g3-canonical-b4-triage/** + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/lc0-artifact-domain-diagnostic.yml b/.github/workflows/lc0-artifact-domain-diagnostic.yml index 9676f71fc..c1a0d95e4 100644 --- a/.github/workflows/lc0-artifact-domain-diagnostic.yml +++ b/.github/workflows/lc0-artifact-domain-diagnostic.yml @@ -5,6 +5,8 @@ on: pull_request: paths: - 'tools/diagnostics/**' + - 'scripts/ci-install-apt-deps.sh' + - 'tests/ci/test_apt_installer.py' - '.github/workflows/lc0-artifact-domain-diagnostic.yml' permissions: @@ -70,9 +72,9 @@ jobs: test "$(git -C source-b rev-parse HEAD)" = "$SOURCE_B" - name: Install one shared build/runtime substrate + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y build-essential meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev + bash scripts/ci-install-apt-deps.sh build-essential meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev echo "ALLFATHER_RUNNER_ENVIRONMENT=${{ runner.environment }}" >> "$GITHUB_ENV" - name: Capture J2 host and runtime-substrate evidence diff --git a/.github/workflows/lc0-strength-qualification.yml b/.github/workflows/lc0-strength-qualification.yml index 983ea4248..8a742c725 100644 --- a/.github/workflows/lc0-strength-qualification.yml +++ b/.github/workflows/lc0-strength-qualification.yml @@ -28,6 +28,8 @@ on: - tests/telemetry/** - engines/lc0/** - vendor.lock.json + - scripts/ci-install-apt-deps.sh + - tests/ci/test_apt_installer.py - .github/workflows/lc0-strength-qualification.yml pull_request: paths: @@ -53,6 +55,8 @@ on: - tests/telemetry/** - engines/lc0/** - vendor.lock.json + - scripts/ci-install-apt-deps.sh + - tests/ci/test_apt_installer.py - .github/workflows/lc0-strength-qualification.yml permissions: @@ -131,11 +135,9 @@ jobs: python3 -m json.tool config/allfather.strength.validation.json >/dev/null - name: Install LC0 real-inference dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y \ - meson ninja-build pkg-config libprotobuf-dev protobuf-compiler \ - zlib1g-dev libopenblas-dev + bash scripts/ci-install-apt-deps.sh meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev - name: Toolchain and hardware probe run: | diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 069a49d2c..b98d94900 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -34,6 +34,9 @@ jobs: make telemetry-adapters make controller-tests make resource-lab-tests + python3 tests/engine_opt/test_aggregate_paths.py + python3 tests/ci/test_apt_installer.py + python3 tests/reckless_repro/test_repro.py python3 -m json.tool qualification/resource-profile-evidence-v1.json >/dev/null python3 -m json.tool qualification/resource-profile-selection-v1.json >/dev/null python3 scripts/qualify-resource-profile-selection.py --output build/test-results/resource-profile-selection/report.json diff --git a/.github/workflows/meta-1.yml b/.github/workflows/meta-1.yml index eb38c58db..b1fd65773 100644 --- a/.github/workflows/meta-1.yml +++ b/.github/workflows/meta-1.yml @@ -12,6 +12,8 @@ on: - 'tests/fixtures/meta1/**' - 'tests/fixtures/meta1-mini/**' - 'Makefile.local-game' + - 'scripts/ci-install-apt-deps.sh' + - 'tests/ci/test_apt_installer.py' - '.github/workflows/meta-1.yml' workflow_dispatch: inputs: @@ -144,9 +146,9 @@ jobs: name: meta1-fastchess-source-test path: build/fastchess-source-test - name: Install build and campaign dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y build-essential git meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev python3-pip python3-setuptools python3-wheel + bash scripts/ci-install-apt-deps.sh build-essential git meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev python3-pip python3-setuptools python3-wheel python3 -m pip install --break-system-packages --no-build-isolation --no-deps --require-hashes -r qualification/local-game-requirements.txt - name: Build exact-head ENGINE-OPT-V2 bundle run: | diff --git a/.github/workflows/online-hybrid-qualification.yml b/.github/workflows/online-hybrid-qualification.yml index b5dff3d0a..9c2cf56c3 100644 --- a/.github/workflows/online-hybrid-qualification.yml +++ b/.github/workflows/online-hybrid-qualification.yml @@ -36,6 +36,8 @@ on: - engines/** - vendor.lock.json - Makefile + - scripts/ci-install-apt-deps.sh + - tests/ci/test_apt_installer.py - .github/workflows/online-hybrid-qualification.yml push: branches: [main] @@ -72,6 +74,8 @@ on: - engines/** - vendor.lock.json - Makefile + - scripts/ci-install-apt-deps.sh + - tests/ci/test_apt_installer.py - .github/workflows/online-hybrid-qualification.yml permissions: contents: read @@ -146,9 +150,9 @@ jobs: python3 -m json.tool qualification/replay-history-sources.json >/dev/null python3 scripts/validate-online-hybrid-resource-calibration.py - name: Install real-inference dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev + bash scripts/ci-install-apt-deps.sh meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev - name: Build frozen ONLINE-2 portable bundle env: JOBS: "2" diff --git a/.github/workflows/online-profile-qualification.yml b/.github/workflows/online-profile-qualification.yml index 793c57e02..347eb06ce 100644 --- a/.github/workflows/online-profile-qualification.yml +++ b/.github/workflows/online-profile-qualification.yml @@ -40,6 +40,8 @@ on: - engines/** - vendor.lock.json - Makefile + - scripts/ci-install-apt-deps.sh + - tests/ci/test_apt_installer.py - .github/workflows/online-profile-qualification.yml pull_request: paths: @@ -77,6 +79,8 @@ on: - engines/** - vendor.lock.json - Makefile + - scripts/ci-install-apt-deps.sh + - tests/ci/test_apt_installer.py - .github/workflows/online-profile-qualification.yml permissions: @@ -160,9 +164,9 @@ jobs: bash -n scripts/build-online-cpu-reference.sh - name: Install real-inference build dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev + bash scripts/ci-install-apt-deps.sh meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev - name: Cache frozen engine networks uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 diff --git a/.github/workflows/reckless-repro-diagnostic.yml b/.github/workflows/reckless-repro-diagnostic.yml new file mode 100644 index 000000000..2aa4bc3c7 --- /dev/null +++ b/.github/workflows/reckless-repro-diagnostic.yml @@ -0,0 +1,168 @@ +name: Reckless same-worker reproducibility diagnostic + +on: + workflow_dispatch: + pull_request: + paths: + - 'tools/reckless_repro/**' + - 'tests/reckless_repro/**' + - 'scripts/reckless-repro.py' + - '.github/workflows/reckless-repro-diagnostic.yml' + +permissions: + contents: read + +env: + REPRO_HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + +concurrency: + group: reckless-repro-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + contracts: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ env.REPRO_HEAD_SHA }} + - name: Verify exact diagnostic HEAD + run: test "$(git rev-parse HEAD)" = "$REPRO_HEAD_SHA" + - name: Diagnostic contract tests + run: | + python3 -m py_compile scripts/reckless-repro.py tools/reckless_repro/*.py tests/reckless_repro/test_repro.py + python3 tests/reckless_repro/test_repro.py + + same-worker-experiment: + needs: contracts + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ env.REPRO_HEAD_SHA }} + - name: Verify exact HEAD and pinned derived engine tree + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$REPRO_HEAD_SHA" + python3 - <<'PY' + import json, subprocess + policy=json.load(open("qualification/engine-derived-lock.json",encoding="utf-8")) + actual=subprocess.check_output(["git","rev-parse","HEAD:engines/reckless"],text=True).strip() + assert actual==policy["engines"]["reckless"]["derived_tree"], "derived Reckless tree not locked" + PY + - name: Install bounded build dependencies + timeout-minutes: 5 + run: bash scripts/ci-install-apt-deps.sh build-essential pkg-config clang binutils + - name: Capture bound J2 hardware report + run: | + set -euo pipefail + mkdir -p evidence/build evidence/bin + echo "ALLFATHER_RUNNER_ENVIRONMENT=${{ runner.environment }}" >> "$GITHUB_ENV" + python3 scripts/lc0-hardware-probe.py > evidence/host.json + python3 -m json.tool evidence/host.json >/dev/null + rustc --version --verbose | tee evidence/build/rustc-version.txt + cargo --version | tee evidence/build/cargo-version.txt + - name: Download locked Reckless NNUE + timeout-minutes: 10 + run: | + set -euo pipefail + NET="$(bash scripts/fetch-reckless-network.sh)" + test -f "$NET" + echo "REPRO_NET=$(realpath "$NET")" >> "$GITHUB_ENV" + python3 - "$NET" <<'PY' + import hashlib,json,sys + from pathlib import Path + path=Path(sys.argv[1]) + lock=json.loads(Path("vendor.lock.json").read_text()) + expected=lock["engines"]["reckless"]["artifacts"]["default_nnue"]["sha256"] + assert hashlib.sha256(path.read_bytes()).hexdigest()==expected,"locked Reckless NNUE mismatch" + PY + - name: Checkout pinned pristine Reckless + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: HMarcusWH/Reckless + ref: 31d9cd6fd2bea6d9f72eeb35e0bac70daa295fb1 + path: build/reckless-repro/pristine-source + fetch-depth: 1 + persist-credentials: false + - name: Verify pinned pristine commit and source tree + run: | + set -euo pipefail + test "$(git -C build/reckless-repro/pristine-source rev-parse HEAD)" = "31d9cd6fd2bea6d9f72eeb35e0bac70daa295fb1" + test "$(git -C build/reckless-repro/pristine-source rev-parse HEAD^{tree})" = "88763d8e81ea45b938403f7f4feee990ad244c2d" + - name: Cleanly compile pristine control and derived A and B serially + timeout-minutes: 35 + run: | + set -euo pipefail + export RUST_BACKTRACE=1 + ROOT_DIR="$PWD" + rm -rf build/reckless-repro/target-pristine build/reckless-repro/target-a build/reckless-repro/target-b + ( + cd build/reckless-repro/pristine-source + EVALFILE="$REPRO_NET" CARGO_TARGET_DIR="$ROOT_DIR/build/reckless-repro/target-pristine" CARGO_ENCODED_RUSTFLAGS='-Ctarget-cpu=x86-64' cargo build --verbose --locked --release --no-default-features --target x86_64-unknown-linux-gnu + ) 2>&1 | tee evidence/build/pristine.log + cp build/reckless-repro/target-pristine/x86_64-unknown-linux-gnu/release/reckless evidence/bin/pristine + ( + cd engines/reckless + EVALFILE="$REPRO_NET" CARGO_TARGET_DIR="$ROOT_DIR/build/reckless-repro/target-a" CARGO_ENCODED_RUSTFLAGS='-Ctarget-cpu=x86-64' cargo build --verbose --locked --release --no-default-features --target x86_64-unknown-linux-gnu + ) 2>&1 | tee evidence/build/derived-a.log + cp build/reckless-repro/target-a/x86_64-unknown-linux-gnu/release/reckless evidence/bin/derived-a + ( + cd engines/reckless + EVALFILE="$REPRO_NET" CARGO_TARGET_DIR="$ROOT_DIR/build/reckless-repro/target-b" CARGO_ENCODED_RUSTFLAGS='-Ctarget-cpu=x86-64' cargo build --verbose --locked --release --no-default-features --target x86_64-unknown-linux-gnu + ) 2>&1 | tee evidence/build/derived-b.log + cp build/reckless-repro/target-b/x86_64-unknown-linux-gnu/release/reckless evidence/bin/derived-b + chmod +x evidence/bin/* + sha256sum evidence/bin/* | tee evidence/build/binaries.sha256 + - name: Verify immediately-post-build binaries before expensive experiment + run: | + python3 - <<'PY' + from pathlib import Path + from tools.reckless_repro.provenance import preflight_binary_hashes + paths = {name: Path("evidence/bin") / name + for name in ("derived-a", "derived-b", "pristine")} + preflight_binary_hashes(paths, Path("evidence/build/binaries.sha256")) + PY + - name: Run complete non-promoting ABBA/BAAB experiment on same CPU + timeout-minutes: 45 + run: | + python3 scripts/reckless-repro.py \ + --derived-a evidence/bin/derived-a \ + --derived-b evidence/bin/derived-b \ + --pristine evidence/bin/pristine \ + --build-hashes evidence/build/binaries.sha256 \ + --build-log-derived-a evidence/build/derived-a.log \ + --build-log-derived-b evidence/build/derived-b.log \ + --build-log-pristine evidence/build/pristine.log \ + --model "$REPRO_NET" \ + --host-report evidence/host.json \ + --expected-head "$REPRO_HEAD_SHA" \ + --output evidence/reckless-repro-report.json + - name: Summarize diagnostic without promotion claims + if: always() + run: | + python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json + from pathlib import Path + p=Path("evidence/reckless-repro-report.json") + print("## Reckless reproducibility diagnostic (NOT promotion evidence)") + if not p.is_file(): + print("Diagnostic report absent; experimental evidence incomplete") + else: + r=json.loads(p.read_text()) + print(f"Source: `{r.get('source_commit')}`") + print(f"Complete: `{r.get('diagnostic_complete')}`") + print(f"Disposition: `{r.get('qualification_disposition')}`") + for name,data in (r.get("analysis") or {}).get("comparisons",{}).items(): + print(f"- {name}: {data.get('per_block_derived_over_pristine')}") + print(f"Errors: {len(r.get('errors',[]))}") + PY + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: reckless-repro-exact-head-diagnostic + path: evidence + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/resource-profile-lab-v2.yml b/.github/workflows/resource-profile-lab-v2.yml index adc0a7116..f257ec87c 100644 --- a/.github/workflows/resource-profile-lab-v2.yml +++ b/.github/workflows/resource-profile-lab-v2.yml @@ -6,10 +6,13 @@ on: paths: - 'engines/**' - 'vendor.lock.json' + - 'qualification/resource-lab-v1.json' - 'qualification/resource-lab-v2.json' - 'qualification/resource-profile-evidence-v2.json' - 'qualification/resource-profile-evidence-v2-source.json' - 'qualification/resource-profile-selection-v2.json' + - 'qualification/resource-profile-catalog-v1-j7-baseline.json' + - 'qualification/online-engine-opt-v2-j7-baseline.json' - 'qualification/engine-opt-v2-candidate-selection.json' - 'qualification/engine-opt-v2-candidate-evidence.json' - 'qualification/online-engine-opt-v2.json' @@ -17,6 +20,7 @@ on: - 'qualification/lc0-strength.lock.json' - 'qualification/lc0-strength-profile.json' - 'config/allfather.online-engine-opt-v2.candidate.json' + - 'config/allfather.online-hybrid-v2.validation-j7-baseline.json' - 'tools/resource_lab/**' - 'tools/engine_opt/**' - 'controller/engine_opt_profile.py' @@ -25,6 +29,8 @@ on: - 'tests/fixtures/engine_opt/**' - 'tests/harness/uci_session.py' - 'Makefile' + - 'scripts/ci-install-apt-deps.sh' + - 'tests/ci/test_apt_installer.py' - '.github/workflows/resource-profile-lab-v2.yml' - 'scripts/build-online-engine-opt-v2.sh' - 'scripts/qualify-engine-opt-v2-candidate.py' @@ -69,6 +75,9 @@ jobs: python3 -m compileall -q tools/resource_lab tests/resource_lab tests/harness/uci_session.py controller/resource_profile_catalog.py python3 -m json.tool qualification/resource-lab-v1.json >/dev/null python3 -m json.tool qualification/resource-lab-v2.json >/dev/null + python3 -m json.tool qualification/resource-profile-catalog-v1-j7-baseline.json >/dev/null + python3 -m json.tool qualification/online-engine-opt-v2-j7-baseline.json >/dev/null + python3 -m json.tool config/allfather.online-hybrid-v2.validation-j7-baseline.json >/dev/null python3 -m json.tool qualification/resource-profile-evidence-v2.json >/dev/null python3 -m json.tool qualification/resource-profile-evidence-v2-source.json >/dev/null python3 -m json.tool qualification/resource-profile-selection-v2.json >/dev/null @@ -87,13 +96,12 @@ jobs: with: ref: ${{ env.ALLFATHER_SOURCE_SHA }} - name: Verify exact source checkout + timeout-minutes: 5 run: test "$(git rev-parse HEAD)" = "$ALLFATHER_SOURCE_SHA" - name: Install build dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y \ - build-essential meson ninja-build pkg-config \ - libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev + bash scripts/ci-install-apt-deps.sh build-essential meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev - name: Cache networks uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: @@ -135,15 +143,16 @@ jobs: name: resource-lab-v2-candidate path: build/candidate-download - name: Restore exact candidate bundle + timeout-minutes: 5 run: | test -d build/candidate-download/online-engine-opt-v2 rm -rf build/online-engine-opt-v2 mv build/candidate-download/online-engine-opt-v2 build/online-engine-opt-v2 chmod +x build/online-engine-opt-v2/bin/* - name: Install runtime measurement dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y libopenblas-dev libprotobuf-dev zlib1g-dev + bash scripts/ci-install-apt-deps.sh libopenblas-dev libprotobuf-dev zlib1g-dev - name: Capture one canonical execution domain run: | mkdir -p build/test-results/resource-lab-v2 diff --git a/.github/workflows/resource-profile-lab.yml b/.github/workflows/resource-profile-lab.yml index 1011a7be2..535bab196 100644 --- a/.github/workflows/resource-profile-lab.yml +++ b/.github/workflows/resource-profile-lab.yml @@ -8,18 +8,23 @@ on: - 'vendor.lock.json' - 'qualification/resource-lab-v1.json' - 'qualification/resource-profile-catalog-v1.json' + - 'qualification/resource-profile-catalog-v1-j7-baseline.json' - 'qualification/online-engine-opt-v2.json' + - 'qualification/online-engine-opt-v2-j7-baseline.json' - 'qualification/engine-opt-v2-selection.json' - 'qualification/engine-derived-lock.json' - 'qualification/lc0-strength.lock.json' - 'qualification/lc0-strength-profile.json' - 'config/allfather.online-hybrid-v2.validation.json' + - 'config/allfather.online-hybrid-v2.validation-j7-baseline.json' - 'tools/resource_lab/**' - 'tools/engine_opt/**' - 'tests/resource_lab/**' - 'tests/fixtures/engine_opt/**' - 'tests/harness/uci_session.py' - 'Makefile' + - 'scripts/ci-install-apt-deps.sh' + - 'tests/ci/test_apt_installer.py' - '.github/workflows/resource-profile-lab.yml' - 'scripts/build-online-engine-opt-v2.sh' - 'scripts/build-lc0-strength.sh' @@ -59,6 +64,9 @@ jobs: run: | python3 -m compileall -q tools/resource_lab tests/resource_lab tests/harness/uci_session.py python3 -m json.tool qualification/resource-lab-v1.json >/dev/null + python3 -m json.tool qualification/resource-profile-catalog-v1-j7-baseline.json >/dev/null + python3 -m json.tool qualification/online-engine-opt-v2-j7-baseline.json >/dev/null + python3 -m json.tool config/allfather.online-hybrid-v2.validation-j7-baseline.json >/dev/null make resource-lab-tests build-candidate: @@ -70,13 +78,12 @@ jobs: with: ref: ${{ env.ALLFATHER_SOURCE_SHA }} - name: Verify exact source checkout + timeout-minutes: 5 run: test "$(git rev-parse HEAD)" = "$ALLFATHER_SOURCE_SHA" - name: Install build dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y \ - build-essential meson ninja-build pkg-config \ - libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev + bash scripts/ci-install-apt-deps.sh build-essential meson ninja-build pkg-config libprotobuf-dev protobuf-compiler zlib1g-dev libopenblas-dev - name: Cache networks uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: @@ -115,15 +122,16 @@ jobs: name: resource-lab-v1-candidate path: build/candidate-download - name: Restore exact candidate bundle + timeout-minutes: 5 run: | test -d build/candidate-download/online-engine-opt-v2 rm -rf build/online-engine-opt-v2 mv build/candidate-download/online-engine-opt-v2 build/online-engine-opt-v2 chmod +x build/online-engine-opt-v2/bin/* - name: Install runtime measurement dependencies + timeout-minutes: 5 run: | - sudo apt-get update - sudo apt-get install -y libopenblas-dev libprotobuf-dev zlib1g-dev + bash scripts/ci-install-apt-deps.sh libopenblas-dev libprotobuf-dev zlib1g-dev - name: Capture one canonical execution domain run: | mkdir -p build/test-results/resource-lab-v1 diff --git a/config/allfather.m14-j-j10.validation.json b/config/allfather.m14-j-j10.validation.json index b22b3fc59..cda7b09ca 100644 --- a/config/allfather.m14-j-j10.validation.json +++ b/config/allfather.m14-j-j10.validation.json @@ -136,14 +136,14 @@ "Backend": "blas", "BackendOptions": "", "NNCacheSize": 262144, - "MinibatchSize": 7, + "MinibatchSize": 4, "MaxConcurrentSearchers": 1, "TaskWorkers": 0, "MultiPV": 1, "UCI_Chess960": false, "Threads": 1, "ScoreType": "WDL_mu", - "MaxPrefetch": 8, + "MaxPrefetch": 0, "AdaptivePrefetch": false, "DefectTelemetry": false }, @@ -157,12 +157,6 @@ "STAGED_VERIFY": { "MultiPV": 3 } - }, - "warmup": { - "enabled": true, - "nodes": 64, - "position": "startpos", - "reset_after": true } } }, diff --git a/config/allfather.m14-j-j8.validation.json b/config/allfather.m14-j-j8.validation.json index e67146745..e58513dc7 100644 --- a/config/allfather.m14-j-j8.validation.json +++ b/config/allfather.m14-j-j8.validation.json @@ -136,14 +136,14 @@ "Backend": "blas", "BackendOptions": "", "NNCacheSize": 262144, - "MinibatchSize": 7, + "MinibatchSize": 4, "MaxConcurrentSearchers": 1, "TaskWorkers": 0, "MultiPV": 1, "UCI_Chess960": false, "Threads": 1, "ScoreType": "WDL_mu", - "MaxPrefetch": 8, + "MaxPrefetch": 0, "AdaptivePrefetch": false, "DefectTelemetry": false }, @@ -157,12 +157,6 @@ "STAGED_VERIFY": { "MultiPV": 3 } - }, - "warmup": { - "enabled": true, - "nodes": 64, - "position": "startpos", - "reset_after": true } } }, diff --git a/config/allfather.m14-j-j9.validation.json b/config/allfather.m14-j-j9.validation.json index 27cc1cba4..fc6f8dc28 100644 --- a/config/allfather.m14-j-j9.validation.json +++ b/config/allfather.m14-j-j9.validation.json @@ -136,14 +136,14 @@ "Backend": "blas", "BackendOptions": "", "NNCacheSize": 262144, - "MinibatchSize": 7, + "MinibatchSize": 4, "MaxConcurrentSearchers": 1, "TaskWorkers": 0, "MultiPV": 1, "UCI_Chess960": false, "Threads": 1, "ScoreType": "WDL_mu", - "MaxPrefetch": 8, + "MaxPrefetch": 0, "AdaptivePrefetch": false, "DefectTelemetry": false }, @@ -157,12 +157,6 @@ "STAGED_VERIFY": { "MultiPV": 3 } - }, - "warmup": { - "enabled": true, - "nodes": 64, - "position": "startpos", - "reset_after": true } } }, diff --git a/config/allfather.online-engine-opt-v2.json b/config/allfather.online-engine-opt-v2.json index 30d8d7415..4399deab0 100644 --- a/config/allfather.online-engine-opt-v2.json +++ b/config/allfather.online-engine-opt-v2.json @@ -136,14 +136,14 @@ "Backend": "blas", "BackendOptions": "", "NNCacheSize": 262144, - "MinibatchSize": 7, + "MinibatchSize": 4, "MaxConcurrentSearchers": 1, "TaskWorkers": 0, "MultiPV": 1, "UCI_Chess960": false, "Threads": 1, "ScoreType": "WDL_mu", - "MaxPrefetch": 8, + "MaxPrefetch": 0, "AdaptivePrefetch": false, "DefectTelemetry": false }, @@ -157,12 +157,6 @@ "STAGED_VERIFY": { "MultiPV": 3 } - }, - "warmup": { - "enabled": true, - "nodes": 64, - "position": "startpos", - "reset_after": true } } }, diff --git a/config/allfather.online-hybrid-v2.g3-triage.json b/config/allfather.online-hybrid-v2.g3-triage.json new file mode 100644 index 000000000..1adb1c94d --- /dev/null +++ b/config/allfather.online-hybrid-v2.g3-triage.json @@ -0,0 +1,217 @@ +{ + "schema_version": 2, + "mode": "active", + "anchor": "stockfish-anchor", + "root": "..", + "shadow": { + "owners": [ + "stockfish", + "reckless", + "lc0" + ], + "instance_by_owner": { + "stockfish": "stockfish-shadow", + "reckless": "reckless-shadow", + "lc0": "lc0-shadow" + }, + "oracle": "stockfish-shadow", + "partition": "root_index_modulo", + "dispatch_limit": { + "nodes": 16 + }, + "lc0_score_type": "WDL_mu", + "replay_root": "build/replays-g3-canonical-b4-triage", + "oracle_timeout_s": 10, + "drain_timeout_s": 5, + "on_anchor_complete": "drain" + }, + "budget": { + "wall_ms": 4000, + "cpu_ms": 12000, + "gpu_ms": 0, + "verification_reserve_fraction": 0.3, + "controller_overhead_reserve_ms": 250 + }, + "routing": { + "policy": "unified_value_v1", + "calibration": null, + "min_observation_nodes": 4000, + "checkpoint_interval_ms": 120, + "max_stages_per_owner": 1, + "extend_nodes": 8000, + "stop_max_reversal_risk": 0.05, + "stop_min_support": 25, + "stop_min_stability_fraction": 0.6, + "stage_cpu_ms_estimate": 500, + "anchor_cpu_ms_estimate": 0, + "verify_stage_cpu_ms_estimate": 750, + "verify_stage_gpu_ms_estimate": 0, + "staged_decision_calibration": null, + "regime_support_calibration": null, + "staged_skip_max_change_probability": 0.1, + "stage_cpu_ms_estimate_by_owner": { + "stockfish": 100, + "reckless": 100, + "lc0": 600 + }, + "verify_stage_cpu_ms_estimate_by_owner": { + "stockfish": 100, + "reckless": 100, + "lc0": 800 + } + }, + "instances": { + "stockfish-anchor": { + "family": "stockfish", + "role": "anchor", + "binary": "build/online-engine-opt-v2/bin/stockfish", + "cwd": ".", + "args": [], + "options": { + "Threads": 1, + "Hash": 16, + "MultiPV": 1, + "UCI_Chess960": false + } + }, + "stockfish-shadow": { + "family": "stockfish", + "role": "shadow", + "binary": "build/online-engine-opt-v2/bin/stockfish", + "cwd": ".", + "args": [], + "options": { + "Threads": 1, + "Hash": 16, + "MultiPV": 1, + "UCI_Chess960": false + }, + "phase_options": { + "EXPLORE": { + "MultiPV": 1 + }, + "VERIFY": { + "MultiPV": 3 + }, + "STAGED_VERIFY": { + "MultiPV": 3 + } + } + }, + "reckless-shadow": { + "family": "reckless", + "role": "shadow", + "binary": "build/online-engine-opt-v2/bin/reckless", + "cwd": ".", + "args": [], + "options": { + "Threads": 1, + "Hash": 16, + "MultiPV": 1, + "Minimal": false, + "UCI_Chess960": false + }, + "phase_options": { + "EXPLORE": { + "MultiPV": 1 + }, + "VERIFY": { + "MultiPV": 3 + }, + "STAGED_VERIFY": { + "MultiPV": 3 + } + } + }, + "lc0-shadow": { + "family": "lc0", + "role": "shadow", + "binary": "build/online-engine-opt-v2/bin/lc0", + "cwd": ".", + "args": [ + "--show-hidden" + ], + "environment": { + "OPENBLAS_NUM_THREADS": "1", + "GOTO_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1" + }, + "options": { + "WeightsFile": "build/online-engine-opt-v2/networks/791556.pb.gz", + "Backend": "blas", + "BackendOptions": "", + "NNCacheSize": 262144, + "MinibatchSize": 4, + "MaxConcurrentSearchers": 1, + "TaskWorkers": 0, + "MultiPV": 1, + "UCI_Chess960": false, + "Threads": 1, + "ScoreType": "WDL_mu", + "MaxPrefetch": 0, + "AdaptivePrefetch": false, + "DefectTelemetry": false + }, + "phase_options": { + "EXPLORE": { + "MultiPV": 1 + }, + "VERIFY": { + "MultiPV": 3 + }, + "STAGED_VERIFY": { + "MultiPV": 3 + } + } + } + }, + "resource_measurement": { + "enabled": true, + "provider": "linux-procfs-v1", + "require_cpu_for_claim": true, + "require_gpu_for_claim": false, + "record_memory": true + }, + "online_time": { + "enabled": true, + "policy": "clock_envelope_v1", + "moves_horizon": 30, + "increment_fraction": 0.75, + "max_move_ms": 4000, + "network_reserve_ms": 100, + "stop_grace_ms": 300, + "output_margin_ms": 50, + "prepare_budget_ms": 100, + "quiesce_budget_ms": 250, + "cpu_parallelism": 4 + }, + "verification": { + "enabled": true, + "nomination_method": "owner_bestmove_union_v1", + "dispatch_limit": { + "nodes": 16 + }, + "staged_extension": { + "enabled": true, + "intervention": "same_process_staged_verify_v1", + "dispatch_limit": { + "nodes": 32 + } + } + }, + "crossfeed": { + "enabled": true, + "policy": "typed_verify_refine_v1" + }, + "counterfactual": { + "enabled": true, + "policy": "unanimous_verify_v1" + }, + "hybrid_authority": { + "enabled": true, + "policy": "clocked_staged_preanchor_v1", + "request_class": "online_time_v1", + "terminal_source_policy": "route_bound_staged_v1", + "allow_skipped_extension_authority": false + } +} diff --git a/config/allfather.online-hybrid-v2.validation-j7-baseline.json b/config/allfather.online-hybrid-v2.validation-j7-baseline.json new file mode 100644 index 000000000..f22005da8 --- /dev/null +++ b/config/allfather.online-hybrid-v2.validation-j7-baseline.json @@ -0,0 +1,223 @@ +{ + "schema_version": 2, + "mode": "active", + "anchor": "stockfish-anchor", + "root": "..", + "shadow": { + "owners": [ + "stockfish", + "reckless", + "lc0" + ], + "instance_by_owner": { + "stockfish": "stockfish-shadow", + "reckless": "reckless-shadow", + "lc0": "lc0-shadow" + }, + "oracle": "stockfish-shadow", + "partition": "root_index_modulo", + "dispatch_limit": { + "nodes": 16 + }, + "lc0_score_type": "WDL_mu", + "replay_root": "build/replays-online-hybrid-v2", + "oracle_timeout_s": 10, + "drain_timeout_s": 5, + "on_anchor_complete": "drain" + }, + "budget": { + "wall_ms": 4000, + "cpu_ms": 12000, + "gpu_ms": 0, + "verification_reserve_fraction": 0.3, + "controller_overhead_reserve_ms": 250 + }, + "routing": { + "policy": "unified_value_v1", + "calibration": null, + "min_observation_nodes": 4000, + "checkpoint_interval_ms": 120, + "max_stages_per_owner": 1, + "extend_nodes": 8000, + "stop_max_reversal_risk": 0.05, + "stop_min_support": 25, + "stop_min_stability_fraction": 0.6, + "stage_cpu_ms_estimate": 500, + "anchor_cpu_ms_estimate": 0, + "verify_stage_cpu_ms_estimate": 750, + "verify_stage_gpu_ms_estimate": 0, + "staged_decision_calibration": null, + "regime_support_calibration": null, + "staged_skip_max_change_probability": 0.1, + "stage_cpu_ms_estimate_by_owner": { + "stockfish": 100, + "reckless": 100, + "lc0": 600 + }, + "verify_stage_cpu_ms_estimate_by_owner": { + "stockfish": 100, + "reckless": 100, + "lc0": 800 + } + }, + "instances": { + "stockfish-anchor": { + "family": "stockfish", + "role": "anchor", + "binary": "build/online-engine-opt-v2/bin/stockfish", + "cwd": ".", + "args": [], + "options": { + "Threads": 1, + "Hash": 16, + "MultiPV": 1, + "UCI_Chess960": false + } + }, + "stockfish-shadow": { + "family": "stockfish", + "role": "shadow", + "binary": "build/online-engine-opt-v2/bin/stockfish", + "cwd": ".", + "args": [], + "options": { + "Threads": 1, + "Hash": 16, + "MultiPV": 1, + "UCI_Chess960": false + }, + "phase_options": { + "EXPLORE": { + "MultiPV": 1 + }, + "VERIFY": { + "MultiPV": 3 + }, + "STAGED_VERIFY": { + "MultiPV": 3 + } + } + }, + "reckless-shadow": { + "family": "reckless", + "role": "shadow", + "binary": "build/online-engine-opt-v2/bin/reckless", + "cwd": ".", + "args": [], + "options": { + "Threads": 1, + "Hash": 16, + "MultiPV": 1, + "Minimal": false, + "UCI_Chess960": false + }, + "phase_options": { + "EXPLORE": { + "MultiPV": 1 + }, + "VERIFY": { + "MultiPV": 3 + }, + "STAGED_VERIFY": { + "MultiPV": 3 + } + } + }, + "lc0-shadow": { + "family": "lc0", + "role": "shadow", + "binary": "build/online-engine-opt-v2/bin/lc0", + "cwd": ".", + "args": [ + "--show-hidden" + ], + "environment": { + "OPENBLAS_NUM_THREADS": "1", + "GOTO_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1" + }, + "options": { + "WeightsFile": "build/online-engine-opt-v2/networks/791556.pb.gz", + "Backend": "blas", + "BackendOptions": "", + "NNCacheSize": 262144, + "MinibatchSize": 7, + "MaxConcurrentSearchers": 1, + "TaskWorkers": 0, + "MultiPV": 1, + "UCI_Chess960": false, + "Threads": 1, + "ScoreType": "WDL_mu", + "MaxPrefetch": 8, + "AdaptivePrefetch": false, + "DefectTelemetry": false + }, + "phase_options": { + "EXPLORE": { + "MultiPV": 1 + }, + "VERIFY": { + "MultiPV": 3 + }, + "STAGED_VERIFY": { + "MultiPV": 3 + } + }, + "warmup": { + "enabled": true, + "nodes": 64, + "position": "startpos", + "reset_after": true + } + } + }, + "resource_measurement": { + "enabled": true, + "provider": "linux-procfs-v1", + "require_cpu_for_claim": true, + "require_gpu_for_claim": false, + "record_memory": true + }, + "online_time": { + "enabled": true, + "policy": "clock_envelope_v1", + "moves_horizon": 30, + "increment_fraction": 0.75, + "max_move_ms": 4000, + "network_reserve_ms": 100, + "stop_grace_ms": 300, + "output_margin_ms": 50, + "prepare_budget_ms": 100, + "quiesce_budget_ms": 250, + "cpu_parallelism": 4 + }, + "verification": { + "enabled": true, + "nomination_method": "owner_bestmove_union_v1", + "dispatch_limit": { + "nodes": 16 + }, + "staged_extension": { + "enabled": true, + "intervention": "same_process_staged_verify_v1", + "dispatch_limit": { + "nodes": 32 + } + } + }, + "crossfeed": { + "enabled": true, + "policy": "typed_verify_refine_v1" + }, + "counterfactual": { + "enabled": true, + "policy": "unanimous_verify_v1" + }, + "hybrid_authority": { + "enabled": true, + "policy": "clocked_staged_preanchor_v1", + "request_class": "online_time_v1", + "terminal_source_policy": "route_bound_staged_v1", + "allow_skipped_extension_authority": false + } +} diff --git a/config/allfather.online-hybrid-v2.validation.json b/config/allfather.online-hybrid-v2.validation.json index f22005da8..e9ad5f4b3 100644 --- a/config/allfather.online-hybrid-v2.validation.json +++ b/config/allfather.online-hybrid-v2.validation.json @@ -141,14 +141,14 @@ "Backend": "blas", "BackendOptions": "", "NNCacheSize": 262144, - "MinibatchSize": 7, + "MinibatchSize": 4, "MaxConcurrentSearchers": 1, "TaskWorkers": 0, "MultiPV": 1, "UCI_Chess960": false, "Threads": 1, "ScoreType": "WDL_mu", - "MaxPrefetch": 8, + "MaxPrefetch": 0, "AdaptivePrefetch": false, "DefectTelemetry": false }, @@ -162,12 +162,6 @@ "STAGED_VERIFY": { "MultiPV": 3 } - }, - "warmup": { - "enabled": true, - "nodes": 64, - "position": "startpos", - "reset_after": true } } }, diff --git a/config/allfather.orchestrated-v1.validation.json b/config/allfather.orchestrated-v1.validation.json index 052dc13cd..308927ef8 100644 --- a/config/allfather.orchestrated-v1.validation.json +++ b/config/allfather.orchestrated-v1.validation.json @@ -136,14 +136,14 @@ "Backend": "blas", "BackendOptions": "", "NNCacheSize": 262144, - "MinibatchSize": 7, + "MinibatchSize": 4, "MaxConcurrentSearchers": 1, "TaskWorkers": 0, "MultiPV": 1, "UCI_Chess960": false, "Threads": 1, "ScoreType": "WDL_mu", - "MaxPrefetch": 8, + "MaxPrefetch": 0, "AdaptivePrefetch": false, "DefectTelemetry": false }, @@ -157,12 +157,6 @@ "STAGED_VERIFY": { "MultiPV": 3 } - }, - "warmup": { - "enabled": true, - "nodes": 64, - "position": "startpos", - "reset_after": true } } }, diff --git a/controller/budget.py b/controller/budget.py index 4fb3f98b9..a1d0457d1 100644 --- a/controller/budget.py +++ b/controller/budget.py @@ -303,9 +303,11 @@ def __init__( *, clock: Callable[[], float] | None = None, started: float | None = None, + cpu_clock: Callable[[], float] | None = None, ) -> None: self.envelope = envelope self._clock = clock or time.monotonic + self._cpu_clock = cpu_clock or time.thread_time self._lock = threading.RLock() self._lanes: dict[str, LaneAccount] = {} self._ids = itertools.count(1) @@ -752,26 +754,29 @@ def native_work_by_semantics(self) -> dict[str, float]: @contextmanager def controller_overhead(self, label: str = "checkpoint") -> Iterator[None]: - """Charge the controller's own metareasoning time to the envelope.""" - started = self._clock() + """Charge this thread's CPU; wall time remains a separate deadline.""" + started_cpu = self._cpu_clock() try: yield finally: - elapsed_ms = (self._clock() - started) * 1000.0 + cpu_ms = self._finite_nonnegative( + (self._cpu_clock() - started_cpu) * 1000.0, + "controller thread CPU", + ) with self._lock: account = self._lane(CONTROLLER_LANE) - account.spent_cpu_ms += elapsed_ms + account.spent_cpu_ms += cpu_ms self._purpose_spent_cpu["controller"] = ( - self._purpose_spent_cpu.get("controller", 0.0) + elapsed_ms + self._purpose_spent_cpu.get("controller", 0.0) + cpu_ms ) account.native_work[f"controller.{label}_ms"] = ( - account.native_work.get(f"controller.{label}_ms", 0.0) + elapsed_ms + account.native_work.get(f"controller.{label}_ms", 0.0) + cpu_ms ) self._append_journal( "controller_charge", lane=CONTROLLER_LANE, purpose="controller", - cpu_ms=elapsed_ms, + cpu_ms=cpu_ms, label=label, ) @@ -780,10 +785,13 @@ def controller_overhead(self, label: str = "checkpoint") -> Iterator[None]: def snapshot(self) -> dict[str, Any]: with self._lock: committed_cpu, committed_gpu = self._committed() + # One observation for both fields. Separate clock samples could + # straddle hard expiry inside the same supposedly atomic snapshot. + elapsed = self.elapsed_ms() return { "envelope": self.envelope.as_dict(), - "elapsed_ms": round(self.elapsed_ms(), 3), - "wall_remaining_ms": round(self.wall_remaining_ms(), 3), + "elapsed_ms": round(elapsed, 3), + "wall_remaining_ms": round(max(0.0, self.envelope.wall_ms - elapsed), 3), "committed_cpu_ms": round(committed_cpu, 3), "committed_gpu_ms": round(committed_gpu, 3), "available_solver_cpu_ms": round(self.available_cpu_ms(), 3), diff --git a/controller/final_decision.py b/controller/final_decision.py index d36d6bfa8..b33fc4fb5 100644 --- a/controller/final_decision.py +++ b/controller/final_decision.py @@ -541,9 +541,11 @@ def _verify_orchestration_provenance( if decision.get("authority") == ANCHOR_CONTROL else verify_orchestrated_composition_integrity ) + require_resource_qualification = authorization.get("authorized") is True for problem in verifier( run_dir, root=repository_root, + require_resource_qualification=require_resource_qualification, ): problems.append( f"final decision orchestration integrity: {problem}" diff --git a/controller/orchestration_integrity.py b/controller/orchestration_integrity.py index 89980dd0b..1050c422d 100644 --- a/controller/orchestration_integrity.py +++ b/controller/orchestration_integrity.py @@ -32,6 +32,9 @@ BUY_BUNDLE, load_allocation_policy, ) +from controller.resource_measurement import ( + RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED, +) from controller.verification import verify_verification_integrity from controller.staged_verification import verify_staged_verification_integrity from controller.work_grant import WorkGrant @@ -967,8 +970,14 @@ def verify_orchestration_integrity( allow_outward_decision: bool = False, allow_config_relocation: bool = False, expected_outward_mode: str | None = None, + require_resource_qualification: bool = True, ) -> list[str]: - """Independently reconstruct J11 provenance. Empty list means valid.""" + """Independently reconstruct J11 provenance. + + Audit-only callers may set ``require_resource_qualification=False``. That + accepts only a typed pre-anchor setup-timeout certificate and never turns + incomplete physical evidence into a resource or move-authority claim. + """ run_dir = Path(run_dir) problems: list[str] = [] @@ -2436,6 +2445,7 @@ def verify_orchestration_integrity( elif ( decision is not None and decision.action == BUY_BUNDLE + and require_resource_qualification ): problems.append( f"J11 BUY_BUNDLE WorkGrant {grant.grant_id} did not settle from physical measurement" @@ -2516,7 +2526,7 @@ def verify_orchestration_integrity( problems.append( "resource CPU coverage does not reconstruct from raw evidence" ) - if not cpu_complete: + if not cpu_complete and require_resource_qualification: problems.append( "J11 qualification requires complete CPU measurement coverage" ) @@ -2538,9 +2548,41 @@ def verify_orchestration_integrity( if resource.get("qualified") is not resource_qualified: problems.append("resource qualified flag does not reconstruct") if not resource_qualified: - problems.append( - "J11 qualification requires complete physical resource evidence" - ) + if require_resource_qualification: + problems.append( + "J11 qualification requires complete physical resource evidence" + ) + else: + failure = resource.get("failure") + if not isinstance(failure, dict): + problems.append( + "audit-only incomplete resource evidence lacks typed failure" + ) + else: + if ( + failure.get("code") + != RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED + ): + problems.append( + "audit-only incomplete resource evidence uses unsupported failure code" + ) + reason = failure.get("reason") + if not isinstance(reason, str) or not reason: + problems.append( + "audit-only incomplete resource evidence lacks failure reason" + ) + if processes or stages: + problems.append( + "preparation-timeout resource evidence fabricated process/stage rows" + ) + if not _approx_equal(resource.get("engine_cpu_ms"), 0.0): + problems.append( + "preparation-timeout resource evidence fabricated engine CPU" + ) + if not _approx_equal(resource.get("stage_engine_cpu_ms"), 0.0): + problems.append( + "preparation-timeout resource evidence fabricated stage CPU" + ) resource_core = { key: value @@ -2606,6 +2648,20 @@ def verify_orchestration_integrity( problems.append( "route physical CPU envelope result does not reconstruct" ) + controller_row = resource.get("controller") if isinstance(resource, dict) else None + controller_cpu = controller_row.get("cpu_ms") if isinstance(controller_row, dict) else None + controller_cap = _finite_nonnegative( + route_envelope.get("controller_overhead_reserve_ms"), "controller CPU partition", + ) + controller_cpu_within = bool( + type(controller_cpu) in (int, float) + and math.isfinite(float(controller_cpu)) + and 0.0 <= float(controller_cpu) <= controller_cap + 1e-9 + ) + if claim.get("controller_cpu_within_partition") is not controller_cpu_within: + problems.append("controller physical CPU partition does not reconstruct") + if claim.get("controller_cpu_accounting_complete") is not True: + problems.append("controller accounting incomplete or unmeasured") clock_outcome = route.get("clock_outcome") if ( @@ -2728,6 +2784,8 @@ def verify_orchestration_integrity( "physical_measurement_required": True, "physical_measurement_qualified": resource_qualified, "physical_cpu_within_envelope": physical_cpu_within, + "controller_cpu_within_partition": controller_cpu_within, + "controller_cpu_accounting_complete": True, } for key, expected_value in expected_claim_fields.items(): if claim.get(key) is not expected_value: @@ -2747,6 +2805,8 @@ def verify_orchestration_integrity( and wall_within and resource_qualified and physical_cpu_within + and controller_cpu_within + and claim.get("controller_cpu_accounting_complete") is True ) if claim.get("claimed") is not expected_claim: problems.append( @@ -2769,6 +2829,7 @@ def verify_orchestrated_composition_integrity( root: Path | str | None = None, expected_source_commit: str | None = None, expected_outward_mode: str | None = None, + require_resource_qualification: bool = True, ) -> list[str]: """Reconstruct the J12 composition without weakening the J11 evidence core.""" @@ -2782,6 +2843,7 @@ def verify_orchestrated_composition_integrity( allow_outward_decision=True, allow_config_relocation=True, expected_outward_mode=expected_outward_mode, + require_resource_qualification=require_resource_qualification, ) run_dir = Path(run_dir) try: @@ -3004,6 +3066,7 @@ def verify_meta1_control_composition_integrity( *, root: Path | str | None = None, expected_source_commit: str | None = None, + require_resource_qualification: bool = True, ) -> list[str]: """J13 control profile: J12 evidence plus exactly one outward control bit.""" @@ -3012,6 +3075,7 @@ def verify_meta1_control_composition_integrity( root=root, expected_source_commit=expected_source_commit, expected_outward_mode="anchor_control_v1", + require_resource_qualification=require_resource_qualification, ) try: manifest = load_manifest(run_dir) diff --git a/controller/resource_measurement.py b/controller/resource_measurement.py index 99a240af4..f46806130 100644 --- a/controller/resource_measurement.py +++ b/controller/resource_measurement.py @@ -28,6 +28,7 @@ RESOURCE_SCHEMA_VERSION = 1 SUPPORTED_RESOURCE_PROVIDERS = ("linux-procfs-v1",) +RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED = "PREPARATION_BUDGET_EXCEEDED" class ResourceMeasurementError(RuntimeError): @@ -188,6 +189,9 @@ def __init__( self._process_endpoints_frozen = False self._interval_frozen = False self._sealed: dict[str, object] | None = None + # Diagnostic timestamps do not participate in the physical certificate. + # Keep them outside resource.json: evidence hashes/claim semantics are unchanged. + self._seal_phase_ns: dict[str, int] | None = None @property def provider_id(self) -> str: @@ -318,7 +322,25 @@ def begin_stage(self, *, key: str, instance: str, phase: str, pid: int | None) - raise ResourceMeasurementError( f"instance {instance!r} already has active measured stage {other!r}" ) + + # The first real stage for an instance is also its run-level process + # baseline. This removes a redundant procfs sample from the + # pre-anchor path while still charging all subsequent work and gaps. + needs_process_baseline = ( + instance not in self._process_starts + and instance not in self._process_totals + ) if self._provider is None: + reason = self._provider_error or "resource provider unavailable" + if needs_process_baseline: + self._process_totals[instance] = { + "instance": instance, + "pid": pid, + "complete": False, + "cpu_ms": None, + "wall_ms": None, + "reason": reason, + } self._measurements[key] = StageResourceMeasurement( key=key, instance=instance, @@ -330,10 +352,20 @@ def begin_stage(self, *, key: str, instance: str, phase: str, pid: int | None) - start_rss_bytes=None, end_rss_bytes=None, vm_hwm_bytes=None, - reason=self._provider_error or "resource provider unavailable", + reason=reason, ) return if pid is None: + reason = "backend pid unavailable at dispatch" + if needs_process_baseline: + self._process_totals[instance] = { + "instance": instance, + "pid": None, + "complete": False, + "cpu_ms": None, + "wall_ms": None, + "reason": reason, + } self._measurements[key] = StageResourceMeasurement( key=key, instance=instance, @@ -345,12 +377,22 @@ def begin_stage(self, *, key: str, instance: str, phase: str, pid: int | None) - start_rss_bytes=None, end_rss_bytes=None, vm_hwm_bytes=None, - reason="backend pid unavailable at dispatch", + reason=reason, ) return try: start = self._provider.snapshot(pid) except Exception as exc: + reason = f"start sample failed: {type(exc).__name__}: {exc}" + if needs_process_baseline: + self._process_totals[instance] = { + "instance": instance, + "pid": pid, + "complete": False, + "cpu_ms": None, + "wall_ms": None, + "reason": f"run baseline {reason}", + } self._measurements[key] = StageResourceMeasurement( key=key, instance=instance, @@ -362,9 +404,11 @@ def begin_stage(self, *, key: str, instance: str, phase: str, pid: int | None) - start_rss_bytes=None, end_rss_bytes=None, vm_hwm_bytes=None, - reason=f"start sample failed: {type(exc).__name__}: {exc}", + reason=reason, ) return + if needs_process_baseline: + self._process_starts[instance] = start self._active[key] = _ActiveStage( key=key, instance=instance, @@ -661,6 +705,25 @@ def _atomic_write(path: Path, payload: str) -> None: except FileNotFoundError: pass + def seal_timing_ms(self, *, started_monotonic: float) -> dict[str, float] | None: + """Non-authorizing diagnostic for freeze, serialization, and durable I/O. + + The caller supplies the same external-go origin as its BudgetLedger. + This trace must never be used as a substitute for a wall claim or CPU + measurement; it is retained only to diagnose post-publication latency. + """ + with self._lock: + marks = None if self._seal_phase_ns is None else dict(self._seal_phase_ns) + if marks is None: + return None + origin_ns = int(started_monotonic * 1_000_000_000) + return { + "freeze_wall_ms": round(max(0, marks["freeze"] - origin_ns) / 1_000_000, 3), + "serialization_ms": round(max(0, marks["serialized"] - marks["freeze"]) / 1_000_000, 3), + "durable_write_ms": round(max(0, marks["persisted"] - marks["serialized"]) / 1_000_000, 3), + "persisted_wall_ms": round(max(0, marks["persisted"] - origin_ns) / 1_000_000, 3), + } + def seal(self, path: Path, *, validity_check: Callable[[], bool] | None = None) -> dict[str, object]: with self._lock: if self._sealed is not None: @@ -673,6 +736,7 @@ def seal(self, path: Path, *, validity_check: Callable[[], bool] | None = None) reason="resource report sealed before a terminal stage sample was observed", ) self.freeze_interval() + frozen_ns = time.monotonic_ns() if validity_check is not None and not validity_check(): self._interval_error = "measurement interval crossed an online anchor generation" payload = self._report_payload() @@ -681,7 +745,14 @@ def seal(self, path: Path, *, validity_check: Callable[[], bool] | None = None) digest = hashlib.sha256(self._canonical_bytes(payload)).hexdigest() payload["report_id"] = f"resource-{digest[:16]}" rendered = json.dumps(payload, indent=2, sort_keys=True) + "\n" + serialized_ns = time.monotonic_ns() self._atomic_write(Path(path), rendered) + persisted_ns = time.monotonic_ns() + self._seal_phase_ns = { + "freeze": frozen_ns, + "serialized": serialized_ns, + "persisted": persisted_ns, + } file_sha = hashlib.sha256(rendered.encode("utf-8")).hexdigest() summary = { "path": Path(path).name, @@ -696,3 +767,94 @@ def seal(self, path: Path, *, validity_check: Callable[[], bool] | None = None) } self._sealed = summary return dict(summary) + + + +def seal_unqualified_resource_report( + path: Path, + *, + run_id: str, + settings: ResourceMeasurementSettings, + failure_code: str, + failure_reason: str, + controller_cpu_started_ns: int, +) -> dict[str, object]: + """Seal explicit negative resource evidence when measurement never started.""" + + if not settings.enabled: + raise ResourceMeasurementError( + "negative resource evidence requires enabled measurement settings" + ) + if failure_code != RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED: + raise ResourceMeasurementError( + f"unsupported negative resource failure code: {failure_code!r}" + ) + if not isinstance(failure_reason, str) or not failure_reason: + raise ResourceMeasurementError("negative resource failure reason is required") + + controller_cpu_ms = max( + 0.0, + (time.process_time_ns() - int(controller_cpu_started_ns)) / 1_000_000.0, + ) + coverage = { + "cpu": { + "required": settings.require_cpu_for_claim, + "complete": False, + }, + "gpu": { + "required": settings.require_gpu_for_claim, + "complete": False, + "provider": None, + }, + "memory": { + "recorded": settings.record_memory, + "semantics": ( + "endpoint RSS plus process-lifetime VmHWM; VmHWM is not a stage-local peak" + if settings.record_memory + else "disabled" + ), + }, + } + payload: dict[str, object] = { + "schema_version": RESOURCE_SCHEMA_VERSION, + "run_id": run_id, + "provider": settings.provider, + "settings": settings.as_dict(), + "provider_error": None, + "coverage": coverage, + "controller": { + "source": "time.process_time_ns", + "cpu_ms": round(controller_cpu_ms, 3), + "scope": ( + "controller process CPU from replay-run creation through negative " + "resource-certificate sealing; backend physical measurement never started" + ), + }, + "engine_cpu_ms": 0.0, + "stage_engine_cpu_ms": 0.0, + "physical_cpu_ms": round(controller_cpu_ms, 3), + "processes": {}, + "stages": [], + "qualified": False, + "failure": { + "code": failure_code, + "reason": failure_reason, + }, + } + digest = hashlib.sha256(ResourceMeasurementRun._canonical_bytes(payload)).hexdigest() + payload["report_id"] = f"resource-{digest[:16]}" + rendered = json.dumps(payload, indent=2, sort_keys=True) + "\n" + ResourceMeasurementRun._atomic_write(Path(path), rendered) + file_sha = hashlib.sha256(rendered.encode("utf-8")).hexdigest() + return { + "path": Path(path).name, + "sha256": file_sha, + "report_id": payload["report_id"], + "provider": payload["provider"], + "coverage": payload["coverage"], + "qualified": False, + "physical_cpu_ms": payload["physical_cpu_ms"], + "engine_cpu_ms": 0.0, + "controller_cpu_ms": payload["controller"]["cpu_ms"], + "failure": payload["failure"], + } diff --git a/controller/routing.py b/controller/routing.py index 34e1ec6ad..d8fd0a72c 100644 --- a/controller/routing.py +++ b/controller/routing.py @@ -32,6 +32,7 @@ import json import math import threading +import time from dataclasses import dataclass, field from enum import Enum from pathlib import Path @@ -768,6 +769,7 @@ def on_run_start(self, context: Any) -> None: self._work_grant_unresolved = False self._fallback = False self._anchor_reserved = False + self._controller_cpu_accounting_complete = True if self.work_scheduler is not None: if not isinstance(resource_plan, MoveResourcePlan): @@ -780,20 +782,35 @@ def on_run_start(self, context: Any) -> None: "exact J8 legacy execution remains in force" ) - # Controller work already done for this run -- run preparation and the - # legal-root oracle -- happened before any reservation existed. Charging - # it now keeps it inside B instead of outside the accounting. - already_elapsed = 0.0 - try: + # Measure process CPU from the external-go receipt until router entry. + # This interval is before the thread-local charges made later. + started_cpu_ns = getattr(context, "controller_cpu_started_ns", None) + valid_origin = ( + type(started_cpu_ns) is int and started_cpu_ns >= 0 + and (clock is None or started_cpu_ns == clock.plan.controller_cpu_started_ns) + ) + if valid_origin: + now_cpu_ns = time.process_time_ns() + if now_cpu_ns >= started_cpu_ns: + measured_cpu_ms = (now_cpu_ns - started_cpu_ns) / 1_000_000.0 + if measured_cpu_ms > 0.0: + self.ledger.charge_elapsed( + "qualification", + cpu_ms=measured_cpu_ms, + note="controller.qualification_ms", + ) + else: + valid_origin = False + if not valid_origin: + # No missing measurement can silently become free work. Synthetic + # non-resource callers keep the conservative historical wall proxy. + self._controller_cpu_accounting_complete = False already_elapsed = max(0.0, float(context.elapsed_ms())) - except Exception: # pragma: no cover - defensive against older contexts - already_elapsed = 0.0 - if already_elapsed > 0.0: - self.ledger.charge_elapsed( - "qualification", - cpu_ms=already_elapsed, - note="controller.qualification_ms", - ) + if already_elapsed > 0.0: + self.ledger.charge_elapsed( + "qualification", cpu_ms=already_elapsed, + note="controller.qualification_ms", + ) if self.calibration is None: self.audit.note( @@ -1392,21 +1409,41 @@ def on_run_end(self, context: Any) -> None: resource_qualified = not resource_required physical_cpu_within = not resource_required + controller_cpu_within = not resource_required cpu_measurement = "stage_wall_ms_x_configured_threads" if isinstance(resource_summary, dict): if resource_required: resource_qualified = bool(resource_summary.get("qualified")) physical_cpu = resource_summary.get("physical_cpu_ms") if isinstance(physical_cpu, (int, float)) and not isinstance(physical_cpu, bool): - physical_cpu_within = float(physical_cpu) <= self.envelope.cpu_ms + physical_cpu_within = math.isfinite(float(physical_cpu)) and ( + 0.0 <= float(physical_cpu) <= self.envelope.cpu_ms + ) elif resource_required: physical_cpu_within = False + # ResourceMeasurementRun.seal() returns a summary; resource.json + # keeps the independently verifiable nested controller field. + controller_cpu = resource_summary.get("controller_cpu_ms") + controller_cpu_within = ( + not resource_required or bool( + type(controller_cpu) in (int, float) + and math.isfinite(float(controller_cpu)) + and 0.0 <= float(controller_cpu) + <= self.envelope.controller_overhead_reserve_ms + ) + ) provider = resource_summary.get("provider") if resource_summary.get("qualified") and isinstance(provider, str): cpu_measurement = provider clock = getattr(context, "clock", None) clock_complete = clock is None or bool(clock.outcome()["output_within_deadline"]) + # Sample the terminal wall clock once. Computing budget.elapsed_ms, + # wall_ms_elapsed, wall_within_envelope and claimed at separate instants + # permitted internally contradictory certificates at hard expiry. + terminal_budget = self.ledger.snapshot() + terminal_wall_ms = terminal_budget["elapsed_ms"] + terminal_wall_within = terminal_wall_ms <= self.envelope.wall_ms payload = { "schema_version": ROUTE_SCHEMA_VERSION, "run_id": audit.run_id, @@ -1414,7 +1451,7 @@ def on_run_end(self, context: Any) -> None: "thresholds": self.policy.as_dict(), "envelope": self.envelope.as_dict(), "calibration": self._calibration_provenance(), - "budget": self.ledger.snapshot(), + "budget": terminal_budget, "resource_measurement": resource_summary, "envelope_claim": { # Reservation accounting staying inside B is necessary but not @@ -1433,14 +1470,16 @@ def on_run_end(self, context: Any) -> None: # wall envelope -- a slow legal-root oracle alone can do it -- # and a claim that ignores the clock is not a claim about the # envelope that was declared. - "wall_ms_elapsed": round(self.ledger.elapsed_ms(), 3), - "wall_within_envelope": ( - self.ledger.elapsed_ms() <= self.envelope.wall_ms - ), + "wall_ms_elapsed": terminal_wall_ms, + "wall_within_envelope": terminal_wall_within, "cpu_measurement": cpu_measurement, "physical_measurement_required": resource_required, "physical_measurement_qualified": resource_qualified, "physical_cpu_within_envelope": physical_cpu_within, + "controller_cpu_within_partition": controller_cpu_within, + "controller_cpu_accounting_complete": ( + self._controller_cpu_accounting_complete or not resource_required + ), "claimed": ( clock_complete and self._anchor_bound[0] @@ -1450,9 +1489,11 @@ def on_run_end(self, context: Any) -> None: and self.ledger.within_partition_caps() and not self._specialist_unresolved and not self._work_grant_unresolved - and self.ledger.elapsed_ms() <= self.envelope.wall_ms + and terminal_wall_within and resource_qualified and physical_cpu_within + and controller_cpu_within + and (self._controller_cpu_accounting_complete or not resource_required) ), }, "decisions": audit.decisions, @@ -1727,10 +1768,9 @@ def release_specialist(self, token: str, *, reason: str) -> None: } ) - def charge_controller_elapsed(self, label: str, elapsed_ms: float) -> None: - """Charge controller-side specialist preparation/restoration work.""" - - value = max(0.0, float(elapsed_ms)) + def charge_controller_elapsed(self, label: str, cpu_ms: float) -> None: + """Charge measured controller thread CPU, not wall delay.""" + value = self.ledger._finite_nonnegative(cpu_ms, "controller thread CPU") self.ledger.charge_elapsed( f"controller:{label}", cpu_ms=value, @@ -1738,6 +1778,11 @@ def charge_controller_elapsed(self, label: str, elapsed_ms: float) -> None: purpose="controller", ) + def invalidate_controller_cpu_accounting(self, reason: str) -> None: + self._controller_cpu_accounting_complete = False + if self.audit is not None: + self.audit.note(f"controller CPU accounting incomplete: {reason}") + def _calibration_provenance(self) -> dict[str, Any] | None: if self.calibration is None: return None diff --git a/controller/shadow.py b/controller/shadow.py index 7abb60331..63dddae3b 100644 --- a/controller/shadow.py +++ b/controller/shadow.py @@ -77,7 +77,12 @@ ) from controller.final_decision import seal_final_decision_artifact from controller.replay import ReplayRun, StageRecord, TelemetryStreamWriter, sha256_file -from controller.resource_measurement import ResourceMeasurementRun, StageResourceMeasurement +from controller.resource_measurement import ( + RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED, + ResourceMeasurementRun, + StageResourceMeasurement, + seal_unqualified_resource_report, +) from controller.prefix_shards import PrefixShardLedger, PrefixShardLedgerError from controller.refinement import ( RefinementError, @@ -199,7 +204,10 @@ def release_work_grant( ) -> None: ... - def charge_controller_elapsed(self, label: str, elapsed_ms: float) -> None: + def charge_controller_elapsed(self, label: str, cpu_ms: float) -> None: + ... + + def invalidate_controller_cpu_accounting(self, reason: str) -> None: ... @property @@ -267,6 +275,7 @@ class RunContext: _coordinator: "ShadowRunCoordinator" clock: ClockSearch | None = None move_resource_plan: MoveResourcePlan | None = None + controller_cpu_started_ns: int | None = None def elapsed_ms(self) -> float: return (time.monotonic() - self.started_monotonic) * 1000.0 @@ -439,6 +448,8 @@ class _ActiveRun: anchor_stage: StageRecord | None = None anchor_stream: TelemetryStreamWriter | None = None resources: ResourceMeasurementRun | None = None + resource_setup_failure: str | None = None + resource_controller_cpu_started_ns: int | None = None ledger: RootShardLedger | None = None verification: VerificationRun | None = None staged_verification: StagedVerificationRun | None = None @@ -1144,6 +1155,7 @@ def prepare_run( _coordinator=self, clock=clock, move_resource_plan=resource_plan, + controller_cpu_started_ns=controller_cpu_started_ns, ) resource_settings = self.runtime.config.resource_measurement resources = ( @@ -1155,24 +1167,42 @@ def prepare_run( controller_cpu_started_ns=controller_cpu_started_ns, ) ) + resource_setup_failure = None if resources is not None and resources.settings.enabled: candidate_resources = resources - def register_resources(): - for instance in sorted(self.runtime.backends): - candidate_resources.register_process(instance=instance, pid=self.runtime.process_pid(instance)) + + def start_anchor_measurement(): + # The first anchor-stage sample is also the run-level process + # baseline. Shadow instances establish their baselines lazily + # immediately before their first real dispatch. if clock is not None: - candidate_resources.begin_stage(key=anchor_search_id, instance=anchor_name, - phase="ANCHOR", pid=self.runtime.process_pid(anchor_name)) + candidate_resources.begin_stage( + key=anchor_search_id, + instance=anchor_name, + phase="ANCHOR", + pid=self.runtime.process_pid(anchor_name), + ) return candidate_resources + if clock is None: - register_resources() + # Non-clocked legacy paths begin the anchor measurement at the + # actual dispatch boundary in note_anchor_dispatch(). + pass else: measured, resources = self._within_prepare_budget( - f"{run_id}-resource-start", deadline=prepare_deadline, work=register_resources, + f"{run_id}-resource-start", + deadline=prepare_deadline, + work=start_anchor_measurement, ) if not measured: resources = None - run.note("clock resource setup missed preparation budget; physical claim unavailable") + resource_setup_failure = ( + RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED + ) + run.note( + "clock resource setup missed preparation budget; " + "physical claim unavailable" + ) active = _ActiveRun( generation=generation, run=run, @@ -1180,6 +1210,8 @@ def register_resources(): anchor_stage=anchor_stage, anchor_stream=anchor_stream, resources=resources, + resource_setup_failure=resource_setup_failure, + resource_controller_cpu_started_ns=controller_cpu_started_ns, started_monotonic=started, ) # Controller overhead is recorded, never hidden. This is the only work @@ -1877,15 +1909,55 @@ def _anchor_resource_measurement( def _seal_resource_report(self, generation: int) -> dict[str, object] | None: with self._lock: active = self._run - if active is None or active.generation != generation or active.resources is None: + if active is None or active.generation != generation: return None resources = active.resources + failure = active.resource_setup_failure + started_ns = active.resource_controller_cpu_started_ns + settings = self.runtime.config.resource_measurement path = active.run.run_dir / "resource.json" + + if resources is None: + if ( + failure == RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED + and settings is not None + and started_ns is not None + ): + try: + return seal_unqualified_resource_report( + path, + run_id=active.run.run_id, + settings=settings, + failure_code=failure, + failure_reason=( + "resource measurement did not complete inside the " + "pre-anchor preparation firewall" + ), + controller_cpu_started_ns=started_ns, + ) + except Exception as exc: + active.run.note( + "negative resource report could not be sealed: " + f"{type(exc).__name__}: {exc}" + ) + return None + try: clock = active.context.clock - return resources.seal(path, **({} if clock is None else { + summary = resources.seal(path, **({} if clock is None else { "validity_check": lambda: not clock.measurement_superseded.is_set(), })) + timing = resources.seal_timing_ms( + started_monotonic=active.started_monotonic, + ) + if timing is not None: + # Manifest note, not a resource/route qualification input. + # In particular, a slow fsync cannot fabricate extra engine CPU. + active.run.note( + "resource seal phases (non-authorizing): " + + ", ".join(f"{name}={value:.3f}" for name, value in timing.items()) + ) + return summary except Exception as exc: active.run.note( f"resource report could not be sealed: {type(exc).__name__}: {exc}" @@ -2663,6 +2735,11 @@ def _run_worker(self, active: _ActiveRun) -> None: if active.anchor_resource_done.is_set(): try: active.resources.freeze_process_endpoints() + active.run.note( + "process endpoints frozen at " + f"{(time.monotonic() - active.started_monotonic) * 1000.0:.3f}ms " + "from external go (non-authorizing)" + ) except Exception as exc: active.run.note( "process resource endpoints could not freeze before engine reuse: " @@ -3078,6 +3155,7 @@ def _prepare_staged_work_grant_bundle( f"{active.run.run_id}:verify-extension:{instance}:0" ) started = time.monotonic() + started_cpu_ns = time.thread_time_ns() expected_options = ( self.runtime.expected_shadow_phase_options( instance, @@ -3091,9 +3169,7 @@ def _prepare_staged_work_grant_bundle( "j10_bundle_expected_options_" f"{owner}" ), - elapsed_ms=( - time.monotonic() - started - ) * 1000.0, + cpu_ms=(time.thread_time_ns() - started_cpu_ns) / 1_000_000.0, ) grant = propose( active.context, @@ -3142,6 +3218,7 @@ def _prepare_staged_work_grant_bundle( for admission in admissions: instance = admission.grant.instance started = time.monotonic() + started_cpu_ns = time.thread_time_ns() try: actual_options = self.runtime.configure_shadow_phase( instance, @@ -3154,11 +3231,10 @@ def _prepare_staged_work_grant_bundle( "j10_bundle_phase_config_" f"{admission.grant.owner}" ), - elapsed_ms=( - time.monotonic() - started - ) * 1000.0, + cpu_ms=(time.thread_time_ns() - started_cpu_ns) / 1_000_000.0, ) digest_started = time.monotonic() + digest_started_cpu_ns = time.thread_time_ns() actual_digest = canonical_digest( dict(sorted(actual_options.items())) ) @@ -3168,9 +3244,7 @@ def _prepare_staged_work_grant_bundle( "j10_bundle_actual_options_" f"{admission.grant.owner}" ), - elapsed_ms=( - time.monotonic() - digest_started - ) * 1000.0, + cpu_ms=(time.thread_time_ns() - digest_started_cpu_ns) / 1_000_000.0, ) if ( actual_digest @@ -3257,6 +3331,7 @@ def _prepare_work_grant( return None try: digest_started = time.monotonic() + digest_started_cpu_ns = time.thread_time_ns() expected_options = self.runtime.expected_shadow_phase_options( instance, phase, @@ -3265,7 +3340,7 @@ def _prepare_work_grant( self._charge_controller_elapsed( active, label=f"work_grant_expected_options_{phase.lower()}_{owner}", - elapsed_ms=(time.monotonic() - digest_started) * 1000.0, + cpu_ms=(time.thread_time_ns() - digest_started_cpu_ns) / 1_000_000.0, ) grant = propose( active.context, @@ -3308,6 +3383,7 @@ def _assert_grant_effective_options( effective_options: dict[str, object], ) -> bool: started = time.monotonic() + started_cpu_ns = time.thread_time_ns() actual = canonical_digest(dict(sorted(effective_options.items()))) self._charge_controller_elapsed( active, @@ -3315,7 +3391,7 @@ def _assert_grant_effective_options( "work_grant_actual_options_" f"{admission.grant.phase.lower()}_{admission.grant.owner}" ), - elapsed_ms=(time.monotonic() - started) * 1000.0, + cpu_ms=(time.thread_time_ns() - started_cpu_ns) / 1_000_000.0, ) if actual == admission.grant.effective_options_digest: return True @@ -3462,20 +3538,26 @@ def _charge_controller_elapsed( active: _ActiveRun, *, label: str, - elapsed_ms: float, + cpu_ms: float, ) -> None: if self.router is None: return charge = getattr(self.router, "charge_controller_elapsed", None) + invalidate = getattr(self.router, "invalidate_controller_cpu_accounting", None) if charge is None: + if callable(invalidate): + invalidate(f"missing charge method: {label}") + active.run.note(f"controller CPU accounting unavailable for {label}") return try: - charge(label, max(0.0, float(elapsed_ms))) + charge(label, cpu_ms) except Exception as exc: # pragma: no cover - router isolation active.run.note( - f"controller overhead accounting failed for {label}: " + f"controller CPU accounting failed for {label}: " f"{type(exc).__name__}: {exc}" ) + if callable(invalidate): + invalidate(f"{label}: {type(exc).__name__}") def _execute(self, active: _ActiveRun) -> tuple[str, str | None]: run = active.run @@ -3681,6 +3763,7 @@ def _build_crossfeed(self, active: _ActiveRun) -> None: return started = time.monotonic() + started_cpu_ns = time.thread_time_ns() try: active.crossfeed_view = build_crossfeed_view( run_id=active.run.run_id, @@ -3696,7 +3779,7 @@ def _build_crossfeed(self, active: _ActiveRun) -> None: self._charge_controller_elapsed( active, label="crossfeed_build", - elapsed_ms=(time.monotonic() - started) * 1000.0, + cpu_ms=(time.thread_time_ns() - started_cpu_ns) / 1_000_000.0, ) def _build_counterfactual(self, active: _ActiveRun) -> None: @@ -3707,6 +3790,7 @@ def _build_counterfactual(self, active: _ActiveRun) -> None: return started = time.monotonic() + started_cpu_ns = time.thread_time_ns() charged = False try: authority = self.runtime.config.hybrid_authority @@ -3760,7 +3844,7 @@ def _build_counterfactual(self, active: _ActiveRun) -> None: self._charge_controller_elapsed( active, label="counterfactual_decision_build", - elapsed_ms=(time.monotonic() - started) * 1000.0, + cpu_ms=(time.thread_time_ns() - started_cpu_ns) / 1_000_000.0, ) charged = True @@ -3791,7 +3875,7 @@ def _build_counterfactual(self, active: _ActiveRun) -> None: self._charge_controller_elapsed( active, label="counterfactual_decision_build", - elapsed_ms=(time.monotonic() - started) * 1000.0, + cpu_ms=(time.thread_time_ns() - started_cpu_ns) / 1_000_000.0, ) def _execute_verification(self, active: _ActiveRun) -> None: @@ -3974,6 +4058,7 @@ def _dispatch_verification_stage( return False phase_config_started = time.monotonic() + phase_config_started_cpu_ns = time.thread_time_ns() try: effective_options = self.runtime.configure_shadow_phase( instance, @@ -4007,7 +4092,7 @@ def _dispatch_verification_stage( self._charge_controller_elapsed( active, label=f"phase_config_verify_{owner}", - elapsed_ms=(time.monotonic() - phase_config_started) * 1000.0, + cpu_ms=(time.thread_time_ns() - phase_config_started_cpu_ns) / 1_000_000.0, ) if admission is not None and not self._assert_grant_effective_options( @@ -4587,6 +4672,7 @@ def _dispatch_staged_verification_stage( return False phase_config_started = time.monotonic() + phase_config_started_cpu_ns = time.thread_time_ns() try: if self._j10_adaptive(active): effective_options = ( @@ -4630,7 +4716,7 @@ def _dispatch_staged_verification_stage( self._charge_controller_elapsed( active, label=f"phase_config_staged_verify_{owner}", - elapsed_ms=(time.monotonic() - phase_config_started) * 1000.0, + cpu_ms=(time.thread_time_ns() - phase_config_started_cpu_ns) / 1_000_000.0, ) if admission is not None and not self._assert_grant_effective_options( @@ -5285,13 +5371,14 @@ def _execute_refinement(self, active: _ActiveRun) -> None: break try: positioned_started = time.monotonic() + positioned_started_cpu_ns = time.thread_time_ns() self.runtime.set_shadow_position( instance, descendant_position.command() ) self._charge_controller_elapsed( active, label="refine_position", - elapsed_ms=(time.monotonic() - positioned_started) * 1000.0, + cpu_ms=(time.thread_time_ns() - positioned_started_cpu_ns) / 1_000_000.0, ) prepared_instances.append(instance) with self._lock: @@ -5753,11 +5840,12 @@ def _execute_refinement_expansion( return None try: positioned_started = time.monotonic() + positioned_started_cpu_ns = time.thread_time_ns() self.runtime.set_shadow_position(instance, oracle_position.command()) self._charge_controller_elapsed( active, label="recursive_refine_position", - elapsed_ms=(time.monotonic() - positioned_started) * 1000.0, + cpu_ms=(time.thread_time_ns() - positioned_started_cpu_ns) / 1_000_000.0, ) prepared_instances.append(instance) with self._lock: @@ -6216,11 +6304,12 @@ def _restore_recursive_refinement_instances( restored = False continue started = time.monotonic() + started_cpu_ns = time.thread_time_ns() self.runtime.restore_shadow_position(instance) self._charge_controller_elapsed( active, label="recursive_refine_restore", - elapsed_ms=(time.monotonic() - started) * 1000.0, + cpu_ms=(time.thread_time_ns() - started_cpu_ns) / 1_000_000.0, ) except ControllerRuntimeError as exc: restored = False @@ -6598,11 +6687,12 @@ def _restore_refinement_instances( restored = False continue restore_started = time.monotonic() + restore_started_cpu_ns = time.thread_time_ns() self.runtime.restore_shadow_position(instance) self._charge_controller_elapsed( active, label="refine_restore", - elapsed_ms=(time.monotonic() - restore_started) * 1000.0, + cpu_ms=(time.thread_time_ns() - restore_started_cpu_ns) / 1_000_000.0, ) except ControllerRuntimeError as exc: restored = False @@ -6631,11 +6721,12 @@ def _restore_all_refinement_positions(self, active: _ActiveRun) -> bool: ok = False continue restore_started = time.monotonic() + restore_started_cpu_ns = time.thread_time_ns() self.runtime.restore_shadow_position(instance) self._charge_controller_elapsed( active, label="refine_restore_cleanup", - elapsed_ms=(time.monotonic() - restore_started) * 1000.0, + cpu_ms=(time.thread_time_ns() - restore_started_cpu_ns) / 1_000_000.0, ) except ControllerRuntimeError as exc: ok = False @@ -6740,6 +6831,7 @@ def _dispatch_stage( return False phase_config_started = time.monotonic() + phase_config_started_cpu_ns = time.thread_time_ns() try: effective_options = self.runtime.configure_shadow_phase( state.instance, @@ -6765,7 +6857,7 @@ def _dispatch_stage( self._charge_controller_elapsed( active, label=f"phase_config_explore_{state.owner}", - elapsed_ms=(time.monotonic() - phase_config_started) * 1000.0, + cpu_ms=(time.thread_time_ns() - phase_config_started_cpu_ns) / 1_000_000.0, ) if admission is not None and not self._assert_grant_effective_options( diff --git a/qualification/adaptive-resource-allocation-v1.json b/qualification/adaptive-resource-allocation-v1.json index 32cde7f32..66993786f 100644 --- a/qualification/adaptive-resource-allocation-v1.json +++ b/qualification/adaptive-resource-allocation-v1.json @@ -4,7 +4,7 @@ "policy_id": "adaptive_resource_v1", "work_scheduler_catalog": { "catalog_id": "work-grant-scheduler-v1", - "catalog_digest": "c5db1463eeb9355368163059e00c8a74bfa83a3c34ef2653365c96f2ef2931f2" + "catalog_digest": "b90c892f8d0c4653009b9bbb0e0972c17a1df65d326ac01b8f5f8b0b80386405" }, "bundle": { "bundle_id": "bundle/staged-verify-v1", diff --git a/qualification/engine-opt-v2-evidence.json b/qualification/engine-opt-v2-evidence.json index 053452bbf..12a05a281 100644 --- a/qualification/engine-opt-v2-evidence.json +++ b/qualification/engine-opt-v2-evidence.json @@ -4,13 +4,17 @@ "measurement_source": { "workflow_runs": [ 36363702414, - 36366449343 + 36366449343, + 37128771497, + 37469940904 ], "source_commits": [ "72fe32567591804061b0489b1bb73743dc0f3c9d", - "9bc81f51ba761f80754e010f95b5778dcdb353b3" + "9bc81f51ba761f80754e010f95b5778dcdb353b3", + "4acce20ec427364e51f5ac51204c8d46a32f81d1", + "56516c34719670054e12218bd3781e9725ae4f65" ], - "note": "Run 36363702414 was the original selection basis. Exact-head run 36366449343 exposed one host-sensitive frozen-corpus bestmove drift in auto-p0-c256k-cold while b7-p8-c256k-warm64 retained the v1 8/8 vector on both runs. The revised selection promotes the cross-run-stable warm b7/p8 candidate to a fresh exact-head repeatability gate; final qualification still requires a new aggregate pass." + "note": "The original b7 selection remains retained as historical evidence. PR #63 head 56516c34719670054e12218bd3781e9725ae4f65 qualified b4-p0-c256k-cold as a promotion-ready exact-host candidate with no engine/network source change. The canonical selection now promotes that candidate and requires a fresh exact-head canonical aggregate." }, "artifacts": { "candidate": { @@ -50,6 +54,20 @@ "artifact_id": 10946819863, "digest": "sha256:7ac2f676e740c28010e3be1c94b31a5b826ca0238669bdd859285757d6c48ab3", "report_sha256": "15f9e4c541aa323194fb473db7f39a646d1b4f518f264cc0ceefcc3ce4c1fdf5" + }, + "b4_candidate_pr63": { + "workflow_run": 37469940904, + "source_commit": "56516c34719670054e12218bd3781e9725ae4f65", + "profile_domain_artifact_id": 11429690812, + "profile_domain_artifact_sha256": "dfd32a14787cd30a4ff29146c063ee459c8223761c00b26e3a8933445630c9f5", + "candidate_aggregate_artifact_id": 11428968112, + "candidate_aggregate_artifact_sha256": "15ddaba2c3276eb51c993619c1c11a8f3142bcbcfcd8c5816775ed80a7882f62", + "candidate_aggregate_report_sha256": "ba12820d8c19556dbe03dbc1ebaca4aeb7427a7b2c8d6ce3c668535a01424b3a", + "final_gate_artifact_id": 11428583715, + "final_gate_artifact_sha256": "fc8a3991e510cae91c7b4b3b5004be91b64e9e6d87451bf88d79f611dc62a035", + "qualification_disposition": "QUALIFIED_EXACT_HOST_ONLY", + "execution_domain_id": "exec-domain/0bf11e027cd419ae8f8a", + "execution_domain_digest": "0bf11e027cd419ae8f8abee52cfa52aef017b525598ca8237acc0a87ae4458ef" } }, "observations": { @@ -115,6 +133,20 @@ 440 ], "note": "Cross-run evidence spans two hosted CPU identities; the revised workflow adds three same-head confirmation repeats." + }, + "b4_promotion_basis": { + "profile": "b4-p0-c256k-cold", + "nn_cache_size": 262144, + "minibatch_size": 4, + "max_prefetch": 0, + "warmup_nodes": null, + "workflow_run": 37469940904, + "candidate_qualified": true, + "promotion_ready": true, + "qualification_disposition": "QUALIFIED_EXACT_HOST_ONLY", + "bestmove_repeatability": "3/3 complete frozen-corpus vectors stable on the qualified candidate run", + "native_work_policy": "lc0-node-stop-contract-v1", + "note": "Promotion changes the canonical profile identity only; engine/network source, resource reservations, move authority, STOP promotion, and deployment/strength claims remain unchanged." } }, "stockfish": { @@ -153,7 +185,7 @@ "prior_selected_reservation_ms": 400, "revised_explore_reservation_ms": 600, "revised_verify_reservation_ms": 800, - "note": "The 600/800 ms reservations are conservative prequalification values for the revised warm b7/p8 profile. Fresh exact-head G3-v2 resource reports must fit inside them before promotion; the aggregate now checks that directly." + "note": "The 600/800 ms LC0 reservations remain the conservative canonical reservations after b4 promotion. PR #63 candidate G3/LOCAL-1 measurements fit inside these bounds; a fresh exact-head canonical aggregate must reconfirm them before merge." } }, "selected": { @@ -168,13 +200,12 @@ "lc0": { "backend": "blas", "network": "791556", - "matrix_profile": "b7-p8-c256k-warm64", "nn_cache_size": 262144, - "minibatch_size": 7, - "max_prefetch": 8, + "minibatch_size": 4, + "max_prefetch": 0, "adaptive_prefetch": false, "defect_telemetry": false, - "warmup_nodes": 64, + "warmup_nodes": null, "phase_multipv": { "EXPLORE": 1, "VERIFY": 3, @@ -182,7 +213,8 @@ }, "uci_args": [ "--show-hidden" - ] + ], + "matrix_profile": "b4-p0-c256k-cold" }, "resource_estimates_ms": { "explore": { diff --git a/qualification/engine-opt-v2-host-binding.json b/qualification/engine-opt-v2-host-binding.json index 3e8a6134e..ec9f19d83 100644 --- a/qualification/engine-opt-v2-host-binding.json +++ b/qualification/engine-opt-v2-host-binding.json @@ -44,18 +44,32 @@ "deployment": false }, "current_domain_bound_qualification": { - "pull_request": 49, - "qualified_head": "3de528e5a1a5a3609e0c293a80296ddc4a006875", - "merge_commit": "0ff86cc53d336780fa0d0566fe0ee414a7b70f3b", - "aggregate_workflow_run": 36589207658, - "aggregate_artifact_id": 11047727810, - "aggregate_artifact_sha256": "efd4f3b0ba3d119ad558d3b81072f3e5f9e81aa1b34d627c6ba890d64303f35b", - "aggregate_report_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", + "pull_request": 64, + "qualified_head": "b90af9f1760c9611826afc2fd26ad0dc438d4557", + "aggregate_workflow_run": 38012334421, + "aggregate_artifact_id": 11657944410, + "aggregate_artifact_sha256": "daf8ff2a3bfb2300de02d60a60baacf52c14638ff4d3aa41d3f0394e5205dbbd", + "aggregate_report_sha256": "f1748fea0de0b8712fc47ae216c2111c525f8a3c645d044f670837ee2e81f9e1", "qualification_disposition": "QUALIFIED_EXACT_HOST_ONLY", "binding_scope": "exact_host_observation", - "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", - "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "execution_domain_id": "exec-domain/2e9673f5cea55cf86e59", + "execution_domain_digest": "2e9673f5cea55cf86e59f34ae1b4d1dddea1c19c4972ac34b323465ab3bd0e2c", "generic_host_portability_established": false, - "note": "This is the first post-J2 qualification whose LC0 matrix, G3-v2 authority evidence, LOCAL-1-v2 lifecycle evidence and candidate artifact identity were all bound to one reconstructed execution domain." + "note": "PR #64 canonical b4 source b90af9f1760c9611826afc2fd26ad0dc438d4557 independently qualified at exact unmerged source parent with aggregate workflow 38012334421 and byte-verified aggregate ZIP/report. Frozen for immediate-child promotion only. Catalog merge_commit remains the actual PR #63 mainline merge cae842801497161f2de37ff43b27d074ac12568a; it is NOT a PR #64 merge. No generic-host, deployment, Elo, or strength authority." + }, + "b4_candidate_promotion_seed": { + "pull_request": 63, + "qualified_head": "56516c34719670054e12218bd3781e9725ae4f65", + "merge_commit": "cae842801497161f2de37ff43b27d074ac12568a", + "workflow_run": 37469940904, + "aggregate_artifact_id": 11428968112, + "aggregate_artifact_sha256": "15ddaba2c3276eb51c993619c1c11a8f3142bcbcfcd8c5816775ed80a7882f62", + "aggregate_report_sha256": "ba12820d8c19556dbe03dbc1ebaca4aeb7427a7b2c8d6ce3c668535a01424b3a", + "qualification_disposition": "QUALIFIED_EXACT_HOST_ONLY", + "binding_scope": "exact_host_observation", + "execution_domain_id": "exec-domain/0bf11e027cd419ae8f8a", + "execution_domain_digest": "0bf11e027cd419ae8f8abee52cfa52aef017b525598ca8237acc0a87ae4458ef", + "canonical_promotion_complete": true, + "note": "Historical PR #63 candidate b4 seed retained unchanged; PR #64 independently qualified canonical b4 at the immediately preceding promotion head and froze that new evidence separately. No generic-host or deployment grant." } } diff --git a/qualification/engine-opt-v2-native-work-policy.json b/qualification/engine-opt-v2-native-work-policy.json new file mode 100644 index 000000000..a814c4237 --- /dev/null +++ b/qualification/engine-opt-v2-native-work-policy.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "policy_id": "lc0-node-stop-contract-v1", + "applies_to": { + "selection_profile_id": "engine-opt-v2-selection", + "selection_status": "selected", + "matrix_profile": "b4-p0-c256k-cold" + }, + "requested_nodes": 16, + "terminal_counter_semantics": "lc0.uci_nodes", + "require_exact_terminal_counter_repeatability": false, + "required_options": { + "MaxConcurrentSearchers": 1, + "TaskWorkers": 0, + "Threads": 1 + }, + "evidence_basis": { + "workflow_run": 37469940904, + "source_commit": "56516c34719670054e12218bd3781e9725ae4f65", + "observation": "PR #63 established that LC0 terminal UCI node counters are threshold/batch diagnostics, not an exact deterministic work vector, while preserving exact bestmove repeatability, fixed go nodes 16 requests, lc0.uci_nodes semantics, completion and resource bounds." + }, + "claim_boundary": { + "qualification_semantics_only": true, + "canonical_profile": true, + "runtime_profile_selection": false, + "resource_authorization": false, + "outward_move": false, + "generic_host_portability": false, + "strength": false, + "elo": false, + "equal_compute": false, + "deployment": false + } +} diff --git a/qualification/engine-opt-v2-selection.json b/qualification/engine-opt-v2-selection.json index 6fac3fb27..4b9c92ec9 100644 --- a/qualification/engine-opt-v2-selection.json +++ b/qualification/engine-opt-v2-selection.json @@ -6,13 +6,17 @@ "path": "qualification/engine-opt-v2-evidence.json", "workflow_runs": [ 36363702414, - 36366449343 + 36366449343, + 37128771497, + 37469940904 ], "measurement_source_commits": [ "72fe32567591804061b0489b1bb73743dc0f3c9d", - "9bc81f51ba761f80754e010f95b5778dcdb353b3" + "9bc81f51ba761f80754e010f95b5778dcdb353b3", + "4acce20ec427364e51f5ac51204c8d46a32f81d1", + "56516c34719670054e12218bd3781e9725ae4f65" ], - "rationale": "The original auto-batch p0 choice was rejected after exact-head cross-host bestmove drift. b7-p8-c256k-warm64 preserved the frozen 8/8 vector on both measured hosts; a fresh exact-head matrix must confirm it repeatedly before promotion." + "rationale": "Canonical promotion of the PR #63-qualified b4-p0-c256k-cold LC0 substrate. PR #63 independently qualified the b4 candidate on one bound execution domain through the frozen LC0 matrix, G3-v2 non-anchor authority witness, LOCAL-1 lifecycle, repeated constituent benchmarks, J8-J12 profile-domain qualification and the host-aware final gate. This selection remains subject to a fresh exact-head canonical aggregate before merge." }, "selected": { "stockfish": { @@ -27,11 +31,11 @@ "backend": "blas", "network": "791556", "nn_cache_size": 262144, - "minibatch_size": 7, - "max_prefetch": 8, + "minibatch_size": 4, + "max_prefetch": 0, "adaptive_prefetch": false, "defect_telemetry": false, - "warmup_nodes": 64, + "warmup_nodes": null, "phase_multipv": { "EXPLORE": 1, "VERIFY": 3, @@ -40,7 +44,7 @@ "uci_args": [ "--show-hidden" ], - "matrix_profile": "b7-p8-c256k-warm64" + "matrix_profile": "b4-p0-c256k-cold" }, "resource_estimates_ms": { "explore": { @@ -55,7 +59,7 @@ } } }, - "claim_boundary": "Selected from cross-run frozen ENGINE-OPT measurement evidence and subject to repeatability confirmation on the exact candidate head. This is a profile/build/resource selection, not an Elo, superiority, equal-compute, GPU, or deployment claim.", + "claim_boundary": "Canonical b4 profile selection promoted from retained PR #63 candidate evidence and subject to fresh exact-head canonical requalification. This is a profile/build/resource selection, not an Elo, superiority, equal-compute, generic-host-portability, GPU, or deployment claim.", "qualification": { "baseline_profile": "v1-current-cold", "corpus_cases": 8, diff --git a/qualification/local-full-game-v2.json b/qualification/local-full-game-v2.json index 24584e316..144fca399 100644 --- a/qualification/local-full-game-v2.json +++ b/qualification/local-full-game-v2.json @@ -90,7 +90,7 @@ }, { "id": "g3-v2", - "script": "scripts/qualify-online-hybrid-v2.py", + "script": "scripts/qualify-online-hybrid-v2-b4.py", "report": "build/test-results/online-hybrid-v2/report.json", "replay_root": "build/replays-online-hybrid-v2", "retain_case_replays": true diff --git a/qualification/online-engine-opt-v2-j7-baseline.json b/qualification/online-engine-opt-v2-j7-baseline.json new file mode 100644 index 000000000..170e5ea31 --- /dev/null +++ b/qualification/online-engine-opt-v2-j7-baseline.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "profile_id": "online-engine-opt-v2", + "qualification_tier": "engine-opt-cpu-reference", + "engine_opt_profile": "engine-opt-v2", + "selection": "qualification/engine-opt-v2-selection.json", + "selection_status_required_for_promotion": "selected", + "bundle_root": "build/online-engine-opt-v2", + "runtime_config": "config/allfather.online-engine-opt-v2.json", + "platform": { + "os": "linux", + "architecture": "x86_64", + "reference_os": "ubuntu-24.04", + "accelerator_kind": "cpu", + "cpu_model": "record-and-bind", + "memory": "record-and-bind" + }, + "builds": { + "stockfish": { + "artifact": "bin/stockfish", + "network_artifact": "networks/nn-134a887f4c8f.nnue", + "arch": "x86-64", + "pgo": true + }, + "reckless": { + "artifact": "bin/reckless", + "network_artifact": "networks/v60-7f587dfb.nnue", + "target": "x86_64-unknown-linux-gnu", + "target_cpu": "x86-64", + "pgo": false + }, + "lc0": { + "artifact": "bin/lc0", + "network_artifact": "networks/791556.pb.gz", + "backend": "blas", + "strength_profile_id": "lc0-strength-v1", + "isa": "x86-64-portable", + "native_arch": false, + "ispc": false, + "ispc_native_only": false, + "popcnt": false, + "f16c": false, + "pext": false + } + }, + "claim_boundary": { + "strength": false, + "elo": false, + "equal_compute": false, + "deployed_bot": false, + "gpu": false + }, + "evidence": "qualification/engine-opt-v2-evidence.json" +} diff --git a/qualification/online-hybrid-v2-b4.json b/qualification/online-hybrid-v2-b4.json new file mode 100644 index 000000000..0961f86fa --- /dev/null +++ b/qualification/online-hybrid-v2-b4.json @@ -0,0 +1,1318 @@ +{ + "schema_version": 1, + "profile_id": "online-hybrid-v2", + "qualification_tier": "online-hybrid-authority-v2", + "source_profile": "online-engine-opt-v2", + "runtime_config": "config/allfather.online-hybrid-v2.validation.json", + "reference_runtime": "config/allfather.online-engine-opt-v2.json", + "bundle_root": "build/online-engine-opt-v2", + "authority_policy": "clocked_staged_preanchor_v1", + "terminal_source_policy": "route_bound_staged_v1", + "allow_skipped_extension_authority": false, + "explore": { + "nodes": 16 + }, + "verification": { + "base": { + "nodes": 16 + }, + "extension": { + "nodes": 32 + }, + "intervention": "same_process_staged_verify_v1" + }, + "claim_boundary": { + "hybrid_authority": true, + "strength": false, + "elo": false, + "equal_compute": false, + "deployed_bot": false + }, + "clock": { + "max_move_ms": 4000, + "reason": "G3 may use the full frozen ONLINE-2 outer wall envelope so real BLAS specialist stages can complete before the soft cutoff" + }, + "positive_cases": [ + { + "id": "startpos", + "moves": [], + "command": "go movetime 4000 searchmoves e2e4 d2d4 g1f3", + "source_set": "legacy_v1" + }, + { + "id": "after-e4", + "moves": [ + "e2e4" + ], + "command": "go movetime 4000 searchmoves e7e5 c7c5 e7e6", + "source_set": "legacy_v1" + }, + { + "id": "after-d4", + "moves": [ + "d2d4" + ], + "command": "go movetime 4000 searchmoves d7d5 g8f6 e7e6", + "source_set": "legacy_v1" + }, + { + "id": "open-game-white", + "moves": [ + "e2e4", + "e7e5" + ], + "command": "go movetime 4000 searchmoves g1f3 f1c4 d2d4", + "source_set": "legacy_v1" + }, + { + "id": "queen-pawn-white", + "moves": [ + "d2d4", + "d7d5" + ], + "command": "go movetime 4000 searchmoves c2c4 g1f3 e2e3", + "source_set": "legacy_v1" + }, + { + "id": "sicilian-white", + "moves": [ + "e2e4", + "c7c5" + ], + "command": "go movetime 4000 searchmoves g1f3 d2d4 c2c3", + "source_set": "legacy_v1" + }, + { + "id": "english-black", + "moves": [ + "c2c4" + ], + "command": "go movetime 4000 searchmoves e7e5 g8f6 c7c5", + "source_set": "legacy_v1" + }, + { + "id": "discovery-base-00-03-g000015", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "b1d2", + "c8f5", + "e1e2" + ], + "command": "go wtime 16625 btime 21993 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "e8d6", + "available_clock_ms": 21993, + "generation": 15, + "hard_budget_ms": 1829, + "job_id": "base-00-03", + "output_within_deadline": true, + "position_id": "pos-3ac7303566ed92e9dffe5f1c", + "proposal_move": "c7c6", + "run_id": "20261003T191555773940Z-g000015-b82b1725" + } + }, + { + "id": "discovery-base-00-03-g000016", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "b1d2", + "c8f5", + "e1e2", + "c7c6", + "a1e1" + ], + "command": "go wtime 16604 btime 21499 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "e8d6", + "available_clock_ms": 21499, + "generation": 16, + "hard_budget_ms": 1813, + "job_id": "base-00-03", + "output_within_deadline": true, + "position_id": "pos-12e158892e482a5252b88a29", + "proposal_move": "h7h6", + "run_id": "20261003T191558293143Z-g000016-9e131ea6" + } + }, + { + "id": "discovery-base-00-03-g000019", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "b1d2", + "c8f5", + "e1e2", + "c7c6", + "a1e1", + "h7h6", + "e2f3", + "f6g5", + "d2b3", + "g5f4", + "f3f4" + ], + "command": "go wtime 13057 btime 20112 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "d8g5", + "available_clock_ms": 20112, + "generation": 19, + "hard_budget_ms": 1767, + "job_id": "base-00-03", + "output_within_deadline": true, + "position_id": "pos-183d75939c59727cd404daaf", + "proposal_move": "e8d6", + "run_id": "20261003T191609239370Z-g000019-d0adfd8a" + } + }, + { + "id": "discovery-base-00-03-g000020", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "b1d2", + "c8f5", + "e1e2", + "c7c6", + "a1e1", + "h7h6", + "e2f3", + "f6g5", + "d2b3", + "g5f4", + "f3f4", + "e8d6", + "h2h4" + ], + "command": "go wtime 6337 btime 19679 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "h6h5", + "available_clock_ms": 19679, + "generation": 20, + "hard_budget_ms": 1752, + "job_id": "base-00-03", + "output_within_deadline": true, + "position_id": "pos-d73f287acde03993bf6eae96", + "proposal_move": "b7b6", + "run_id": "20261003T191618395980Z-g000020-44f80194" + } + }, + { + "id": "discovery-base-00-06-g000006", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5" + ], + "command": "go wtime 27824 btime 27283 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "c6e5", + "available_clock_ms": 27283, + "generation": 6, + "hard_budget_ms": 2006, + "job_id": "base-00-06", + "output_within_deadline": true, + "position_id": "pos-861474f8f2058c1168a38657", + "proposal_move": "f8e7", + "run_id": "20261003T193357876882Z-g000006-b2a7fb67" + } + }, + { + "id": "discovery-base-00-06-g000021", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c1f4", + "e8e1", + "d1e1", + "d6e8", + "e1e3", + "d7d5", + "b1d2", + "c8f5", + "c2c3", + "e8d6", + "e3f3", + "f5e6", + "a1e1", + "f6g5", + "f1d3", + "h7h6", + "h2h3", + "g5f4", + "f3f4", + "d8d7", + "d2f1" + ], + "command": "go wtime 17948 btime 19258 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "e6f5", + "available_clock_ms": 19258, + "generation": 21, + "hard_budget_ms": 1738, + "job_id": "base-00-06", + "output_within_deadline": true, + "position_id": "pos-90dcb8e08ce628acd1d08f0a", + "proposal_move": "a8e8", + "run_id": "20261003T193445837846Z-g000021-518d2a7d" + } + }, + { + "id": "discovery-base-00-06-g000023", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c1f4", + "e8e1", + "d1e1", + "d6e8", + "e1e3", + "d7d5", + "b1d2", + "c8f5", + "c2c3", + "e8d6", + "e3f3", + "f5e6", + "a1e1", + "f6g5", + "f1d3", + "h7h6", + "h2h3", + "g5f4", + "f3f4", + "d8d7", + "d2f1", + "a8e8", + "f1g3", + "f7f6", + "b2b3" + ], + "command": "go wtime 18266 btime 18466 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "b7b6", + "available_clock_ms": 18466, + "generation": 23, + "hard_budget_ms": 1712, + "job_id": "base-00-06", + "output_within_deadline": true, + "position_id": "pos-d6ab8374cb274c1c295642b5", + "proposal_move": "e6f7", + "run_id": "20261003T193450319712Z-g000023-5f9bf86a" + } + }, + { + "id": "discovery-base-00-06-g000029", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c1f4", + "e8e1", + "d1e1", + "d6e8", + "e1e3", + "d7d5", + "b1d2", + "c8f5", + "c2c3", + "e8d6", + "e3f3", + "f5e6", + "a1e1", + "f6g5", + "f1d3", + "h7h6", + "h2h3", + "g5f4", + "f3f4", + "d8d7", + "d2f1", + "a8e8", + "f1g3", + "f7f6", + "b2b3", + "e6f7", + "e1c1", + "b7b6", + "h3h4", + "g8f8", + "g3f1", + "a7a5", + "f1e3", + "d6e4", + "g2g4", + "h6h5", + "g4g5" + ], + "command": "go wtime 2704 btime 16387 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "d7d8", + "available_clock_ms": 16387, + "generation": 29, + "hard_budget_ms": 1642, + "job_id": "base-00-06", + "output_within_deadline": true, + "position_id": "pos-57d7558ad1ef1886875329a4", + "proposal_move": "f6g5", + "run_id": "20261003T193519983981Z-g000029-2fc85a97" + } + }, + { + "id": "discovery-base-00-08-g000033", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "f1d3", + "g7g6", + "b1d2", + "e8g7", + "d2f3", + "c8f5", + "d3f5", + "g7f5", + "e1d2", + "f5g7", + "a1e1", + "d8d7", + "f4e5", + "f6e5", + "f3e5", + "d7d6", + "d2f4", + "f7f6", + "e5d3", + "d6f4", + "d3f4", + "c7c6", + "e1e7", + "a8e8", + "e7e8", + "g7e8", + "f4e6", + "e8d6", + "e6c5", + "g8f7", + "g1f1", + "b7b6", + "c5d3", + "f7e6", + "f1e2", + "d6f7", + "d3f4", + "e6d6", + "f2f3" + ], + "command": "go wtime 9265 btime 15212 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "c6c5", + "available_clock_ms": 15212, + "generation": 33, + "hard_budget_ms": 1603, + "job_id": "base-00-08", + "output_within_deadline": true, + "position_id": "pos-7072f4b51048871585f5e73d", + "proposal_move": "f7d8", + "run_id": "20261003T194654516591Z-g000033-a9ec4d72" + } + }, + { + "id": "discovery-base-00-08-g000040", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "f1d3", + "g7g6", + "b1d2", + "e8g7", + "d2f3", + "c8f5", + "d3f5", + "g7f5", + "e1d2", + "f5g7", + "a1e1", + "d8d7", + "f4e5", + "f6e5", + "f3e5", + "d7d6", + "d2f4", + "f7f6", + "e5d3", + "d6f4", + "d3f4", + "c7c6", + "e1e7", + "a8e8", + "e7e8", + "g7e8", + "f4e6", + "e8d6", + "e6c5", + "g8f7", + "g1f1", + "b7b6", + "c5d3", + "f7e6", + "f1e2", + "d6f7", + "d3f4", + "e6d6", + "f2f3", + "f7d8", + "e2e3", + "c6c5", + "g2g4", + "d8c6", + "f4e2", + "c5c4", + "f3f4", + "c6e7", + "e2g3", + "d6e6", + "e3f3", + "f6f5", + "g4g5" + ], + "command": "go wtime 6225 btime 13500 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "b6b5", + "available_clock_ms": 13500, + "generation": 40, + "hard_budget_ms": 1546, + "job_id": "base-00-08", + "output_within_deadline": true, + "position_id": "pos-5f4095dcb1caf13db008dc0f", + "proposal_move": "e7c8", + "run_id": "20261003T194713297257Z-g000040-afa5859e" + } + }, + { + "id": "discovery-base-00-08-g000043", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "f1d3", + "g7g6", + "b1d2", + "e8g7", + "d2f3", + "c8f5", + "d3f5", + "g7f5", + "e1d2", + "f5g7", + "a1e1", + "d8d7", + "f4e5", + "f6e5", + "f3e5", + "d7d6", + "d2f4", + "f7f6", + "e5d3", + "d6f4", + "d3f4", + "c7c6", + "e1e7", + "a8e8", + "e7e8", + "g7e8", + "f4e6", + "e8d6", + "e6c5", + "g8f7", + "g1f1", + "b7b6", + "c5d3", + "f7e6", + "f1e2", + "d6f7", + "d3f4", + "e6d6", + "f2f3", + "f7d8", + "e2e3", + "c6c5", + "g2g4", + "d8c6", + "f4e2", + "c5c4", + "f3f4", + "c6e7", + "e2g3", + "d6e6", + "e3f3", + "f6f5", + "g4g5", + "e7c8", + "g3f1", + "b6b5", + "f3e2", + "a7a5", + "a2a3" + ], + "command": "go wtime 5028 btime 12883 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "e6d6", + "available_clock_ms": 12883, + "generation": 43, + "hard_budget_ms": 1526, + "job_id": "base-00-08", + "output_within_deadline": true, + "position_id": "pos-0b5db6fd4099f22823836049", + "proposal_move": "c8b6", + "run_id": "20261003T194721123733Z-g000043-b0907f0f" + } + }, + { + "id": "discovery-base-00-08-g000052", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "f1d3", + "g7g6", + "b1d2", + "e8g7", + "d2f3", + "c8f5", + "d3f5", + "g7f5", + "e1d2", + "f5g7", + "a1e1", + "d8d7", + "f4e5", + "f6e5", + "f3e5", + "d7d6", + "d2f4", + "f7f6", + "e5d3", + "d6f4", + "d3f4", + "c7c6", + "e1e7", + "a8e8", + "e7e8", + "g7e8", + "f4e6", + "e8d6", + "e6c5", + "g8f7", + "g1f1", + "b7b6", + "c5d3", + "f7e6", + "f1e2", + "d6f7", + "d3f4", + "e6d6", + "f2f3", + "f7d8", + "e2e3", + "c6c5", + "g2g4", + "d8c6", + "f4e2", + "c5c4", + "f3f4", + "c6e7", + "e2g3", + "d6e6", + "e3f3", + "f6f5", + "g4g5", + "e7c8", + "g3f1", + "b6b5", + "f3e2", + "a7a5", + "a2a3", + "c8b6", + "e2d2", + "b6a4", + "d2c2", + "a4b6", + "f1d2", + "e6d6", + "d2f3", + "b6a8", + "c2d2", + "a8c7", + "d2e3", + "b5b4", + "a3b4", + "a5b4", + "f3e5", + "b4c3", + "b2c3" + ], + "command": "go wtime 2806 btime 11370 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "c7e6", + "available_clock_ms": 11370, + "generation": 52, + "hard_budget_ms": 1475, + "job_id": "base-00-08", + "output_within_deadline": true, + "position_id": "pos-b0daeb52af9a1e19b9cf9903", + "proposal_move": "c7b5", + "run_id": "20261003T194742895112Z-g000052-24dcb614" + } + }, + { + "id": "discovery-base-00-09-g000017", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "b1d2", + "e8d6", + "h2h3", + "c8f5", + "a2a4", + "a7a5", + "e1d1" + ], + "command": "go wtime 19324 btime 21014 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "c7c6", + "available_clock_ms": 21014, + "generation": 17, + "hard_budget_ms": 1797, + "job_id": "base-00-09", + "output_within_deadline": true, + "position_id": "pos-7c89388270e7170027d18000", + "proposal_move": "f6g5", + "run_id": "20261003T195403196308Z-g000017-aef6462f" + } + }, + { + "id": "discovery-base-00-09-g000023", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "b1d2", + "e8d6", + "h2h3", + "c8f5", + "a2a4", + "a7a5", + "e1d1", + "f6g5", + "f4h2", + "c7c6", + "d2f3", + "g5e7", + "d1b3", + "h7h6", + "f3e5", + "e7f8", + "a1e1", + "f5h7", + "e1e3" + ], + "command": "go wtime 12922 btime 18466 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "g8h8", + "available_clock_ms": 18466, + "generation": 23, + "hard_budget_ms": 1712, + "job_id": "base-00-09", + "output_within_deadline": true, + "position_id": "pos-5fd38348e14cf67631c80fbd", + "proposal_move": "f7f6", + "run_id": "20261003T195424176479Z-g000023-8cfb288a" + } + }, + { + "id": "discovery-base-00-09-g000034", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "b1d2", + "e8d6", + "h2h3", + "c8f5", + "a2a4", + "a7a5", + "e1d1", + "f6g5", + "f4h2", + "c7c6", + "d2f3", + "g5e7", + "d1b3", + "h7h6", + "f3e5", + "e7f8", + "a1e1", + "f5h7", + "e1e3", + "f7f6", + "e5f3", + "d8d7", + "h2d6", + "f8d6", + "c3c4", + "d5c4", + "f1c4", + "g8h8", + "c4e6", + "d7c7", + "f3h4", + "g7g6", + "b3d3", + "h8g7", + "e6b3", + "a8d8", + "g2g4", + "d6h2", + "g1f1", + "h2d6", + "e3e6" + ], + "command": "go wtime 9609 btime 14942 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "d6c5", + "available_clock_ms": 14942, + "generation": 34, + "hard_budget_ms": 1594, + "job_id": "base-00-09", + "output_within_deadline": true, + "position_id": "pos-0d0442f521175c29b704cac2", + "proposal_move": "d6b4", + "run_id": "20261003T195453065572Z-g000034-1e58b296" + } + }, + { + "id": "discovery-base-00-09-g000038", + "moves": [ + "g1f3", + "g8f6", + "f3g1", + "f6g8", + "e2e4", + "e7e5", + "g1f3", + "b8c6", + "f1b5", + "g8f6", + "e1g1", + "f6e4", + "f1e1", + "e4d6", + "f3e5", + "f8e7", + "b5f1", + "c6e5", + "e1e5", + "e8g8", + "d2d4", + "e7f6", + "e5e1", + "f8e8", + "c2c3", + "e8e1", + "d1e1", + "d6e8", + "c1f4", + "d7d5", + "b1d2", + "e8d6", + "h2h3", + "c8f5", + "a2a4", + "a7a5", + "e1d1", + "f6g5", + "f4h2", + "c7c6", + "d2f3", + "g5e7", + "d1b3", + "h7h6", + "f3e5", + "e7f8", + "a1e1", + "f5h7", + "e1e3", + "f7f6", + "e5f3", + "d8d7", + "h2d6", + "f8d6", + "c3c4", + "d5c4", + "f1c4", + "g8h8", + "c4e6", + "d7c7", + "f3h4", + "g7g6", + "b3d3", + "h8g7", + "e6b3", + "a8d8", + "g2g4", + "d6h2", + "g1f1", + "h2d6", + "e3e6", + "d6b4", + "d3f3", + "d8f8", + "f3e4", + "f8d8", + "b3c4", + "d8d7", + "e6e8" + ], + "command": "go wtime 9436 btime 13953 winc 1000 binc 1000", + "source_set": "discovery_local1_v1", + "discovery": { + "anchor_move": "b4f8", + "available_clock_ms": 13953, + "generation": 38, + "hard_budget_ms": 1561, + "job_id": "base-00-09", + "output_within_deadline": true, + "position_id": "pos-08f066969931cd29311f47b9", + "proposal_move": "d7e7", + "run_id": "20261003T195502247479Z-g000038-f0d8854d" + } + } + ], + "positive_requirement": { + "require_authority": "HYBRID", + "require_terminal_source": "staged_verification", + "require_non_anchor_move": true, + "semantics": "At least one predeclared real-backend case must actually emit a specialist-selected move different from the Stockfish anchor." + }, + "candidate_witnesses": { + "path": "qualification/online-hybrid-v2-candidate-witnesses.json", + "sha256": "7418f77ae73df08b1f323bc7714d741cb5215c05232ff081132d0911d2b40e2f", + "corpus_id": "g3-candidate-local1-discovery-v1", + "discovery_source": { + "artifact_digest": "sha256:932c2d83d5fc3f2dff0a9e341542c8c245cc002c196149e7a805090781318336", + "artifact_id": 11283786124, + "artifact_name": "engine-opt-v2-profile-domain", + "repository": "HMarcusWH/AllfatherChess", + "source_commit": "03a5f958691924fa859586f961087d9c9746569f", + "workflow_run": 37144819528 + }, + "selection_algorithm": { + "case_count": 16, + "description": "From candidate LOCAL-1 base-* allfather-g3 replays, retain only valid non-anchor HYBRID decisions with granted authorization, BUY_STAGED_VERIFY, completed staged verification, claimed envelope, qualified resource evidence, output within deadline, startpos history, and available_clock_ms >= 10000. Deduplicate position_id, sort each job by (generation, run_id), sort job ids lexicographically, and take the earliest four distinct positions from each override-bearing base job. No validation-run result may alter this frozen set.", + "id": "qualified-natural-override-base-round-robin-v1", + "minimum_available_clock_ms": 10000, + "required_per_job": 4 + }, + "legacy_policy_path": "qualification/online-hybrid-v2.json", + "legacy_case_count": 7, + "discovery_case_count": 16, + "source_metadata": "qualification/g3-candidate-discovery-source.json" + } +} diff --git a/qualification/resource-profile-catalog-v1-j7-baseline.json b/qualification/resource-profile-catalog-v1-j7-baseline.json new file mode 100644 index 000000000..364864055 --- /dev/null +++ b/qualification/resource-profile-catalog-v1-j7-baseline.json @@ -0,0 +1,438 @@ +{ + "schema_version": 1, + "catalog_version": "resource-profile-catalog-v1", + "catalog_id": "resource-profile-catalog-v1", + "source_profile": "engine-opt-v2", + "selection_enabled": false, + "fallback_profile": "engine-opt-v2", + "profiles": [ + { + "schema_version": 1, + "profile_id": "stockfish/anchor-engine-opt-v2", + "family": "stockfish", + "process_identity": { + "binary_sha256": "850a080f7e5325403243a2c010b7843db8f9620f37850fcb73f63556b26640e6", + "artifacts": [], + "backend": null, + "args": [], + "environment": {} + }, + "options": [ + { + "name": "Threads", + "value": 1, + "boundary": "game", + "phase": null + }, + { + "name": "Hash", + "value": 16, + "boundary": "game", + "phase": null + }, + { + "name": "MultiPV", + "value": 1, + "boundary": "game", + "phase": null + } + ], + "cpu_slots": 1, + "expected_memory_mib": 261, + "accelerator": "cpu", + "accelerator_memory_mib": 0, + "work_chunk_ids": [], + "qualification": { + "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", + "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", + "evidence_id": "engine-opt-v2/pr49-exact-host", + "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", + "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "binding_scope": "exact_host_observation" + }, + "authority": { + "resource_profile": true, + "resource_authorization": false, + "outward_move": false + } + }, + { + "schema_version": 1, + "profile_id": "stockfish/specialist-engine-opt-v2", + "family": "stockfish", + "process_identity": { + "binary_sha256": "850a080f7e5325403243a2c010b7843db8f9620f37850fcb73f63556b26640e6", + "artifacts": [], + "backend": null, + "args": [], + "environment": {} + }, + "options": [ + { + "name": "Threads", + "value": 1, + "boundary": "game", + "phase": null + }, + { + "name": "Hash", + "value": 16, + "boundary": "game", + "phase": null + }, + { + "name": "MultiPV", + "value": 1, + "boundary": "search", + "phase": "EXPLORE" + }, + { + "name": "MultiPV", + "value": 3, + "boundary": "search", + "phase": "VERIFY" + }, + { + "name": "MultiPV", + "value": 3, + "boundary": "search", + "phase": "STAGED_VERIFY" + } + ], + "cpu_slots": 1, + "expected_memory_mib": 261, + "accelerator": "cpu", + "accelerator_memory_mib": 0, + "work_chunk_ids": [], + "qualification": { + "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", + "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", + "evidence_id": "engine-opt-v2/pr49-exact-host", + "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", + "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "binding_scope": "exact_host_observation" + }, + "authority": { + "resource_profile": true, + "resource_authorization": false, + "outward_move": false + } + }, + { + "schema_version": 1, + "profile_id": "reckless/specialist-engine-opt-v2", + "family": "reckless", + "process_identity": { + "binary_sha256": "a54a4241dbcd501058a9a1cd06e1e1ca4d0dc08111d9231348eaf32b599c6b1d", + "artifacts": [], + "backend": null, + "args": [], + "environment": {} + }, + "options": [ + { + "name": "Threads", + "value": 1, + "boundary": "game", + "phase": null + }, + { + "name": "Hash", + "value": 16, + "boundary": "game", + "phase": null + }, + { + "name": "MultiPV", + "value": 1, + "boundary": "search", + "phase": "EXPLORE" + }, + { + "name": "MultiPV", + "value": 3, + "boundary": "search", + "phase": "VERIFY" + }, + { + "name": "MultiPV", + "value": 3, + "boundary": "search", + "phase": "STAGED_VERIFY" + } + ], + "cpu_slots": 1, + "expected_memory_mib": 97, + "accelerator": "cpu", + "accelerator_memory_mib": 0, + "work_chunk_ids": [], + "qualification": { + "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", + "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", + "evidence_id": "engine-opt-v2/pr49-exact-host", + "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", + "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "binding_scope": "exact_host_observation" + }, + "authority": { + "resource_profile": true, + "resource_authorization": false, + "outward_move": false + } + }, + { + "schema_version": 1, + "profile_id": "lc0/specialist-engine-opt-v2", + "family": "lc0", + "process_identity": { + "binary_sha256": "10e91df0afa6be3a61d8354d0065246014e99a37192532f61c15f50e10169d90", + "artifacts": [ + { + "name": "network", + "sha256": "b1c7047582a8ad37620849bf328935cb46f2ccbbecb7f2696c30c6b913ed690d" + } + ], + "backend": "blas", + "args": [ + "--show-hidden" + ], + "environment": { + "OPENBLAS_NUM_THREADS": "1", + "GOTO_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1" + } + }, + "options": [ + { + "name": "WeightsFile", + "value": "build/online-engine-opt-v2/networks/791556.pb.gz", + "boundary": "process", + "phase": null + }, + { + "name": "Backend", + "value": "blas", + "boundary": "process", + "phase": null + }, + { + "name": "BackendOptions", + "value": "", + "boundary": "process", + "phase": null + }, + { + "name": "NNCacheSize", + "value": 262144, + "boundary": "game", + "phase": null + }, + { + "name": "MinibatchSize", + "value": 7, + "boundary": "game", + "phase": null + }, + { + "name": "MaxConcurrentSearchers", + "value": 1, + "boundary": "game", + "phase": null + }, + { + "name": "TaskWorkers", + "value": 0, + "boundary": "game", + "phase": null + }, + { + "name": "Threads", + "value": 1, + "boundary": "game", + "phase": null + }, + { + "name": "MaxPrefetch", + "value": 8, + "boundary": "game", + "phase": null + }, + { + "name": "AdaptivePrefetch", + "value": false, + "boundary": "game", + "phase": null + }, + { + "name": "MultiPV", + "value": 1, + "boundary": "search", + "phase": "EXPLORE" + }, + { + "name": "MultiPV", + "value": 3, + "boundary": "search", + "phase": "VERIFY" + }, + { + "name": "MultiPV", + "value": 3, + "boundary": "search", + "phase": "STAGED_VERIFY" + } + ], + "cpu_slots": 1, + "expected_memory_mib": 154, + "accelerator": "cpu", + "accelerator_memory_mib": 0, + "work_chunk_ids": [], + "qualification": { + "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", + "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", + "evidence_id": "engine-opt-v2/pr49-exact-host", + "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", + "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "binding_scope": "exact_host_observation" + }, + "authority": { + "resource_profile": true, + "resource_authorization": false, + "outward_move": false + } + } + ], + "compositions": [ + { + "schema_version": 1, + "composition_id": "composition/engine-opt-v2-exact-host", + "declared_cpu_slots": 4, + "expected_memory_mib": 773, + "enforcement_required": "observed", + "bindings": [ + { + "instance": "stockfish-anchor", + "role": "anchor", + "family": "stockfish", + "profile_id": "stockfish/anchor-engine-opt-v2", + "cpu_slots": 1, + "concurrency_group": "move" + }, + { + "instance": "stockfish-shadow", + "role": "specialist", + "family": "stockfish", + "profile_id": "stockfish/specialist-engine-opt-v2", + "cpu_slots": 1, + "concurrency_group": "move" + }, + { + "instance": "reckless-shadow", + "role": "specialist", + "family": "reckless", + "profile_id": "reckless/specialist-engine-opt-v2", + "cpu_slots": 1, + "concurrency_group": "move" + }, + { + "instance": "lc0-shadow", + "role": "specialist", + "family": "lc0", + "profile_id": "lc0/specialist-engine-opt-v2", + "cpu_slots": 1, + "concurrency_group": "move" + } + ], + "qualification": { + "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", + "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", + "evidence_id": "engine-opt-v2/pr49-exact-host", + "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", + "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "binding_scope": "exact_host_observation" + }, + "authority": { + "resource_profile": true, + "resource_authorization": false, + "outward_move": false + } + } + ], + "default_composition_id": "composition/engine-opt-v2-exact-host", + "profile_runtime": { + "stockfish/anchor-engine-opt-v2": { + "frozen_options": { + "UCI_Chess960": false + }, + "warmup": null + }, + "stockfish/specialist-engine-opt-v2": { + "frozen_options": { + "UCI_Chess960": false + }, + "warmup": null + }, + "reckless/specialist-engine-opt-v2": { + "frozen_options": { + "Minimal": false, + "UCI_Chess960": false + }, + "warmup": null + }, + "lc0/specialist-engine-opt-v2": { + "frozen_options": { + "ScoreType": "WDL_mu", + "DefectTelemetry": false, + "UCI_Chess960": false + }, + "warmup": { + "enabled": true, + "nodes": 64, + "position": "startpos", + "reset_after": true + } + } + }, + "qualification_snapshot": { + "qualified_head": "3de528e5a1a5a3609e0c293a80296ddc4a006875", + "merge_commit": "0ff86cc53d336780fa0d0566fe0ee414a7b70f3b", + "workflow_run": 36589207658, + "aggregate_artifact_id": 11047727810, + "aggregate_artifact_sha256": "efd4f3b0ba3d119ad558d3b81072f3e5f9e81aa1b34d627c6ba890d64303f35b", + "aggregate_report_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", + "qualification_disposition": "QUALIFIED_EXACT_HOST_ONLY", + "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", + "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "binding_scope": "exact_host_observation" + }, + "legacy_policy": { + "dispatch_limits": { + "EXPLORE": { + "nodes": 16 + }, + "VERIFY": { + "nodes": 16 + }, + "STAGED_VERIFY": { + "nodes": 32 + } + }, + "resource_estimates_ms": { + "explore": { + "stockfish": 100, + "reckless": 100, + "lc0": 600 + }, + "verify": { + "stockfish": 100, + "reckless": 100, + "lc0": 800 + } + } + }, + "authority": { + "resource_profile_catalog": true, + "resource_authorization": false, + "outward_move": false + } +} diff --git a/qualification/resource-profile-catalog-v1.json b/qualification/resource-profile-catalog-v1.json index 364864055..7b1efe9b4 100644 --- a/qualification/resource-profile-catalog-v1.json +++ b/qualification/resource-profile-catalog-v1.json @@ -11,7 +11,7 @@ "profile_id": "stockfish/anchor-engine-opt-v2", "family": "stockfish", "process_identity": { - "binary_sha256": "850a080f7e5325403243a2c010b7843db8f9620f37850fcb73f63556b26640e6", + "binary_sha256": "01b94f31282e5cdc55aa031a7e69c6bfaf0eb4ffabe215a79d8632184f39f956", "artifacts": [], "backend": null, "args": [], @@ -43,11 +43,11 @@ "accelerator_memory_mib": 0, "work_chunk_ids": [], "qualification": { - "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", - "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", - "evidence_id": "engine-opt-v2/pr49-exact-host", - "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", - "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "source_commit": "b90af9f1760c9611826afc2fd26ad0dc438d4557", + "evidence_sha256": "f1748fea0de0b8712fc47ae216c2111c525f8a3c645d044f670837ee2e81f9e1", + "evidence_id": "engine-opt-v2/pr64-b4-canonical-exact-host", + "execution_domain_id": "exec-domain/2e9673f5cea55cf86e59", + "execution_domain_digest": "2e9673f5cea55cf86e59f34ae1b4d1dddea1c19c4972ac34b323465ab3bd0e2c", "binding_scope": "exact_host_observation" }, "authority": { @@ -61,7 +61,7 @@ "profile_id": "stockfish/specialist-engine-opt-v2", "family": "stockfish", "process_identity": { - "binary_sha256": "850a080f7e5325403243a2c010b7843db8f9620f37850fcb73f63556b26640e6", + "binary_sha256": "01b94f31282e5cdc55aa031a7e69c6bfaf0eb4ffabe215a79d8632184f39f956", "artifacts": [], "backend": null, "args": [], @@ -105,11 +105,11 @@ "accelerator_memory_mib": 0, "work_chunk_ids": [], "qualification": { - "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", - "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", - "evidence_id": "engine-opt-v2/pr49-exact-host", - "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", - "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "source_commit": "b90af9f1760c9611826afc2fd26ad0dc438d4557", + "evidence_sha256": "f1748fea0de0b8712fc47ae216c2111c525f8a3c645d044f670837ee2e81f9e1", + "evidence_id": "engine-opt-v2/pr64-b4-canonical-exact-host", + "execution_domain_id": "exec-domain/2e9673f5cea55cf86e59", + "execution_domain_digest": "2e9673f5cea55cf86e59f34ae1b4d1dddea1c19c4972ac34b323465ab3bd0e2c", "binding_scope": "exact_host_observation" }, "authority": { @@ -123,7 +123,7 @@ "profile_id": "reckless/specialist-engine-opt-v2", "family": "reckless", "process_identity": { - "binary_sha256": "a54a4241dbcd501058a9a1cd06e1e1ca4d0dc08111d9231348eaf32b599c6b1d", + "binary_sha256": "a97ee2ce63d5b568780815a2011a169d4c5a6552f114a673a6fe51aa34b27a37", "artifacts": [], "backend": null, "args": [], @@ -167,11 +167,11 @@ "accelerator_memory_mib": 0, "work_chunk_ids": [], "qualification": { - "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", - "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", - "evidence_id": "engine-opt-v2/pr49-exact-host", - "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", - "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "source_commit": "b90af9f1760c9611826afc2fd26ad0dc438d4557", + "evidence_sha256": "f1748fea0de0b8712fc47ae216c2111c525f8a3c645d044f670837ee2e81f9e1", + "evidence_id": "engine-opt-v2/pr64-b4-canonical-exact-host", + "execution_domain_id": "exec-domain/2e9673f5cea55cf86e59", + "execution_domain_digest": "2e9673f5cea55cf86e59f34ae1b4d1dddea1c19c4972ac34b323465ab3bd0e2c", "binding_scope": "exact_host_observation" }, "authority": { @@ -185,7 +185,7 @@ "profile_id": "lc0/specialist-engine-opt-v2", "family": "lc0", "process_identity": { - "binary_sha256": "10e91df0afa6be3a61d8354d0065246014e99a37192532f61c15f50e10169d90", + "binary_sha256": "97ce6e942c1621715a0fae29489d133a9df09e24290575689f13899d942b86ab", "artifacts": [ { "name": "network", @@ -229,7 +229,7 @@ }, { "name": "MinibatchSize", - "value": 7, + "value": 4, "boundary": "game", "phase": null }, @@ -253,7 +253,7 @@ }, { "name": "MaxPrefetch", - "value": 8, + "value": 0, "boundary": "game", "phase": null }, @@ -288,11 +288,11 @@ "accelerator_memory_mib": 0, "work_chunk_ids": [], "qualification": { - "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", - "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", - "evidence_id": "engine-opt-v2/pr49-exact-host", - "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", - "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "source_commit": "b90af9f1760c9611826afc2fd26ad0dc438d4557", + "evidence_sha256": "f1748fea0de0b8712fc47ae216c2111c525f8a3c645d044f670837ee2e81f9e1", + "evidence_id": "engine-opt-v2/pr64-b4-canonical-exact-host", + "execution_domain_id": "exec-domain/2e9673f5cea55cf86e59", + "execution_domain_digest": "2e9673f5cea55cf86e59f34ae1b4d1dddea1c19c4972ac34b323465ab3bd0e2c", "binding_scope": "exact_host_observation" }, "authority": { @@ -344,11 +344,11 @@ } ], "qualification": { - "source_commit": "3de528e5a1a5a3609e0c293a80296ddc4a006875", - "evidence_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", - "evidence_id": "engine-opt-v2/pr49-exact-host", - "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", - "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "source_commit": "b90af9f1760c9611826afc2fd26ad0dc438d4557", + "evidence_sha256": "f1748fea0de0b8712fc47ae216c2111c525f8a3c645d044f670837ee2e81f9e1", + "evidence_id": "engine-opt-v2/pr64-b4-canonical-exact-host", + "execution_domain_id": "exec-domain/2e9673f5cea55cf86e59", + "execution_domain_digest": "2e9673f5cea55cf86e59f34ae1b4d1dddea1c19c4972ac34b323465ab3bd0e2c", "binding_scope": "exact_host_observation" }, "authority": { @@ -385,24 +385,19 @@ "DefectTelemetry": false, "UCI_Chess960": false }, - "warmup": { - "enabled": true, - "nodes": 64, - "position": "startpos", - "reset_after": true - } + "warmup": null } }, "qualification_snapshot": { - "qualified_head": "3de528e5a1a5a3609e0c293a80296ddc4a006875", - "merge_commit": "0ff86cc53d336780fa0d0566fe0ee414a7b70f3b", - "workflow_run": 36589207658, - "aggregate_artifact_id": 11047727810, - "aggregate_artifact_sha256": "efd4f3b0ba3d119ad558d3b81072f3e5f9e81aa1b34d627c6ba890d64303f35b", - "aggregate_report_sha256": "b4f1de8f4ae338ba19350acdddf570c43b55fa08d7274e791c1a4eeab5bf6943", + "qualified_head": "b90af9f1760c9611826afc2fd26ad0dc438d4557", + "merge_commit": "cae842801497161f2de37ff43b27d074ac12568a", + "workflow_run": 38012334421, + "aggregate_artifact_id": 11657944410, + "aggregate_artifact_sha256": "daf8ff2a3bfb2300de02d60a60baacf52c14638ff4d3aa41d3f0394e5205dbbd", + "aggregate_report_sha256": "f1748fea0de0b8712fc47ae216c2111c525f8a3c645d044f670837ee2e81f9e1", "qualification_disposition": "QUALIFIED_EXACT_HOST_ONLY", - "execution_domain_id": "exec-domain/9284bde03da0ebd1677f", - "execution_domain_digest": "9284bde03da0ebd1677fafa4a5078c00cc842f0b3b5ea8079c6f7989189199af", + "execution_domain_id": "exec-domain/2e9673f5cea55cf86e59", + "execution_domain_digest": "2e9673f5cea55cf86e59f34ae1b4d1dddea1c19c4972ac34b323465ab3bd0e2c", "binding_scope": "exact_host_observation" }, "legacy_policy": { diff --git a/qualification/work-grant-scheduler-v1.json b/qualification/work-grant-scheduler-v1.json index bcfa53c20..fe6d7a4a7 100644 --- a/qualification/work-grant-scheduler-v1.json +++ b/qualification/work-grant-scheduler-v1.json @@ -5,7 +5,7 @@ "resource_catalog": { "path": "qualification/resource-profile-catalog-v1.json", "catalog_id": "resource-profile-catalog-v1", - "catalog_digest": "0adc5e6e71aab984a7df6802a20283a74627090418f11b53ce458fe3bd831621" + "catalog_digest": "dc98b3dff303b0be9d3d195b298cf3c10dbb9635e4446b3e5413a020d96e8f0c" }, "round_count": 3, "max_grants_per_round": 3, diff --git a/scripts/ci-install-apt-deps.sh b/scripts/ci-install-apt-deps.sh new file mode 100644 index 000000000..c2fff02cc --- /dev/null +++ b/scripts/ci-install-apt-deps.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Bound Ubuntu package installation for the hosted qualification workflows. +# The official signed Ubuntu archive is used in an isolated APT sources/list +# namespace; runner-provided Azure mirror lists are not consulted. +set -euo pipefail + +APT_UPDATE_TIMEOUT_SEC=90 +APT_INSTALL_TIMEOUT_SEC=180 +APT_NETWORK_TIMEOUT_SEC=15 +APT_MAX_RETRIES=1 +APT_KEYRING=/usr/share/keyrings/ubuntu-archive-keyring.gpg + +render_sources() { + local codename="${1:?Ubuntu codename required}" + local arch="${2:?DPKG architecture required}" + case "$codename" in + jammy|noble) ;; + *) echo "unsupported Ubuntu codename: $codename" >&2; return 2 ;; + esac + local archive security + case "$arch" in + amd64) + archive=https://archive.ubuntu.com/ubuntu + security=https://security.ubuntu.com/ubuntu + ;; + arm64) + archive=https://ports.ubuntu.com/ubuntu-ports + security=https://ports.ubuntu.com/ubuntu-ports + ;; + *) echo "unsupported Ubuntu architecture: $arch" >&2; return 2 ;; + esac + cat <&2 + return 2 + fi + for package in "$@"; do + if [[ ! "$package" =~ ^[a-z0-9][a-z0-9+.-]*$ ]]; then + echo "Invalid APT package argument: $package" >&2 + return 2 + fi + done + if [[ ! -s "$APT_KEYRING" ]]; then + echo "Ubuntu archive signing keyring missing: $APT_KEYRING" >&2 + return 2 + fi + local state_dir + state_dir="$(mktemp -d)" + trap 'rm -rf "$state_dir"' EXIT + mkdir -p "$state_dir/sources.list.d" "$state_dir/lists/partial" + render_sources "$codename" "$arch" > "$state_dir/sources.list.d/ubuntu.sources" + echo "APT qualification bootstrap: Ubuntu=$codename arch=$arch" + echo "APT source: signed official Ubuntu HTTPS archive (isolated lists)" + echo "APT update timeout: ${APT_UPDATE_TIMEOUT_SEC}s; install timeout: ${APT_INSTALL_TIMEOUT_SEC}s" + if apt_run "$state_dir" "$APT_UPDATE_TIMEOUT_SEC" update; then + echo "APT official package indexes refreshed successfully" + else + status=$? + echo "APT update failed/expired (exit $status); no stale-index fallback" >&2 + return "$status" + fi + if apt_run "$state_dir" "$APT_INSTALL_TIMEOUT_SEC" install -y "$@"; then + echo "APT dependencies installed" + else + status=$? + echo "APT installation failed/expired (exit $status)" >&2 + return "$status" + fi + echo "APT installed package versions:" + dpkg-query -W --showformat='${Package}=${Version}\n' "$@" +) + +main() { + if [[ "${1:-}" == "--render-sources" ]]; then + if (($# != 3)); then + echo "Usage: $0 --render-sources {jammy|noble} {amd64|arm64}" >&2 + return 2 + fi + render_sources "$2" "$3" + return + fi + if (($# == 0)); then + echo "Usage: $0 [ ...]" >&2 + return 2 + fi + # GitHub-hosted runners must be supported Ubuntu; never guess a release. + # shellcheck source=/etc/os-release + source /etc/os-release + if [[ "${ID:-}" != "ubuntu" ]]; then + echo "APT qualification bootstrap only supports Ubuntu runners" >&2 + return 2 + fi + install_packages "${VERSION_CODENAME:-}" "$(dpkg --print-architecture)" "$@" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/scripts/diagnose-g3-canonical-b4.py b/scripts/diagnose-g3-canonical-b4.py new file mode 100644 index 000000000..1897c3cdf --- /dev/null +++ b/scripts/diagnose-g3-canonical-b4.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Bounded source-locked G3 23-case triage against canonical b4; NEVER promotion evidence.""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from tools.engine_opt.domain import candidate_bundle_identity, load_execution_domain + +POLICY = "qualification/online-hybrid-v2-candidate.json" +LEGACY = "qualification/online-hybrid-v2.json" +SELECTION = "qualification/engine-opt-v2-selection.json" +REFERENCE = "config/allfather.online-engine-opt-v2.json" +CANONICAL = "config/allfather.online-hybrid-v2.validation.json" +TRIAGE = "config/allfather.online-hybrid-v2.g3-triage.json" +REPLAY = "build/replays-g3-canonical-b4-triage" +RESULT = "build/test-results/g3-canonical-b4-triage" + + +class TriageError(RuntimeError): + pass + + +def require(value: bool, why: str) -> None: + if not value: + raise TriageError(why) + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def json_at(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + require(isinstance(value, dict), f"invalid JSON object: {path}") + return value + + +def verify_overlay(canonical: dict, triage: dict, policy: dict, + legacy: dict, selection: dict) -> None: + expected = copy.deepcopy(canonical) + require(expected["shadow"]["replay_root"] != REPLAY, "canonical replay already triage") + expected["shadow"]["replay_root"] = REPLAY + require(triage == expected, "triage runtime changed more than shadow.replay_root") + require(((selection.get("selected") or {}).get("lc0") or {}).get("matrix_profile") + == "b4-p0-c256k-cold", "not canonical b4 selection") + require(policy.get("runtime_config") == "config/allfather.online-hybrid-v2.candidate.validation.json", + "candidate-policy declared runtime drift") + require(policy.get("reference_runtime") == "config/allfather.online-engine-opt-v2.candidate.json", + "candidate-policy declared reference drift") + meta = policy.get("candidate_witnesses") or {} + require(meta.get("legacy_policy_path") == LEGACY, "unapproved legacy policy migration") + require(meta.get("legacy_case_count") == 7 and meta.get("discovery_case_count") == 16, + "frozen witness counts drift") + cases = policy.get("positive_cases") or [] + require(len(cases) == 23, "expected exactly 23 frozen diagnostic positions") + actual_legacy = [{k: v for k, v in case.items() if k != "source_set"} + for case in cases if case.get("source_set") == "legacy_v1"] + require(actual_legacy == legacy.get("positive_cases") and len(actual_legacy) == 7, + "immutable seven-case legacy baseline drift") + require(policy.get("positive_requirement") == legacy.get("positive_requirement"), + "frozen non-anchor G3 authority requirement drift") + require(sum(case.get("source_set") == "discovery_local1_v1" for case in cases) == 16, + "historical discovery corpus drift") + + +def write_sealed(path: Path, body: dict) -> None: + core = dict(body) + core.pop("content_sha256", None) + encoded = json.dumps(core, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + core["content_sha256"] = hashlib.sha256(encoded).hexdigest() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(core, sort_keys=True, indent=2, allow_nan=False) + "\n", + encoding="utf-8") + + +def main() -> int: + args = argparse.ArgumentParser(description=__doc__) + args.add_argument("--execution-domain", type=Path) + args.add_argument("--expected-head") + args.add_argument("--static", action="store_true") + options = args.parse_args() + paths = [POLICY, LEGACY, SELECTION, REFERENCE, CANONICAL, TRIAGE, + "qualification/online-hybrid-v2-candidate-witnesses.json", + "qualification/g3-candidate-discovery-source.json"] + contracts = {name: digest(ROOT / name) for name in paths} + policy = json_at(ROOT / POLICY) + legacy = json_at(ROOT / LEGACY) + canonical = json_at(ROOT / CANONICAL) + triage = json_at(ROOT / TRIAGE) + selection = json_at(ROOT / SELECTION) + verify_overlay(canonical, triage, policy, legacy, selection) + if options.static: + print(json.dumps({"static_contract": True, "diagnostic_only": True, + "contract_hashes": contracts}, sort_keys=True)) + return 0 + require(options.execution_domain is not None and options.expected_head is not None, + "execution domain and expected head required") + head = subprocess.check_output(["git", "-C", str(ROOT), "rev-parse", "HEAD"], + text=True).strip() + require(head == options.expected_head, "diagnostic is not on exact Git HEAD") + domain = load_execution_domain(options.execution_domain.resolve()) + require(domain["source_commit"] == head, "execution-domain source drift") + bundle = candidate_bundle_identity(ROOT / "build/online-engine-opt-v2", + expected_source_commit=head) + out = ROOT / RESULT + raw_root = out / "raw-qualifier" + replay_root = ROOT / REPLAY + require(not replay_root.exists(), "triage replay path must be initially empty") + require(not raw_root.exists(), "triage qualifier output path must be initially empty") + env = os.environ.copy() + env.update({ + "ALLFATHER_G3_POLICY": POLICY, + "ALLFATHER_G3_SELECTION": SELECTION, + "ALLFATHER_G3_REFERENCE": REFERENCE, + "ALLFATHER_G3_CONFIG": TRIAGE, + "ALLFATHER_G3_RESULT": str(raw_root), + "ALLFATHER_EXECUTION_DOMAIN_PATH": str(options.execution_domain.resolve()), + }) + proc = subprocess.run([sys.executable, str(ROOT / "scripts/qualify-online-hybrid-v2.py"), + "--record-disposition"], cwd=ROOT, env=env, + capture_output=True, text=True, check=False) + out.mkdir(parents=True, exist_ok=True) + (out / "qualifier.stdout.txt").write_text(proc.stdout, encoding="utf-8") + (out / "qualifier.stderr.txt").write_text(proc.stderr, encoding="utf-8") + raw_path = raw_root / "report.json" + raw = json_at(raw_path) if raw_path.is_file() else {} + actual = raw.get("cases") or [] + complete = (proc.returncode == 0 and raw.get("evidence_valid") is True + and raw.get("source_commit") == head and raw.get("execution_domain") == domain + and raw.get("candidate_bundle") == bundle + and len(actual) <= 23 + and len({c.get("case") for c in actual}) == len(actual)) + found = bool(raw.get("authority_qualified") is True and raw.get("positive_case")) + report = { + "schema_version": 1, + "kind": "g3-canonical-b4-diagnostic", + "source_commit": head, + "diagnostic_complete": bool(complete), + "diagnostic_positive_witness": found if complete else False, + "cases_exercised": len(actual), + "first_positive_stops_run": True, + "qualifier_returncode": proc.returncode, + "raw_qualifier_report_sha256": digest(raw_path) if raw_path.is_file() else None, + "contracts": contracts, + "candidate_bundle": bundle, + "execution_domain": domain, + "runtime_override": { + "policy_declares_candidate_files": True, + "reference_actually_used": REFERENCE, + "selection_actually_used": SELECTION, + "runtime_actually_used": TRIAGE, + "only_allowed_runtime_difference": "shadow.replay_root", + }, + "qualification_disposition": ("DIAGNOSTIC_POSITIVE_NOT_PROMOTION" + if complete and found else + "DIAGNOSTIC_NEGATIVE_NOT_PROMOTION" if complete else + "DIAGNOSTIC_INCOMPLETE"), + "promotion_evidence": False, + "promotable": False, + "canonical_policy_changed": False, + "claim_boundary": {"canonical_b4_promotion": False, + "strength": False, "elo": False, + "equal_compute": False, "deployment": False}, + } + write_sealed(out / "report.json", report) + print(json.dumps({"disposition": report["qualification_disposition"], + "cases_exercised": len(actual)}, sort_keys=True)) + return 0 if complete else 2 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (TriageError, OSError, ValueError, KeyError) as exc: + out = ROOT / RESULT + write_sealed(out / "report.json", { + "schema_version": 1, "kind": "g3-canonical-b4-diagnostic", + "diagnostic_complete": False, "promotion_evidence": False, + "promotable": False, "canonical_policy_changed": False, + "qualification_disposition": "DIAGNOSTIC_INCOMPLETE", + "error": f"{type(exc).__name__}: {exc}", + }) + print(f"G3 canonical-b4 triage failed: {exc}", file=sys.stderr) + raise SystemExit(2) diff --git a/scripts/qualify-adaptive-resource-allocator.py b/scripts/qualify-adaptive-resource-allocator.py index 2c9dba04b..03376a799 100755 --- a/scripts/qualify-adaptive-resource-allocator.py +++ b/scripts/qualify-adaptive-resource-allocator.py @@ -39,6 +39,7 @@ STAGED_STATUS = ROOT / "qualification/j10-staged-decision-model-v1.json" REGIME_STATUS = ROOT / "qualification/j10-regime-support-model-v1.json" J7 = ROOT / "qualification/resource-profile-selection-v1.json" +ENGINE_SELECTION = ROOT / "qualification/engine-opt-v2-selection.json" SCHEDULER = ROOT / "qualification/work-grant-scheduler-v1.json" @@ -84,6 +85,7 @@ def main() -> int: staged = load(STAGED_STATUS) regime = load(REGIME_STATUS) j7 = load(J7) + engine_selection = load(ENGINE_SELECTION) allocation = load_allocation_policy(POLICY) scheduler, _ = load_work_scheduler_catalog(SCHEDULER) @@ -238,13 +240,30 @@ def main() -> int: verify == staged_opts, f"{name}: J10 VERIFY/STAGED_VERIFY options differ", ) + selected = engine_selection.get("selected") or {} if spec["family"] == "stockfish": - require(spec["options"].get("Hash") == 16, f"{name}: J7 Hash32 leaked into J10") - if spec["family"] == "lc0": require( - spec["options"].get("MaxPrefetch") == 8, - f"{name}: J7 MaxPrefetch0 leaked into J10", + spec["options"].get("Hash") == (selected.get("stockfish") or {}).get("hash_mb"), + f"{name}: Stockfish Hash differs from canonical ENGINE-OPT selection", + ) + if spec["family"] == "reckless": + require( + spec["options"].get("Hash") == (selected.get("reckless") or {}).get("hash_mb"), + f"{name}: Reckless Hash differs from canonical ENGINE-OPT selection", ) + if spec["family"] == "lc0": + lc0_selected = selected.get("lc0") or {} + for option, value in { + "NNCacheSize": lc0_selected.get("nn_cache_size"), + "MinibatchSize": lc0_selected.get("minibatch_size"), + "MaxPrefetch": lc0_selected.get("max_prefetch"), + "AdaptivePrefetch": lc0_selected.get("adaptive_prefetch"), + "DefectTelemetry": lc0_selected.get("defect_telemetry"), + }.items(): + require( + spec["options"].get(option) == value, + f"{name}: LC0 {option} differs from canonical ENGINE-OPT selection", + ) require( (j7.get("claim_boundary") or {}).get("runtime_profile_selection") diff --git a/scripts/qualify-adaptive-time.py b/scripts/qualify-adaptive-time.py index 9a12927a7..d16355a65 100755 --- a/scripts/qualify-adaptive-time.py +++ b/scripts/qualify-adaptive-time.py @@ -215,13 +215,37 @@ def qualify() -> dict[str, Any]: and (j7.get("claim_boundary") or {}).get("adaptive_allocation") is False, "J8 may not consume J7 selections", ) + canonical_selected = engine_selection.get("selected") or {} + canonical_lc0 = canonical_selected.get("lc0") or {} for name, spec in runtime["instances"].items(): if spec["family"] == "stockfish": - require(spec["options"].get("Hash") == 16, f"{name}: J7 Hash=32 leaked into runtime") + require( + spec["options"].get("Hash") == (canonical_selected.get("stockfish") or {}).get("hash_mb"), + f"{name}: runtime Stockfish Hash differs from canonical ENGINE-OPT selection", + ) + if spec["family"] == "reckless": + require( + spec["options"].get("Hash") == (canonical_selected.get("reckless") or {}).get("hash_mb"), + f"{name}: runtime Reckless Hash differs from canonical ENGINE-OPT selection", + ) if spec["family"] == "lc0": + expected = { + "NNCacheSize": canonical_lc0.get("nn_cache_size"), + "MinibatchSize": canonical_lc0.get("minibatch_size"), + "MaxPrefetch": canonical_lc0.get("max_prefetch"), + "AdaptivePrefetch": canonical_lc0.get("adaptive_prefetch"), + "DefectTelemetry": canonical_lc0.get("defect_telemetry"), + } + for option, value in expected.items(): + require( + spec["options"].get(option) == value, + f"{name}: runtime {option} differs from canonical ENGINE-OPT selection", + ) + warmup = spec.get("warmup") + warmup_nodes = canonical_lc0.get("warmup_nodes") require( - spec["options"].get("MaxPrefetch") == 8, - f"{name}: J7 MaxPrefetch=0 leaked into runtime", + (warmup is None if warmup_nodes is None else isinstance(warmup, dict) and warmup.get("nodes") == warmup_nodes), + f"{name}: runtime warmup differs from canonical ENGINE-OPT selection", ) composition = catalog.default_composition diff --git a/scripts/qualify-engine-opt-evidence.py b/scripts/qualify-engine-opt-evidence.py index 90e16ca28..c000ddba3 100644 --- a/scripts/qualify-engine-opt-evidence.py +++ b/scripts/qualify-engine-opt-evidence.py @@ -24,6 +24,10 @@ ) from tools.engine_opt.matrix_qualification import qualify_lc0_matrix from tools.engine_opt.constituent_hash import load_policy as load_constituent_policy, qualify_hash_matrix +from tools.engine_opt.g3_b4 import ( + B4_POLICY, WITNESS_CORPUS, file_hash as b4_file_hash, validate_b4_policy, + require_positive_g3, +) class QualificationError(RuntimeError): @@ -50,6 +54,66 @@ def find_one(root: Path, pattern: str) -> Path: return found[0] +def local1_campaign_pattern(*, candidate_mode: bool) -> str: + """Select the candidate-era or promoted-canonical campaign explicitly.""" + directory = "local1" if candidate_mode else "canonical-local1" + return f"engine-opt-v2-profile-domain/**/{directory}/campaign/report.json" + + +def j12_campaign_pattern() -> str: + return "engine-opt-v2-profile-domain/**/j12-local1/campaign/report.json" + + +def validate_j12_campaign_metadata( + local: dict[str, Any], + manifest: dict[str, Any], + prerequisite: dict[str, Any], + *, + source: str, + candidate_bundle: dict[str, Any], + policy_sha256: str, +) -> None: + """Reject inconsistent J12 lifecycle evidence without granting hybrid authority.""" + require(local.get("passed") is True, f"J12 LOCAL-1 failed: {local.get('errors')}") + require(local.get("errors") == [], "J12 lifecycle has retained errors") + require(local.get("execution_scope") == "required_local1", "J12 lifecycle scope drift") + require(local.get("validated_games") == 28, "J12 lifecycle did not validate 28 games") + require(local.get("observed_games") == 28, "J12 lifecycle did not observe 28 games") + require(local.get("claim_boundary", {}).get("full_game_lifecycle") is True, + "J12 lifecycle lacks full-game evidence") + require((manifest.get("source") or {}).get("commit") == source, + "J12 lifecycle source is not exact head") + require(manifest.get("status") == "completed" and not manifest.get("failures"), + "J12 lifecycle manifest is incomplete") + require(manifest.get("mode") == "required", "J12 lifecycle mode is not required") + require((manifest.get("policy") or {}).get("path") + == "qualification/local-full-game-orchestrated-v1.json", + "J12 lifecycle policy path drift") + require((manifest.get("policy") or {}).get("sha256") == policy_sha256, + "J12 lifecycle policy bytes drift") + require(local.get("candidate_bundle") == candidate_bundle + and manifest.get("candidate_bundle") == candidate_bundle, + "J12 lifecycle bundle differs from exact-head candidate") + require((manifest.get("prerequisites") or []) and + [row.get("id") for row in manifest["prerequisites"]] == ["engine-opt-v2", "j12"] + and all(row.get("returncode") == 0 and row.get("timed_out") is False + for row in manifest["prerequisites"]), + "J12 lifecycle prerequisites missing, reordered or failed") + require(prerequisite.get("schema_version") == 1 + and prerequisite.get("profile_id") == "allfather.orchestrated-v1" + and prerequisite.get("mechanism_valid") is True + and prerequisite.get("source_commit") == source + and prerequisite.get("candidate_bundle") == candidate_bundle, + "J12 prerequisite mechanism identity is invalid") + require(prerequisite.get("work_grants_authorized") == 9 + and prerequisite.get("work_grants_settled") == 9 + and prerequisite.get("open_reservations") == 0 + and prerequisite.get("allocation_action") == "BUY_BUNDLE", + "J12 prerequisite did not settle its frozen WorkGrant mechanism") + # J12 may validly remain ANCHOR_FALLBACK on a host with synthetic capacity. + # Its prerequisite's authority_qualified=false is not an invalid lifecycle. + + def current_source(root: Path) -> str: return subprocess.check_output( ["git", "-C", str(root), "rev-parse", "HEAD"], @@ -347,7 +411,7 @@ def check_lc0() -> dict[str, Any]: def check_local1_and_g3() -> dict[str, Any]: report_path = find_one( artifact_root, - "engine-opt-v2-profile-domain/**/local1/campaign/report.json", + local1_campaign_pattern(candidate_mode=args.candidate_mode), ) local = load(report_path) manifest = load(report_path.parent / "manifest.json") @@ -383,6 +447,15 @@ def check_local1_and_g3() -> dict[str, Any]: require(g3.get("evidence_valid") is True, "G3-v2 evidence is invalid") require(g3.get("source_commit") == source, "G3-v2 source is not exact head") require(g3.get("candidate_bundle") == candidate, "G3-v2 candidate binding mismatch") + if not args.candidate_mode: + validate_b4_policy(repo) + contracts = g3.get("contracts") or {} + require(contracts.get("policy_sha256") == b4_file_hash(repo, B4_POLICY) + and contracts.get("selection_sha256") == sha256(selection_path) + and contracts.get("reference_runtime_sha256") == sha256(repo / "config/allfather.online-engine-opt-v2.json") + and contracts.get("hybrid_runtime_sha256") == sha256(repo / "config/allfather.online-hybrid-v2.validation.json") + and contracts.get("witnesses_sha256") == b4_file_hash(repo, WITNESS_CORPUS), + "canonical G3 prerequisite differs from frozen b4 source contracts") if args.candidate_mode: overlay=(details.get("candidate") or {}).get("candidate_overlay") or {} contracts=g3.get("contracts") or {} @@ -410,13 +483,16 @@ def check_local1_and_g3() -> dict[str, Any]: if g3.get("authority_qualified") is not True: fail("NOT_QUALIFIED_AUTHORITY", "G3-v2 produced valid evidence but no frozen non-anchor HYBRID witness") else: - positive = g3.get("positive_case") or {} - require( - positive.get("authority") == "HYBRID" - and positive.get("emitted_move") - and positive.get("emitted_move") != positive.get("anchor_move"), - "G3-v2 qualified flag lacks a genuine non-anchor witness", - ) + if args.candidate_mode: + positive = g3.get("positive_case") or {} + require( + positive.get("authority") == "HYBRID" + and positive.get("emitted_move") + and positive.get("emitted_move") != positive.get("anchor_move"), + "G3-v2 qualified flag lacks a genuine non-anchor witness", + ) + else: + require_positive_g3(g3, policy_sha256=b4_file_hash(repo, B4_POLICY)) resource_docs = [ load(path) @@ -463,12 +539,52 @@ def check_local1_and_g3() -> dict[str, Any]: "actual_anchor_overrides": local.get("actual_anchor_overrides"), "g3_authority_qualified": g3.get("authority_qualified"), "g3_positive_case": g3.get("positive_case"), + "g3_policy_sha256": (None if args.candidate_mode else b4_file_hash(repo, B4_POLICY)), "lc0_explore_cpu_ms_max": max(explore) if explore else None, "lc0_verify_cpu_ms_max": max(verify) if verify else None, "lc0_explore_reserved_ms": explore_reserved, "lc0_verify_reserved_ms": verify_reserved, } + def check_j12_lifecycle() -> dict[str, Any]: + report_path = find_one(artifact_root, j12_campaign_pattern()) + local = load(report_path) + manifest = load(report_path.parent / "manifest.json") + prerequisite = load(report_path.parent / "prerequisites" / "j12.json") + candidate = details["candidate"]["bundle"] + validate_j12_campaign_metadata( + local, manifest, prerequisite, + source=source, + candidate_bundle=candidate, + policy_sha256=sha256(repo / "qualification/local-full-game-orchestrated-v1.json"), + ) + local_domain = validate_execution_domain( + local.get("execution_domain"), expected_source_commit=source, + ) + manifest_domain = validate_execution_domain( + manifest.get("execution_domain"), expected_source_commit=source, + ) + prerequisite_domain = validate_execution_domain( + prerequisite.get("execution_domain"), expected_source_commit=source, + ) + require_same_execution_domain( + { + "profile_domain": details["execution_domain"], + "j12_report": local_domain, + "j12_manifest": manifest_domain, + "j12_prerequisite": prerequisite_domain, + }, + expected_source_commit=source, + ) + return { + "execution_domain": local_domain, + "campaign_id": local.get("campaign_id"), + "validated_games": local.get("validated_games"), + "mechanism_valid": prerequisite.get("mechanism_valid"), + "authority_qualified": prerequisite.get("authority_qualified"), + "qualification_disposition": prerequisite.get("qualification_disposition"), + } + invalid_gate("execution_domain", check_domain) invalid_gate("candidate", check_candidate) invalid_gate("constituent_ab", check_ab) @@ -476,6 +592,8 @@ def check_local1_and_g3() -> dict[str, Any]: invalid_gate("lc0", check_lc0) if not invalid: invalid_gate("local1_g3", check_local1_and_g3) + if not invalid and not args.candidate_mode: + invalid_gate("j12_lifecycle", check_j12_lifecycle) evidence_valid = not invalid profile_qualified = bool(evidence_valid and not qualification_failures) @@ -527,7 +645,12 @@ def check_local1_and_g3() -> dict[str, Any]: "engine_profile_qualified": False if args.candidate_mode else profile_qualified, "candidate_overlay_qualified": profile_qualified if args.candidate_mode else False, "full_game_lifecycle": bool( - evidence_valid and (details.get("local1_g3") or {}).get("validated_games") == 28 + evidence_valid + and (details.get("local1_g3") or {}).get("validated_games") == 28 + and ( + args.candidate_mode + or (details.get("j12_lifecycle") or {}).get("validated_games") == 28 + ) ), "generic_host_portability": bool( profile_qualified diff --git a/scripts/qualify-engine-opt-v2-promotion-gate.py b/scripts/qualify-engine-opt-v2-promotion-gate.py new file mode 100755 index 000000000..7507f5c2d --- /dev/null +++ b/scripts/qualify-engine-opt-v2-promotion-gate.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse, hashlib, json, sys +from pathlib import Path +from typing import Any +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT)) +from tools.engine_opt.promotion_gate import PromotionGateError, compare_promotion_surface, evaluate_promotion_gate + +def load(path: Path) -> dict[str,Any]: + value=json.loads(path.read_text(encoding='utf-8')) + if not isinstance(value,dict): raise PromotionGateError(f'{path}: root must be object') + return value + +def seal(report: dict[str,Any]) -> dict[str,Any]: + core=dict(report); core.pop('content_sha256',None) + encoded=json.dumps(core,sort_keys=True,separators=(',',':'),allow_nan=False).encode() + core['content_sha256']=hashlib.sha256(encoded).hexdigest(); return core + +def write(path: Path, report: dict[str,Any]) -> None: + path.parent.mkdir(parents=True,exist_ok=True) + payload=json.dumps(seal(report),indent=2,sort_keys=True,allow_nan=False)+'\n' + path.write_text(payload,encoding='utf-8'); print(payload,end='') + +def main() -> int: + ap=argparse.ArgumentParser(description=__doc__) + ap.add_argument('--aggregate',type=Path,required=True) + ap.add_argument('--host-binding',type=Path,required=True) + ap.add_argument('--catalog',type=Path,required=True) + ap.add_argument('--base-sha',required=True) + ap.add_argument('--repo-root',type=Path,default=ROOT) + ap.add_argument('--output',type=Path,required=True) + args=ap.parse_args() + surface={"base_sha":args.base_sha,"head_sha":None,"authorized_profile_change":False} + aggregate=None + try: + aggregate=load(args.aggregate); host_binding=load(args.host_binding); catalog=load(args.catalog) + surface=compare_promotion_surface(args.repo_root.resolve(),args.base_sha) + current_head=surface['head_sha'] + report=evaluate_promotion_gate(aggregate=aggregate,host_binding=host_binding,catalog=catalog,promotion_surface=surface,current_head=current_head) + except (PromotionGateError,OSError,ValueError,KeyError,json.JSONDecodeError) as exc: + report={ + "schema_version":1,"passed":False,"disposition":"BLOCKED_FAIL_CLOSED", + "failure":{"type":type(exc).__name__,"message":str(exc)}, + "aggregate_disposition":None if aggregate is None else aggregate.get('qualification_disposition'), + "promotion_surface":surface, + "claim_boundary":{"canonical_b4_promotion":False,"runtime_profile_selection":False,"adaptive_stop_promoted":False,"generic_host_portability":False,"strength":False,"elo":False,"equal_compute":False,"deployment":False}, + } + write(args.output,report) + print(f'ENGINE-OPT-V2 canonical b4 promotion gate FAILED: {exc}',file=sys.stderr) + return 2 + write(args.output,report); return 0 + +if __name__=='__main__': raise SystemExit(main()) diff --git a/scripts/qualify-online-hybrid-v2-b4.py b/scripts/qualify-online-hybrid-v2-b4.py new file mode 100644 index 000000000..ee834ef7c --- /dev/null +++ b/scripts/qualify-online-hybrid-v2-b4.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Canonical-b4 G3 witness qualification using the frozen 7+16 corpus.""" +from __future__ import annotations + +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from tools.engine_opt.g3_b4 import ( + B4_POLICY, CANONICAL_REFERENCE, CANONICAL_RUNTIME, validate_b4_policy, +) + + +def main() -> None: + validate_b4_policy(ROOT) + env = os.environ.copy() + env.update({ + "ALLFATHER_G3_POLICY": B4_POLICY, + "ALLFATHER_G3_SELECTION": "qualification/engine-opt-v2-selection.json", + "ALLFATHER_G3_REFERENCE": CANONICAL_REFERENCE, + "ALLFATHER_G3_CONFIG": CANONICAL_RUNTIME, + "ALLFATHER_G3_RESULT": "build/test-results/online-hybrid-v2", + }) + os.execvpe(sys.executable, + [sys.executable, str(ROOT / "scripts/qualify-online-hybrid-v2.py"), *sys.argv[1:]], + env) + + +if __name__ == "__main__": + main() diff --git a/scripts/qualify-resource-profile-selection.py b/scripts/qualify-resource-profile-selection.py index 79888b9df..b37fe5c6c 100755 --- a/scripts/qualify-resource-profile-selection.py +++ b/scripts/qualify-resource-profile-selection.py @@ -22,6 +22,7 @@ EVIDENCE = Path("qualification/resource-profile-evidence-v1.json") SELECTION = Path("qualification/resource-profile-selection-v1.json") CATALOG = Path("qualification/resource-profile-catalog-v1.json") +HISTORICAL_CATALOG = Path("qualification/resource-profile-catalog-v1-j7-baseline.json") AUTH = {"runtime_authority": False, "resource_authorization": False, "outward_move": False} FAMILIES = {"stockfish": (16, 64, 256), "reckless": (16, 64, 256), "lc0": (16, 32, 64)} STAGE_B_IDS = {"c0-four-way-v2-current", "c1-anchor2-reckless-lc0", "c2-anchor2-lc0-threads2"} @@ -76,8 +77,14 @@ def candidate_key(row: dict[str, Any]) -> tuple[Any, ...]: return tuple(metric(row, name) for name in METRICS) + (row["id"],) -def validate(evidence_path: Path = EVIDENCE, selection_path: Path = SELECTION, catalog_path: Path = CATALOG) -> dict[str, Any]: +def validate( + evidence_path: Path = EVIDENCE, + selection_path: Path = SELECTION, + catalog_path: Path = CATALOG, + historical_catalog_path: Path = HISTORICAL_CATALOG, +) -> dict[str, Any]: ev, sel, catalog = load(evidence_path), load(selection_path), load(catalog_path) + historical_catalog = load(historical_catalog_path) require(ev.get("schema_version") == 1 and ev.get("evidence_id") == "resource-profile-evidence-v1", "wrong evidence schema/id") require(sel.get("schema_version") == 1 and sel.get("selection_id") == "resource-profile-selection-v1", "wrong selection schema/id") @@ -102,10 +109,26 @@ def validate(evidence_path: Path = EVIDENCE, selection_path: Path = SELECTION, c clock = sel.get("catalog") ref = source.get("reference_contract") require(isinstance(clock, dict) and isinstance(ref, dict), "catalog/reference lock missing") - require(clock.get("path") == "qualification/resource-profile-catalog-v1.json", "wrong catalog path") - require(clock.get("sha256") == ref.get("catalog_sha256") == sha(catalog_path), "J3 catalog hash changed") - require(catalog.get("schema_version") == 1 and catalog.get("catalog_version") == "resource-profile-catalog-v1", "J3 catalog schema changed") - require(catalog.get("catalog_id") == clock.get("catalog_id") == "resource-profile-catalog-v1", "J3 catalog identity changed") + require(clock.get("path") == "qualification/resource-profile-catalog-v1.json", "wrong historical catalog path") + historical_sha = sha(historical_catalog_path) + require( + clock.get("sha256") == ref.get("catalog_sha256") == historical_sha, + "J7 historical catalog snapshot hash changed", + ) + require( + historical_catalog.get("schema_version") == 1 + and historical_catalog.get("catalog_version") == "resource-profile-catalog-v1", + "J7 historical catalog schema changed", + ) + require( + historical_catalog.get("catalog_id") == clock.get("catalog_id") == "resource-profile-catalog-v1", + "J7 historical catalog identity changed", + ) + # The canonical catalog is allowed to advance after J7, but J7 still proves its + # original source bytes through the immutable snapshot above. Current runtime + # authority boundaries remain fail-closed. + require(catalog.get("schema_version") == 1 and catalog.get("catalog_version") == "resource-profile-catalog-v1", "current J3 catalog schema changed") + require(catalog.get("catalog_id") == "resource-profile-catalog-v1", "current J3 catalog identity changed") require(catalog.get("selection_enabled") is False and clock.get("selection_enabled") is False, "runtime profile selection became enabled") require(catalog.get("fallback_profile") == clock.get("fallback_profile") == "engine-opt-v2", "fallback profile changed") require(catalog.get("authority") == {"resource_profile_catalog": True, "resource_authorization": False, "outward_move": False}, "J3 catalog authority changed") diff --git a/scripts/qualify-work-grant-scheduler.py b/scripts/qualify-work-grant-scheduler.py index d97aebdc3..59bdc7bab 100755 --- a/scripts/qualify-work-grant-scheduler.py +++ b/scripts/qualify-work-grant-scheduler.py @@ -22,6 +22,7 @@ J3 = ROOT / "qualification/resource-profile-catalog-v1.json" J7 = ROOT / "qualification/resource-profile-selection-v1.json" +ENGINE_SELECTION = ROOT / "qualification/engine-opt-v2-selection.json" J8 = ROOT / "config/allfather.m14-j-j8.validation.json" J9 = ROOT / "config/allfather.m14-j-j9.validation.json" J9_CATALOG = ROOT / "qualification/work-grant-scheduler-v1.json" @@ -52,6 +53,7 @@ def qualify() -> dict: resource_catalog=resource_catalog, ) j7 = load(J7) + engine_selection = load(ENGINE_SELECTION) j8 = load(J8) j9 = load(J9) @@ -176,23 +178,31 @@ def qualify() -> dict: "J9 must remain fixed conservative compatibility scheduling", ) - stockfish = [ - spec - for spec in j9["instances"].values() - if spec["family"] == "stockfish" - ] - require( - all(spec["options"].get("Hash") == 16 for spec in stockfish), - "J7 Stockfish Hash=32 leaked into J9 runtime", - ) - lc0 = next( - spec for spec in j9["instances"].values() - if spec["family"] == "lc0" - ) - require( - lc0["options"].get("MaxPrefetch") == 8, - "J7 LC0 MaxPrefetch=0 leaked into J9 runtime", - ) + selected = engine_selection.get("selected") or {} + for spec in j9["instances"].values(): + if spec["family"] == "stockfish": + require( + spec["options"].get("Hash") == (selected.get("stockfish") or {}).get("hash_mb"), + "J9 Stockfish Hash differs from canonical ENGINE-OPT selection", + ) + elif spec["family"] == "reckless": + require( + spec["options"].get("Hash") == (selected.get("reckless") or {}).get("hash_mb"), + "J9 Reckless Hash differs from canonical ENGINE-OPT selection", + ) + elif spec["family"] == "lc0": + lc0_selected = selected.get("lc0") or {} + for option, value in { + "NNCacheSize": lc0_selected.get("nn_cache_size"), + "MinibatchSize": lc0_selected.get("minibatch_size"), + "MaxPrefetch": lc0_selected.get("max_prefetch"), + "AdaptivePrefetch": lc0_selected.get("adaptive_prefetch"), + "DefectTelemetry": lc0_selected.get("defect_telemetry"), + }.items(): + require( + spec["options"].get(option) == value, + f"J9 LC0 {option} differs from canonical ENGINE-OPT selection", + ) require( j7.get("authority") == { diff --git a/scripts/reckless-repro.py b/scripts/reckless-repro.py new file mode 100644 index 000000000..24f42227f --- /dev/null +++ b/scripts/reckless-repro.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Exact-head, single-worker Reckless A/B/A-build diagnostic; NEVER promotion evidence.""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from tools.engine_opt.corpus import load_epd +from tools.engine_opt.runner import run_case +from tools.engine_opt.report import source_identity, write_report +from tools.reckless_repro.analysis import DiagnosticError, NODES, PROTOCOL_ID, REPEATS, analyze, schedule +from tools.reckless_repro.provenance import ( + BinaryMutationError, assert_identity, binary_identity, + preflight_binary_hashes, provenance, sha256, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + for label in ("derived-a", "derived-b", "pristine"): + parser.add_argument("--" + label, type=Path, required=True) + parser.add_argument("--build-log-" + label, type=Path, required=True) + parser.add_argument("--build-hashes", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--host-report", type=Path, required=True) + parser.add_argument("--expected-head", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + paths = {label: getattr(args, label.replace("-", "_")).resolve() + for label in ("derived-a", "derived-b", "pristine")} + logs = {label: getattr(args, "build_log_" + label.replace("-", "_")).resolve() + for label in paths} + report = { + "schema_version": 1, + "kind": "reckless-repro-diagnostic", + "protocol_id": PROTOCOL_ID, + "source_commit": args.expected_head, + "diagnostic_complete": False, + "promotable": False, + "promotion_evidence": False, + "qualification_disposition": "DIAGNOSTIC_INCOMPLETE", + "claim_boundary": {"canonical_b4_promotion": False, "engine_profile_qualified": False, + "generic_host_portability": False, "elo": False, + "strength": False, "equal_compute": False, "deployment": False}, + "rows": [], + "errors": [], + } + try: + source = source_identity(ROOT) + if source["commit"] != args.expected_head: + raise DiagnosticError("diagnostic source is not exact expected HEAD") + lock = json.loads((ROOT / "vendor.lock.json").read_text()) + engine_lock = json.loads((ROOT / "qualification/engine-derived-lock.json").read_text()) + actual_tree = subprocess.check_output( + ["git", "-C", str(ROOT), "rev-parse", "HEAD:engines/reckless"], text=True + ).strip() + if actual_tree != engine_lock["engines"]["reckless"]["derived_tree"]: + raise DiagnosticError("derived Reckless source tree differs from frozen lock") + if sha256(args.model) != lock["engines"]["reckless"]["artifacts"]["default_nnue"]["sha256"]: + raise DiagnosticError("Reckless model is not the locked NNUE") + report["source"] = source + report["derived_reckless_tree"] = actual_tree + report["pristine_reckless_commit"] = lock["engines"]["reckless"]["commit"] + report["pristine_reckless_tree"] = lock["engines"]["reckless"]["tree"] + report["immediate_build_hashes"] = preflight_binary_hashes(paths, args.build_hashes) + report["provenance"] = provenance(paths, args.model, logs, args.host_report) + preflight_binary_hashes(paths, args.build_hashes) + derived_a = report["provenance"]["binaries"]["derived-a"] + derived_b = report["provenance"]["binaries"]["derived-b"] + a_text, b_text = derived_a["elf_text"], derived_b["elf_text"] + report["build_reproducibility"] = { + "derived_bytes_equal": derived_a["sha256"] == derived_b["sha256"], + "derived_text_digest_available": a_text.get("available") is True and b_text.get("available") is True, + "derived_text_equal": ( + a_text["sha256"] == b_text["sha256"] + if a_text.get("available") is True and b_text.get("available") is True + else None + ), + "claim": "diagnostic only; binary metadata may vary even with identical source trees", + } + + affinity = sorted(os.sched_getaffinity(0)) + if not affinity: + raise DiagnosticError("CPU affinity mask empty") + target_cpu = affinity[0] + os.sched_setaffinity(0, {target_cpu}) + if os.sched_getaffinity(0) != {target_cpu}: + raise DiagnosticError("could not enforce stable single-logical-CPU placement") + report["affinity"] = {"original_allowed_cpus": affinity, + "pinned_logical_cpu": target_cpu, + "effective_cpus": sorted(os.sched_getaffinity(0)), + "claim": "same-worker-single-logical-cpu; not a fixed physical host"} + cases = load_epd(ROOT / "tests/fixtures/engine_opt/positions.epd") + ids = [c.case_id for c in cases] + case_map = {c.case_id: c for c in cases} + slots = schedule(ids) + report["plan"] = {"repeats": REPEATS, "nodes": NODES, + "attempt_policy": "single-predeclared-schedule-no-retry", + "ordering": "per-case-ABBA/BAAB", + "planned_rows": len(slots), + "cases": ids, + "pairs": [["derived-a", "pristine"], ["derived-b", "pristine"]]} + options = {"Threads": 1, "Hash": 16, "MultiPV": 1, + "Minimal": False, "UCI_Chess960": False} + original_identities = {label: binary_identity(path) for label, path in paths.items()} + for slot in slots: + owner = slot["binary"] + assert_identity(paths[owner], original_identities[owner], "before search") + try: + row = run_case(binary=paths[slot["binary"]], cwd=ROOT, + family="reckless", case=case_map[slot["case_id"]], + options=dict(options), nodes=NODES, deadline_ms=10000.0) + assert_identity(paths[owner], original_identities[owner], "after search") + if row["binary_sha256"] != original_identities[owner]["sha256"]: + raise BinaryMutationError("binary changed during diagnostic run") + row.update(slot) + report["rows"].append(row) + except BinaryMutationError: + raise # never continue with 320 identical executable-integrity failures + except Exception as exc: + assert_identity(paths[owner], original_identities[owner], "after failed search") + report["errors"].append({**slot, "error": f"{type(exc).__name__}: {exc}"}) + # No selective retries or cherry-picked partial pair. + if report["errors"]: + raise DiagnosticError(f"predeclared schedule has {len(report['errors'])} execution errors") + result = analyze(report["rows"], ids) + report["analysis"] = result + divergent = any(item["bestmove_mismatches"] or item["native_work_mismatches"] + for item in result["comparisons"].values()) + if divergent: + raise DiagnosticError("search work/bestmove differs; performance not comparable") + report["diagnostic_complete"] = True + report["qualification_disposition"] = "DIAGNOSTIC_COMPLETE_NOT_PROMOTION" + except (OSError, ValueError, KeyError, RuntimeError, json.JSONDecodeError, + subprocess.CalledProcessError) as exc: + report["errors"].append({"error": f"{type(exc).__name__}: {exc}"}) + report["qualification_disposition"] = "DIAGNOSTIC_INCOMPLETE" + finally: + write_report(args.output, report) + return 0 if report["diagnostic_complete"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/recursive-refinement-contract.py b/scripts/recursive-refinement-contract.py index 662910512..d7f02de8d 100644 --- a/scripts/recursive-refinement-contract.py +++ b/scripts/recursive-refinement-contract.py @@ -99,7 +99,16 @@ def main() -> int: lines = run_shell( config, - ["go movetime 8000", "await:bestmove "], + [ + "go movetime 8000", + "await:bestmove ", + # The outward bestmove may precede shadow/VERIFY/REFINE finalization. + # A same-position UCI synchronization forces the exact-generation + # quiesce barrier before the harness quits the controller. + "position startpos", + "isready", + "await:readyok", + ], timeout=45.0, ) outward = [line for line in lines if line.startswith("bestmove ")] @@ -118,6 +127,37 @@ def main() -> int: if parent_problems: raise ContractError(f"parent replay integrity failed: {parent_problems}") if refine_problems: + # Preserve original cause and primitive replay state. The temporary + # run root is about to disappear; never manufacture missing evidence. + def optional(path: Path): + if not path.is_file(): + return {"exists": False} + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + return {"parse_error": f"{type(exc).__name__}: {exc}"} + diagnostic = { + "problem": refine_problems, + "parent_manifest": optional(run_dir / "manifest.json"), + "route": optional(run_dir / "route.json"), + "verification_manifest": optional(run_dir / "verification" / "manifest.json"), + "refinement_manifest": optional(run_dir / "refinement" / "manifest.json"), + "children": sorted(str(p.relative_to(run_dir)) + for p in run_dir.rglob("manifest.json")), + "outward": outward, + } + RESULT_DIR.mkdir(parents=True, exist_ok=True) + (RESULT_DIR / "failed-replay.json").write_text( + json.dumps(diagnostic, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print("REFINE negative replay diagnostic: " + json.dumps({ + "error": refine_problems, + "parent_status": diagnostic["parent_manifest"].get("status"), + "parent_notes": diagnostic["parent_manifest"].get("notes"), + "verification_present": diagnostic["verification_manifest"].get("exists", True), + "refinement_present": diagnostic["refinement_manifest"].get("exists", True), + "children": diagnostic["children"], + }, sort_keys=True), file=sys.stderr) raise ContractError(f"recursive REFINE integrity failed: {refine_problems}") manifest = load_refinement_manifest(run_dir) diff --git a/tests/ci/test_apt_installer.py b/tests/ci/test_apt_installer.py new file mode 100644 index 000000000..0ac8f7768 --- /dev/null +++ b/tests/ci/test_apt_installer.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Fail-closed offline contract tests for the hosted-runner APT bootstrap.""" +from __future__ import annotations + +import subprocess +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts/ci-install-apt-deps.sh" +WORKFLOWS = ROOT / ".github/workflows" +EXPECTED_STEPS = { + "baseline.yml": 1, + "engine-optimization.yml": 3, + "full-game-qualification.yml": 4, + "lc0-artifact-domain-diagnostic.yml": 1, + "lc0-strength-qualification.yml": 1, + "meta-1.yml": 1, + "online-hybrid-qualification.yml": 1, + "online-profile-qualification.yml": 1, + "resource-profile-lab-v2.yml": 2, + "resource-profile-lab.yml": 2, +} + + +class AptInstallerTests(unittest.TestCase): + def invoke(self, *args): + return subprocess.run( + ["bash", str(SCRIPT), *args], + capture_output=True, text=True, check=False, + ) + + def test_supported_official_sources_use_archive_signature_verification(self): + for codename in ("jammy", "noble"): + for arch in ("amd64", "arm64"): + with self.subTest(codename=codename, arch=arch): + result = self.invoke("--render-sources", codename, arch) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg", result.stdout) + self.assertIn(f"Suites: {codename} {codename}-updates {codename}-backports", result.stdout) + self.assertIn(f"Suites: {codename}-security", result.stdout) + self.assertIn(f"Architectures: {arch}", result.stdout) + self.assertNotIn("azure.archive.ubuntu.com", result.stdout) + self.assertNotIn("trusted=yes", result.stdout.lower()) + self.assertNotIn("http://", result.stdout) + if arch == "amd64": + self.assertIn("https://archive.ubuntu.com/ubuntu", result.stdout) + self.assertIn("https://security.ubuntu.com/ubuntu", result.stdout) + else: + self.assertIn("https://ports.ubuntu.com/ubuntu-ports", result.stdout) + + def test_unknown_releases_and_architectures_fail_closed(self): + for codename, arch in (("focal", "amd64"), ("noble", "riscv64"), ("x", "x")): + with self.subTest(codename=codename, arch=arch): + self.assertNotEqual( + self.invoke("--render-sources", codename, arch).returncode, 0, + ) + self.assertNotEqual(self.invoke().returncode, 0) + + def test_update_timeout_and_install_failure_do_not_continue(self): + # Mock *all* timeout invocations. No networking, root, or actual apt. + for code, label in ((124, "update"), (100, "install")): + script = f""" +source "{SCRIPT}" +APT_KEYRING="$(mktemp)" +printf 'offline-test-keyring' > "$APT_KEYRING" +trap 'rm -f "$APT_KEYRING"' EXIT +timeout() {{ + if [[ "{label}" == "update" ]]; then + return 124 + fi + if [[ "$*" == *" update" ]]; then + return 0 + fi + return 100 +}} +if install_packages noble amd64 zlib1g-dev; then + echo "unexpected success" >&2 + exit 1 +else + status=$? + test "$status" -eq {code} +fi +""" + result = subprocess.run( + ["bash", "-c", script], capture_output=True, text=True, check=False, + ) + self.assertEqual(result.returncode, 0, f"{label}: {result.stdout}\n{result.stderr}") + expected_error = "APT update failed/expired" if label == "update" else "APT installation failed/expired" + self.assertIn(expected_error, result.stderr) + self.assertNotIn("APT dependencies installed", result.stdout) + + def test_workflows_have_no_direct_unbounded_apt_calls(self): + observed = 0 + for name, expected in EXPECTED_STEPS.items(): + path = WORKFLOWS / name + doc = path.read_text(encoding="utf-8") + self.assertNotIn("sudo apt-get update", doc, name) + self.assertNotIn("sudo apt-get install", doc, name) + actual = doc.count("bash scripts/ci-install-apt-deps.sh ") + self.assertEqual(actual, expected, name) + self.assertGreaterEqual(doc.count("timeout-minutes: 5"), 1, name) + self.assertIn("scripts/ci-install-apt-deps.sh", doc, name) + self.assertIn("tests/ci/test_apt_installer.py", doc, name) + observed += actual + self.assertEqual(observed, 17) + + def test_bootstrap_has_independent_limits_and_strict_indexes(self): + doc = SCRIPT.read_text(encoding="utf-8") + for literal in ( + "APT_UPDATE_TIMEOUT_SEC=90", + "APT_INSTALL_TIMEOUT_SEC=180", + "APT::Update::Error-Mode=any", + "Dir::Etc::sourceparts=", + "Dir::State::lists=", + "timeout --signal=TERM --kill-after=5s", + "APT update failed/expired", + "APT installation failed/expired", + ): + self.assertIn(literal, doc) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/controller/test_adaptive_time.py b/tests/controller/test_adaptive_time.py index c96e1bf4b..6687ccc0f 100644 --- a/tests/controller/test_adaptive_time.py +++ b/tests/controller/test_adaptive_time.py @@ -383,7 +383,7 @@ def test_j7_operating_points_are_not_runtime_consumed(self): spec for spec in runtime["instances"].values() if spec["family"] == "lc0" ) - self.assertEqual(lc0["options"]["MaxPrefetch"], 8) + self.assertEqual(lc0["options"]["MaxPrefetch"], 0) if __name__ == "__main__": diff --git a/tests/controller/test_budget_routing.py b/tests/controller/test_budget_routing.py index d3385610d..5d6f88f83 100644 --- a/tests/controller/test_budget_routing.py +++ b/tests/controller/test_budget_routing.py @@ -252,6 +252,43 @@ def test_controller_overhead_is_charged_to_the_same_envelope(self): self.assertGreater(lane["spent_cpu_ms"], 0.0) self.assertIn("controller.unit_ms", lane["native_work"]) + def test_thread_cpu_clock_ignores_wall_scheduler_stall(self): + wall = [0.0] + cpu = [0.0] + ledger = BudgetLedger( + envelope(controller_overhead_reserve_ms=25.0), + clock=lambda: wall[0], started=0.0, + cpu_clock=lambda: cpu[0], + ) + with ledger.controller_overhead("stall"): + wall[0] = 0.3 + cpu[0] = 0.02 + self.assertAlmostEqual( + ledger.snapshot()["lanes"]["controller"]["spent_cpu_ms"], 20.0, + ) + self.assertTrue(ledger.within_partition_caps()) + self.assertAlmostEqual(ledger.elapsed_ms(), 300.0) + + def test_genuine_controller_thread_cpu_overrun_fails(self): + cpu = [0.0] + ledger = BudgetLedger( + envelope(controller_overhead_reserve_ms=25.0), + cpu_clock=lambda: cpu[0], + ) + with ledger.controller_overhead("busy"): + cpu[0] = 0.026 + self.assertFalse(ledger.within_partition_caps()) + + def test_invalid_thread_cpu_sample_does_not_charge_zero(self): + cpu = [0.0] + ledger = BudgetLedger( + envelope(controller_overhead_reserve_ms=25.0), + cpu_clock=lambda: cpu[0], + ) + with self.assertRaises(BudgetError): + with ledger.controller_overhead("bad"): + cpu[0] = float("nan") + def test_native_work_is_never_summed_across_engine_semantics(self): ledger = BudgetLedger(envelope()) ledger.record_native_work("shadow:stockfish", value=1000, semantics="stockfish.uci_nodes") @@ -566,6 +603,71 @@ def test_qualification_time_is_charged_to_the_envelope(self): self.assertEqual(lane["spent_cpu_ms"], 250.0) self.assertIn("controller.qualification_ms", lane["native_work"]) + def test_pre_router_process_cpu_is_not_elapsed_wall_time(self): + router = ConservativeRouter(envelope=envelope(), policy=policy(), clock=lambda: 0.0) + context = _FakeContext() + context.controller_cpu_started_ns = 1_000_000_000 + context.elapsed = 309.682 # scheduler pause before the router opens + with mock.patch("controller.routing.time.process_time_ns", return_value=1_020_000_000): + router.on_run_start(context) + lane = router.ledger.snapshot()["lanes"]["qualification"] + self.assertAlmostEqual(lane["spent_cpu_ms"], 20.0) + self.assertTrue(router._controller_cpu_accounting_complete) + self.assertIn("controller.qualification_ms", lane["native_work"]) + + def test_physical_controller_overrun_rejects_even_when_ledger_undercounts(self): + router = ConservativeRouter( + envelope=envelope(controller_overhead_reserve_ms=250.0), policy=policy(), + ) + context = _FakeContext() + context.controller_cpu_started_ns = 1_000_000_000 + context.resource_measurement_required = lambda: True + context.seal_resource_report = lambda: { + "qualified": True, + "physical_cpu_ms": 1624.074, + "provider": "linux-procfs-v1", + "controller_cpu_ms": 260.0, + } + with mock.patch("controller.routing.time.process_time_ns", return_value=1_020_000_000): + router.on_run_start(context) + claim = _end_and_read_claim(router, context) + self.assertTrue(claim["physical_cpu_within_envelope"]) + self.assertFalse(claim["controller_cpu_within_partition"]) + self.assertFalse(claim["claimed"]) + + def test_missing_controller_cpu_origin_rejects_measured_claim(self): + router = ConservativeRouter(envelope=envelope(), policy=policy()) + context = _FakeContext() + context.resource_measurement_required = lambda: True + context.seal_resource_report = lambda: { + "qualified": True, "physical_cpu_ms": 900.0, + "provider": "linux-procfs-v1", + "controller_cpu_ms": 20.0, + } + router.on_run_start(context) + claim = _end_and_read_claim(router, context) + self.assertFalse(claim["controller_cpu_accounting_complete"]) + self.assertFalse(claim["claimed"]) + + def test_controller_cpu_charge_error_poisons_qualification(self): + router = ConservativeRouter(envelope=envelope(), policy=policy()) + context = _FakeContext() + context.controller_cpu_started_ns = 1_000_000_000 + context.resource_measurement_required = lambda: True + context.seal_resource_report = lambda: { + "qualified": True, "physical_cpu_ms": 900.0, + "provider": "linux-procfs-v1", + "controller_cpu_ms": 20.0, + } + with mock.patch("controller.routing.time.process_time_ns", return_value=1_020_000_000): + router.on_run_start(context) + with self.assertRaises(BudgetError): + router.charge_controller_elapsed("bad", float("nan")) + router.invalidate_controller_cpu_accounting("bad CPU sample") + claim = _end_and_read_claim(router, context) + self.assertFalse(claim["controller_cpu_accounting_complete"]) + self.assertFalse(claim["claimed"]) + def test_a_stage_that_outruns_its_estimate_is_charged_in_full(self): router = ConservativeRouter( envelope=envelope(), @@ -899,6 +1001,43 @@ def test_a_run_past_its_wall_envelope_cannot_claim_compliance(self): "a run that outlasted its wall envelope reported itself compliant", ) + def test_budget_snapshot_uses_one_terminal_wall_observation(self): + clock = _Clock(0.0) + ledger = BudgetLedger(envelope(wall_ms=100.0), clock=clock) + with mock.patch.object(ledger, "elapsed_ms", side_effect=[99.8, 101.0]) as elapsed: + snapshot = ledger.snapshot() + self.assertEqual(elapsed.call_count, 1) + self.assertEqual(snapshot["elapsed_ms"], 99.8) + self.assertEqual(snapshot["wall_remaining_ms"], 0.2) + + def test_route_claim_agrees_with_atomic_budget_timestamp(self): + clock = _Clock(0.0) + router = ConservativeRouter( + envelope=envelope(wall_ms=100.0, cpu_ms=100000.0), + policy=policy(anchor_cpu_ms_estimate=10.0), + clock=clock, + ) + context = _FakeContext() + context.external_go_command = "go movetime 10" + router.on_run_start(context) + clock.now = 0.09 + original_snapshot = router.ledger.snapshot + + def snapshot_then_cross_deadline(): + snapshot = original_snapshot() + clock.now = 0.20 + return snapshot + + with tempfile.TemporaryDirectory() as tmp: + context.run_dir = Path(tmp) + with mock.patch.object(router.ledger, "snapshot", side_effect=snapshot_then_cross_deadline): + router.on_run_end(context) + doc = json.loads((Path(tmp) / "route.json").read_text()) + claim = doc["envelope_claim"] + self.assertEqual(doc["budget"]["elapsed_ms"], claim["wall_ms_elapsed"]) + self.assertTrue(claim["wall_within_envelope"]) + self.assertTrue(claim["claimed"]) + def test_a_run_inside_its_wall_envelope_still_claims(self): router = ConservativeRouter( envelope=envelope(wall_ms=100000.0, cpu_ms=100000.0), diff --git a/tests/controller/test_online_deadlines.py b/tests/controller/test_online_deadlines.py index f005f47cd..a57bdba4f 100644 --- a/tests/controller/test_online_deadlines.py +++ b/tests/controller/test_online_deadlines.py @@ -85,7 +85,15 @@ def test_normal_outward_move_and_bound_replay(self): route=json.loads((run/'route.json').read_text()) self.assertEqual(route['time_plan']['plan_id'],manifest['time_plan']['plan_id']) self.assertTrue(route['envelope_claim']['clock_output_complete']) - self.assertTrue(route['envelope_claim']['claimed']) + self.assertTrue( + route['envelope_claim']['claimed'], + { + "claim": route['envelope_claim'], + "controller_resource": (json.loads((run / 'resource.json').read_text()) + .get('controller')), + "controller_ledger": route['budget']['purpose_totals'].get('controller'), + }, + ) def test_quiesce_grace_does_not_shrink_protocol_readiness_timeout(self): with shell_fixture(settings={"quiesce_budget_ms": 200}) as ( diff --git a/tests/controller/test_orchestrated_authority.py b/tests/controller/test_orchestrated_authority.py index e0d3d55d1..48a00085e 100644 --- a/tests/controller/test_orchestrated_authority.py +++ b/tests/controller/test_orchestrated_authority.py @@ -8,6 +8,7 @@ import tempfile import unittest from dataclasses import replace +from unittest.mock import patch from pathlib import Path ROOT = Path(__file__).resolve().parents[2] @@ -29,6 +30,7 @@ from controller.orchestration_integrity import ( runtime_config_matches_frozen, ) +from controller.final_decision import _verify_orchestration_provenance from controller.resource_allocator import ( BUY_BUNDLE, FALLBACK, @@ -208,6 +210,78 @@ def allocation(action: str) -> AllocationDecision: ) +class OrchestrationAuditBoundaryTests(unittest.TestCase): + def test_denied_fallback_uses_audit_only_resource_reconstruction(self): + bound = provenance() + snap = snapshot(orchestration_provenance=bound).as_dict() + authorization = { + "policy": ORCHESTRATED_CLOCKED_AUTHORIZATION_POLICY, + "authorized": False, + } + decision = { + "authority": "ANCHOR_FALLBACK", + "anchor_move": "d2d4", + "emitted_move": "d2d4", + } + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + (run / "orchestration.json").write_text( + json.dumps({"authority_binding": bound.as_dict()}), + encoding="utf-8", + ) + problems = [] + with patch( + "controller.orchestration_integrity.verify_orchestrated_composition_integrity", + return_value=[], + ) as verifier: + _verify_orchestration_provenance( + run, + decision, + authorization, + snap, + problems, + ) + self.assertEqual(problems, []) + self.assertFalse( + verifier.call_args.kwargs["require_resource_qualification"] + ) + + def test_authorized_hybrid_keeps_strict_resource_reconstruction(self): + bound = provenance() + snap = snapshot(orchestration_provenance=bound).as_dict() + authorization = { + "policy": ORCHESTRATED_CLOCKED_AUTHORIZATION_POLICY, + "authorized": True, + } + decision = { + "authority": "HYBRID", + "anchor_move": "d2d4", + "emitted_move": "e2e4", + } + with tempfile.TemporaryDirectory() as tmp: + run = Path(tmp) + (run / "orchestration.json").write_text( + json.dumps({"authority_binding": bound.as_dict()}), + encoding="utf-8", + ) + problems = [] + with patch( + "controller.orchestration_integrity.verify_orchestrated_composition_integrity", + return_value=[], + ) as verifier: + _verify_orchestration_provenance( + run, + decision, + authorization, + snap, + problems, + ) + self.assertEqual(problems, []) + self.assertTrue( + verifier.call_args.kwargs["require_resource_qualification"] + ) + + class OrchestratedAuthorityTests(unittest.TestCase): def test_historical_g3_policy_does_not_require_orchestration(self): ev = evidence() diff --git a/tests/controller/test_resource_measurement.py b/tests/controller/test_resource_measurement.py index 1744b9ba7..b7719d4be 100644 --- a/tests/controller/test_resource_measurement.py +++ b/tests/controller/test_resource_measurement.py @@ -1,9 +1,12 @@ from __future__ import annotations +import hashlib import json import sys import tempfile +import time import unittest +from unittest import mock from pathlib import Path ROOT = Path(__file__).resolve().parents[2] @@ -11,9 +14,11 @@ from adapters.resource import ProcessSnapshot from controller.resource_measurement import ( + RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED, ResourceMeasurementError, ResourceMeasurementRun, ResourceMeasurementSettings, + seal_unqualified_resource_report, ) @@ -71,6 +76,71 @@ def settings(self): record_memory=True, ) + def test_first_stage_snapshot_is_also_the_lazy_process_baseline(self): + provider = _Provider() + run = ResourceMeasurementRun( + run_id="lazy-baseline", + settings=self.settings(), + provider=provider, + ) + run.begin_stage( + key="anchor", + instance="stockfish-anchor", + phase="ANCHOR", + pid=55, + ) + self.assertEqual( + provider.count[55], + 1, + "first-stage process baseline took a redundant procfs sample", + ) + run.finish_stage("anchor") + self.assertEqual(provider.count[55], 2) + with tempfile.TemporaryDirectory() as tmp: + run.seal(Path(tmp) / "resource.json") + doc = json.loads((Path(tmp) / "resource.json").read_text()) + self.assertEqual(provider.count[55], 3) + self.assertTrue(doc["qualified"]) + self.assertIn("stockfish-anchor", doc["processes"]) + + def test_negative_setup_certificate_is_hash_bound_and_non_claiming(self): + settings = self.settings() + started = time.process_time_ns() + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "resource.json" + summary = seal_unqualified_resource_report( + path, + run_id="negative", + settings=settings, + failure_code=RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED, + failure_reason="pre-anchor resource setup timed out", + controller_cpu_started_ns=started, + ) + doc = json.loads(path.read_text()) + self.assertEqual( + summary["sha256"], + hashlib.sha256(path.read_bytes()).hexdigest(), + ) + self.assertFalse(summary["qualified"]) + self.assertFalse(doc["qualified"]) + self.assertEqual(doc["processes"], {}) + self.assertEqual(doc["stages"], []) + self.assertFalse(doc["coverage"]["cpu"]["complete"]) + self.assertEqual( + doc["failure"]["code"], + RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED, + ) + core = {key: value for key, value in doc.items() if key != "report_id"} + digest = hashlib.sha256( + json.dumps( + core, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + ).hexdigest() + self.assertEqual(doc["report_id"], f"resource-{digest[:16]}") + def test_stage_and_run_process_cpu_are_separate(self): provider = _Provider() run = ResourceMeasurementRun( @@ -165,6 +235,38 @@ def test_unfinished_stage_fails_cpu_coverage(self): class IntervalFreezeTests(unittest.TestCase): + def test_seal_diagnostics_separate_freeze_from_slow_fsync(self): + run = ResourceMeasurementRun( + run_id="durable-latency", settings=ResourceMeasurementSettings( + enabled=True, provider="linux-procfs-v1", + require_cpu_for_claim=True, require_gpu_for_claim=False, + record_memory=False, + ), provider=_Provider(), + ) + run.begin_stage(key="a", instance="anchor", phase="ANCHOR", pid=7) + run.finish_stage("a") + origin = time.monotonic() + writer = run._atomic_write + + def slow_persistence(path, payload): + time.sleep(0.035) + return writer(path, payload) + + with tempfile.TemporaryDirectory() as tmp: + with mock.patch.object(run, "_atomic_write", side_effect=slow_persistence): + result = run.seal(Path(tmp) / "resource.json") + diag = run.seal_timing_ms(started_monotonic=origin) + doc = json.loads((Path(tmp) / "resource.json").read_text()) + self.assertTrue(result["qualified"]) + self.assertTrue(doc["qualified"]) + self.assertIsNotNone(diag) + self.assertGreaterEqual(diag["durable_write_ms"], 30.0) + self.assertGreaterEqual( + diag["persisted_wall_ms"] - diag["freeze_wall_ms"], 30.0, + ) + self.assertNotIn("seal_phase_ns", doc) + self.assertEqual(diag, run.seal_timing_ms(started_monotonic=origin)) + def test_freeze_interval_stops_controller_and_process_endpoint_growth(self): settings = ResourceMeasurementSettings( enabled=True, diff --git a/tests/controller/test_resource_profile_catalog.py b/tests/controller/test_resource_profile_catalog.py index 4dfbacf20..5761858e2 100644 --- a/tests/controller/test_resource_profile_catalog.py +++ b/tests/controller/test_resource_profile_catalog.py @@ -71,42 +71,82 @@ def test_equivalence_contract_accepts_explicit_no_warmup_selection(self): ) self.assertIsNone(catalog.warmup("lc0/specialist-engine-opt-v2")) - def test_seed_is_bound_to_pr49_exact_host_qualification(self): + def test_promoted_catalog_freezes_pr64_immediate_parent_exact_host_evidence(self): catalog = load_resource_profile_catalog(CATALOG) snapshot = catalog.qualification_snapshot + history = load(HOST_BINDING) + bound = history["current_domain_bound_qualification"] self.assertEqual( snapshot["qualified_head"], - "3de528e5a1a5a3609e0c293a80296ddc4a006875", + "b90af9f1760c9611826afc2fd26ad0dc438d4557", ) + self.assertEqual(snapshot["qualification_disposition"], "QUALIFIED_EXACT_HOST_ONLY") + self.assertEqual(snapshot["binding_scope"], "exact_host_observation") + self.assertEqual(snapshot["workflow_run"], 38012334421) + self.assertEqual(snapshot["aggregate_artifact_id"], 11657944410) self.assertEqual( - snapshot["qualification_disposition"], - "QUALIFIED_EXACT_HOST_ONLY", + snapshot["merge_commit"], + "cae842801497161f2de37ff43b27d074ac12568a", + "frozen schema retains the actual PR63 mainline merge, not a fictional PR64 merge", + ) + self.assertEqual( + snapshot["aggregate_artifact_sha256"], + "daf8ff2a3bfb2300de02d60a60baacf52c14638ff4d3aa41d3f0394e5205dbbd", + ) + self.assertEqual( + snapshot["aggregate_report_sha256"], + "f1748fea0de0b8712fc47ae216c2111c525f8a3c645d044f670837ee2e81f9e1", + ) + self.assertEqual( + snapshot["execution_domain_digest"], + "2e9673f5cea55cf86e59f34ae1b4d1dddea1c19c4972ac34b323465ab3bd0e2c", + ) + self.assertEqual(snapshot["qualified_head"], bound["qualified_head"]) + self.assertEqual(snapshot["workflow_run"], bound["aggregate_workflow_run"]) + for field in ( + "aggregate_artifact_id", + "aggregate_artifact_sha256", "aggregate_report_sha256", + "qualification_disposition", "execution_domain_id", + "execution_domain_digest", "binding_scope", + ): + self.assertEqual(snapshot[field], bound[field]) + self.assertFalse(bound["generic_host_portability_established"]) + self.assertEqual(catalog.digest, "dc98b3dff303b0be9d3d195b298cf3c10dbb9635e4446b3e5413a020d96e8f0c") + scheduler_policy = load(ROOT / "qualification/work-grant-scheduler-v1.json") + allocator_policy = load(ROOT / "qualification/adaptive-resource-allocation-v1.json") + self.assertEqual(scheduler_policy["resource_catalog"]["catalog_digest"], catalog.digest) + self.assertEqual( + allocator_policy["work_scheduler_catalog"]["catalog_digest"], + "b90c892f8d0c4653009b9bbb0e0972c17a1df65d326ac01b8f5f8b0b80386405", ) - self.assertEqual(snapshot["binding_scope"], "exact_host_observation") self.assertFalse(catalog.selection_enabled) for binding in catalog.default_composition.bindings: profile = catalog.profile(binding.profile_id) - self.assertEqual( - profile.qualification.execution_domain_id, - snapshot["execution_domain_id"], - ) - self.assertEqual( - profile.qualification.execution_domain_digest, - snapshot["execution_domain_digest"], - ) + self.assertEqual(profile.qualification.source_commit, snapshot["qualified_head"]) + self.assertEqual(profile.qualification.evidence_sha256, snapshot["aggregate_report_sha256"]) + self.assertEqual(profile.qualification.execution_domain_id, snapshot["execution_domain_id"]) + self.assertEqual(profile.qualification.execution_domain_digest, snapshot["execution_domain_digest"]) self.assertEqual(profile.qualification.binding_scope, snapshot["binding_scope"]) self.assertEqual(profile.work_chunk_ids, ()) + composition = catalog.default_composition + self.assertEqual(composition.qualification.source_commit, snapshot["qualified_head"]) + self.assertEqual(composition.qualification.evidence_sha256, snapshot["aggregate_report_sha256"]) + self.assertEqual(composition.qualification.execution_domain_digest, snapshot["execution_domain_digest"]) + def test_pr63_candidate_seed_is_retained_but_no_longer_active_snapshot(self): history = load(HOST_BINDING) - current = history["current_domain_bound_qualification"] - self.assertEqual(current["qualified_head"], snapshot["qualified_head"]) - self.assertEqual( - current["aggregate_report_sha256"], - snapshot["aggregate_report_sha256"], + seed = history["b4_candidate_promotion_seed"] + self.assertEqual(seed["pull_request"], 63) + self.assertEqual(seed["qualified_head"], "56516c34719670054e12218bd3781e9725ae4f65") + self.assertEqual(seed["qualification_disposition"], "QUALIFIED_EXACT_HOST_ONLY") + self.assertTrue(seed["canonical_promotion_complete"]) + self.assertNotEqual( + history["current_domain_bound_qualification"]["qualified_head"], + seed["qualified_head"], ) - self.assertEqual( - current["execution_domain_digest"], - snapshot["execution_domain_digest"], + self.assertNotEqual( + load_resource_profile_catalog(CATALOG).qualification_snapshot["qualified_head"], + seed["qualified_head"], ) def test_catalog_records_measured_seed_memory_without_enforcement_claim(self): @@ -172,7 +212,12 @@ def test_one_unit_profile_or_policy_drift_breaks_v2_equivalence(self): mutations.append(profile) warmup = copy.deepcopy(raw) - warmup["profile_runtime"]["lc0/specialist-engine-opt-v2"]["warmup"]["nodes"] = 63 + warmup["profile_runtime"]["lc0/specialist-engine-opt-v2"]["warmup"] = { + "enabled": True, + "nodes": 64, + "position": "startpos", + "reset_after": True, + } mutations.append(warmup) reservation = copy.deepcopy(raw) diff --git a/tests/controller/test_resource_profile_selection.py b/tests/controller/test_resource_profile_selection.py index 4ccbc2704..9242cf60d 100644 --- a/tests/controller/test_resource_profile_selection.py +++ b/tests/controller/test_resource_profile_selection.py @@ -23,6 +23,7 @@ EVIDENCE = ROOT / "qualification/resource-profile-evidence-v1.json" SELECTION = ROOT / "qualification/resource-profile-selection-v1.json" CATALOG = ROOT / "qualification/resource-profile-catalog-v1.json" +HISTORICAL_CATALOG = ROOT / "qualification/resource-profile-catalog-v1-j7-baseline.json" def load(path: Path) -> dict: @@ -38,25 +39,40 @@ def setUp(self): self.evidence = load(EVIDENCE) self.selection = load(SELECTION) self.catalog = load(CATALOG) - - def _validate(self, evidence=None, selection=None, catalog=None, *, rebind_evidence=True, rebind_catalog=True): + self.historical_catalog = load(HISTORICAL_CATALOG) + + def _validate( + self, + evidence=None, + selection=None, + catalog=None, + historical_catalog=None, + *, + rebind_evidence=True, + rebind_historical_catalog=True, + ): evidence = copy.deepcopy(self.evidence if evidence is None else evidence) selection = copy.deepcopy(self.selection if selection is None else selection) catalog = copy.deepcopy(self.catalog if catalog is None else catalog) + historical_catalog = copy.deepcopy( + self.historical_catalog if historical_catalog is None else historical_catalog + ) with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) - ep, sp, cp = root / "evidence.json", root / "selection.json", root / "catalog.json" + ep, sp = root / "evidence.json", root / "selection.json" + cp, hp = root / "catalog.json", root / "historical-catalog.json" dump(ep, evidence) dump(cp, catalog) + dump(hp, historical_catalog) if rebind_evidence: selection["evidence_sha256"] = hashlib.sha256(ep.read_bytes()).hexdigest() - if rebind_catalog: - selection["catalog"]["sha256"] = hashlib.sha256(cp.read_bytes()).hexdigest() + if rebind_historical_catalog: + selection["catalog"]["sha256"] = hashlib.sha256(hp.read_bytes()).hexdigest() dump(sp, selection) - return mod.validate(ep, sp, cp) + return mod.validate(ep, sp, cp, hp) def test_frozen_selection_qualifies(self): - report = mod.validate(EVIDENCE, SELECTION, CATALOG) + report = mod.validate(EVIDENCE, SELECTION, CATALOG, HISTORICAL_CATALOG) self.assertTrue(report["qualified"]) self.assertEqual(report["groups"], 9) self.assertEqual(report["pareto_candidates"], 18) @@ -136,7 +152,7 @@ def test_catalog_digest_mismatch_is_rejected(self): catalog = copy.deepcopy(self.catalog) catalog["fallback_profile"] = "wrong" with self.assertRaises(mod.SelectionQualificationError): - self._validate(catalog=catalog, rebind_catalog=False) + self._validate(historical_catalog=catalog, rebind_historical_catalog=False) if __name__ == "__main__": diff --git a/tests/controller/test_work_scheduler.py b/tests/controller/test_work_scheduler.py index 08a0cae8f..7eb7ed22b 100644 --- a/tests/controller/test_work_scheduler.py +++ b/tests/controller/test_work_scheduler.py @@ -458,7 +458,7 @@ def test_j9_runtime_preserves_j7_nonconsumption_and_authority_boundary(self): spec for spec in doc["instances"].values() if spec["family"] == "lc0" ) - self.assertEqual(lc0["options"]["MaxPrefetch"], 8) + self.assertEqual(lc0["options"]["MaxPrefetch"], 0) if __name__ == "__main__": diff --git a/tests/engine_opt/test_aggregate_paths.py b/tests/engine_opt/test_aggregate_paths.py new file mode 100644 index 000000000..78c941025 --- /dev/null +++ b/tests/engine_opt/test_aggregate_paths.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Regression tests for promotion-vs-candidate evidence path and J12 identity gates.""" +from __future__ import annotations + +import copy +import runpy +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +api = runpy.run_path(str(ROOT / "scripts/qualify-engine-opt-evidence.py")) +find_one = api["find_one"] +local1_campaign_pattern = api["local1_campaign_pattern"] +j12_campaign_pattern = api["j12_campaign_pattern"] +validate_j12_campaign_metadata = api["validate_j12_campaign_metadata"] +QualificationError = api["QualificationError"] + +SOURCE = "a" * 40 +POLICY = "b" * 64 +BUNDLE = {"source_commit": SOURCE, "build_manifest_sha256": "c" * 64} + + +def fixtures(): + local = { + "passed": True, "errors": [], "execution_scope": "required_local1", + "validated_games": 28, "observed_games": 28, + "claim_boundary": {"full_game_lifecycle": True}, + "candidate_bundle": copy.deepcopy(BUNDLE), + } + manifest = { + "source": {"commit": SOURCE}, "status": "completed", + "failures": [], "mode": "required", + "policy": { + "path": "qualification/local-full-game-orchestrated-v1.json", + "sha256": POLICY, + }, + "candidate_bundle": copy.deepcopy(BUNDLE), + "prerequisites": [ + {"id": "engine-opt-v2", "returncode": 0, "timed_out": False}, + {"id": "j12", "returncode": 0, "timed_out": False}, + ], + } + prerequisite = { + "schema_version": 1, "profile_id": "allfather.orchestrated-v1", + "mechanism_valid": True, "source_commit": SOURCE, + "candidate_bundle": copy.deepcopy(BUNDLE), + "work_grants_authorized": 9, "work_grants_settled": 9, + "open_reservations": 0, "allocation_action": "BUY_BUNDLE", + "authority_qualified": False, "qualification_disposition": "NOT_QUALIFIED_HOST_CAPACITY", + } + return local, manifest, prerequisite + + +class EngineOptAggregatePathTests(unittest.TestCase): + def test_candidate_and_canonical_campaign_paths_are_distinct(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for dir_name in ("local1", "canonical-local1", "j12-local1"): + report = root / "engine-opt-v2-profile-domain" / "artifact" / dir_name / "campaign" / "report.json" + report.parent.mkdir(parents=True) + report.write_text("{}", encoding="utf-8") + self.assertIn( + "canonical-local1", + str(find_one(root, local1_campaign_pattern(candidate_mode=False))), + ) + self.assertIn( + "/local1/", + str(find_one(root, local1_campaign_pattern(candidate_mode=True))), + ) + self.assertIn("j12-local1", str(find_one(root, j12_campaign_pattern()))) + (root / "engine-opt-v2-profile-domain" / "artifact" / "local1" / "campaign" / "report.json").unlink() + with self.assertRaisesRegex(QualificationError, "expected one file, found 0"): + find_one(root, local1_campaign_pattern(candidate_mode=True)) + duplicate = root / "engine-opt-v2-profile-domain" / "artifact2" / "canonical-local1" / "campaign" / "report.json" + duplicate.parent.mkdir(parents=True) + duplicate.write_text("{}", encoding="utf-8") + with self.assertRaisesRegex(QualificationError, "expected one file, found 2"): + find_one(root, local1_campaign_pattern(candidate_mode=False)) + + def test_qualified_j12_lifecycle_does_not_require_non_anchor_hybrid(self): + validate_j12_campaign_metadata( + *fixtures(), source=SOURCE, candidate_bundle=BUNDLE, policy_sha256=POLICY, + ) + + def test_wrong_exact_head_bundle_or_frozen_policy_rejected(self): + mutations = [ + ("source", lambda l, m, j: m["source"].__setitem__("commit", "d" * 40)), + ("bundle", lambda l, m, j: j["candidate_bundle"].__setitem__("source_commit", "d" * 40)), + ("policy", lambda l, m, j: m["policy"].__setitem__("sha256", "d" * 64)), + ("games", lambda l, m, j: l.__setitem__("validated_games", 27)), + ("errors", lambda l, m, j: l["errors"].append("invalid")), + ("authority", lambda l, m, j: j.__setitem__("mechanism_valid", False)), + ("grants", lambda l, m, j: j.__setitem__("work_grants_settled", 8)), + ("prerequisite", lambda l, m, j: m["prerequisites"][1].__setitem__("timed_out", True)), + ] + for name, mutate in mutations: + with self.subTest(name=name): + local, manifest, prerequisite = fixtures() + mutate(local, manifest, prerequisite) + with self.assertRaises(QualificationError): + validate_j12_campaign_metadata( + local, manifest, prerequisite, + source=SOURCE, candidate_bundle=BUNDLE, policy_sha256=POLICY, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/engine_opt/test_contracts.py b/tests/engine_opt/test_contracts.py index c82973648..ad8469203 100644 --- a/tests/engine_opt/test_contracts.py +++ b/tests/engine_opt/test_contracts.py @@ -36,19 +36,20 @@ def test_hybrid_rejects_reference_envelope_and_clock_drift(self): def test_selection_state_is_explicit(self): s=load_json(ROOT/"qualification/engine-opt-v2-selection.json") self.assertIn(s["status"],("provisional","selected")) - self.assertEqual(s["selected"]["lc0"]["matrix_profile"],"b7-p8-c256k-warm64") - self.assertEqual(s["selected"]["lc0"]["warmup_nodes"],64) + self.assertEqual(s["selected"]["lc0"]["matrix_profile"],"b4-p0-c256k-cold") + self.assertIsNone(s["selected"]["lc0"]["warmup_nodes"]) self.assertGreaterEqual(s["qualification"]["confirmation_repeats"],2) self.assertEqual(s["qualification"]["baseline_profile"],"v1-current-cold") self.assertEqual(s["qualification"]["corpus_cases"],8) - def test_candidate_overlay_is_non_authoritative_and_distinct(self): + def test_candidate_overlay_remains_non_authoritative_after_canonical_promotion(self): canonical=load_json(ROOT/"qualification/engine-opt-v2-selection.json") candidate=load_json(ROOT/"qualification/engine-opt-v2-candidate-selection.json") self.assertEqual(canonical["status"],"selected") self.assertEqual(candidate["status"],"qualification_candidate") - self.assertEqual(canonical["selected"]["lc0"]["matrix_profile"],"b7-p8-c256k-warm64") + self.assertEqual(canonical["selected"]["lc0"]["matrix_profile"],"b4-p0-c256k-cold") self.assertEqual(candidate["selected"]["lc0"]["matrix_profile"],"b4-p0-c256k-cold") + self.assertEqual(canonical["selected"], candidate["selected"]) self.assertFalse(candidate["authority"]["runtime_profile_selection"]) self.assertFalse(candidate["authority"]["promotion_authority"]) self.assertIsNone(candidate["selected"]["lc0"]["warmup_nodes"]) @@ -59,7 +60,7 @@ def test_candidate_overlay_is_non_authoritative_and_distinct(self): validate_reference(policy,candidate,reference,require_selected=False) validate_hybrid(hybrid_policy,reference,hybrid) - def test_candidate_does_not_mutate_historical_j8_to_j12_configs(self): + def test_canonical_b4_is_propagated_through_j8_to_j12_configs(self): for relative in ( "config/allfather.m14-j-j8.validation.json", "config/allfather.m14-j-j9.validation.json", @@ -69,9 +70,9 @@ def test_candidate_does_not_mutate_historical_j8_to_j12_configs(self): with self.subTest(path=relative): doc=load_json(ROOT/relative) lc0=doc["instances"]["lc0-shadow"] - self.assertEqual(lc0["options"]["MinibatchSize"],7) - self.assertEqual(lc0["options"]["MaxPrefetch"],8) - self.assertEqual(lc0["warmup"]["nodes"],64) + self.assertEqual(lc0["options"]["MinibatchSize"],4) + self.assertEqual(lc0["options"]["MaxPrefetch"],0) + self.assertNotIn("warmup",lc0) def test_candidate_cold_profile_row_contract(self): selected=load_json(ROOT/"qualification/engine-opt-v2-candidate-selection.json")["selected"]["lc0"] @@ -89,6 +90,34 @@ def test_candidate_cold_profile_row_contract(self): with self.assertRaises(AssertionError): validate_selected_lc0_rows([bad],selected,lambda ok,msg:self.assertTrue(ok,msg)) + + def test_canonical_b4_native_work_policy_is_explicit(self): + policy=load_json(ROOT/"qualification/engine-opt-v2-native-work-policy.json") + self.assertEqual(policy["policy_id"],"lc0-node-stop-contract-v1") + self.assertEqual(policy["applies_to"],{ + "selection_profile_id":"engine-opt-v2-selection", + "selection_status":"selected", + "matrix_profile":"b4-p0-c256k-cold", + }) + self.assertEqual(policy["requested_nodes"],16) + self.assertEqual(policy["terminal_counter_semantics"],"lc0.uci_nodes") + self.assertFalse(policy["require_exact_terminal_counter_repeatability"]) + + def test_canonical_matrix_qualifier_uses_canonical_identity_report(self): + workflow=(ROOT/".github/workflows/engine-optimization.yml").read_text() + marker=" - name: Qualify canonical LC0 matrix independently" + self.assertIn(marker,workflow) + segment=workflow.split(marker,1)[1].split(" - uses:",1)[0] + self.assertIn("--candidate-report-pattern",segment) + self.assertIn( + "'engine-opt-v2-profile-domain/**/canonical/report.json'", + segment, + ) + self.assertNotIn( + "engine-opt-v2-profile-domain/**/candidate/report.json", + segment, + ) + def test_candidate_g3_witness_corpus_preserves_legacy_requirement(self): canonical=load_json(ROOT/"qualification/online-hybrid-v2.json") candidate=load_json(ROOT/"qualification/online-hybrid-v2-candidate.json") diff --git a/tests/engine_opt/test_g3_b4.py b/tests/engine_opt/test_g3_b4.py new file mode 100644 index 000000000..f85751674 --- /dev/null +++ b/tests/engine_opt/test_g3_b4.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Fail-closed contracts for canonical b4's source-frozen 23-case G3 authority.""" +from __future__ import annotations + +import copy +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +from tools.engine_opt.g3_b4 import ( + B4G3ContractError, B4_POLICY, CANDIDATE_POLICY, LEGACY_POLICY, + WITNESS_CORPUS, file_hash, require_positive_g3, validate_b4_policy, +) + + +def witness_report(): + row = { + "case": "discovery-base-00-03-g000016", "run_id": "g000016", + "authority": "HYBRID", + "authorization_policy": "clocked_staged_preanchor_v1", + "authorization_granted": True, + "terminal_source": "staged_verification", + "route_action": "BUY_STAGED_VERIFY", + "staged_complete": True, "anchor_move": "e8d6", + "proposal_move": "h7h6", "emitted_move": "h7h6", + "envelope_claimed": True, "resource_qualified": True, + "route_resource_qualified": True, + "clock_outcome": {"output_within_deadline": True}, + } + return {"evidence_valid": True, "authority_qualified": True, + "passed": True, "positive_case": row, "cases": [row], + "contracts": {"policy_sha256": file_hash(ROOT, B4_POLICY)}} + + +class B4PolicyTests(unittest.TestCase): + def test_frozen_canonical_policy_preserves_case_order_and_runtime(self): + p = validate_b4_policy(ROOT) + self.assertEqual(len(p["positive_cases"]), 23) + self.assertEqual(p["positive_cases"][7]["source_set"], "discovery_local1_v1") + self.assertEqual(p["runtime_config"], "config/allfather.online-hybrid-v2.validation.json") + + def test_mutations_to_corpus_authority_budget_or_provenance_fail(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for path in (B4_POLICY, CANDIDATE_POLICY, LEGACY_POLICY, WITNESS_CORPUS): + dest = root / path + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes((ROOT / path).read_bytes()) + original = json.loads((root / B4_POLICY).read_text()) + for mutation in ("witness", "authority", "budget", "legacy", "path", "order", "missing"): + with self.subTest(mutation=mutation): + p = copy.deepcopy(original) + if mutation == "witness": + p["positive_cases"][8]["command"] = "go movetime 4000" + elif mutation == "authority": + p["positive_requirement"]["require_non_anchor_move"] = False + elif mutation == "budget": + p["clock"]["max_move_ms"] = 5000 + elif mutation == "legacy": + p["positive_cases"][0]["moves"] = ["e2e4"] + elif mutation == "path": + p["runtime_config"] = "config/allfather.online-hybrid-v2.candidate.validation.json" + elif mutation == "order": + p["positive_cases"][7:9] = list(reversed(p["positive_cases"][7:9])) + elif mutation == "missing": + p["positive_cases"].pop() + (root / B4_POLICY).write_text(json.dumps(p)) + with self.assertRaises(B4G3ContractError): + validate_b4_policy(root) + (root / B4_POLICY).write_text(json.dumps(original)) + corpus = json.loads((root / WITNESS_CORPUS).read_text()) + corpus["cases"][0]["command"] = "go movetime 4000" + (root / WITNESS_CORPUS).write_text(json.dumps(corpus)) + with self.assertRaises(B4G3ContractError): + validate_b4_policy(root) + + +class WitnessTests(unittest.TestCase): + def test_real_non_anchor_witness(self): + self.assertEqual( + require_positive_g3(witness_report(), policy_sha256=file_hash(ROOT, B4_POLICY))["proposal_move"], + "h7h6", + ) + + def test_no_witness_and_diagnostic_report_are_not_qualification(self): + report = witness_report() + report.update(passed=False, authority_qualified=False, positive_case=None) + with self.assertRaisesRegex(B4G3ContractError, "positive authority witness missing"): + require_positive_g3(report, policy_sha256=file_hash(ROOT, B4_POLICY)) + for name in ("diagnostic", "orphan", "same-move", "no-staged", "expired", "no-resource", "no-authority"): + with self.subTest(name=name): + r = witness_report() + p = r["positive_case"] + if name == "diagnostic": + r["contracts"]["policy_sha256"] = "f" * 64 + elif name == "orphan": + r["cases"] = [] + elif name == "same-move": + p["anchor_move"] = p["proposal_move"] + elif name == "no-staged": + p["staged_complete"] = False + elif name == "expired": + p["clock_outcome"]["output_within_deadline"] = False + elif name == "no-resource": + p["resource_qualified"] = False + elif name == "no-authority": + p["authorization_granted"] = False + with self.assertRaises(B4G3ContractError): + require_positive_g3(r, policy_sha256=file_hash(ROOT, B4_POLICY)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/engine_opt/test_g3_canonical_b4_triage.py b/tests/engine_opt/test_g3_canonical_b4_triage.py new file mode 100644 index 000000000..338d0045c --- /dev/null +++ b/tests/engine_opt/test_g3_canonical_b4_triage.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Fail-closed source and runtime tests for diagnostic-only canonical b4 G3 triage.""" +from __future__ import annotations + +import copy +import importlib.util +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +script = ROOT / "scripts/diagnose-g3-canonical-b4.py" +spec = importlib.util.spec_from_file_location("g3_b4_triage", script) +assert spec and spec.loader +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def load(path): + return json.loads((ROOT / path).read_text(encoding="utf-8")) + + +class TriageContracts(unittest.TestCase): + def setUp(self): + self.canonical = load(module.CANONICAL) + self.triage = load(module.TRIAGE) + self.policy = load(module.POLICY) + self.legacy = load(module.LEGACY) + self.selection = load(module.SELECTION) + + def check(self): + module.verify_overlay(self.canonical, self.triage, + self.policy, self.legacy, self.selection) + + def test_exact_real_source_overlay(self): + self.check() + self.assertEqual(self.triage["shadow"]["replay_root"], module.REPLAY) + self.assertEqual(len(self.policy["positive_cases"]), 23) + + def test_no_change_to_budget_and_control_authority(self): + for category in ("budget", "online_time", "routing", "instances", + "verification", "crossfeed", "counterfactual", "hybrid_authority"): + with self.subTest(category=category): + self.assertEqual(self.triage[category], self.canonical[category]) + + def test_runtime_drift_fails_closed(self): + for field, new in (("MaxPrefetch", 8), ("MinibatchSize", 32)): + with self.subTest(field=field): + triage = copy.deepcopy(self.triage) + triage["instances"]["lc0-shadow"]["options"][field] = new + with self.assertRaises(module.TriageError): + module.verify_overlay(self.canonical, triage, self.policy, + self.legacy, self.selection) + triage = copy.deepcopy(self.triage) + triage["budget"]["wall_ms"] += 100 + with self.assertRaises(module.TriageError): + module.verify_overlay(self.canonical, triage, self.policy, + self.legacy, self.selection) + + def test_policy_migration_and_legacy_changes_fail_closed(self): + altered = copy.deepcopy(self.policy) + altered["candidate_witnesses"]["legacy_policy_path"] = "elsewhere.json" + with self.assertRaises(module.TriageError): + module.verify_overlay(self.canonical, self.triage, altered, + self.legacy, self.selection) + altered = copy.deepcopy(self.policy) + altered["positive_cases"].pop() + with self.assertRaises(module.TriageError): + module.verify_overlay(self.canonical, self.triage, altered, + self.legacy, self.selection) + selection = copy.deepcopy(self.selection) + selection["selected"]["lc0"]["matrix_profile"] = "b7-p8-c256k-warm64" + with self.assertRaises(module.TriageError): + module.verify_overlay(self.canonical, self.triage, self.policy, + self.legacy, selection) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/engine_opt/test_matrix_scope.py b/tests/engine_opt/test_matrix_scope.py index 6116f2c6a..c907f60e2 100644 --- a/tests/engine_opt/test_matrix_scope.py +++ b/tests/engine_opt/test_matrix_scope.py @@ -22,13 +22,13 @@ class MatrixScopeTests(unittest.TestCase): def test_qualification_only_runs_exactly_baseline_and_selected(self): rows = matrix.profiles_for_run( - "b7-p8-c256k-warm64", + "b4-p0-c256k-cold", "v1-current-cold", True, ) self.assertEqual( [row[0] for row in rows], - ["v1-current-cold", "b7-p8-c256k-warm64"], + ["v1-current-cold", "b4-p0-c256k-cold"], ) def test_candidate_qualification_only_ignores_unrelated_exploratory_profiles(self): diff --git a/tests/engine_opt/test_promotion_gate.py b/tests/engine_opt/test_promotion_gate.py new file mode 100644 index 000000000..ba52ac042 --- /dev/null +++ b/tests/engine_opt/test_promotion_gate.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import copy, unittest, sys, subprocess, tempfile, json, hashlib +from pathlib import Path +ROOT=Path(__file__).resolve().parents[2] +sys.path.insert(0,str(ROOT)) +from tools.engine_opt.promotion_gate import PromotionGateError, evaluate_promotion_gate, verify_promotion_git_history + +HEAD='a'*40 +PARENT='c'*40 +DOMAIN='b'*64 +BOUND_DOMAIN='d'*64 + +def aggregate(): + return { + 'source_commit':HEAD,'evidence_valid':True,'invalid_evidence':[],'profile_qualified':True,'passed':True, + 'qualification_disposition':'QUALIFIED_EXACT_HOST_ONLY','qualification_failures':[], + 'details':{ + 'execution_domain':{'execution_domain_digest':DOMAIN}, + 'lc0':{'selected_profile':'b4-p0-c256k-cold','native_work_policy':'lc0-node-stop-contract-v1'}, + 'local1_g3':{'validated_games':28,'g3_authority_qualified':True,'g3_policy_sha256':'1'*64}, + }, + } + +def binding(): + return { + 'current_domain_bound_qualification':{ + 'qualified_head':PARENT,'qualification_disposition':'QUALIFIED_EXACT_HOST_ONLY','execution_domain_digest':BOUND_DOMAIN, + }, + 'b4_candidate_promotion_seed':{'canonical_promotion_complete':True}, + } + +def catalog(): + return {'qualification_snapshot':{'qualified_head':PARENT,'qualification_disposition':'QUALIFIED_EXACT_HOST_ONLY','execution_domain_digest':BOUND_DOMAIN}} + +class PromotionGateTests(unittest.TestCase): + def test_prior_head_binding_plus_exact_current_aggregate_closes_promotion(self): + report=evaluate_promotion_gate(aggregate=aggregate(),host_binding=binding(),catalog=catalog(),promotion_surface={'authorized_profile_change':True,'parent_sha':PARENT,'g3_b4_contract_verified':True,'g3_b4_policy_sha256':'1'*64},current_head=HEAD) + self.assertTrue(report['passed']) + self.assertEqual(report['disposition'],'CANONICAL_B4_PROMOTION_QUALIFIED') + self.assertTrue(report['claim_boundary']['canonical_b4_promotion']) + self.assertFalse(report['claim_boundary']['runtime_profile_selection']) + + def test_unfrozen_promotion_head_fails_closed(self): + bad=binding(); bad['current_domain_bound_qualification']['qualified_head']='e'*40 + with self.assertRaisesRegex(PromotionGateError,'preceding promotion head'): + evaluate_promotion_gate(aggregate=aggregate(),host_binding=bad,catalog=catalog(),promotion_surface={'authorized_profile_change':True,'parent_sha':PARENT,'g3_b4_contract_verified':True,'g3_b4_policy_sha256':'1'*64},current_head=HEAD) + + def test_non_b4_aggregate_is_rejected(self): + bad=aggregate(); bad['details']['lc0']['selected_profile']='b7-p8-c256k-warm64' + with self.assertRaises(PromotionGateError): + evaluate_promotion_gate(aggregate=bad,host_binding=binding(),catalog=catalog(),promotion_surface={'authorized_profile_change':True,'parent_sha':PARENT,'g3_b4_contract_verified':True,'g3_b4_policy_sha256':'1'*64},current_head=HEAD) + + +class GitHistoryIntegrationTests(unittest.TestCase): + def git(self, root, *args): + return subprocess.check_output(["git", "-C", str(root), *args], text=True, stderr=subprocess.STDOUT).strip() + + def test_full_history_resolves_exact_parent_and_base(self): + with tempfile.TemporaryDirectory() as temp: + root=Path(temp) + self.git(root,"init","-q") + self.git(root,"config","user.name","Test") + self.git(root,"config","user.email","test@example.invalid") + (root/"sentinel").write_text("a") + self.git(root,"add",".") + self.git(root,"commit","-qm","one") + base=self.git(root,"rev-parse","HEAD") + (root/"sentinel").write_text("b") + self.git(root,"commit","-qam","two") + parent=self.git(root,"rev-parse","HEAD") + (root/"sentinel").write_text("c") + self.git(root,"commit","-qam","three") + head=self.git(root,"rev-parse","HEAD") + self.assertEqual(verify_promotion_git_history(root,base), + {"head_sha":head,"parent_sha":parent,"base_sha":base}) + with self.assertRaises(PromotionGateError): + verify_promotion_git_history(root,"f"*40) + + def test_shallow_parent_failure_produces_sealed_negative_report(self): + with tempfile.TemporaryDirectory() as temp: + src=Path(temp)/"source" + src.mkdir() + self.git(src,"init","-q") + self.git(src,"config","user.name","Test") + self.git(src,"config","user.email","test@example.invalid") + (src/"sentinel").write_text("a") + self.git(src,"add",".") + self.git(src,"commit","-qm","one") + base=self.git(src,"rev-parse","HEAD") + (src/"sentinel").write_text("b") + self.git(src,"commit","-qam","two") + clone=Path(temp)/"shallow" + subprocess.run(["git","clone","-q","--depth=1",src.resolve().as_uri(),str(clone)],check=True) + with self.assertRaisesRegex(PromotionGateError,"required Git history/object unavailable"): + verify_promotion_git_history(clone,base) + for name in ("aggregate.json","host.json","catalog.json"): + (clone/name).write_text("{}") + report_path=clone/"report.json" + process=subprocess.run( + [sys.executable,str(ROOT/"scripts/qualify-engine-opt-v2-promotion-gate.py"), + "--aggregate",str(clone/"aggregate.json"), + "--host-binding",str(clone/"host.json"), + "--catalog",str(clone/"catalog.json"), + "--base-sha",base,"--repo-root",str(clone), + "--output",str(report_path)], + capture_output=True,text=True) + self.assertEqual(process.returncode,2,process.stdout+process.stderr) + report=json.loads(report_path.read_text()) + self.assertEqual(report["disposition"],"BLOCKED_FAIL_CLOSED") + self.assertEqual(report["failure"]["type"],"PromotionGateError") + self.assertFalse(report["claim_boundary"]["canonical_b4_promotion"]) + digest=report.pop("content_sha256") + self.assertEqual(digest,hashlib.sha256( + json.dumps(report,sort_keys=True,separators=(",",":"),allow_nan=False).encode() + ).hexdigest()) + +if __name__=='__main__': unittest.main() diff --git a/tests/local_game/test_g3_b4_preflight.py b/tests/local_game/test_g3_b4_preflight.py new file mode 100644 index 000000000..44f85a213 --- /dev/null +++ b/tests/local_game/test_g3_b4_preflight.py @@ -0,0 +1,105 @@ +"""Integration regression: frozen positive G3 prerequisite gates LOCAL-1 games.""" +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import tools.local_game.runner as runner +from tools.engine_opt.g3_b4 import B4G3ContractError, B4_POLICY, CANDIDATE_POLICY, LEGACY_POLICY, WITNESS_CORPUS, file_hash + +ROOT = Path(__file__).resolve().parents[2] + + +class CanonicalG3PreflightTests(unittest.TestCase): + def exercise(self, *, positive: bool): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name in (B4_POLICY, CANDIDATE_POLICY, LEGACY_POLICY, WITNESS_CORPUS): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((ROOT / name).read_bytes()) + campaign = root / "build/attempt/prerequisites" + campaign.mkdir(parents=True) + replay = root / "build/replays-online-hybrid-v2" + replay.mkdir(parents=True) + case = { + "case": "discovery-base-00-03-g000016", + "run_id": "g000016", + "authority": "HYBRID", + "authorization_policy": "clocked_staged_preanchor_v1", + "authorization_granted": True, + "terminal_source": "staged_verification", + "route_action": "BUY_STAGED_VERIFY", + "staged_complete": True, + "anchor_move": "e8d6", + "proposal_move": "h7h6", + "emitted_move": "h7h6", + "envelope_claimed": True, + "resource_qualified": True, + "route_resource_qualified": True, + "clock_outcome": {"output_within_deadline": True}, + } + g3 = { + "evidence_valid": True, "authority_qualified": positive, + "passed": positive, + "cases": [case], "positive_case": case if positive else None, + "contracts": {"policy_sha256": file_hash(root, B4_POLICY)}, + } + def bounded(argv, cwd, log, timeout, **kwargs): + self.assertTrue(str(log).startswith(str(campaign))) + name = Path(argv[1]).name + if name == "qualify-engine-opt-v2.py": + target = root / "build/test-results/engine-opt-v2/report.json" + data = {"passed": True, "candidate_identity_valid": True} + elif name == "qualify-online-hybrid-v2-b4.py": + target = root / "build/test-results/online-hybrid-v2/report.json" + data = g3 + (replay / case["run_id"]).mkdir() + else: + raise AssertionError(f"unexpected prerequisite {name}") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(data)) + return {"returncode": 0, "timed_out": False} + def retain(source_root, destination, run_ids, prior, label): + self.assertEqual(run_ids, ["g000016"]) + self.assertEqual(label, "g3-v2") + destination.mkdir() + (destination / "g000016").mkdir() + + spec = { + "qualification": {"require_g3_positive_prerequisite": True}, + "prerequisites": [ + {"id": "engine-opt-v2", "script": "scripts/qualify-engine-opt-v2.py", + "report": "build/test-results/engine-opt-v2/report.json"}, + {"id": "g3-v2", "script": "scripts/qualify-online-hybrid-v2-b4.py", + "report": "build/test-results/online-hybrid-v2/report.json", + "replay_root": "build/replays-online-hybrid-v2", + "retain_case_replays": True}, + ], + } + with (mock.patch.object(runner, "ROOT", root), + mock.patch.object(runner, "bounded", side_effect=bounded), + mock.patch.object(runner, "retain_report_runs", side_effect=retain)): + if positive: + rows = runner.prerequisites(campaign, spec, {"commit": "a" * 40}) + self.assertEqual([row["id"] for row in rows], ["engine-opt-v2", "g3-v2"]) + else: + with self.assertRaisesRegex(B4G3ContractError, "positive authority witness missing"): + runner.prerequisites(campaign, spec, {"commit": "a" * 40}) + # The negative attempt must remain auditable when the campaign stops. + self.assertTrue((campaign / "g3-v2.json").is_file()) + self.assertTrue((campaign / "g3-v2-replays/g000016").is_dir()) + self.assertEqual(len(json.loads((campaign / "prerequisites.json").read_text())), 2) + + def test_negative_g3_stops_before_any_games_after_replay_retention(self): + self.exercise(positive=False) + + def test_positive_g3_reaches_local1_campaign(self): + self.exercise(positive=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/local_game/test_integrity.py b/tests/local_game/test_integrity.py index 89fb12447..503cbcd97 100644 --- a/tests/local_game/test_integrity.py +++ b/tests/local_game/test_integrity.py @@ -15,7 +15,33 @@ verify_session_commands, verify_specialist_settlements, ) -from tools.local_game.validate import apply_scope_flags, verify_game_clocks, verify_runner_log +from tools.local_game.validate import ( + apply_scope_flags, describe_resource_denial, verify_game_clocks, + verify_runner_log, +) + + +class ResourceDenialDiagnosticsTests(unittest.TestCase): + def test_wall_denial_reports_replay_and_exact_false_claim(self): + route = { + "budget": {"elapsed_ms": 1517.479}, + "envelope": {"wall_ms": 1446.0}, + "envelope_claim": { + "wall_within_envelope": False, + "physical_measurement_qualified": True, + "physical_cpu_within_envelope": True, + "controller_cpu_within_partition": True, + }, + } + details = describe_resource_denial( + Path("/replays/20261009T192450382505Z-g000060-408b38c9"), + route, + {"qualified": True, "claimed": False}, + ) + self.assertIn("g000060", details) + self.assertIn("false_claims=['wall_within_envelope']", details) + self.assertIn("excess_ms=71.479", details) + self.assertIn("resource_qualified=True", details) class G3PrerequisiteSeparationTests(unittest.TestCase): diff --git a/tests/reckless_repro/test_repro.py b/tests/reckless_repro/test_repro.py new file mode 100644 index 000000000..5e1e682dd --- /dev/null +++ b/tests/reckless_repro/test_repro.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Destructive offline tests: diagnostic ordering, provenance, and no-promotion boundary.""" +from __future__ import annotations + +import copy +import hashlib +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from tools.reckless_repro.analysis import ( + REPEATS, NODES, PROTOCOL_ID, DiagnosticError, analyze, schedule, +) + + +from tools.reckless_repro.provenance import ( + BinaryMutationError, binary_identity, elf_text, binaries, first_pass_mutator_probe, + preflight_binary_hashes, +) + + +class RealElfIntegrityTests(unittest.TestCase): + def test_disposable_objcopy_and_readelf_leave_real_elf_unchanged(self): + compiler = shutil.which("cc") + if compiler is None or shutil.which("objcopy") is None or shutil.which("readelf") is None: + self.skipTest("C compiler and binutils required") + with tempfile.TemporaryDirectory() as tmp: + executable = Path(tmp) / "freshly-built.elf" + subprocess.run([compiler, "-x", "c", "-o", str(executable), "-"], + input="int main(void) { return 0; }\n", + text=True, check=True, capture_output=True) + before = binary_identity(executable) + attribution = first_pass_mutator_probe(executable) + self.assertTrue(attribution["copy_only"]) + self.assertEqual(set(attribution["operations"]), {"objcopy", "readelf"}) + self.assertIn(attribution["first_mutator"], (None, "objcopy", "readelf")) + self.assertEqual(binary_identity(executable), before) + section = elf_text(executable) + self.assertTrue(section["available"], section) + self.assertEqual(before, binary_identity(executable)) + info = binaries({"fixture": executable})["fixture"] + self.assertTrue(info["inspection_preserved_original"]) + self.assertEqual(before, binary_identity(executable)) + self.assertEqual(info["sha256"], before["sha256"]) + self.assertEqual(info["size_bytes"], before["size_bytes"]) + self.assertEqual(info["mode"], before["mode"]) + + def test_immediate_post_build_hash_manifest_rejects_mutated_elf(self): + source = shutil.which("true") + if source is None: + self.skipTest("system ELF required") + with tempfile.TemporaryDirectory() as tmp: + paths = {} + for label in ("derived-a", "derived-b", "pristine"): + executable = Path(tmp) / label + shutil.copy2(source, executable) + paths[label] = executable + manifest = Path(tmp) / "binaries.sha256" + manifest.write_text("".join( + f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path}\n" + for path in paths.values() + ), encoding="utf-8") + self.assertEqual(len(preflight_binary_hashes(paths, manifest)), 3) + paths["pristine"].write_bytes(paths["pristine"].read_bytes() + b"x") + with self.assertRaises(BinaryMutationError): + preflight_binary_hashes(paths, manifest) + + +CASES = [f"case-{x}" for x in range(8)] + + +def rows(): + result=[] + for slot in schedule(CASES): + metric={ + "wall_ms": 900.0 if slot["binary"] == "pristine" else 920.0, + "cpu_ms": 900.0, + "native_work_value": NODES + 1, + "completed_before_deadline": True, + "bestmove": "e2e4", + } + result.append({**slot,"metrics":metric}) + return result + + +class ScheduleTests(unittest.TestCase): + def test_predeclared_balanced_abba_baab_complete(self): + plan=schedule(CASES) + self.assertEqual(len(plan),5*2*8*4) + for i in range(0,len(plan),4): + group=plan[i:i+4] + names=[x["binary"] for x in group] + self.assertTrue(names in ( + ["derived-a","pristine","pristine","derived-a"], + ["pristine","derived-a","derived-a","pristine"], + ["derived-b","pristine","pristine","derived-b"], + ["pristine","derived-b","derived-b","pristine"], + )) + self.assertEqual([x["order_index"] for x in group],[0,1,2,3]) + for block in range(REPEATS): + sub=[x for x in plan if x["repeat_index"]==block] + self.assertEqual(len(sub),64) + + def test_valid_diagnostic_comparison_does_not_authorize_promotion(self): + report=analyze(rows(),CASES) + self.assertTrue(report["diagnostic_complete"]) + self.assertFalse(report["promotion_evidence"]) + self.assertEqual(report["planned"],320) + self.assertEqual(report["observed"],320) + for data in report["comparisons"].values(): + self.assertGreater(data["mean_log_ratio_backtransformed"],1.0) + self.assertFalse(data["frozen_gate_applied"]) + self.assertFalse(data["qualification_claim"]) + self.assertFalse(data["bestmove_mismatches"]) + + def test_missing_rows_order_changes_bad_metrics_and_deadlines_rejected(self): + baseline=rows() + with self.assertRaises(DiagnosticError): + analyze(baseline[:-1],CASES) + edits=[ + ("order",lambda x:x[0].__setitem__("binary","pristine")), + ("block",lambda x:x[1].__setitem__("repeat_index",99)), + ("timeout",lambda x:x[2]["metrics"].__setitem__("completed_before_deadline",False)), + ("nodes",lambda x:x[3]["metrics"].__setitem__("native_work_value",None)), + ("cpu",lambda x:x[4]["metrics"].__setitem__("cpu_ms",float("nan"))), + ("move",lambda x:x[5]["metrics"].__setitem__("bestmove","")), + ] + for name,mutation in edits: + with self.subTest(name=name): + data=copy.deepcopy(baseline) + mutation(data) + with self.assertRaises(DiagnosticError): + analyze(data,CASES) + + def test_search_work_and_move_drift_are_visible_not_silently_averaged(self): + data=rows() + data[0]["metrics"]["bestmove"]="d2d4" + data[1]["metrics"]["native_work_value"]=NODES+2 + result=analyze(data,CASES) + self.assertTrue(result["comparisons"]["derived-a"]["bestmove_mismatches"]) + self.assertTrue(result["comparisons"]["derived-a"]["native_work_mismatches"]) + self.assertFalse(result["promotion_evidence"]) + + def test_source_files_are_not_in_frozen_engine_opt_pattern(self): + workflow=(ROOT/".github/workflows/reckless-repro-diagnostic.yml").read_text() + self.assertIn("workflow_dispatch:",workflow) + self.assertIn("tools/reckless_repro/**",workflow) + self.assertIn("tests/reckless_repro/**",workflow) + self.assertIn("scripts/reckless-repro.py",workflow) + self.assertIn("cancel-in-progress: false",workflow) + self.assertIn("actions/upload-artifact@",workflow) + engine=(ROOT/".github/workflows/engine-optimization.yml").read_text() + for path in ("tools/reckless_repro/**","tests/reckless_repro/**","scripts/reckless-repro.py"): + self.assertNotIn(path,engine) + cli=(ROOT/"scripts/reckless-repro.py").read_text() + self.assertIn('"promotable": False',cli) + self.assertIn('"promotion_evidence": False',cli) + self.assertIn('lock["engines"]["reckless"]["artifacts"]["default_nnue"]["sha256"]',cli) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/resource_lab/test_candidate_matrix.py b/tests/resource_lab/test_candidate_matrix.py index 9f446d63c..15c177840 100644 --- a/tests/resource_lab/test_candidate_matrix.py +++ b/tests/resource_lab/test_candidate_matrix.py @@ -2,6 +2,7 @@ from __future__ import annotations import copy +import hashlib import json import sys import unittest @@ -10,12 +11,15 @@ ROOT=Path(__file__).resolve().parents[2] sys.path.insert(0,str(ROOT)) +import tools.resource_lab.candidate_matrix as candidate_matrix_mod from tools.resource_lab.candidate_matrix import ( LabSpec, ResourceLabSpecError, expand_candidates, expand_compositions, load_lab_spec, + reference_contract_manifest, + resolve_reference_contract_paths, stage_a_attempt_plan, stage_b_attempt_plan, validate_lab_spec, @@ -24,6 +28,7 @@ SPEC=ROOT/"qualification/resource-lab-v1.json" SPEC_V2=ROOT/"qualification/resource-lab-v2.json" +EVIDENCE_V1=ROOT/"qualification/resource-profile-evidence-v1.json" def fake_manifest(): @@ -103,8 +108,62 @@ def test_candidate_identity_binds_artifact_hashes(self): self.assertNotEqual(first.candidate_id,second.candidate_id) self.assertNotEqual(first.digest,second.digest) - def test_v2_reference_contract_matches_runtime_catalog_and_build_policy(self): - validate_reference_contract(load_lab_spec(SPEC),ROOT) + def test_v1_reference_contract_matches_frozen_historical_archive(self): + spec=load_lab_spec(SPEC) + declared=dict(spec.raw["reference_contract"]) + resolved=resolve_reference_contract_paths(spec) + self.assertEqual( + declared, + { + "catalog":"qualification/resource-profile-catalog-v1.json", + "runtime":"config/allfather.online-hybrid-v2.validation.json", + "build_policy":"qualification/online-engine-opt-v2.json", + }, + ) + self.assertEqual( + resolved, + { + "catalog":"qualification/resource-profile-catalog-v1-j7-baseline.json", + "runtime":"config/allfather.online-hybrid-v2.validation-j7-baseline.json", + "build_policy":"qualification/online-engine-opt-v2-j7-baseline.json", + }, + ) + + evidence=json.loads(EVIDENCE_V1.read_text())["source"] + self.assertEqual( + hashlib.sha256(SPEC.read_bytes()).hexdigest(), + evidence["lab_spec"]["sha256"], + ) + manifest=reference_contract_manifest(spec,ROOT) + for key,row in manifest.items(): + self.assertEqual(row["path"],evidence["reference_contract"][f"{key}_path"]) + self.assertEqual(row["sha256"],evidence["reference_contract"][f"{key}_sha256"]) + self.assertEqual(row["resolved_path"],resolved[key]) + + live=json.loads( + (ROOT/"config/allfather.online-hybrid-v2.validation.json").read_text() + )["instances"]["lc0-shadow"] + archived=json.loads( + (ROOT/"config/allfather.online-hybrid-v2.validation-j7-baseline.json").read_text() + )["instances"]["lc0-shadow"] + self.assertEqual(live["options"]["MinibatchSize"],4) + self.assertEqual(live["options"]["MaxPrefetch"],0) + self.assertNotIn("warmup",live) + self.assertEqual(archived["options"]["MinibatchSize"],7) + self.assertEqual(archived["options"]["MaxPrefetch"],8) + self.assertEqual(archived["warmup"]["nodes"],64) + validate_reference_contract(spec,ROOT) + + def test_v1_reference_archive_cannot_fall_back_to_live_b4_runtime(self): + spec=load_lab_spec(SPEC) + archive=candidate_matrix_mod.REFERENCE_CONTRACT_ARCHIVE_PATHS_BY_LAB["resource-lab-v1"] + original=archive["runtime"] + try: + archive["runtime"]="config/allfather.online-hybrid-v2.validation.json" + with self.assertRaisesRegex(ResourceLabSpecError,"archived runtime SHA"): + validate_reference_contract(spec,ROOT) + finally: + archive["runtime"]=original def test_b4_v2_spec_binds_candidate_runtime_without_mutating_v1_catalog(self): spec=load_lab_spec(SPEC_V2) diff --git a/tools/diagnostics/compare_lc0_artifact_domain.py b/tools/diagnostics/compare_lc0_artifact_domain.py index ff2f3d220..83b041a6e 100755 --- a/tools/diagnostics/compare_lc0_artifact_domain.py +++ b/tools/diagnostics/compare_lc0_artifact_domain.py @@ -629,6 +629,73 @@ def command_prove_surface(args: argparse.Namespace) -> int: return 0 if proof["equivalent"] else 1 +def write_execution_error_report( + output: Path, + matrices: dict[str, dict[str, Any]], + matrix_paths: dict[str, Path], + binary_hashes: dict[str, str], + weights_hash: str, + *, + expected_a: str, + expected_b: str, + proof_sha: str, +) -> bool: + """Preserve valid sealed matrix execution errors without pretending equivalence.""" + errors_by_matrix: dict[str, list[dict[str, Any]]] = {} + for key, matrix in matrices.items(): + issues = matrix.get("errors") + require(isinstance(issues, list), f"matrix {key}: errors must be a list") + if issues: + require(all(isinstance(issue, dict) for issue in issues), + f"matrix {key}: error record is malformed") + errors_by_matrix[key] = issues + if not errors_by_matrix: + return False + + identities: dict[str, Any] = {} + for key, matrix in matrices.items(): + expected_source = expected_a if key == "a" else expected_b + require(matrix.get("schema_version") == 1 + and matrix.get("kind") == "lc0-cpu-runtime-matrix", + f"matrix {key}: schema drift on negative path") + require((matrix.get("source") or {}).get("commit") == expected_source, + f"matrix {key}: source identity drift on negative path") + require((matrix.get("binary") or {}).get("sha256") == binary_hashes[key], + f"matrix {key}: binary identity drift on negative path") + require((matrix.get("weights") or {}).get("sha256") == weights_hash, + f"matrix {key}: network identity drift on negative path") + require(matrix.get("nodes") == 16 + and float(matrix.get("deadline_ms")) == 3500.0, + f"matrix {key}: frozen workload drift on negative path") + identities[key] = { + "source_commit": expected_source, + "binary_sha256": binary_hashes[key], + "matrix_file_sha256": sha256_file(matrix_paths[key]), + "matrix_content_sha256": matrix["content_sha256"], + "matrix_rows_recorded": len(matrix.get("rows") or []), + "errors": matrix.get("errors"), + } + + write_sealed_json(output, { + "schema_version": 1, + "kind": "lc0-artifact-domain-negative-diagnostic", + "diagnostic_complete": False, + "passed": False, + "qualification_disposition": "RETAINED_NEGATIVE_EXECUTION_EVIDENCE", + "reason": "one or more historical LC0 matrices contain execution errors", + "source_equivalence_proof_sha256": proof_sha, + "network_sha256": weights_hash, + "matrices": identities, + "error_matrices": sorted(errors_by_matrix), + "claim_boundary": { + "diagnostic_only": True, "profile_promotion": False, + "strength": False, "elo": False, "equal_compute": False, + "generic_host_portability": False, + }, + }) + return True + + def command_compare(args: argparse.Namespace) -> int: proof = load_sealed_json(args.surface_proof, label="surface proof") validate_surface_proof( @@ -649,6 +716,16 @@ def command_compare(args: argparse.Namespace) -> int: binary_hashes = {label: sha256_file(path) for label, path in binaries.items()} weights_hash = sha256_file(args.weights) + if write_execution_error_report( + args.output, matrices, + {"a": args.matrix_a, "b": args.matrix_b, "c": args.matrix_c}, + binary_hashes, weights_hash, + expected_a=args.expected_a, expected_b=args.expected_b, + proof_sha=proof["content_sha256"], + ): + print("diagnostic evidence error: matrix contains execution errors", file=sys.stderr) + return 2 + validated = { "a": validate_matrix_document( matrices["a"], diff --git a/tools/diagnostics/test_lc0_artifact_domain.py b/tools/diagnostics/test_lc0_artifact_domain.py index 4cdd87067..d5142d383 100755 --- a/tools/diagnostics/test_lc0_artifact_domain.py +++ b/tools/diagnostics/test_lc0_artifact_domain.py @@ -449,5 +449,68 @@ def test_classifier_table(self): ) + +class NegativeDiagnosticReportTests(unittest.TestCase): + def test_historical_missing_bestmove_is_sealed_and_remains_negative(self): + from tools.diagnostics.compare_lc0_artifact_domain import ( + write_execution_error_report, sha256_file, + ) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + inputs = { + "a": matrix(source=EXPECTED_A, binary_hash="a" * 64), + "b": matrix(source=EXPECTED_B, binary_hash="b" * 64), + "c": matrix(source=EXPECTED_B, binary_hash="c" * 64), + } + failed = { + "profile": PROFILES[0], "case_id": CASES[-1], "repeat_index": 0, + "error": "UCI response wait expired without bestmove", + } + inputs["c"]["errors"] = [failed] + paths = {} + for name, doc in inputs.items(): + path = root / f"matrix-{name}.json" + write_sealed_json(path, doc) + paths[name] = path + inputs[name] = load_sealed_json(path, label=name) + output = root / "negative.json" + changed = write_execution_error_report( + output, inputs, paths, + {"a": "a" * 64, "b": "b" * 64, "c": "c" * 64}, + inputs["a"]["weights"]["sha256"], + expected_a=EXPECTED_A, expected_b=EXPECTED_B, + proof_sha="d" * 64, + ) + self.assertTrue(changed) + report = load_sealed_json(output, label="negative") + self.assertFalse(report["passed"]) + self.assertFalse(report["diagnostic_complete"]) + self.assertFalse(report["claim_boundary"]["profile_promotion"]) + self.assertEqual(report["matrices"]["c"]["errors"], [failed]) + self.assertEqual(report["matrices"]["c"]["matrix_file_sha256"], + sha256_file(paths["c"])) + self.assertEqual(report["error_matrices"], ["c"]) + + def test_clean_matrices_do_not_produce_negative_report(self): + from tools.diagnostics.compare_lc0_artifact_domain import write_execution_error_report + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + inputs = { + "a": matrix(source=EXPECTED_A, binary_hash="a" * 64), + "b": matrix(source=EXPECTED_B, binary_hash="b" * 64), + "c": matrix(source=EXPECTED_B, binary_hash="c" * 64), + } + paths = {name: root / f"{name}.json" for name in inputs} + output = root / "negative.json" + self.assertFalse(write_execution_error_report( + output, inputs, paths, + {"a": "a" * 64, "b": "b" * 64, "c": "c" * 64}, + "f" * 64, + expected_a=EXPECTED_A, expected_b=EXPECTED_B, + proof_sha="d" * 64, + )) + self.assertFalse(output.exists()) + + if __name__ == "__main__": unittest.main() diff --git a/tools/engine_opt/g3_b4.py b/tools/engine_opt/g3_b4.py new file mode 100644 index 000000000..d6966ecba --- /dev/null +++ b/tools/engine_opt/g3_b4.py @@ -0,0 +1,115 @@ +"""Canonical b4's frozen 7+16 G3 qualification corpus; no runtime authority.""" +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path +from typing import Any + +B4_POLICY = "qualification/online-hybrid-v2-b4.json" +LEGACY_POLICY = "qualification/online-hybrid-v2.json" +CANDIDATE_POLICY = "qualification/online-hybrid-v2-candidate.json" +WITNESS_CORPUS = "qualification/online-hybrid-v2-candidate-witnesses.json" +CANONICAL_RUNTIME = "config/allfather.online-hybrid-v2.validation.json" +CANONICAL_REFERENCE = "config/allfather.online-engine-opt-v2.json" + + +class B4G3ContractError(ValueError): + pass + + +def require(ok: bool, why: str) -> None: + if not ok: + raise B4G3ContractError(why) + + +def load_object(root: Path, path: str) -> dict[str, Any]: + data = json.loads((root / path).read_text(encoding="utf-8")) + require(isinstance(data, dict), f"{path}: expected object") + return data + + +def file_hash(root: Path, path: str) -> str: + return hashlib.sha256((root / path).read_bytes()).hexdigest() + + +def validate_b4_policy(root: Path) -> dict[str, Any]: + """Mechanically bind the historic witness set to canonical b4. + + Only the runtime/reference paths and candidate-only claim metadata change. + The seven baseline cases, discovery provenance, authority requirements, + clock budgets, node limits and frozen sixteen-case set must be identical. + """ + root = Path(root) + policy = load_object(root, B4_POLICY) + candidate = load_object(root, CANDIDATE_POLICY) + legacy = load_object(root, LEGACY_POLICY) + corpus = load_object(root, WITNESS_CORPUS) + require(candidate.get("profile_id") == legacy.get("profile_id") == "online-hybrid-v2", + "G3 source policies have incompatible identities") + expected = copy.deepcopy(candidate) + expected["runtime_config"] = CANONICAL_RUNTIME + expected["reference_runtime"] = CANONICAL_REFERENCE + expected["claim_boundary"] = copy.deepcopy(legacy["claim_boundary"]) + require(policy == expected, "canonical b4 G3 policy differs from frozen corpus transformation") + cases = policy.get("positive_cases") or [] + original = [{key: value for key, value in c.items() if key != "source_set"} + for c in cases if c.get("source_set") == "legacy_v1"] + discovered = [c for c in cases if c.get("source_set") == "discovery_local1_v1"] + archived_form = [{**c, "moves": " ".join(c["moves"])} for c in discovered] + require(len(cases) == 23 and len(original) == 7 and len(discovered) == 16, + "canonical b4 case count or source classification drift") + require(original == legacy.get("positive_cases"), + "canonical b4 changed frozen seven-case baseline") + require(policy.get("positive_requirement") == legacy.get("positive_requirement"), + "canonical b4 weakened G3 non-anchor witness requirement") + meta = policy.get("candidate_witnesses") or {} + require(meta.get("path") == WITNESS_CORPUS and meta.get("legacy_policy_path") == LEGACY_POLICY, + "G3 corpus provenance path drift") + require(meta.get("sha256") == file_hash(root, WITNESS_CORPUS), + "G3 witness corpus bytes no longer match frozen policy") + require(meta.get("discovery_source") == corpus.get("source"), + "G3 historical discovery source changed") + require(meta.get("selection_algorithm") == corpus.get("selection_algorithm"), + "G3 deterministic witness selection changed") + require(meta.get("corpus_id") == corpus.get("corpus_id"), + "G3 corpus identity changed") + require(meta.get("legacy_case_count") == 7 and meta.get("discovery_case_count") == 16, + "G3 case count declaration changed") + require(archived_form == corpus.get("cases"), "G3 witness cases differ from frozen archive") + return policy + + +def require_positive_g3(report: dict[str, Any], *, policy_sha256: str) -> dict[str, Any]: + """Require a real qualifying witness, not a diagnostic or negative report.""" + require(isinstance(report, dict), "G3 report must be an object") + require(report.get("evidence_valid") is True and + report.get("authority_qualified") is True and + report.get("passed") is True, "canonical b4 G3 positive authority witness missing") + require((report.get("contracts") or {}).get("policy_sha256") == policy_sha256, + "G3 report is not bound to canonical b4 policy bytes") + positive = report.get("positive_case") + require(isinstance(positive, dict), "G3 positive witness record missing") + rows = report.get("cases") or [] + require(isinstance(rows, list) and positive in rows and + isinstance(positive.get("run_id"), str) and bool(positive["run_id"]), + "G3 positive witness is not retained among tested cases") + require(positive.get("authority") == "HYBRID" and + positive.get("authorization_policy") == "clocked_staged_preanchor_v1" and + positive.get("authorization_granted") is True and + positive.get("terminal_source") == "staged_verification" and + positive.get("route_action") == "BUY_STAGED_VERIFY" and + positive.get("staged_complete") is True, + "G3 positive witness lacks frozen staged HYBRID authorization") + proposal = positive.get("proposal_move") + require(isinstance(proposal, str) and bool(proposal) and + proposal != positive.get("anchor_move") and + proposal == positive.get("emitted_move"), + "G3 positive witness is not a distinct emitted specialist move") + require(positive.get("envelope_claimed") is True and + positive.get("resource_qualified") is True and + positive.get("route_resource_qualified") is True and + (positive.get("clock_outcome") or {}).get("output_within_deadline") is True, + "G3 positive witness did not qualify its physical/outward envelope") + return positive diff --git a/tools/engine_opt/matrix_qualification.py b/tools/engine_opt/matrix_qualification.py index bf8bbb284..149d7b651 100644 --- a/tools/engine_opt/matrix_qualification.py +++ b/tools/engine_opt/matrix_qualification.py @@ -14,6 +14,9 @@ from tools.engine_opt.selection import validate_selected_lc0_rows ROOT = Path(__file__).resolve().parents[2] +CANONICAL_NATIVE_WORK_POLICY_PATH = ( + ROOT / "qualification/engine-opt-v2-native-work-policy.json" +) CANDIDATE_NATIVE_WORK_POLICY_PATH = ( ROOT / "qualification/engine-opt-v2-candidate-native-work-policy.json" ) @@ -82,26 +85,34 @@ def repeat_evidence( } -def _load_candidate_native_work_policy( +def _load_native_work_policy( selection: dict[str, Any], ) -> tuple[dict[str, Any] | None, str | None]: - if selection.get("status") != "qualification_candidate": + status = selection.get("status") + matrix_profile = ((selection.get("selected") or {}).get("lc0") or {}).get("matrix_profile") + policy_path: Path | None = None + label = "LC0 native-work policy" + if status == "selected" and matrix_profile == "b4-p0-c256k-cold": + policy_path = CANONICAL_NATIVE_WORK_POLICY_PATH + label = "canonical LC0 native-work policy" + elif status == "qualification_candidate" and matrix_profile == "b4-p0-c256k-cold": + policy_path = CANDIDATE_NATIVE_WORK_POLICY_PATH + label = "candidate LC0 native-work policy" + if policy_path is None: return None, None try: - raw = CANDIDATE_NATIVE_WORK_POLICY_PATH.read_bytes() + raw = policy_path.read_bytes() policy = json.loads(raw.decode("utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise MatrixQualificationError( - f"cannot load candidate native-work policy: {exc}" - ) from exc - require(isinstance(policy, dict), "candidate native-work policy root must be an object") + raise MatrixQualificationError(f"cannot load {label}: {exc}") from exc + require(isinstance(policy, dict), f"{label} root must be an object") applies = policy.get("applies_to") expected = { "selection_profile_id": selection.get("profile_id"), - "selection_status": selection.get("status"), - "matrix_profile": ((selection.get("selected") or {}).get("lc0") or {}).get("matrix_profile"), + "selection_status": status, + "matrix_profile": matrix_profile, } - require(applies == expected, "candidate native-work policy applicability drift") + require(applies == expected, f"{label} applicability drift") return policy, hashlib.sha256(raw).hexdigest() @@ -179,7 +190,7 @@ def qualify_lc0_matrix( policy_sha256: str | None = None if native_work_policy is None: - native_work_policy, policy_sha256 = _load_candidate_native_work_policy(selection) + native_work_policy, policy_sha256 = _load_native_work_policy(selection) native_work_policy_id, selected_native_work_exact = _selected_native_work_policy(native_work_policy) confirmation = matrix.get("confirmation") or {} diff --git a/tools/engine_opt/promotion_gate.py b/tools/engine_opt/promotion_gate.py new file mode 100644 index 000000000..acc374047 --- /dev/null +++ b/tools/engine_opt/promotion_gate.py @@ -0,0 +1,285 @@ +"""Fail-closed canonical b4 promotion gate for ENGINE-OPT-V2.""" +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + + +class PromotionGateError(ValueError): + pass + + +ALLOWED_DISPOSITIONS = {"QUALIFIED_EXACT_HOST_ONLY", "QUALIFIED_REUSABLE_DOMAIN"} +EXPECTED_LC0 = { + "matrix_profile": "b4-p0-c256k-cold", + "nn_cache_size": 262144, + "minibatch_size": 4, + "max_prefetch": 0, + "adaptive_prefetch": False, + "defect_telemetry": False, + "warmup_nodes": None, +} +RUNTIME_FILES = ( + "config/allfather.online-engine-opt-v2.json", + "config/allfather.online-hybrid-v2.validation.json", + "config/allfather.m14-j-j8.validation.json", + "config/allfather.m14-j-j9.validation.json", + "config/allfather.m14-j-j10.validation.json", + "config/allfather.orchestrated-v1.validation.json", +) +UNCHANGED_GUARD_FILES = ( + "qualification/online-engine-opt-v2.json", + "qualification/engine-derived-lock.json", + "qualification/lc0-strength.lock.json", + "qualification/lc0-strength-profile.json", + "vendor.lock.json", + "scripts/build-online-engine-opt-v2.sh", + "scripts/build-lc0-strength.sh", + "scripts/lc0-strength-lock.py", +) +ENGINE_TREES = ("engines/stockfish", "engines/reckless", "engines/lc0") + + +def require(ok: bool, message: str) -> None: + if not ok: + raise PromotionGateError(message) + + +def _git(repo_root: Path, *args: str, text: bool = True) -> str | bytes: + """Require accessible Git objects; never guess missing shallow-history identity.""" + try: + result = subprocess.run( + ["git", "-C", str(repo_root), *args], + check=True, capture_output=True, text=text, + ) + except (OSError, subprocess.CalledProcessError) as exc: + detail = getattr(exc, "stderr", None) or str(exc) + if isinstance(detail, bytes): + detail = detail.decode("utf-8", "replace") + raise PromotionGateError( + f"required Git history/object unavailable ({' '.join(args)}): {str(detail).strip()}" + ) from exc + return result.stdout.strip() if text else result.stdout + + +def verify_promotion_git_history(repo_root: Path, base_sha: str) -> dict[str, str]: + """Resolve exact HEAD, its immediate parent and the PR base before checks.""" + require(isinstance(base_sha, str) and len(base_sha) == 40 + and all(c in "0123456789abcdef" for c in base_sha.lower()), + "PR base SHA missing or malformed") + head = str(_git(repo_root, "rev-parse", "--verify", "HEAD^{commit}")) + parent = str(_git(repo_root, "rev-parse", "--verify", "HEAD^")) + base = str(_git(repo_root, "rev-parse", "--verify", f"{base_sha}^{{commit}}")) + require(base == base_sha.lower(), "PR base commit resolution drift") + return {"head_sha": head, "parent_sha": parent, "base_sha": base} + + +def _base_json(repo_root: Path, base_sha: str, path: str) -> dict[str, Any]: + try: + raw = _git(repo_root, "show", f"{base_sha}:{path}", text=False) + assert isinstance(raw, bytes) + value = json.loads(raw.decode("utf-8")) + except (subprocess.CalledProcessError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise PromotionGateError(f"cannot reconstruct PR-base file {path}: {exc}") from exc + require(isinstance(value, dict), f"PR-base file {path} must contain an object") + return value + + +def _load(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PromotionGateError(f"cannot load {path}: {exc}") from exc + require(isinstance(value, dict), f"{path}: root must be object") + return value + + +def _assert_b4_runtime(doc: dict[str, Any], label: str) -> None: + lc0 = ((doc.get("instances") or {}).get("lc0-shadow") or {}) + options = lc0.get("options") or {} + require(options.get("NNCacheSize") == 262144, f"{label}: NNCacheSize drift") + require(options.get("MinibatchSize") == 4, f"{label}: MinibatchSize drift") + require(options.get("MaxPrefetch") == 0, f"{label}: MaxPrefetch drift") + require(options.get("AdaptivePrefetch") is False, f"{label}: AdaptivePrefetch drift") + require(options.get("DefectTelemetry") is False, f"{label}: DefectTelemetry drift") + require("warmup" not in lc0, f"{label}: cold b4 profile unexpectedly has warmup") + + +def compare_promotion_surface(repo_root: Path, base_sha: str) -> dict[str, Any]: + identities = verify_promotion_git_history(repo_root, base_sha) + head = identities["head_sha"] + parent = identities["parent_sha"] + + base_selection = _base_json(repo_root, base_sha, "qualification/engine-opt-v2-selection.json") + current_selection = _load(repo_root / "qualification/engine-opt-v2-selection.json") + base_selected = base_selection.get("selected") or {} + current_selected = current_selection.get("selected") or {} + require( + ((base_selected.get("lc0") or {}).get("matrix_profile")) == "b7-p8-c256k-warm64", + "PR base is not the expected b7 canonical profile", + ) + current_lc0 = current_selected.get("lc0") or {} + for key, value in EXPECTED_LC0.items(): + require(current_lc0.get(key) == value, f"canonical b4 selection {key} drift") + for key in ("stockfish", "reckless", "resource_estimates_ms"): + require(current_selected.get(key) == base_selected.get(key), f"promotion changed {key}") + require( + current_selection.get("qualification") == base_selection.get("qualification"), + "promotion changed matrix qualification thresholds", + ) + + changed_runtime: list[str] = [] + for rel in RUNTIME_FILES: + base = _base_json(repo_root, base_sha, rel) + current = _load(repo_root / rel) + _assert_b4_runtime(current, rel) + normalized = json.loads(json.dumps(current)) + normalized_lc0 = normalized["instances"]["lc0-shadow"] + base_lc0 = base["instances"]["lc0-shadow"] + for option in ("NNCacheSize", "MinibatchSize", "MaxPrefetch", "AdaptivePrefetch", "DefectTelemetry"): + normalized_lc0["options"][option] = base_lc0["options"][option] + if "warmup" in base_lc0: + normalized_lc0["warmup"] = base_lc0["warmup"] + else: + normalized_lc0.pop("warmup", None) + require(normalized == base, f"{rel}: promotion changed behavior outside canonical LC0 profile") + changed_runtime.append(rel) + + unchanged_files: list[str] = [] + for rel in UNCHANGED_GUARD_FILES: + current = (repo_root / rel).read_bytes() + base = _git(repo_root, "show", f"{base_sha}:{rel}", text=False) + require(current == base, f"promotion changed guarded build/source contract {rel}") + unchanged_files.append(rel) + + unchanged_trees: list[str] = [] + for rel in ENGINE_TREES: + base_tree = str(_git(repo_root, "rev-parse", f"{base_sha}:{rel}")) + current_tree = str(_git(repo_root, "rev-parse", f"HEAD:{rel}")) + require(base_tree == current_tree, f"promotion changed engine tree {rel}") + unchanged_trees.append(rel) + + catalog = _load(repo_root / "qualification/resource-profile-catalog-v1.json") + require(catalog.get("selection_enabled") is False, "promotion enabled runtime profile selection") + require(catalog.get("fallback_profile") == "engine-opt-v2", "promotion changed fallback profile") + # Promotion must recognize source-frozen 23-case qualification, not a + # diagnostic-only replay or a loosened witness predicate. + from tools.engine_opt.g3_b4 import B4_POLICY, file_hash, validate_b4_policy + try: + validate_b4_policy(repo_root) + except (ValueError, OSError, KeyError) as exc: + raise PromotionGateError(f"canonical G3 b4 policy invalid: {exc}") from exc + g3_policy_hash = file_hash(repo_root, B4_POLICY) + lc0_profiles = [row for row in catalog.get("profiles", []) if row.get("profile_id") == "lc0/specialist-engine-opt-v2"] + require(len(lc0_profiles) == 1, "catalog LC0 canonical profile missing") + options = {row.get("name"): row.get("value") for row in lc0_profiles[0].get("options", [])} + require(options.get("MinibatchSize") == 4 and options.get("MaxPrefetch") == 0, "catalog LC0 b4 options drift") + runtime_meta = (catalog.get("profile_runtime") or {}).get("lc0/specialist-engine-opt-v2") or {} + require(runtime_meta.get("warmup") is None, "catalog LC0 b4 profile is not cold") + + policy = _load(repo_root / "qualification/engine-opt-v2-native-work-policy.json") + require(policy.get("policy_id") == "lc0-node-stop-contract-v1", "canonical node-stop policy drift") + require(policy.get("applies_to") == { + "selection_profile_id": "engine-opt-v2-selection", + "selection_status": "selected", + "matrix_profile": "b4-p0-c256k-cold", + }, "canonical node-stop policy applicability drift") + require(policy.get("requested_nodes") == 16, "canonical node-stop request drift") + require(policy.get("terminal_counter_semantics") == "lc0.uci_nodes", "canonical node-stop semantics drift") + require(policy.get("require_exact_terminal_counter_repeatability") is False, "canonical node-stop policy re-enabled exact terminal counter repeatability") + + return { + "base_sha": base_sha, + "head_sha": head, + "parent_sha": parent, + "authorized_profile_change": True, + "runtime_files_checked": changed_runtime, + "guard_files_unchanged": unchanged_files, + "engine_trees_unchanged": unchanged_trees, + "runtime_profile_selection_enabled": False, + "g3_b4_contract_verified": True, + "g3_b4_policy_sha256": g3_policy_hash, + } + + +def _domain_digest(report: dict[str, Any]) -> str | None: + domain = ((report.get("details") or {}).get("execution_domain") or {}) + value = domain.get("execution_domain_digest") + return value if isinstance(value, str) else None + + +def evaluate_promotion_gate( + *, + aggregate: dict[str, Any], + host_binding: dict[str, Any], + catalog: dict[str, Any], + promotion_surface: dict[str, Any], + current_head: str, +) -> dict[str, Any]: + require(promotion_surface.get("authorized_profile_change") is True, "promotion surface is not authorized") + require(aggregate.get("source_commit") == current_head, "canonical aggregate is not bound to the exact current head") + require(aggregate.get("evidence_valid") is True, "canonical aggregate evidence invalid") + require(not (aggregate.get("invalid_evidence") or []), "canonical aggregate contains invalid evidence") + require(aggregate.get("profile_qualified") is True, "canonical b4 profile is not qualified") + require(aggregate.get("passed") is True, "canonical b4 aggregate did not pass") + require(aggregate.get("qualification_disposition") in ALLOWED_DISPOSITIONS, "canonical b4 disposition is not promotable") + require(not (aggregate.get("qualification_failures") or []), "canonical b4 aggregate has qualification failures") + lc0 = (aggregate.get("details") or {}).get("lc0") or {} + require(lc0.get("selected_profile") == "b4-p0-c256k-cold", "aggregate did not qualify canonical b4") + require(lc0.get("native_work_policy") == "lc0-node-stop-contract-v1", "aggregate did not use canonical node-stop policy") + lifecycle = (aggregate.get("details") or {}).get("local1_g3") or {} + require(lifecycle.get("validated_games") == 28, "canonical b4 LOCAL-1 lifecycle incomplete") + require(lifecycle.get("g3_authority_qualified") is True, "canonical b4 lacks G3 non-anchor authority qualification") + require(promotion_surface.get("g3_b4_contract_verified") is True and + lifecycle.get("g3_policy_sha256") == promotion_surface.get("g3_b4_policy_sha256"), + "canonical G3 b4 witness policy is not bound to promotion surface") + + digest = _domain_digest(aggregate) + require(isinstance(digest, str) and len(digest) == 64, "canonical aggregate execution-domain digest missing") + + prior_head = promotion_surface.get("parent_sha") + require( + isinstance(prior_head, str) and len(prior_head) == 40, + "promotion surface does not identify the immediately preceding qualification head", + ) + bound = host_binding.get("current_domain_bound_qualification") or {} + require( + bound.get("qualified_head") == prior_head, + "immediately preceding promotion head has not been frozen into host binding", + ) + require(bound.get("qualification_disposition") in ALLOWED_DISPOSITIONS, "promotion host binding is not qualified") + + snapshot = catalog.get("qualification_snapshot") or {} + require( + snapshot.get("qualified_head") == prior_head, + "immediately preceding promotion head has not been frozen into catalog qualification snapshot", + ) + require(snapshot.get("qualification_disposition") in ALLOWED_DISPOSITIONS, "catalog promotion snapshot is not qualified") + require( + snapshot.get("execution_domain_digest") == bound.get("execution_domain_digest"), + "catalog promotion snapshot differs from the frozen promotion-head domain", + ) + + seed = host_binding.get("b4_candidate_promotion_seed") or {} + require(seed.get("canonical_promotion_complete") is True, "promotion seed has not been closed by exact-head requalification") + + return { + "schema_version": 1, + "passed": True, + "disposition": "CANONICAL_B4_PROMOTION_QUALIFIED", + "source_commit": current_head, + "execution_domain_digest": digest, + "promotion_surface": promotion_surface, + "claim_boundary": { + "canonical_b4_promotion": True, + "runtime_profile_selection": False, + "adaptive_stop_promoted": False, + "generic_host_portability": aggregate.get("qualification_disposition") == "QUALIFIED_REUSABLE_DOMAIN", + "strength": False, + "elo": False, + "equal_compute": False, + "deployment": False, + }, + } diff --git a/tools/local_game/integrity.py b/tools/local_game/integrity.py index 615075172..0dce7abf1 100644 --- a/tools/local_game/integrity.py +++ b/tools/local_game/integrity.py @@ -169,6 +169,12 @@ def verify_prerequisites(output: Path, source: dict, p: dict | None = None) -> N if label == "g3-v2": require(report.get("evidence_valid") is True, f"{label}: prerequisite evidence is invalid") + if (p.get("qualification") or {}).get("require_g3_positive_prerequisite") is True: + from tools.engine_opt.g3_b4 import ( + B4_POLICY, file_hash, require_positive_g3, validate_b4_policy, + ) + validate_b4_policy(ROOT) + require_positive_g3(report, policy_sha256=file_hash(ROOT, B4_POLICY)) elif label == "j12": require( report.get("mechanism_valid") is True @@ -500,6 +506,9 @@ def verify_resource_claim(run: Path, manifest: dict) -> dict: """ from common.search_request import parse_go_request from controller.budget import ResourceEnvelope + from controller.resource_measurement import ( + RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED, + ) resource_path = run / "resource.json" route_path = run / "route.json" @@ -516,11 +525,29 @@ def verify_resource_claim(run: Path, manifest: dict) -> dict: stages = resource.get("stages") processes = resource.get("processes") - require(isinstance(stages, list) and stages, "resource stages are missing") - require(isinstance(processes, dict) and processes, "resource process totals are missing") + require(isinstance(stages, list), "resource stages are not an array") + require(isinstance(processes, dict), "resource process totals are not an object") + + failure = resource.get("failure") + preparation_timeout = bool( + isinstance(failure, dict) + and failure.get("code") + == RESOURCE_FAILURE_PREPARATION_BUDGET_EXCEEDED + ) + if preparation_timeout: + require( + isinstance(failure.get("reason"), str) and failure["reason"], + "preparation-timeout resource evidence lacks failure reason", + ) + require(not stages and not processes, + "preparation-timeout resource evidence fabricated primitive rows") + else: + require(stages, "resource stages are missing") + require(processes, "resource process totals are missing") + require(failure is None, "qualified resource evidence carries unexpected failure") stage_cpu = 0.0 - stage_complete = True + stage_complete = bool(stages) for row in stages: require(isinstance(row, dict) and type(row.get("complete")) is bool, "malformed measured stage") @@ -532,7 +559,7 @@ def verify_resource_claim(run: Path, manifest: dict) -> dict: require(row.get("cpu_ms") is None, "incomplete stage carries fabricated CPU") process_cpu = 0.0 - process_complete = True + process_complete = bool(processes) for name, row in processes.items(): require(isinstance(name, str) and name and isinstance(row, dict) and type(row.get("complete")) is bool, "malformed process total") @@ -552,6 +579,8 @@ def verify_resource_claim(run: Path, manifest: dict) -> dict: cpu_complete = bool( resource.get("provider_error") is None and resource.get("interval_error") is None + and bool(stages) + and bool(processes) and stage_complete and process_complete ) @@ -694,6 +723,7 @@ def verify_resource_claim(run: Path, manifest: dict) -> dict: ) wall_within = _number(budget.get("elapsed_ms"), "budget elapsed_ms") <= envelope.wall_ms + 1e-6 physical_within = physical <= envelope.cpu_ms + 1e-6 + controller_cpu_within = controller_cpu <= envelope.controller_overhead_reserve_ms + 1e-6 clock_complete = (manifest.get("clock_outcome") or {}).get("output_within_deadline") is True claim = route.get("envelope_claim") or {} @@ -708,6 +738,8 @@ def verify_resource_claim(run: Path, manifest: dict) -> dict: "physical_measurement_required": bool(settings.get("require_cpu_for_claim") or settings.get("require_gpu_for_claim")), "physical_measurement_qualified": expected_qualified, "physical_cpu_within_envelope": physical_within, + "controller_cpu_within_partition": controller_cpu_within, + "controller_cpu_accounting_complete": True, "clock_output_complete": clock_complete, } for key, value in derived.items(): @@ -715,7 +747,8 @@ def verify_resource_claim(run: Path, manifest: dict) -> dict: expected_claim = all(( clock_complete, anchor_bounded, anchor_reserved, gpu_accounted, within_envelope, within_partitions, settlement_complete, wall_within, - expected_qualified, physical_within, + expected_qualified, physical_within, controller_cpu_within, + claim.get("controller_cpu_accounting_complete") is True, )) require(claim.get("claimed") is expected_claim, "envelope claimed flag contradicts reconstructed evidence") diff --git a/tools/local_game/runner.py b/tools/local_game/runner.py index b2d69e187..eb5be099b 100644 --- a/tools/local_game/runner.py +++ b/tools/local_game/runner.py @@ -225,6 +225,15 @@ def prerequisites(output: Path, p: dict, source_id: dict, execution_domain_path: replay_before, label, ) + if (label == "g3-v2" and + (p.get("qualification") or {}).get("require_g3_positive_prerequisite") is True): + # Keep negative replays, then stop before the expensive 28-game + # campaign if the exact candidate cannot prove non-anchor G3. + from tools.engine_opt.g3_b4 import ( + B4_POLICY, file_hash, require_positive_g3, validate_b4_policy, + ) + validate_b4_policy(ROOT) + require_positive_g3(report_doc, policy_sha256=file_hash(ROOT, B4_POLICY)) return records online_runtime = load(ROOT / "config/allfather.online.cpu-reference.json") diff --git a/tools/local_game/validate.py b/tools/local_game/validate.py index 03ac96fd2..462e9fcc1 100644 --- a/tools/local_game/validate.py +++ b/tools/local_game/validate.py @@ -324,6 +324,37 @@ def session_games( return groups, summaries +def describe_resource_denial(run: Path, route: dict, derived: dict) -> str: + """Explain a LOCAL-1 denial without changing its fail-closed predicate. + + The independent resource verifier has already checked consistency. Report + primitive wall values and the specifically denied claim bits, so a rare + long-tail event can be traced to its exact retained replay directory. + """ + claim = route.get("envelope_claim") or {} + required = ( + "anchor_request_bounded", "anchor_cost_reserved", "gpu_accounted", + "reservations_within_envelope", "specialist_partitions_within_caps", + "specialist_settlement_complete", "work_grant_settlement_complete", + "wall_within_envelope", "physical_measurement_qualified", + "physical_cpu_within_envelope", "controller_cpu_within_partition", + "controller_cpu_accounting_complete", "clock_output_complete", + ) + denied = [key for key in required if claim.get(key) is False] + budget = route.get("budget") or {} + envelope = route.get("envelope") or {} + elapsed = float(budget.get("elapsed_ms", 0.0)) + wall_limit = float(envelope.get("wall_ms", 0.0)) + excess = max(0.0, elapsed - wall_limit) + return ( + "unexpected physical/envelope failure: " + f"replay={run.name}; false_claims={denied}; " + f"resource_qualified={derived.get('qualified')}; " + f"elapsed_ms={elapsed:.3f}; wall_limit_ms={wall_limit:.3f}; " + f"excess_ms={excess:.3f}" + ) + + def validate_g3(item: dict, allow_denial: bool) -> dict: # Existing integrity verifiers reconstruct sealed evidence, not just file hashes. from controller.replay import verify_bundle_integrity @@ -345,7 +376,7 @@ def validate_g3(item: dict, allow_denial: bool) -> dict: claimed = derived["claimed"] if not allow_denial: require(derived["qualified"] is True and claimed, - "unexpected physical/envelope failure") + describe_resource_denial(run, load(run / "route.json"), derived)) authority = final["authority"] require( authority in ("HYBRID", "ANCHOR_FALLBACK", "ANCHOR_CONTROL"), diff --git a/tools/reckless_repro/__init__.py b/tools/reckless_repro/__init__.py new file mode 100644 index 000000000..8605fa785 --- /dev/null +++ b/tools/reckless_repro/__init__.py @@ -0,0 +1 @@ +"""Bounded, non-promoting Reckless reproducibility experiment.""" diff --git a/tools/reckless_repro/analysis.py b/tools/reckless_repro/analysis.py new file mode 100644 index 000000000..430f813dc --- /dev/null +++ b/tools/reckless_repro/analysis.py @@ -0,0 +1,107 @@ +"""Pure, offline verification for non-promoting Reckless build/runtime diagnostics.""" +from __future__ import annotations + +import math +import statistics +from collections import defaultdict +from typing import Any + +PROTOCOL_ID = "reckless-derived-pristine-repro-v1" +REPEATS = 5 +NODES = 200000 + + +class DiagnosticError(ValueError): + """Malformed or incomplete diagnostic evidence; never a promotion decision.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise DiagnosticError(message) + + +def schedule(case_ids: list[str], *, repeats: int = REPEATS) -> list[dict[str, Any]]: + require(repeats == REPEATS, "diagnostic repeat count drift") + require(len(case_ids) == 8 and len(set(case_ids)) == 8, "expected eight distinct frozen positions") + slots: list[dict[str, Any]] = [] + for block in range(repeats): + for variant in ("derived-a", "derived-b"): + for case_index, case_id in enumerate(case_ids): + # ABBA and BAAB cancel linear first/last-mover drift. + order = ([variant, "pristine", "pristine", variant] + if (block + case_index) % 2 == 0 + else ["pristine", variant, variant, "pristine"]) + for order_index, binary in enumerate(order): + slots.append({ + "repeat_index": block, + "variant": variant, + "case_index": case_index, + "case_id": case_id, + "order_index": order_index, + "binary": binary, + }) + return slots + + +def analyze(rows: list[dict[str, Any]], case_ids: list[str]) -> dict[str, Any]: + expected = schedule(case_ids) + require(len(rows) == len(expected), "diagnostic incomplete: planned case count differs") + pairs: dict[tuple[int, str, str], dict[str, list[dict[str, Any]]]] = defaultdict( + lambda: defaultdict(list) + ) + for index, (row, slot) in enumerate(zip(rows, expected)): + require(isinstance(row, dict), f"row {index}: not a mapping") + require(all(row.get(key) == value for key, value in slot.items()), + f"row {index}: ordering/identity drift") + metric = row.get("metrics") + require(isinstance(metric, dict), f"row {index}: missing metrics") + require(metric.get("completed_before_deadline") is True, + f"row {index}: timed out") + work = metric.get("native_work_value") + require(type(work) is int and work >= NODES, + f"row {index}: missing/insufficient native work") + wall = metric.get("wall_ms") + cpu = metric.get("cpu_ms") + require(type(wall) in (int, float) and math.isfinite(wall) and wall > 0, + f"row {index}: invalid wall") + require(type(cpu) in (int, float) and math.isfinite(cpu) and cpu >= 0, + f"row {index}: invalid cpu") + require(isinstance(metric.get("bestmove"), str) and metric["bestmove"], + f"row {index}: bestmove missing") + pairs[(slot["repeat_index"],slot["variant"],slot["case_id"])][slot["binary"]].append(metric) + + out: dict[str, Any] = {} + for variant in ("derived-a", "derived-b"): + block_ratios = [] + mismatches = [] + work_mismatches = [] + for block in range(REPEATS): + ratios = [] + for case in case_ids: + group = pairs[(block, variant, case)] + require(len(group[variant]) == 2 and len(group["pristine"]) == 2, + f"{variant}/{block}/{case}: incomplete pair") + measured = group[variant] + group["pristine"] + moves = {x["bestmove"] for x in measured} + if len(moves) > 1: + mismatches.append(f"{block}/{case}") + counts = {x["native_work_value"] for x in measured} + if len(counts) > 1: + work_mismatches.append(f"{block}/{case}") + left = statistics.median(x["wall_ms"] for x in group[variant]) + right = statistics.median(x["wall_ms"] for x in group["pristine"]) + ratios.append(math.log(left / right)) + block_ratios.append(math.exp(statistics.mean(ratios))) + out[variant] = { + "per_block_derived_over_pristine": block_ratios, + "mean_log_ratio_backtransformed": math.exp(statistics.mean(map(math.log, block_ratios))), + "minimum_block_ratio": min(block_ratios), + "maximum_block_ratio": max(block_ratios), + "bestmove_mismatches": mismatches, + "native_work_mismatches": work_mismatches, + "frozen_gate_applied": False, + "qualification_claim": False, + } + return {"protocol_id": PROTOCOL_ID, "planned": len(expected), + "observed": len(rows), "comparisons": out, + "diagnostic_complete": True, "promotion_evidence": False} diff --git a/tools/reckless_repro/provenance.py b/tools/reckless_repro/provenance.py new file mode 100644 index 000000000..91a1e04e7 --- /dev/null +++ b/tools/reckless_repro/provenance.py @@ -0,0 +1,215 @@ +"""Collect read-only binary, toolchain and host identity for a diagnostic run.""" +from __future__ import annotations + +import hashlib +import json +import os +import platform +import re +import shutil +import stat +import subprocess +import tempfile +from pathlib import Path +from typing import Any + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as fh: + for piece in iter(lambda: fh.read(1024 * 1024), b""): + h.update(piece) + return h.hexdigest() + + + +class BinaryMutationError(RuntimeError): + """An original executable changed after its immediately-post-build hash.""" + + +def binary_identity(path: Path) -> dict[str, Any]: + if not path.is_file(): + raise BinaryMutationError(f"missing executable: {path}") + info = path.stat() + mode = stat.S_IMODE(info.st_mode) + if not (mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)): + raise BinaryMutationError(f"not executable: {path}") + return {"sha256": sha256(path), "size_bytes": info.st_size, "mode": mode} + + +def assert_identity(path: Path, original: dict[str, Any], operation: str) -> None: + after = binary_identity(path) + if after != original: + raise BinaryMutationError( + f"binary mutated {operation}: {path}; before={original}, after={after}" + ) + + +def preflight_binary_hashes(paths: dict[str, Path], manifest: Path) -> dict[str, str]: + """Verify exact executable bytes against immediately-post-build sha256sum.""" + if not manifest.is_file(): + raise BinaryMutationError(f"missing build hash manifest: {manifest}") + resolved = {str(p.resolve()): label for label, p in paths.items()} + recorded: dict[str, str] = {} + for line in manifest.read_text(encoding="utf-8").splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) [ *](.+)", line) + if match is None: + raise BinaryMutationError("malformed build hash manifest") + digest, name = match.groups() + full = str(Path(name).resolve()) + if full not in resolved or full in recorded: + raise BinaryMutationError(f"unexpected or duplicate build hash path: {name}") + recorded[full] = digest + if set(recorded) != set(resolved): + raise BinaryMutationError("build hash manifest must cover all three executables") + for full, label in resolved.items(): + if binary_identity(paths[label])["sha256"] != recorded[full]: + raise BinaryMutationError(f"built executable mutated before run: {label}") + return {label: recorded[full] for full, label in resolved.items()} + + +def version(command: list[str]) -> dict[str, Any]: + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=15, check=False) + return { + "command": command, + "returncode": result.returncode, + "stdout": result.stdout.strip()[:12000], + "stderr": result.stderr.strip()[:4000], + } + except (OSError, subprocess.TimeoutExpired) as exc: + return {"command": command, "error": f"{type(exc).__name__}: {exc}"} + + +def elf_text(path: Path) -> dict[str, Any]: + original = binary_identity(path) + try: + with tempfile.TemporaryDirectory() as tmp: + copy = Path(tmp) / "inspection-elf" + section = Path(tmp) / "text.bin" + shutil.copy2(path, copy) + try: + result = subprocess.run( + ["objcopy", "--dump-section", f".text={section}", str(copy)], + capture_output=True, text=True, timeout=20, check=False, + ) + if result.returncode != 0 or not section.is_file(): + return {"available": False, "reason": result.stderr.strip()[:500]} + return {"available": True, "sha256": sha256(section), + "size_bytes": section.stat().st_size} + except (OSError, subprocess.TimeoutExpired) as exc: + return {"available": False, "reason": str(exc)[:500]} + finally: + assert_identity(path, original, "during disposable-copy objcopy") + + +def first_pass_mutator_probe(path: Path) -> dict[str, Any]: + """Attribute possible first-pass ELF rewriting on a throwaway fresh copy only.""" + original = binary_identity(path) + try: + with tempfile.TemporaryDirectory() as tmp: + probe = Path(tmp) / "fresh-probe.elf" + section = Path(tmp) / "text.bin" + shutil.copy2(path, probe) + def probe_identity() -> dict[str, Any]: + info = probe.stat() + return {"sha256": sha256(probe), "size_bytes": info.st_size, + "mode": stat.S_IMODE(info.st_mode)} + before = probe_identity() + operations = {} + for name, command in ( + ("objcopy", ["objcopy", "--dump-section", f".text={section}", str(probe)]), + ("readelf", ["readelf", "-n", str(probe)]), + ): + try: + result = subprocess.run( + command, capture_output=True, text=True, timeout=20, check=False, + ) + status = {"returncode": result.returncode, + "stderr": result.stderr.strip()[:500]} + except (OSError, subprocess.TimeoutExpired) as exc: + status = {"error": f"{type(exc).__name__}: {exc}"} + after = probe_identity() + operations[name] = { + **status, "before": before, "after": after, + "mutated_probe": before != after, + } + before = after + return {"copy_only": True, "operations": operations, + "first_mutator": next( + (name for name in ("objcopy", "readelf") + if operations[name]["mutated_probe"]), None, + )} + finally: + assert_identity(path, original, "during first-pass disposable attribution probe") + + +def binaries(paths: dict[str, Path]) -> dict[str, Any]: + result = {} + for label, file in sorted(paths.items()): + original = binary_identity(file) + probe = first_pass_mutator_probe(file) + assert_identity(file, original, "after first-pass probe") + text = elf_text(file) + assert_identity(file, original, "after elf_text") + notes = version(["readelf", "-n", str(file)]) + assert_identity(file, original, "after readelf") + result[label] = { + "path": str(file.resolve()), + **original, + "elf_text": text, + "elf_notes": notes, + "first_pass_mutator_probe": probe, + "inspection_preserved_original": True, + } + return result + + +def host() -> dict[str, Any]: + cpu_model = None + microcode = None + try: + for line in Path("/proc/cpuinfo").read_text().splitlines(): + if cpu_model is None and line.startswith("model name"): + cpu_model = line.split(":", 1)[1].strip() + if microcode is None and line.startswith("microcode"): + microcode = line.split(":", 1)[1].strip() + except OSError: + pass + return { + "platform": platform.platform(), + "machine": platform.machine(), + "cpu_model_name": cpu_model, + "microcode": microcode, + "logical_cpus": os.cpu_count(), + "allowed_cpus_before_pin": sorted(os.sched_getaffinity(0)), + "runner_image_os": os.getenv("ImageOS"), + "runner_image_version": os.getenv("ImageVersion"), + "runner_environment": os.getenv("ALLFATHER_RUNNER_ENVIRONMENT"), + "kernel": platform.release(), + "rustc": version(["rustc", "--version", "--verbose"]), + "cargo": version(["cargo", "--version"]), + "linker": version(["ld", "--version"]), + "resolved_build_rustflags": "-Ctarget-cpu=x86-64", + } + + +def provenance(paths: dict[str, Path], model: Path, + build_logs: dict[str, Path], host_report: Path) -> dict[str, Any]: + if not model.is_file() or not host_report.is_file(): + raise ValueError("verified Reckless model or J2 host report missing") + doc = json.loads(host_report.read_text(encoding="utf-8")) + if not isinstance(doc, dict): + raise ValueError("J2 host report root is not an object") + return { + "binaries": binaries(paths), + "network": {"sha256": sha256(model), "size_bytes": model.stat().st_size}, + "host": host(), + "j2_host_report_sha256": sha256(host_report), + "build_logs": { + key: {"sha256": sha256(value), "size_bytes": value.stat().st_size} + for key, value in build_logs.items() + }, + "exact_binary_identity_required": True, + "historical_artifacts_used": False, + } diff --git a/tools/resource_lab/candidate_matrix.py b/tools/resource_lab/candidate_matrix.py index 8a2bf8e11..b17b049ae 100644 --- a/tools/resource_lab/candidate_matrix.py +++ b/tools/resource_lab/candidate_matrix.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from dataclasses import dataclass from pathlib import Path @@ -51,6 +52,19 @@ }, } +# Historical specifications keep their original logical reference paths. Once a +# live canonical input advances, those paths resolve to immutable snapshots. +REFERENCE_CONTRACT_ARCHIVE_PATHS_BY_LAB = { + "resource-lab-v1": { + "catalog": "qualification/resource-profile-catalog-v1-j7-baseline.json", + "runtime": "config/allfather.online-hybrid-v2.validation-j7-baseline.json", + "build_policy": "qualification/online-engine-opt-v2-j7-baseline.json", + }, +} +HISTORICAL_REFERENCE_EVIDENCE_BY_LAB = { + "resource-lab-v1": "qualification/resource-profile-evidence-v1.json", +} + class ResourceLabSpecError(ValueError): pass @@ -654,12 +668,94 @@ def _normalized_reference_options( return options +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def resolve_reference_contract_paths(spec: LabSpec) -> dict[str, str]: + """Resolve logical reference paths without mutating historical specifications.""" + declared = dict(_object(spec.raw["reference_contract"], "reference_contract")) + expected = REFERENCE_CONTRACT_PATHS_BY_LAB.get(spec.lab_id) + require(expected is not None, f"unsupported reference lab id: {spec.lab_id!r}") + require(declared == expected, f"{spec.lab_id} reference contract paths drift") + resolved = REFERENCE_CONTRACT_ARCHIVE_PATHS_BY_LAB.get(spec.lab_id, expected) + require( + set(resolved) == set(declared), + f"{spec.lab_id} resolved reference contract keys drift", + ) + return dict(resolved) + + +def reference_contract_manifest(spec: LabSpec, root: Path) -> dict[str, Any]: + """Bind logical paths to the exact source-controlled bytes being validated.""" + declared = dict(_object(spec.raw["reference_contract"], "reference_contract")) + resolved = resolve_reference_contract_paths(spec) + return { + key: { + "path": declared[key], + "resolved_path": resolved[key], + "sha256": _sha256_file(root / resolved[key]), + } + for key in declared + } + + +def _validate_historical_reference_archive( + spec: LabSpec, + root: Path, + resolved: Mapping[str, str], +) -> None: + evidence_rel = HISTORICAL_REFERENCE_EVIDENCE_BY_LAB.get(spec.lab_id) + if evidence_rel is None: + return + try: + evidence = json.loads((root / evidence_rel).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ResourceLabSpecError( + f"{spec.lab_id}: cannot load frozen historical evidence: {exc}" + ) from exc + + source = _object(evidence.get("source"), "historical evidence source") + frozen_spec = _object(source.get("lab_spec"), "historical lab spec binding") + spec_path = frozen_spec.get("path") + spec_sha = frozen_spec.get("sha256") + require( + isinstance(spec_path, str) and isinstance(spec_sha, str), + f"{spec.lab_id}: historical lab spec binding missing", + ) + require( + _sha256_file(root / spec_path) == spec_sha, + f"{spec.lab_id}: historical lab spec SHA differs from frozen evidence", + ) + + frozen_ref = _object( + source.get("reference_contract"), + "historical reference contract binding", + ) + for key, logical_path in spec.raw["reference_contract"].items(): + require( + frozen_ref.get(f"{key}_path") == logical_path, + f"{spec.lab_id}: historical {key} logical path differs from frozen evidence", + ) + expected_sha = frozen_ref.get(f"{key}_sha256") + require( + isinstance(expected_sha, str) + and _sha256_file(root / resolved[key]) == expected_sha, + f"{spec.lab_id}: archived {key} SHA differs from frozen evidence", + ) + + def validate_reference_contract(spec: LabSpec, root: Path) -> None: - """Prove the hand-written laboratory reference matches its declared runtime.""" - ref = _object(spec.raw["reference_contract"], "reference_contract") - runtime = json.loads((root / str(ref["runtime"])).read_text(encoding="utf-8")) + """Prove the laboratory reference matches the correct frozen runtime.""" + resolved = resolve_reference_contract_paths(spec) + _validate_historical_reference_archive(spec, root, resolved) + runtime = json.loads((root / resolved["runtime"]).read_text(encoding="utf-8")) build_policy = json.loads( - (root / str(ref["build_policy"])).read_text(encoding="utf-8") + (root / resolved["build_policy"]).read_text(encoding="utf-8") ) instances = _object(runtime.get("instances"), "reference runtime instances") builds = _object(build_policy.get("builds"), "reference build policy") @@ -668,12 +764,12 @@ def validate_reference_contract(spec: LabSpec, root: Path) -> None: if spec.lab_id == "resource-lab-v1": from controller.resource_profile_catalog import load_resource_profile_catalog - catalog = load_resource_profile_catalog(root / str(ref["catalog"])) + catalog = load_resource_profile_catalog(root / resolved["catalog"]) elif spec.lab_id == "resource-lab-v2": from controller.engine_opt_profile import validate_reference selection = json.loads( - (root / str(ref["selection"])).read_text(encoding="utf-8") + (root / resolved["selection"]).read_text(encoding="utf-8") ) validate_reference( build_policy, diff --git a/tools/resource_lab/preflight.py b/tools/resource_lab/preflight.py index 64e34d2ff..d026c8065 100644 --- a/tools/resource_lab/preflight.py +++ b/tools/resource_lab/preflight.py @@ -18,6 +18,7 @@ expand_candidates, expand_compositions, load_lab_spec, + reference_contract_manifest, validate_reference_contract, ) from .qualify import ( @@ -40,6 +41,10 @@ def validate_preflight(root: Path, spec_path: Path) -> dict[str, Any]: spec = load_lab_spec(spec_path) validate_reference_contract(spec, ROOT) manifest = load(root / "manifest.json") + require( + manifest.get("reference_contract") == reference_contract_manifest(spec, ROOT), + "preflight reference contract manifest drift", + ) source = source_identity(ROOT) require(manifest.get("source_commit") == source["commit"], "preflight source is not exact head") require(manifest.get("cpu_measurement") == spec.raw["cpu_measurement"], "preflight CPU policy drift") diff --git a/tools/resource_lab/qualify.py b/tools/resource_lab/qualify.py index d1683cdf9..a68b735d1 100644 --- a/tools/resource_lab/qualify.py +++ b/tools/resource_lab/qualify.py @@ -25,6 +25,7 @@ expand_candidates, expand_compositions, load_lab_spec, + reference_contract_manifest, stage_a_attempt_plan, stage_b_attempt_plan, validate_reference_contract, @@ -305,14 +306,10 @@ def qualify(root: Path, spec_path: Path) -> dict[str, Any]: reference_manifest = manifest.get("reference_contract") require(isinstance(reference_manifest, dict), "reference contract manifest missing") - for key, path in spec.raw["reference_contract"].items(): - retained = reference_manifest.get(key) - require(isinstance(retained, dict), f"reference contract {key} missing") - require(retained.get("path") == path, f"reference contract {key} path drift") - require( - retained.get("sha256") == sha256(ROOT / path), - f"reference contract {key} SHA drift", - ) + require( + reference_manifest == reference_contract_manifest(spec, ROOT), + "reference contract manifest drift", + ) corpus_path = ROOT / spec.corpus cases = load_epd(corpus_path) diff --git a/tools/resource_lab/runner.py b/tools/resource_lab/runner.py index 332a6f74b..9155517c1 100644 --- a/tools/resource_lab/runner.py +++ b/tools/resource_lab/runner.py @@ -27,6 +27,7 @@ expand_candidates, expand_compositions, load_lab_spec, + reference_contract_manifest, stage_a_attempt_plan, stage_b_attempt_plan, validate_reference_contract, @@ -237,17 +238,6 @@ def run_isolated( return row -def _reference_contract_manifest(spec, spec_path: Path) -> dict[str, Any]: - root = spec_path.parents[1] - return { - key: { - "path": path, - "sha256": sha256(root / path), - } - for key, path in spec.raw["reference_contract"].items() - } - - def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -326,7 +316,7 @@ def main() -> int: "sha256": sha256(ROOT / spec.corpus), "cases": [case.case_id for case in cases], }, - "reference_contract": _reference_contract_manifest(spec, spec_path), + "reference_contract": reference_contract_manifest(spec, ROOT), "candidate_bundle": candidate_bundle, "execution_domain": execution_domain, "attempt_policy": spec.raw["attempt_policy"],