diff --git a/.github/workflows/receipt-gate.yml b/.github/workflows/receipt-gate.yml deleted file mode 100644 index bea642e..0000000 --- a/.github/workflows/receipt-gate.yml +++ /dev/null @@ -1,120 +0,0 @@ -name: Commit CI Preflight - -on: - pull_request_target: - types: [opened, synchronize, reopened, ready_for_review] - -permissions: - contents: read - statuses: write - -concurrency: - group: commit-ci-preflight-${{ github.event.pull_request.number }} - cancel-in-progress: true - -env: - CCP_EXPECTED_COMMIT: ${{ github.event.pull_request.head.sha }} - CCP_STATUS_CONTEXT: commit-ci-preflight/receipt - -jobs: - receipt: - name: Verify local receipt - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - timeout-minutes: 6 - steps: - - name: Publish pending status on the exact pull-request commit - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - [[ "$CCP_EXPECTED_COMMIT" =~ ^[0-9a-f]{40}$ ]] - gh api --method POST \ - "repos/${GITHUB_REPOSITORY}/statuses/${CCP_EXPECTED_COMMIT}" \ - -f state=pending \ - -f context="$CCP_STATUS_CONTEXT" \ - -f description="Local receipt verification started" \ - -f target_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - >/dev/null - - - name: Check out the trusted base revision - id: trusted - continue-on-error: true - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ github.event.pull_request.base.sha }} - path: trusted - fetch-depth: 1 - persist-credentials: false - lfs: false - submodules: false - - - name: Retrieve the commit-bound evidence branch - id: evidence - if: steps.trusted.outcome == 'success' - continue-on-error: true - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ccp-evidence/${{ github.event.pull_request.head.sha }} - path: evidence - fetch-depth: 1 - persist-credentials: false - lfs: false - submodules: false - sparse-checkout: | - .ccp/receipt.json - sparse-checkout-cone-mode: false - - - name: Build the trusted Rust verifier only - id: build - if: steps.evidence.outcome == 'success' - continue-on-error: true - working-directory: trusted - run: cargo build --locked --release --bin commit-ci-preflight - - - name: Verify receipt integrity and repository policy - id: verify - if: steps.build.outcome == 'success' - continue-on-error: true - working-directory: trusted - run: | - scripts/github-receipt-gate.sh \ - target/release/commit-ci-preflight \ - ../evidence/.ccp/receipt.json \ - .commit-ci-policy.toml \ - "$CCP_EXPECTED_COMMIT" - - - name: Publish final commit status - if: always() - env: - GH_TOKEN: ${{ github.token }} - TRUSTED_OUTCOME: ${{ steps.trusted.outcome }} - EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} - BUILD_OUTCOME: ${{ steps.build.outcome }} - VERIFY_OUTCOME: ${{ steps.verify.outcome }} - run: | - set -euo pipefail - state=failure - description="Receipt missing or verification failed" - if [[ "$TRUSTED_OUTCOME" == "success" && - "$EVIDENCE_OUTCOME" == "success" && - "$BUILD_OUTCOME" == "success" && - "$VERIFY_OUTCOME" == "success" ]]; then - state=success - description="Commit-bound local receipt accepted" - elif [[ "$VERIFY_OUTCOME" != "failure" ]]; then - { - echo "### Commit CI Preflight" - echo - echo "FAIL: trusted verifier build or evidence retrieval did not complete." - } >> "$GITHUB_STEP_SUMMARY" - echo "::error title=Commit CI Preflight::Receipt retrieval or trusted verifier build failed; see the job summary." - fi - gh api --method POST \ - "repos/${GITHUB_REPOSITORY}/statuses/${CCP_EXPECTED_COMMIT}" \ - -f state="$state" \ - -f context="$CCP_STATUS_CONTEXT" \ - -f description="$description" \ - -f target_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - >/dev/null - [[ "$state" == "success" ]] diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml new file mode 100644 index 0000000..d0aa0cd --- /dev/null +++ b/.github/workflows/rust-ci.yml @@ -0,0 +1,109 @@ +name: Rust CI + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: rust-ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + quality: + name: Format, lint, docs, and metadata + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Check out the exact source commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + fetch-depth: 1 + lfs: false + submodules: false + - name: Resolve the pinned Rust toolchain + run: rustup show active-toolchain + - name: Check formatting + run: cargo fmt --all -- --check + - name: Deny Clippy warnings + run: cargo clippy --locked --workspace --all-targets --all-features -- -D warnings + - name: Build documentation without dependency docs + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --locked --workspace --all-features --no-deps + - name: Verify release metadata + run: cargo run --locked --quiet --example generate_release_metadata -- --check + + test: + name: Test (${{ matrix.runner }}) + strategy: + fail-fast: false + matrix: + runner: [ubuntu-24.04, macos-15] + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + steps: + - name: Check out the exact source commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + fetch-depth: 1 + lfs: false + submodules: false + - name: Resolve the pinned Rust toolchain + run: rustup show active-toolchain + - name: Run the complete deterministic suite + env: + CCP_TEST_ROOT: ${{ runner.temp }}/ccp-tests + run: cargo test --locked --workspace --all-targets --all-features + + windows_compile: + name: Compile tests (windows-2025) + runs-on: windows-2025 + timeout-minutes: 45 + steps: + - name: Check out the exact source commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + fetch-depth: 1 + lfs: false + submodules: false + - name: Resolve the pinned Rust toolchain + run: rustup show active-toolchain + - name: Compile every test target without execution + run: cargo test --locked --workspace --all-targets --all-features --no-run + + gate: + name: Hosted CI gate + if: always() + needs: [quality, test, windows_compile] + permissions: {} + runs-on: ubuntu-24.04 + timeout-minutes: 1 + steps: + - name: Require every hosted lane + env: + QUALITY_RESULT: ${{ needs.quality.result }} + TEST_RESULT: ${{ needs.test.result }} + WINDOWS_COMPILE_RESULT: ${{ needs.windows_compile.result }} + run: | + set -euo pipefail + [[ "$QUALITY_RESULT" == "success" ]] + [[ "$TEST_RESULT" == "success" ]] + [[ "$WINDOWS_COMPILE_RESULT" == "success" ]] diff --git a/CHANGELOG.md b/CHANGELOG.md index e9548e0..ef1c886 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,18 @@ Semantic Versioning after its first public release. ### Added +- Added auditable economic qualification guidance with machine-readable August + 2026 case-study inputs for Matryca-Knowledge and the private case published as + Matryca-Brain. The documentation separates quota preservation and GitHub + compute avoided from still-uncertified net savings. + +- Replaced this public repository's ordinary per-PR receipt gate with standard + GitHub-hosted Rust CI: Linux and macOS execute the complete deterministic + suite, while Windows compiles every test target without claiming runtime + qualification. Receipt verification remains an optional product capability + for economically qualified private workloads or separately justified + non-economic evidence. + - Unix cache generations now preserve ordinary opaque payload links during inventory and reuse; strict control paths remain link-free, failed preparation owns cleanup before reuse, and native candidate qualification remains pending. - Added the adoption-help issue-form source and a 1280 × 640 social-preview PNG diff --git a/README.md b/README.md index 6b0e25a..c4e0a47 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ growing without weakening review, security, or platform coverage. Start here: - [PR #71 case study](docs/CASE_STUDY_PR71.md) — a bounded, public example. +- [Economic qualification](docs/ECONOMIC_QUALIFICATION.md) — measured private-repository savings and limits. - [Clean-room tutorial](docs/TUTORIAL.md) — produce and verify a first receipt. - [Adoption guide](docs/ADOPTION_GUIDE.md) — decide whether CCP fits your repository. @@ -128,11 +129,22 @@ repository policy. For installation, checksum verification, and local candidate archives, see [the installation guide](docs/INSTALLATION.md). -## Dogfooding proof +## Repository CI policy -This repository uses CCP for its own bounded qualification workflow. See the -[PR #71 case study](docs/CASE_STUDY_PR71.md) for the exact public anchors and -the limits of what that evidence proves. +This public repository uses standard GitHub-hosted CI for ordinary pull +requests. Linux and macOS execute the complete deterministic suite; Windows +compiles every test target without executing runtime and cache paths whose +native qualification remains pending. Public standard runners are free. +Replacing those jobs with local execution produces no billable public-runner +savings. It would consume maintainer time and Mac resources without an +economic return, so the per-PR receipt requirement is retired here. + +The historical CCP receipts remain valid +evidence for their exact commits; they are not current pull-request gates. The +[PR #71 case study](docs/CASE_STUDY_PR71.md) +records the exact public anchors and limits. CCP can still qualify its own +release candidates or bounded native behavior when a separate non-economic +reason and exact authorization justify that work. ## What makes it different @@ -157,10 +169,29 @@ Official project descriptions used for this comparison: These projects solve overlapping but different problems. Commit CI Preflight does not claim feature superiority or full GitHub Actions parity. -## Cost example (assumptions only) +## When CCP actually saves money + +CCP produces no billable runner saving when it replaces standard hosted CI in +a public repository. In a private repository it can preserve included quota +and avoid billed compute, but only when the same required checks move off the +hosted runners and the retained receipt gate remains cheaper. + +Measured August 2026 examples: + +| Case | Observed result | +|---|---| +| This public repository | `$0` billable saving from replacing ordinary standard hosted PR jobs; hosted CI remains the default | +| Matryca-Knowledge | 28 rounded runner-minutes avoided across 30 gate events, approximately 46.7% of the former minimum and `$0.168` at the billed Linux rate | +| Matryca-Brain | 22 CCP-guarded local attempts correspond to an estimated 635.1 hosted Linux minutes and `$3.81` of GitHub compute avoided | + +These are bounded observations, not a universal savings promise. The detailed +inputs, arithmetic, privacy boundary, and limits are in +[Economic qualification and measured savings](docs/ECONOMIC_QUALIFICATION.md). + +## Cost model -Assumption (example only): pricing and quotas vary by account and date, so treat -the formula below as a planning aid. +Pricing and quotas vary by account and date, so treat the formula below as a +planning aid and replace every input with current measurements. Remote bill estimate (assumption): diff --git a/docs/BENCHMARK_AND_PARITY.md b/docs/BENCHMARK_AND_PARITY.md index ed64481..789c549 100644 --- a/docs/BENCHMARK_AND_PARITY.md +++ b/docs/BENCHMARK_AND_PARITY.md @@ -125,6 +125,10 @@ their current plan, runner multiplier, included quota, taxes, and local energy or hardware assumptions. Commit CI Preflight does not hard-code a currency or claim guaranteed savings. +For measured private-repository examples, including the distinction between +remote compute avoided and uncertified net savings, see +[Economic qualification and measured savings](ECONOMIC_QUALIFICATION.md). + Quality-critical remote responsibilities remain separate: event identity, review/permission policy, secret-backed integration, deployment environments, and native platforms not covered by accepted local receipts. diff --git a/docs/BETA_SUPPORT.md b/docs/BETA_SUPPORT.md index e13a666..cae0ecc 100644 --- a/docs/BETA_SUPPORT.md +++ b/docs/BETA_SUPPORT.md @@ -14,6 +14,7 @@ without an operator review of the threat model, policy, and local runtime. | Fixed benchmark on native macOS arm64 | `QUALIFIED` | Deterministic correctness digest matched | | Fixed benchmark on native Linux x86_64 | `QUALIFIED` | Standard `ubuntu-24.04` runner receipt independently verified | | Fixed benchmark on native Windows x86_64 | `QUALIFIED` | Standard `windows-2025` runner receipt independently verified | +| Ordinary hosted pull-request CI | `ACTIVE_BOUNDARY` | Linux and macOS execute the complete deterministic suite; Windows compiles every test target without execution | | Unsigned macOS arm64 archive in GitHub prerelease `v0.1.0-rc.1` | `PUBLISHED_RC` | Exact tagged source, archive checksum, SBOM, notices, and rollback evidence | | Complete project `run` path on Linux x86_64 | `PENDING` | Benchmark evidence is not full runtime qualification | | Complete project `run` path on Windows x86_64 | `PENDING` | Benchmark evidence is not full runtime qualification | diff --git a/docs/ECONOMIC_QUALIFICATION.md b/docs/ECONOMIC_QUALIFICATION.md new file mode 100644 index 0000000..e2df341 --- /dev/null +++ b/docs/ECONOMIC_QUALIFICATION.md @@ -0,0 +1,173 @@ +# Economic qualification and measured savings + +Commit CI Preflight saves GitHub Actions money only when it replaces billable +hosted work. Standard GitHub-hosted runners are free for public repositories, +and self-hosted runners and standard hosted Dependabot work do not consume +billable Actions minutes. Private repositories receive an included allowance; +usage beyond it is billed. + +The current official references are: + +- [GitHub Actions billing](https://docs.github.com/en/billing/concepts/product-billing/github-actions); +- [job execution time and per-job rounding](https://docs.github.com/en/actions/how-tos/monitor-workflows/view-job-execution-time); +- [included product usage](https://docs.github.com/en/billing/reference/product-usage-included). + +At the observation date, GitHub documented 3,000 included Actions minutes for +GitHub Pro, `$0.006` per minute for a standard two-core Linux runner, and +rounding of every private hosted job up to the next whole minute. Pricing and +allowances can change; recheck them before applying these examples. + +## What counts as savings + +Keep four quantities separate: + +1. **Remote compute avoided**: hosted runner minutes not executed. +2. **Included quota preserved**: avoided minutes that would have consumed the + monthly allowance. +3. **GitHub charges avoided**: avoided minutes after the allowance was + exhausted, multiplied by the applicable runner rate. +4. **Net savings**: GitHub charges avoided minus electricity, hardware + amortization, operator time, maintenance, and local failures. + +The first three are measured or bounded-estimated below, as labeled. Net +savings are not yet certified. CCP should not be run solely to claim savings +when local operating cost is unknown or greater than the avoided GitHub charge. + +## Results at a glance + +| Case | Measured comparison | GitHub compute avoided | Claim | +|---|---:|---:|---| +| This public repository | Standard public hosted CI versus local replacement | `$0` billable savings | Keep hosted CI for ordinary PRs | +| Matryca-Knowledge | 30 receipt-gate events versus a two-job hosted matrix | 28 rounded minutes; about `$0.168` | Measured billed-phase saving | +| Matryca-Brain | 22 CCP-guarded self-hosted attempts scaled from 194 executed hosted PR validations | about 635.1 minutes; about `$3.81` | Bounded counterfactual estimate | + +The machine-readable inputs and hand-checked expected results are in +[`economic-case-studies-2026-08.json`](evidence/economic-case-studies-2026-08.json). + +## Case study: Matryca-Knowledge + +The private repository qualified exact commit +`2dae025a3dde112b7b35a24ddd9b0514d1f1ee7c` locally on Python 3.11 and 3.13. +Each runtime completed 279 tests, 83% aggregate coverage, and all 19 declared +checks. The independently verified receipt IDs were: + +- Python 3.11: `sha256:a9465bf170eb110bdd4ed29b3f977bd4dc3bc21a617ad01ca54d0c357d85350d`; +- Python 3.13: `sha256:cee6416805b7b78ad23d385fa8415cd64f4f2615dd9c8df04b016fa6ee58273a`. + +The August observation recorded 111 Linux minutes, `$0.666` gross compute, +`$0.096` of allowance or discount, and `$0.570` billed. The old path normally +created two hosted jobs per event; the receipt gate retains one short hosted +job. Because GitHub rounds each job separately, the ordinary event moves from +at least two rounded minutes to one. + +Across 30 observed receipt-gate runs, the gate used 538 runner-seconds. Its +successful median was 15 seconds and it consumed approximately 32 rounded +runner-minutes, compared with at least 60 for the former two-job matrix: + +```text +avoided_minutes = 60 - 32 = 28 +quota_reduction = 28 / 60 = 46.7% +avoided_billed_compute = 28 * $0.006 = $0.168 +``` + +This proves a small but real GitHub-side saving during a billed phase. It does +not prove that operating the Mac solely for this saving is economical. The +local qualification also served provenance and reproducibility needs. + +## Case study: Matryca-Brain + +“Matryca-Brain” is the approved public name for a private-repository case +study. The underlying private repository and raw billing records are not +published. + +### Observed August usage + +The GitHub billing dashboard and Actions API were inspected on 2026-08-30: + +| Observation | Value | +|---|---:| +| Total Actions runs | 358 | +| PR Validation runs | 314 | +| Dependency-update runs | 22 | +| CCP-guarded self-hosted runs | 22 | +| Linux runner minutes | 5,600 | +| Gross Linux compute | `$33.60` | +| Gross storage | `$0.12` | +| Gross total | `$33.72` | +| Included allowance or discounts | `$18.34` | +| Billed total | `$15.38` | + +Before the cutover there were 286 PR Validation runs across 207 distinct +heads: 139 succeeded, 43 failed, 12 were cancelled, and 92 were skipped. The +194 non-skipped executions are the comparable hosted-workflow population. + +The 22 local/self-hosted attempts were protected by CCP and covered 19 distinct +commits. Seventeen succeeded, one failed, and four were cancelled. Failed and +cancelled attempts remain in the denominator: hiding them would overstate the +economics. The operator reports that the local ten-stage Ready mirror preserved +the number and quality of tests during the move offline. This is the +comparability input for the estimate, not a claim that a local container is a +GitHub-hosted runner. + +### Savings calculation + +Dependency-update runs are reported for completeness but excluded from this +comparison because they are not the PR Validation workload replaced by CCP; +GitHub also documents standard hosted Dependabot work as free. No other paid +workflow family was present in the observed run inventory. The observed hosted +average includes failed and cancelled PR runs, so it is more conservative than +using successful full-suite runs only: + +```text +observed_minutes_per_executed_pr = 5,600 / 194 + = 28.866 minutes + +estimated_avoided_minutes = 28.866 * 22 + = 635.1 minutes + +estimated_avoided_github_compute = 635.1 * $0.006 + = $3.81 +``` + +From 2026-08-20 through 2026-08-30 the billing dashboard recorded `$0.00` of +new gross usage for Matryca-Brain. That is a useful operational observation, +not permission to extrapolate ten quiet days into a fabricated monthly saving. +The `$3.81` figure is instead tied to the number of comparable CCP attempts. + +The repository's observed `$15.38` billed total is historical spend, not an +amount wholly saved by CCP. The study claims approximately `$3.81` of avoided +GitHub Linux compute under the documented comparison. Net savings remain +unknown until local operating costs are priced. + +## Reusable decision rule + +For a private repository, measure rather than assume: + +```text +avoided_hosted_cost + = replaced_rounded_job_minutes * applicable_runner_rate + +net_savings + = avoided_hosted_cost + - retained_remote_gate_cost + - local_energy_cost + - hardware_amortization + - operator_and_maintenance_cost +``` + +Adopt CCP for economic reasons only when the same required checks remain +covered, the receipt gate is materially cheaper than the replaced hosted jobs, +and the conservative net result is positive. Otherwise use ordinary hosted CI, +especially for public repositories where standard runners already cost zero. + +## Evidence limits + +- Billing-dashboard values are an observed current-cycle snapshot, not a final + tax invoice. +- Private run URLs and raw logs are intentionally omitted. +- Receipt integrity proves the declared checks for an exact commit; it does not + prove producer identity or universal GitHub Actions parity. +- Per-job rounding means workflow wall time cannot be substituted for billable + runner-minutes. +- Future pricing, included quota, runner type, workflow shape, or local costs + require a fresh calculation. diff --git a/docs/GITHUB_GATE.md b/docs/GITHUB_GATE.md index 752f7ea..77214e3 100644 --- a/docs/GITHUB_GATE.md +++ b/docs/GITHUB_GATE.md @@ -1,8 +1,8 @@ -# Lightweight GitHub receipt gate +# Optional lightweight GitHub receipt gate ## Purpose -The repository-native workflow verifies locally produced evidence without +This product capability verifies locally produced evidence without re-running the project's heavy checks on GitHub-hosted compute. It retains only the remote facts that the local runner cannot assert: the pull-request event, the exact head commit, the base repository, and publication of a GitHub commit @@ -14,6 +14,16 @@ evidence checkout is treated as untrusted data and is limited to the canonical receipt path. The verifier rejects malformed, oversized, stale, digest-invalid, or policy-invalid input. +It is not active for this public repository's ordinary pull requests, which use +the complete standard GitHub-hosted Rust workflow instead. An adopting +repository should enable a receipt gate only when CCP is economically qualified +from measured billing, replaced workflow minutes, and local cost, or when a +separately documented non-economic requirement justifies it. + +The public [economic qualification guide](ECONOMIC_QUALIFICATION.md) records +the current decision rule and two bounded private-repository measurements. It +does not convert those observations into a universal savings guarantee. + ## Why the receipt uses a separate branch A receipt for commit X cannot be added to commit X: adding the file creates a @@ -36,10 +46,10 @@ identity. A repository writer could replace or fabricate unsigned evidence. Identity-bound signing remains a later plan tranche and must not be inferred from a green v1 status. -## Repository setup +## Adopting-repository setup -The active workflow is -[receipt-gate.yml](../.github/workflows/receipt-gate.yml). It uses: +Start from the +[cross-repository template](../examples/github/receipt-gate.yml.example). It uses: - pull_request_target, so the workflow definition comes from the default branch; @@ -57,12 +67,10 @@ end-to-end trial has proved that the status is attached to the latest PR head commit. Keep review, permission, secret-backed, deployment, and uncovered platform checks as separate GitHub rules or workflows. -The active workflow is specific to this repository because its trusted base -contains the CCP Rust verifier source. An adopting repository must use the -[cross-repository template](../examples/github/receipt-gate.yml.example), pin -an exact reviewed CCP source commit, and keep its own policy in the adopting -repository's trusted base. Follow the [adoption guide](ADOPTION_GUIDE.md); -copying this repository-native workflow unchanged will not work safely. +An adopting repository must pin an exact reviewed CCP source commit and keep +its own policy in the adopting repository's trusted base. Follow the +[adoption guide](ADOPTION_GUIDE.md); copying historical repository-native +workflow text unchanged will not work safely. This boundary follows GitHub's official guidance for [`pull_request_target`](https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target): diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index 17c8db8..975e986 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -36,6 +36,18 @@ cargo install --locked --path . commit-ci-preflight --version ``` +The full test command above is the current hosted contract on Linux and macOS. +On Windows, compile every test target without executing the still-pending +native runtime and cache paths: + +```powershell +cargo test --locked --workspace --all-targets --all-features --no-run +``` + +That compile-only result proves source portability, not Windows runtime +qualification. The current platform boundary is recorded in +[`BETA_SUPPORT.md`](BETA_SUPPORT.md). + `cargo install --path .` installs only the `commit-ci-preflight` binary. It does not register a GitHub runner, start a daemon, alter repository settings, or upload a receipt. diff --git a/docs/evidence/economic-case-studies-2026-08.json b/docs/evidence/economic-case-studies-2026-08.json new file mode 100644 index 0000000..ec76711 --- /dev/null +++ b/docs/evidence/economic-case-studies-2026-08.json @@ -0,0 +1,94 @@ +{ + "schema_version": "1.0", + "observed_at_utc": "2026-08-30T00:00:00Z", + "currency": "USD", + "github_pricing": { + "linux_2_core_usd_per_minute": 0.006, + "github_pro_included_minutes_per_month": 3000, + "job_rounding": "Each private-repository hosted job is rounded up to the next whole minute.", + "public_standard_hosted_is_free": true, + "self_hosted_is_free": true, + "dependabot_standard_hosted_is_free": true, + "source": "https://docs.github.com/en/billing/concepts/product-billing/github-actions" + }, + "case_studies": { + "matryca_knowledge": { + "public_name": "Matryca-Knowledge", + "repository_visibility": "private", + "observation_period": "2026-08-01..2026-08-30", + "qualified_commit": "2dae025a3dde112b7b35a24ddd9b0514d1f1ee7c", + "receipt_python_311": "sha256:a9465bf170eb110bdd4ed29b3f977bd4dc3bc21a617ad01ca54d0c357d85350d", + "receipt_python_313": "sha256:cee6416805b7b78ad23d385fa8415cd64f4f2615dd9c8df04b016fa6ee58273a", + "qualified_tests_per_runtime": 279, + "qualified_coverage_percent": 83, + "qualified_check_count": 19, + "observed_linux_minutes": 111, + "observed_gross_compute_usd": 0.666, + "observed_discount_usd": 0.096, + "observed_billed_usd": 0.570, + "sample_gate_runs": 30, + "sample_gate_runner_seconds": 538, + "successful_gate_median_seconds": 15, + "baseline_rounded_runner_minutes": 60, + "retained_rounded_runner_minutes": 32, + "avoided_rounded_runner_minutes": 28, + "avoided_runner_minutes_percent": 46.7, + "estimated_avoided_github_compute_usd": 0.168, + "method": "Two historical hosted matrix jobs per event are compared with one retained hosted receipt-gate job. GitHub rounds each private hosted job separately." + }, + "matryca_brain": { + "public_name": "Matryca-Brain", + "repository_visibility": "private", + "observation_period": "2026-08-01..2026-08-30", + "billing_period_label": "2026-08-01..2026-08-31", + "observed_total_actions_runs": 358, + "observed_pr_validation_runs": 314, + "observed_dependency_update_runs": 22, + "observed_ccp_guarded_self_hosted_runs": 22, + "pre_cutover_pr_runs": 286, + "pre_cutover_distinct_pr_heads": 207, + "pre_cutover_pr_outcomes": { + "success": 139, + "failure": 43, + "cancelled": 12, + "skipped": 92 + }, + "hosted_pr_executions": 194, + "hosted_dependency_update_executions": 22, + "ccp_guarded_self_hosted_attempts": 22, + "ccp_guarded_distinct_commits": 19, + "ccp_guarded_outcomes": { + "success": 17, + "failure": 1, + "cancelled": 4 + }, + "observed_linux_minutes": 5600, + "observed_gross_compute_usd": 33.60, + "observed_gross_storage_usd": 0.12, + "observed_gross_total_usd": 33.72, + "observed_discount_usd": 18.34, + "observed_billed_total_usd": 15.38, + "post_cutover_period": "2026-08-20..2026-08-30", + "post_cutover_gross_usd": 0.00, + "average_observed_linux_minutes_per_executed_pr": 28.866, + "estimated_avoided_github_compute_minutes": 635.1, + "estimated_avoided_github_compute_usd": 3.81, + "estimate_kind": "bounded counterfactual from observed repository billing and comparable workflow counts", + "method": "The 5,600 observed Linux minutes are divided by the 194 non-skipped PR Validation executions, then multiplied by 22 CCP-guarded self-hosted attempts. Dependency-update runs are reported but excluded because they are not the PR workflow replaced by CCP.", + "comparability_boundary": "The operator reports that the local ten-stage Ready mirror preserved the number and quality of tests. The public study treats this as a bounded comparability input, not cryptographic proof of runner identity." + } + }, + "claim_boundary": { + "github_billed_savings_demonstrated_by": [ + "matryca_knowledge" + ], + "github_billed_savings_estimated_for": [ + "matryca_brain" + ], + "included_quota_preservation_demonstrated": true, + "avoided_remote_compute_demonstrated": true, + "net_savings_certified": false, + "reason": "Energy, hardware amortization, operator time, maintenance, and failed local attempts have not been priced." + }, + "privacy_boundary": "Private billing and run records were inspected by the operator. This public record contains aggregates, exact qualification anchors that were approved for publication, and the public case-study name Matryca-Brain; it excludes account identity, private repository URLs, raw logs, local paths, and secrets." +} diff --git a/src/cache_payload.rs b/src/cache_payload.rs index 6b8d8ba..1c151d0 100644 --- a/src/cache_payload.rs +++ b/src/cache_payload.rs @@ -613,7 +613,11 @@ mod tests { #[test] fn payload_link_recreation_is_explicitly_unsupported() { assert!(matches!( - recreate_payload_link(Path::new("source"), Path::new("destination")), + recreate_payload_link( + Path::new("source"), + Path::new("destination"), + PayloadLinkPolicy::Reject, + ), Err(CacheError::PayloadSymlinkUnsupported(_)) )); } diff --git a/tests/github_gate_contract.rs b/tests/github_gate_contract.rs index a339df1..89e5e5a 100644 --- a/tests/github_gate_contract.rs +++ b/tests/github_gate_contract.rs @@ -20,45 +20,11 @@ use commit_ci_preflight::config::ConfigV1; use commit_ci_preflight::verify::{VerificationPolicyDocument, load_verification_policy_document}; use saphyr::LoadableYamlNode; -const WORKFLOW: &str = include_str!("../.github/workflows/receipt-gate.yml"); const CROSS_REPOSITORY_TEMPLATE: &str = include_str!("../examples/github/receipt-gate.yml.example"); const GATE_SCRIPT: &str = include_str!("../scripts/github-receipt-gate.sh"); const CHECKOUT_SHA: &str = "de0fac2e4500dabe0009e67214ff5f5447ce83dd"; const FIXTURE_COMMIT: &str = "0123456789abcdef0123456789abcdef01234567"; -#[test] -fn workflow_uses_a_trusted_minimal_fail_closed_boundary() { - assert!(WORKFLOW.contains("pull_request_target:")); - assert!(WORKFLOW.contains("contents: read")); - assert!(WORKFLOW.contains("statuses: write")); - assert!(WORKFLOW.contains("cancel-in-progress: true")); - assert!(WORKFLOW.contains("timeout-minutes: 6")); - assert!(WORKFLOW.contains("github.event.pull_request.head.sha")); - assert!(WORKFLOW.contains("github.event.pull_request.base.sha")); - assert!(WORKFLOW.contains("ccp-evidence/")); - assert!(WORKFLOW.contains(&format!("actions/checkout@{CHECKOUT_SHA}"))); - assert!(WORKFLOW.contains("persist-credentials: false")); - assert!(WORKFLOW.contains("cargo build --locked --release --bin commit-ci-preflight")); - assert!(WORKFLOW.contains("commit-ci-preflight/receipt")); - - for forbidden in [ - "pull_request:\n", - "pull_request.head.ref", - "pull_request.head.repo", - "actions/cache", - "cargo test", - "docker run", - "github.step_summary", - "secrets.", - "permissions: write-all", - ] { - assert!( - !WORKFLOW.contains(forbidden), - "forbidden workflow surface: {forbidden}" - ); - } -} - #[test] fn cross_repository_template_separates_and_pins_all_trust_inputs() { let documents = saphyr::YamlOwned::load_from_str(CROSS_REPOSITORY_TEMPLATE) diff --git a/tests/hosted_ci_workflow.rs b/tests/hosted_ci_workflow.rs new file mode 100644 index 0000000..f6dd783 --- /dev/null +++ b/tests/hosted_ci_workflow.rs @@ -0,0 +1,280 @@ +// Copyright 2026 Marco Porcellato +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 + +use std::{fs, path::PathBuf}; + +use saphyr::{LoadableYamlNode, MappingOwned, YamlOwned}; + +const WORKFLOW: &str = include_str!("../.github/workflows/rust-ci.yml"); +const README: &str = include_str!("../README.md"); +const GITHUB_GATE: &str = include_str!("../docs/GITHUB_GATE.md"); +const CHANGELOG: &str = include_str!("../CHANGELOG.md"); +const RUST_TOOLCHAIN: &str = include_str!("../rust-toolchain.toml"); +const CHECKOUT_SHA: &str = "3d3c42e5aac5ba805825da76410c181273ba90b1"; + +#[test] +fn public_repository_uses_full_standard_hosted_rust_ci() { + let documents = saphyr::YamlOwned::load_from_str(WORKFLOW).expect("hosted CI YAML"); + assert_eq!(documents.len(), 1); + + for required in [ + "pull_request:", + "push:", + "branches: [main]", + "workflow_dispatch:", + "permissions:\n contents: read", + "cancel-in-progress: true", + "ubuntu-24.04", + "macos-15", + "windows-2025", + &format!("actions/checkout@{CHECKOUT_SHA}"), + "github.event.pull_request.head.repo.full_name || github.repository", + "github.event.pull_request.head.sha || github.sha", + "persist-credentials: false", + "rustup show active-toolchain", + "cargo fmt --all -- --check", + "cargo clippy --locked --workspace --all-targets --all-features -- -D warnings", + "cargo test --locked --workspace --all-targets --all-features", + "cargo test --locked --workspace --all-targets --all-features --no-run", + "cargo doc --locked --workspace --all-features --no-deps", + "cargo run --locked --quiet --example generate_release_metadata -- --check", + "CCP_TEST_ROOT: ${{ runner.temp }}/ccp-tests", + "needs: [quality, test, windows_compile]", + "QUALITY_RESULT: ${{ needs.quality.result }}", + "TEST_RESULT: ${{ needs.test.result }}", + "WINDOWS_COMPILE_RESULT: ${{ needs.windows_compile.result }}", + ] { + assert!( + WORKFLOW.contains(required), + "missing hosted CI boundary: {required}" + ); + } + + for forbidden in [ + "pull_request_target:", + "statuses: write", + "ccp-evidence/", + "commit-ci-preflight/receipt", + "commit-ci-preflight run", + "guard exec", + "docker ", + "self-hosted", + "actions/cache", + "cache:", + "secrets.", + ] { + assert!( + !WORKFLOW.contains(forbidden), + "forbidden public hosted CI surface: {forbidden}" + ); + } + + assert!(RUST_TOOLCHAIN.contains("channel = \"1.96.0\"")); + assert!(RUST_TOOLCHAIN.contains("components = [\"clippy\", \"rustfmt\"]")); + assert!(RUST_TOOLCHAIN.contains("profile = \"minimal\"")); +} + +#[test] +fn repository_has_no_ordinary_per_pr_receipt_workflow() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + assert!( + !root.join(".github/workflows/receipt-gate.yml").exists(), + "public repository must not require local CCP receipts for ordinary pull requests" + ); +} + +#[test] +fn runner_temp_is_resolved_inside_the_test_step() { + let documents = saphyr::YamlOwned::load_from_str(WORKFLOW).expect("hosted CI YAML"); + let root = documents[0].as_mapping().expect("workflow mapping"); + let jobs = mapping_get(root, "jobs") + .and_then(YamlOwned::as_mapping) + .expect("jobs mapping"); + let test_job = mapping_get(jobs, "test") + .and_then(YamlOwned::as_mapping) + .expect("test job mapping"); + + let job_env_has_test_root = mapping_get(test_job, "env") + .and_then(YamlOwned::as_mapping) + .and_then(|env| mapping_get(env, "CCP_TEST_ROOT")) + .is_some(); + assert!( + !job_env_has_test_root, + "runner context is unavailable in job-level env" + ); + + let steps = mapping_get(test_job, "steps") + .and_then(YamlOwned::as_sequence) + .expect("test steps"); + let suite_step = steps + .iter() + .filter_map(YamlOwned::as_mapping) + .find(|step| { + mapping_get(step, "name").and_then(YamlOwned::as_str) + == Some("Run the complete deterministic suite") + }) + .expect("complete deterministic suite step"); + let test_root = mapping_get(suite_step, "env") + .and_then(YamlOwned::as_mapping) + .and_then(|env| mapping_get(env, "CCP_TEST_ROOT")) + .and_then(YamlOwned::as_str); + assert_eq!(test_root, Some("${{ runner.temp }}/ccp-tests")); +} + +#[test] +fn windows_compiles_all_tests_without_executing_unqualified_runtime_paths() { + let documents = saphyr::YamlOwned::load_from_str(WORKFLOW).expect("hosted CI YAML"); + let root = documents[0].as_mapping().expect("workflow mapping"); + let jobs = mapping_get(root, "jobs") + .and_then(YamlOwned::as_mapping) + .expect("jobs mapping"); + + let test_job = mapping_get(jobs, "test") + .and_then(YamlOwned::as_mapping) + .expect("test job mapping"); + let runners = mapping_get(test_job, "strategy") + .and_then(YamlOwned::as_mapping) + .and_then(|strategy| mapping_get(strategy, "matrix")) + .and_then(YamlOwned::as_mapping) + .and_then(|matrix| mapping_get(matrix, "runner")) + .and_then(YamlOwned::as_sequence) + .expect("runtime test runner matrix") + .iter() + .map(|runner| runner.as_str().expect("runner name")) + .collect::>(); + assert_eq!(runners, ["ubuntu-24.04", "macos-15"]); + + let windows_job = mapping_get(jobs, "windows_compile") + .and_then(YamlOwned::as_mapping) + .expect("Windows compile-only job"); + assert_eq!( + mapping_get(windows_job, "runs-on").and_then(YamlOwned::as_str), + Some("windows-2025") + ); + let windows_steps = mapping_get(windows_job, "steps") + .and_then(YamlOwned::as_sequence) + .expect("Windows compile steps"); + let compile_step = windows_steps + .iter() + .filter_map(YamlOwned::as_mapping) + .find(|step| { + mapping_get(step, "name").and_then(YamlOwned::as_str) + == Some("Compile every test target without execution") + }) + .expect("Windows compile-only step"); + assert_eq!( + mapping_get(compile_step, "run").and_then(YamlOwned::as_str), + Some("cargo test --locked --workspace --all-targets --all-features --no-run") + ); + + let gate_job = mapping_get(jobs, "gate") + .and_then(YamlOwned::as_mapping) + .expect("hosted gate job"); + let gate_dependencies = mapping_get(gate_job, "needs") + .and_then(YamlOwned::as_sequence) + .expect("hosted gate dependencies") + .iter() + .map(|dependency| dependency.as_str().expect("gate dependency")) + .collect::>(); + assert_eq!(gate_dependencies, ["quality", "test", "windows_compile"]); +} + +#[test] +fn public_hosted_policy_and_optional_receipt_product_are_documented_separately() { + let normalized_readme = README.split_whitespace().collect::>().join(" "); + for required in [ + "This public repository uses standard GitHub-hosted CI", + "no billable public-runner savings", + "historical CCP receipts remain valid", + ] { + assert!( + normalized_readme.contains(required), + "README boundary missing: {required}" + ); + } + for required in [ + "not active for this public repository's ordinary pull requests", + "cross-repository template", + "economically qualified", + ] { + assert!( + GITHUB_GATE.contains(required), + "GitHub gate boundary missing: {required}" + ); + } + assert!(CHANGELOG.contains("Replaced this public repository's ordinary per-PR receipt gate")); +} + +#[test] +fn economic_case_studies_recompute_remote_savings_from_observed_inputs() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let evidence_path = root.join("docs/evidence/economic-case-studies-2026-08.json"); + assert!( + evidence_path.is_file(), + "public economic evidence must exist at {}", + evidence_path.display() + ); + + let evidence: serde_json::Value = + serde_json::from_slice(&fs::read(&evidence_path).expect("read public economic evidence")) + .expect("parse public economic evidence"); + + let knowledge = &evidence["case_studies"]["matryca_knowledge"]; + assert_eq!(knowledge["observed_linux_minutes"], 111); + assert_eq!(knowledge["observed_gross_compute_usd"], 0.666); + assert_eq!(knowledge["observed_billed_usd"], 0.570); + assert_eq!(knowledge["baseline_rounded_runner_minutes"], 60); + assert_eq!(knowledge["retained_rounded_runner_minutes"], 32); + assert_eq!(knowledge["avoided_rounded_runner_minutes"], 28); + assert_eq!(knowledge["estimated_avoided_github_compute_usd"], 0.168); + + let brain = &evidence["case_studies"]["matryca_brain"]; + assert_eq!(brain["public_name"], "Matryca-Brain"); + assert_eq!(brain["observed_linux_minutes"], 5_600); + assert_eq!(brain["observed_gross_compute_usd"], 33.60); + assert_eq!(brain["observed_gross_storage_usd"], 0.12); + assert_eq!(brain["observed_gross_total_usd"], 33.72); + assert_eq!(brain["observed_billed_total_usd"], 15.38); + assert_eq!(brain["hosted_pr_executions"], 194); + assert_eq!(brain["hosted_dependency_update_executions"], 22); + assert_eq!(brain["ccp_guarded_self_hosted_attempts"], 22); + assert_eq!(brain["ccp_guarded_distinct_commits"], 19); + assert_eq!(brain["ccp_guarded_outcomes"]["success"], 17); + assert_eq!(brain["ccp_guarded_outcomes"]["failure"], 1); + assert_eq!(brain["ccp_guarded_outcomes"]["cancelled"], 4); + assert_eq!(brain["post_cutover_gross_usd"], 0.00); + + let comparable_hosted_executions = brain["hosted_pr_executions"] + .as_u64() + .expect("hosted PR executions"); + let average_minutes = brain["observed_linux_minutes"] + .as_u64() + .expect("observed Linux minutes") as f64 + / comparable_hosted_executions as f64; + let avoided_minutes = average_minutes + * brain["ccp_guarded_self_hosted_attempts"] + .as_u64() + .expect("CCP guarded attempts") as f64; + let avoided_usd = avoided_minutes + * evidence["github_pricing"]["linux_2_core_usd_per_minute"] + .as_f64() + .expect("Linux runner rate"); + + assert!((average_minutes - 28.865_979_381_4).abs() < 1e-9); + assert!((avoided_minutes - 635.051_546_391_8).abs() < 1e-9); + assert!((avoided_usd - 3.810_309_278_4).abs() < 1e-9); + assert_eq!(brain["estimated_avoided_github_compute_minutes"], 635.1); + assert_eq!(brain["estimated_avoided_github_compute_usd"], 3.81); + assert_eq!(evidence["claim_boundary"]["net_savings_certified"], false); +} + +fn mapping_get<'a>(mapping: &'a MappingOwned, key: &str) -> Option<&'a YamlOwned> { + mapping + .iter() + .find_map(|(candidate, value)| (candidate.as_str() == Some(key)).then_some(value)) +} diff --git a/tests/release_hardening_contract.rs b/tests/release_hardening_contract.rs index ce69054..d3c3a7b 100644 --- a/tests/release_hardening_contract.rs +++ b/tests/release_hardening_contract.rs @@ -54,7 +54,18 @@ fn public_readme_is_human_first_and_truthfully_differentiated() { "missing official comparison source: {official_source}" ); } - assert!(!README.to_ascii_lowercase().contains("matryca")); + for public_case_study in ["Matryca-Knowledge", "Matryca-Brain"] { + assert!( + README.contains(public_case_study), + "missing approved public case-study name: {public_case_study}" + ); + } + for private_context in ["Matryca-per-Delineat", "MarcoPorcellato/Matryca"] { + assert!( + !README.contains(private_context), + "README leaked private case-study context: {private_context}" + ); + } assert!(README.contains("not an identity attestation")); assert!(README.contains("does not execute marketplace actions")); }