From fd98e01c744c68ccdc3cf48cbbee8eb529d98a1a Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:29:12 +0100 Subject: [PATCH 1/3] ci: coverage measures on Python 3.14, where branch coverage runs on sys.monitoring The Coverage & Audit job went from ~15 to 37-41 minutes after the fragment suite landed (#760): with `branch = true`, coverage.py on 3.12 can only use the C tracer, and that tracer took tests/fragments/test_split (390 in-process tests) from 6.7 to 31.3 minutes in CI. coverage.py's sys.monitoring core ("sysmon") measures branches only on 3.14+, where it is the default (https://coverage.readthedocs.io/en/latest/config.html). Measured locally on that module: 3.12 226s plain, 546s with coverage; 3.14 217s plain, 220s with coverage. The job now runs on 3.14 with COVERAGE_CORE=sysmon set (a fallback then at least logs no-sysmon), and its timeout drops from 45 to 30, under the ~40 min a fallback would take. Gate margins on main are 5 to 49 points (core 82/70, V1 85/80, local 80/50, cloud 69/20). Nothing outside ci.yml uses the job or artifact name; neither is a required check. --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22684e24..314eeffe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,27 +258,39 @@ jobs: # the same ~5-min profile (uniform legs are far less likely to be # singled out by runner-infra SIGTERM cancellations) while this job # runs concurrently, so overall PR wall-clock does not regress. - # Step order, commands, env, and Codecov behavior are preserved - # verbatim from the old 3.12 leg. The perf-budget suite is NOT - # duplicated here — it runs serially on every matrix leg above. + # The perf-budget suite is NOT duplicated here — it runs serially on + # every matrix leg above. + # + # Python 3.14, not 3.12: branch coverage (`branch = true`) can only use + # coverage.py's sys.monitoring core ("sysmon") on 3.14+, where it is the + # default; on 3.12 and 3.13 it falls back to the C tracer + # (https://coverage.readthedocs.io/en/latest/config.html, `[run] core`). + # Measured on tests/fragments/test_split.py, the heaviest module (390 + # tests): 3.12 226s plain, 546s with coverage; 3.14 217s plain, 220s + # with coverage. The tracer's 2.4x took this job from ~15 to ~40 min + # once the fragment suite landed; under sysmon it costs about nothing. coverage-and-audit: - name: Coverage & Audit (Python 3.12) + name: Coverage & Audit (Python 3.14) runs-on: ubuntu-latest - # Observed ~20 min for the old overloaded 3.12 leg; this job carries - # the same coverage run minus the perf step. 45 gives headroom while - # still bounding a true hang to minutes, not GitHub's 6-hour default; - # it was 30 until the fragment scenario suite added about 12 minutes - # under coverage (the job ran 19, then past 30). - timeout-minutes: 45 + # Under sysmon the run takes about as long as a plain test leg. 30 + # bounds a true hang to minutes, not GitHub's 6-hour default, and is + # under the ~40 min a fallback to the C tracer takes, so a silent + # fallback fails the job instead of only slowing it. + timeout-minutes: 30 env: # Same glibc-arena cap as the test matrix (see comment there) — # this is the identical thread-heavy pytest run, plus coverage. MALLOC_ARENA_MAX: "2" + # Set, not left to the 3.14 default: a default sysmon that meets a + # setting it cannot serve (a plugin, dynamic contexts) falls back to + # the C tracer without a word; set explicitly, coverage.py 7.17 + # still falls back but logs a `no-sysmon` CoverageWarning. + COVERAGE_CORE: sysmon steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: "3.12" + python-version: "3.14" cache: pip - name: Install dependencies @@ -368,14 +380,14 @@ jobs: run: python -m coverage report || true # ── Artifacts ─────────────────────────────────────────────── - # Distinct artifact name — the 3.12 matrix leg above still owns - # `test-results-py3.12` (its plain-run junit.xml), and duplicate + # Distinct artifact name — the 3.14 matrix leg above still owns + # `test-results-py3.14` (its plain-run junit.xml), and duplicate # artifact names fail under upload-artifact v4+. - name: Upload coverage results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: coverage-results-py3.12 + name: coverage-results-py3.14 path: | junit.xml coverage.xml From b5b70ae48621f185ffb205557ca9570c7699680f Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:29:12 +0100 Subject: [PATCH 2/3] ci(iac-tests): Stage 1's tofu lanes run as parallel jobs, and its unit step under xdist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 1 ran its six tofu lanes one after another in a single job, about 27 minutes on main for a month: the job took the sum of them. Each lane runs its own files against moto or an in-process stand-in and asserts its own junit, so each is now a matrix job, and Stage 1 takes the longest lane plus setup. A lane shares one TF_PLUGIN_CACHE_DIR across its modules (the tests honour an external one), one provider download per lane instead of per module; a lane is one pytest process, so nothing writes that cache concurrently. The unit step ran ~7.4k tests in one process (five minutes); it now runs with `-n auto --dist loadscope`, as ci.yml's test matrix runs the same tests on every PR, under the same MALLOC_ARENA_MAX cap. The emulator extra stays out of that job, so it still cannot change what the unit step runs. Check names: "Stage 1 — unit + tofu validate" stays; the lanes report as "Stage 1 — ". None is a required check. --- .github/workflows/iac-tests.yml | 202 +++++++++++++++++++++----------- 1 file changed, 131 insertions(+), 71 deletions(-) diff --git a/.github/workflows/iac-tests.yml b/.github/workflows/iac-tests.yml index 4bdf8337..0963a219 100644 --- a/.github/workflows/iac-tests.yml +++ b/.github/workflows/iac-tests.yml @@ -3,7 +3,8 @@ name: iac-tests # Three-stage test ladder for the IaC autogenerator (see # HONESTLY_TESTED.md for the full coverage matrix): # -# * Stage 1 — unit + tofu validate. No creds, runs on every PR + push. +# * Stage 1 — unit + tofu validate. No creds, runs on every PR + push; +# its tofu lanes (moto, in-process stand-ins) run as parallel jobs. # * Stage 2 — docker emulators (LocalStack for AWS, # goccy/bigquery-emulator + fsouza/fake-gcs-server + gcloud pubsub # for GCP). Runs on every PR + push. @@ -76,103 +77,162 @@ jobs: name: "Stage 1 — unit + tofu validate" runs-on: ubuntu-latest if: github.event_name != 'schedule' + env: + # The unit step runs under xdist: the same glibc-arena cap as ci.yml's + # test matrix, which keeps a thread-heavy run's virtual memory flat. + MALLOC_ARENA_MAX: "2" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.12" + cache: pip - name: Install OpenTofu uses: opentofu/setup-opentofu@a1320f892987e89d278cc92dc5adc984fb93aca4 # v2.0.2 with: tofu_version: 1.12.0 + # The emulator extra is never installed in this job, so it cannot change + # what the unit step runs; the tofu lanes below install it in their own. - name: Install Python deps run: | python -m venv .venv .venv/bin/pip install --quiet --upgrade pip .venv/bin/pip install --quiet -e '.[dev]' + # Same distribution as the ci.yml test matrix, which runs these tests + # under xdist on every PR: one process took five minutes of this job. - name: Stage 1 — unit (pure-function emit) - run: .venv/bin/python -m pytest -m "unit and not slow" -q + run: .venv/bin/python -m pytest -m "unit and not slow" -q -n auto --dist loadscope - name: Stage 1 — tofu validate (creds-free) run: .venv/bin/python -m pytest tests/iac/test_iac_tofu_validate.py -q - # The Lake Formation bucket-policy filter is only decided when `tofu plan` - # reads the caller identity, so it is proven by a real plan against moto's - # STS. Installed after the unit step so the extra cannot change what that - # step runs; the coverage assert fails the step if the file only skipped. - # The column-grant file rides the same step: `tofu validate` passes a - # table_with_columns block the provider refuses at plan, so only a plan - # proves it, and the assert fails the step if it only skipped. - - name: Stage 1 — Lake Formation bucket-policy + column-grant plans (tofu plan vs moto, creds-free) + # The tofu lanes are independent (each runs its own files against moto or an + # in-process stand-in, and asserts its own junit), so each is its own job: + # run one after another they made Stage 1 the sum of them, ~27 min; in + # parallel it is the longest one. Each lane fails if a file it requires only + # skipped (scripts/ci/assert_lane_coverage.py). + stage1-lanes: + name: "Stage 1 — ${{ matrix.lane.name }}" + runs-on: ubuntu-latest + if: github.event_name != 'schedule' + strategy: + fail-fast: false + matrix: + lane: + # The Lake Formation bucket-policy filter is only decided when `tofu + # plan` reads the caller identity, so it is proven by a real plan + # against moto's STS. The column-grant file rides the same lane: + # `tofu validate` passes a table_with_columns block the provider + # refuses at plan, so only a plan proves it. + - name: "Lake Formation bucket-policy + column-grant plans (tofu plan vs moto, creds-free)" + tests: tests/iac/test_iac_lakeformation_bucket_policy_plan.py tests/iac/test_iac_lakeformation_column_grants_plan.py + pytest-args: "" + junit: lf-bucket-policy-plan.xml + require: | + LF bucket-policy plan=tests/iac/test_iac_lakeformation_bucket_policy_plan.py + LF column-grant plan=tests/iac/test_iac_lakeformation_column_grants_plan.py + # Retention and encryption at rest: the key policy is only evaluated + # at plan time, and the lifecycle rule, default SSE-KMS, key rotation + # and the key's scheduled deletion only exist after a real apply / + # destroy, so both run against moto (S3, KMS, STS). + - name: "retention + KMS plan/apply/destroy (tofu vs moto, creds-free)" + tests: tests/iac/test_iac_aws_retention_encryption_moto.py + pytest-args: "" + junit: retention-kms-moto.xml + require: | + retention + KMS moto=tests/iac/test_iac_aws_retention_encryption_moto.py + # `fluid diff` refreshes the apply's OpenTofu state: a real apply, + # then the refresh plan, against moto. Whether a clean apply reads as + # no drift depends on what the provider reads back, so only a real + # run proves it. + - name: "state drift (fluid diff reads the apply's state vs moto, creds-free)" + tests: tests/iac/test_iac_state_drift_moto.py + pytest-args: "-p no:randomly" + junit: state-drift.xml + require: | + state drift=tests/iac/test_iac_state_drift_moto.py + # Lake Formation tags: a tag the contract defines is named by + # reference, one the platform owns by its literal key. Only a real + # apply and plan show that an association applied with depends_on + # plans no change under the reference, and that a platform-owned tag + # is associated and survives the destroy. + - name: "Lake Formation tags apply + plan (tofu vs moto, creds-free)" + tests: tests/iac/test_iac_lakeformation_tags_moto.py + pytest-args: "-p no:randomly" + junit: lf-tags-moto.xml + require: | + LF tags moto=tests/iac/test_iac_lakeformation_tags_moto.py + # The default state key names the provider, and the first apply + # after the upgrade moves the old key's state with `tofu init + # -migrate-state`. Only a real apply, move and plan prove the plan + # after it changes nothing. + - name: "per-provider state key and its migration (tofu vs moto, creds-free)" + tests: tests/iac/test_iac_state_key_migration_moto.py + pytest-args: "-p no:randomly" + junit: state-key-migration.xml + require: | + state key migration=tests/iac/test_iac_state_key_migration_moto.py + # Governance parity (retention, CMEK, column restrictions, mapped + # principals): tofu validate of each governed GCP shape; a real plan + # and apply with terraform-provider-google against an in-process + # stand-in for the BigQuery REST API (the goccy emulator crashes the + # provider on apply), which is where the data-loss gate decides; and + # the Lake Formation excluded columns planned against moto. + - name: "governance parity (tofu vs BigQuery stand-in and moto, creds-free)" + tests: tests/iac/test_iac_gcp_governance_plan.py tests/iac/test_iac_aws_column_restrictions.py tests/iac/test_iac_gcp_governance.py + pytest-args: "-p no:randomly" + junit: governance-parity.xml + require: | + GCP governance plan=tests/iac/test_iac_gcp_governance_plan.py + AWS column restrictions=tests/iac/test_iac_aws_column_restrictions.py + GCP governance validate=tests/iac/test_iac_gcp_governance.py + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + cache: pip + + - name: Install OpenTofu + uses: opentofu/setup-opentofu@a1320f892987e89d278cc92dc5adc984fb93aca4 # v2.0.2 + with: + tofu_version: 1.12.0 + + - name: Install Python deps run: | + python -m venv .venv + .venv/bin/pip install --quiet --upgrade pip .venv/bin/pip install --quiet -e '.[dev,test-emulators]' - .venv/bin/python -m pytest tests/iac/test_iac_lakeformation_bucket_policy_plan.py \ - tests/iac/test_iac_lakeformation_column_grants_plan.py \ - -q --junitxml=lf-bucket-policy-plan.xml - .venv/bin/python scripts/ci/assert_lane_coverage.py lf-bucket-policy-plan.xml \ - --require "LF bucket-policy plan=tests/iac/test_iac_lakeformation_bucket_policy_plan.py" \ - --require "LF column-grant plan=tests/iac/test_iac_lakeformation_column_grants_plan.py" - - # Retention and encryption at rest: the key policy is only evaluated at - # plan time, and the lifecycle rule, default SSE-KMS, key rotation and the - # key's scheduled deletion only exist after a real apply / destroy, so - # both run against moto (S3, KMS, STS). Fails if the file only skipped. - - name: Stage 1 — retention + KMS plan/apply/destroy (tofu vs moto, creds-free) - run: | - .venv/bin/python -m pytest tests/iac/test_iac_aws_retention_encryption_moto.py \ - -q --junitxml=retention-kms-moto.xml - .venv/bin/python scripts/ci/assert_lane_coverage.py retention-kms-moto.xml \ - --require "retention + KMS moto=tests/iac/test_iac_aws_retention_encryption_moto.py" - # `fluid diff` refreshes the apply's OpenTofu state: a real apply, then - # the refresh plan, against moto. Whether a clean apply reads as no drift - # depends on what the provider reads back, so only a real run proves it. - # The emulator extra is already installed by the step above. - - name: Stage 1 — state drift (fluid diff reads the apply's state vs moto, creds-free) - run: | - .venv/bin/python -m pytest tests/iac/test_iac_state_drift_moto.py \ - -q -p no:randomly --junitxml=state-drift.xml - .venv/bin/python scripts/ci/assert_lane_coverage.py state-drift.xml \ - --require "state drift=tests/iac/test_iac_state_drift_moto.py" - # Lake Formation tags: a tag the contract defines is named by reference, one - # the platform owns by its literal key. Only a real apply and plan show that - # an association applied with depends_on plans no change under the - # reference, and that a platform-owned tag is associated and survives the - # destroy. Fails if the file only skipped. - - name: Stage 1 — Lake Formation tags apply + plan (tofu vs moto, creds-free) - run: | - .venv/bin/python -m pytest tests/iac/test_iac_lakeformation_tags_moto.py \ - -q -p no:randomly --junitxml=lf-tags-moto.xml - .venv/bin/python scripts/ci/assert_lane_coverage.py lf-tags-moto.xml \ - --require "LF tags moto=tests/iac/test_iac_lakeformation_tags_moto.py" - # The default state key names the provider, and the first apply after the - # upgrade moves the old key's state with `tofu init -migrate-state`. Only a - # real apply, move and plan prove the plan after it changes nothing. - - name: Stage 1 — per-provider state key and its migration (tofu vs moto, creds-free) + + # One provider download per lane: the tests use TF_PLUGIN_CACHE_DIR when + # it is set (env.setdefault) and a fresh temporary cache per module when + # it is not. A lane runs one pytest process, so nothing writes the cache + # concurrently (OpenTofu's plugin cache is not safe for that). + - name: Share one provider cache across the lane's modules run: | - .venv/bin/python -m pytest tests/iac/test_iac_state_key_migration_moto.py \ - -q -p no:randomly --junitxml=state-key-migration.xml - .venv/bin/python scripts/ci/assert_lane_coverage.py state-key-migration.xml \ - --require "state key migration=tests/iac/test_iac_state_key_migration_moto.py" - # Governance parity (retention, CMEK, column restrictions, mapped - # principals): tofu validate of each governed GCP shape; a real plan and - # apply with terraform-provider-google against an in-process stand-in for - # the BigQuery REST API (the goccy emulator crashes the provider on - # apply), which is where the data-loss gate decides; and the Lake - # Formation excluded columns planned against moto. Fails if a file only - # skipped. - - name: Stage 1 — governance parity (tofu vs BigQuery stand-in and moto, creds-free) + mkdir -p "$RUNNER_TEMP/tofu-plugin-cache" + echo "TF_PLUGIN_CACHE_DIR=$RUNNER_TEMP/tofu-plugin-cache" >> "$GITHUB_ENV" + + - name: Stage 1 — ${{ matrix.lane.name }} + env: + LANE_TESTS: ${{ matrix.lane.tests }} + LANE_PYTEST_ARGS: ${{ matrix.lane.pytest-args }} + LANE_JUNIT: ${{ matrix.lane.junit }} + LANE_REQUIRE: ${{ matrix.lane.require }} run: | - .venv/bin/python -m pytest tests/iac/test_iac_gcp_governance_plan.py \ - tests/iac/test_iac_aws_column_restrictions.py tests/iac/test_iac_gcp_governance.py \ - -q -p no:randomly --junitxml=governance-parity.xml - .venv/bin/python scripts/ci/assert_lane_coverage.py governance-parity.xml \ - --require "GCP governance plan=tests/iac/test_iac_gcp_governance_plan.py" \ - --require "AWS column restrictions=tests/iac/test_iac_aws_column_restrictions.py" \ - --require "GCP governance validate=tests/iac/test_iac_gcp_governance.py" + read -ra tests <<< "$LANE_TESTS" + read -ra extra <<< "$LANE_PYTEST_ARGS" + .venv/bin/python -m pytest "${tests[@]}" -q "${extra[@]}" --junitxml="$LANE_JUNIT" + requires=() + while IFS= read -r label; do + [ -n "$label" ] && requires+=(--require "$label") + done <<< "$LANE_REQUIRE" + .venv/bin/python scripts/ci/assert_lane_coverage.py "$LANE_JUNIT" "${requires[@]}" # ------------------------------------------------------------------- # Stage 2 — docker emulators. PR + push. From cd7d1238e0cb22e9482caff9d15cec6a70214a1a Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:38:21 +0100 Subject: [PATCH 3/3] ci: the secrets baseline follows the test credential iac-tests.yml moved Splitting Stage 1 into lanes moved `AWS_SECRET_ACCESS_KEY: test` from line 258 to 318, so the changed-files detect-secrets step failed on the stale line number. Regenerated with detect-secrets 1.5.0, the CI pin. --- .secrets.baseline | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 439b7b3e..a719e6c9 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -139,7 +139,7 @@ "filename": ".github/workflows/iac-tests.yml", "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 258 + "line_number": 318 } ], ".github/workflows/integration.yml": [ @@ -2154,5 +2154,5 @@ } ] }, - "generated_at": "2026-10-29T15:59:02Z" + "generated_at": "2026-10-30T16:38:10Z" }