diff --git a/.github/scripts/test-native-queue-workflows.py b/.github/scripts/test-native-queue-workflows.py index 5b7f40d040..e48c2c83a1 100644 --- a/.github/scripts/test-native-queue-workflows.py +++ b/.github/scripts/test-native-queue-workflows.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Structural contracts for required native queue workflows (not a Go runner).""" import pathlib +import re import unittest import yaml @@ -69,6 +70,30 @@ def test_dependabot_runs_the_same_trusted_pr_verifier(self): self.assertEqual(job["name"], "verify PR contents") self.assertIn("docker run --rm", job["steps"][-1]["run"]) + def test_gosec_concurrency_fits_the_runner_cpu_limit(self): + # gosec defaults -concurrency to runtime.NumCPU(), which inside a + # runner pod is the NODE's vCPU count, not the pod's cgroup limit. + # Unbounded, every worker type-checks and builds SSA for its own + # package at once, and the pod is OOM-killed or evicted mid-Scan: + # "runner lost communication", hivecommons/hive#16040. The pod CPU + # limit is a ceiling, not a memory proof: it only guarantees gosec + # never runs more workers than the pod can schedule. + values = yaml.safe_load( + (ROOT / "src/deploy/ci-runners/lke/hive-runners-lke-values.yaml").read_text()) + runner = next(c for c in values["template"]["spec"]["containers"] + if c["name"] == "runner") + cpu = str(runner["resources"]["limits"]["cpu"]) + cpu_limit = int(cpu[:-1]) // 1000 if cpu.endswith("m") else int(float(cpu)) + for file in ("go-security-analysis.yml", + "integration-technical-go-security-analysis.yml"): + with self.subTest(file=file): + steps = workflow(file)["jobs"]["gosec"]["steps"] + scan = next(s for s in steps if s.get("name") == "Scan")["run"] + match = re.search(r"\bgosec\b.*\s-concurrency[ =](\d+)\b", scan) + self.assertIsNotNone(match, f"gosec must pass an explicit -concurrency: {scan!r}") + self.assertGreaterEqual(int(match.group(1)), 1) + self.assertLessEqual(int(match.group(1)), cpu_limit) + if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/go-security-analysis.yml b/.github/workflows/go-security-analysis.yml index f134fffa8f..3aa7507434 100644 --- a/.github/workflows/go-security-analysis.yml +++ b/.github/workflows/go-security-analysis.yml @@ -252,10 +252,18 @@ jobs: # of the triage issue #4903; do not flip it before the # HIGH-severity findings are resolved, and do not delete the step to make # the summary look clean. + # + # -concurrency 4: gosec defaults to runtime.NumCPU(), which in a runner + # pod is the NODE's 32 vCPUs, not the pod's 6-CPU limit + # (src/deploy/ci-runners/lke/hive-runners-lke-values.yaml). Each worker + # type-checks and builds SSA for its own package, so 32 at once pushed + # the pod far past its 3Gi request and it was killed mid-Scan — the + # "runner lost communication" failures of #16040. Keep this at or below + # the runner CPU limit (test-native-queue-workflows.py enforces it). - name: Scan working-directory: src continue-on-error: true - run: gosec -exclude-generated ./... ./crux/... + run: gosec -concurrency 4 -exclude-generated ./... ./crux/... golangci-lint: needs: required-context-scope diff --git a/.github/workflows/integration-candidate.yml b/.github/workflows/integration-candidate.yml index 544777afa0..abfa20723b 100644 --- a/.github/workflows/integration-candidate.yml +++ b/.github/workflows/integration-candidate.yml @@ -1856,7 +1856,7 @@ jobs: - name: Scan working-directory: src continue-on-error: true - run: gosec -exclude-generated ./... ./crux/... + run: gosec -concurrency 4 -exclude-generated ./... ./crux/... env: GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} security-golangci-lint: diff --git a/.github/workflows/integration-technical-go-security-analysis.yml b/.github/workflows/integration-technical-go-security-analysis.yml index 0710fed719..c78faa8b7c 100644 --- a/.github/workflows/integration-technical-go-security-analysis.yml +++ b/.github/workflows/integration-technical-go-security-analysis.yml @@ -119,7 +119,7 @@ jobs: - name: Scan working-directory: src continue-on-error: true - run: gosec -exclude-generated ./... ./crux/... + run: gosec -concurrency 4 -exclude-generated ./... ./crux/... golangci-lint: needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }}