From f300596a89772d63c767f695fe69f372b5a2a925 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:17:13 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20fix(ci):=20bound=20gosec=20c?= =?UTF-8?q?oncurrency=20so=20Scan=20stops=20losing=20its=20runner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gosec v2.29.0 defaults -concurrency to runtime.NumCPU(). Inside a self-hosted runner pod that is the node's 32 vCPUs, not the pod's 6-CPU limit, and every worker type-checks and builds SSA for its own package. Thirty-two of those at once push the pod far past its 3Gi request on an overcommitted node, and it is killed mid-Scan: "The self-hosted runner lost communication with the server", logs BlobNotFound. Evidence: in the four failing jobs cited by the issue (124530335184, 124538397043, 124527124413, 124542607459) the job died in the Scan step every time, on four different runner pods, while the green runs finish Scan in about four minutes. Pin -concurrency 4 (at or below the runner CPU limit), regenerate the integration twin, and add a workflow contract asserting both gosec invocations pass an explicit -concurrency no higher than the runner container CPU limit in hive-runners-lke-values.yaml. Fixes #16040 Hive-Run: hivecommons/hive#16040 Hive-Plan: issue-16040 asks 1-2 (runner-side evidence, gosec resource profile) Hive-Spec: issue-16040#ask-2 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HRH6kn7bGF1abH5x72U7zL Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --- .../scripts/test-native-queue-workflows.py | 25 +++++++++++++++++++ .github/workflows/go-security-analysis.yml | 10 +++++++- ...gration-technical-go-security-analysis.yml | 2 +- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test-native-queue-workflows.py b/.github/scripts/test-native-queue-workflows.py index 5b7f40d040..e48c2c83a1 100644 --- a/.github/scripts/test-native-queue-workflows.py +++ b/.github/scripts/test-native-queue-workflows.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Structural contracts for required native queue workflows (not a Go runner).""" import pathlib +import re import unittest import yaml @@ -69,6 +70,30 @@ def test_dependabot_runs_the_same_trusted_pr_verifier(self): self.assertEqual(job["name"], "verify PR contents") self.assertIn("docker run --rm", job["steps"][-1]["run"]) + def test_gosec_concurrency_fits_the_runner_cpu_limit(self): + # gosec defaults -concurrency to runtime.NumCPU(), which inside a + # runner pod is the NODE's vCPU count, not the pod's cgroup limit. + # Unbounded, every worker type-checks and builds SSA for its own + # package at once, and the pod is OOM-killed or evicted mid-Scan: + # "runner lost communication", hivecommons/hive#16040. The pod CPU + # limit is a ceiling, not a memory proof: it only guarantees gosec + # never runs more workers than the pod can schedule. + values = yaml.safe_load( + (ROOT / "src/deploy/ci-runners/lke/hive-runners-lke-values.yaml").read_text()) + runner = next(c for c in values["template"]["spec"]["containers"] + if c["name"] == "runner") + cpu = str(runner["resources"]["limits"]["cpu"]) + cpu_limit = int(cpu[:-1]) // 1000 if cpu.endswith("m") else int(float(cpu)) + for file in ("go-security-analysis.yml", + "integration-technical-go-security-analysis.yml"): + with self.subTest(file=file): + steps = workflow(file)["jobs"]["gosec"]["steps"] + scan = next(s for s in steps if s.get("name") == "Scan")["run"] + match = re.search(r"\bgosec\b.*\s-concurrency[ =](\d+)\b", scan) + self.assertIsNotNone(match, f"gosec must pass an explicit -concurrency: {scan!r}") + self.assertGreaterEqual(int(match.group(1)), 1) + self.assertLessEqual(int(match.group(1)), cpu_limit) + if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/go-security-analysis.yml b/.github/workflows/go-security-analysis.yml index f134fffa8f..3aa7507434 100644 --- a/.github/workflows/go-security-analysis.yml +++ b/.github/workflows/go-security-analysis.yml @@ -252,10 +252,18 @@ jobs: # of the triage issue #4903; do not flip it before the # HIGH-severity findings are resolved, and do not delete the step to make # the summary look clean. + # + # -concurrency 4: gosec defaults to runtime.NumCPU(), which in a runner + # pod is the NODE's 32 vCPUs, not the pod's 6-CPU limit + # (src/deploy/ci-runners/lke/hive-runners-lke-values.yaml). Each worker + # type-checks and builds SSA for its own package, so 32 at once pushed + # the pod far past its 3Gi request and it was killed mid-Scan — the + # "runner lost communication" failures of #16040. Keep this at or below + # the runner CPU limit (test-native-queue-workflows.py enforces it). - name: Scan working-directory: src continue-on-error: true - run: gosec -exclude-generated ./... ./crux/... + run: gosec -concurrency 4 -exclude-generated ./... ./crux/... golangci-lint: needs: required-context-scope diff --git a/.github/workflows/integration-technical-go-security-analysis.yml b/.github/workflows/integration-technical-go-security-analysis.yml index 0710fed719..c78faa8b7c 100644 --- a/.github/workflows/integration-technical-go-security-analysis.yml +++ b/.github/workflows/integration-technical-go-security-analysis.yml @@ -119,7 +119,7 @@ jobs: - name: Scan working-directory: src continue-on-error: true - run: gosec -exclude-generated ./... ./crux/... + run: gosec -concurrency 4 -exclude-generated ./... ./crux/... golangci-lint: needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} From 20eb62938d0e17168276789eaa45de39b36bdb42 Mon Sep 17 00:00:00 2001 From: Karibhou <157208370+Karibhou@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:20:02 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20fix(ci):=20regenerate=20inte?= =?UTF-8?q?gration-candidate.yml=20for=20the=20gosec=20concurrency=20pin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #16035 composed the gosec Scan job into integration-candidate.yml, so after rebasing onto it the generator --check (build-and-test, "Check readonly candidate CI generation") flagged the candidate as stale. Rerun generate-integration-ci.py so the composed Scan step carries -concurrency 4 too. Refs #16040 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Karibhou <157208370+Karibhou@users.noreply.github.com> --- .github/workflows/integration-candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-candidate.yml b/.github/workflows/integration-candidate.yml index 544777afa0..abfa20723b 100644 --- a/.github/workflows/integration-candidate.yml +++ b/.github/workflows/integration-candidate.yml @@ -1856,7 +1856,7 @@ jobs: - name: Scan working-directory: src continue-on-error: true - run: gosec -exclude-generated ./... ./crux/... + run: gosec -concurrency 4 -exclude-generated ./... ./crux/... env: GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} security-golangci-lint: