From a0df1c579ed44df0baa07f9ff36f9304a978929f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n-Otero?= Date: Tue, 28 Jul 2026 09:45:22 -0400 Subject: [PATCH 1/3] perf(ci): gate the three cross-cutting semgrep scans on changed paths semgrep-general, semgrep-devex and semgrep-test-rules run on every PR. Measured against the file lists of all 136 PRs merged in a trailing 24-hour window, they have nothing in scope on 89%, 13% and 99% of them respectively, which is roughly 1,600 job-minutes a day. Each new filter mirrors what its job actually reads: - general: '**' minus the same twelve trees the job passes as --exclude flags. Deliberately expressed as excludes rather than positive includes so a new top-level directory fails open and still triggers the scan. - devex: the directory list the job hands to semgrep. - test-rules: '.semgrep/**' only, since `semgrep --test .semgrep/` exercises the rules against their own committed fixtures and cannot be affected by anything outside that tree. Coverage this deliberately preserves: - master pushes are untouched. The filter step is gated on `github.event_name != 'push'`, so every scan still runs unconditionally on master, which is what covers repo-wide drift and newly published rules against unchanged code. - .github/** always triggers general. That is the surface CODEOWNERS singles out as untrusted, and the job loads p/github-actions to scan it. - editing this workflow triggers every scan, the mitigation from #68640. - outputs keep their `|| 'true'` default, so a skipped or failed filter step fails open. Raising this as a proposal rather than folding it into a CI cost pass: #68640 left these three unconditional on purpose, and #70907 hit the same cost problem and chose `--jobs 4` over path filtering, explicitly to avoid weakening the check. Narrowing them is team-security's call. Generated-By: PostHog Code Task-Id: 7bad9f9c-04b8-40b0-b44c-e967a3f8c731 --- .github/workflows/ci-security.yaml | 57 ++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-security.yaml b/.github/workflows/ci-security.yaml index a2fe30993fe8..063bb73e8200 100644 --- a/.github/workflows/ci-security.yaml +++ b/.github/workflows/ci-security.yaml @@ -14,11 +14,17 @@ env: SEMGREP_IMAGE: semgrep/semgrep:1.167.0@sha256:06938c1f365d3f67b8cedd8bc117607ae64253f88a0e768e9da9408548927dd6 jobs: - # Job to decide if we should run language-specific semgrep scans. + # Job to decide which semgrep scans need to run. # See .github/actions/paths-filter/README.md for filter semantics # NOTE: With token auth, paths-filter uses pulls.listFiles which caps at # 3000 changed files. PRs exceeding that will silently truncate, but the # outputs default to 'true' so scans run unconditionally (safe fallback). + # That default only covers an absent output (filter step skipped or failed). + # The include-only filters below also truncate toward 'true', since any + # returned page will contain a file matching them. `general` is the + # exception: it includes '**' and is decided purely by its excludes, so a + # truncated >3000-file PR whose returned page sits entirely inside those + # excluded trees resolves to 'false'. changes: runs-on: ubuntu-latest timeout-minutes: 5 @@ -33,6 +39,9 @@ jobs: rust: ${{ steps.filter.outputs.rust || 'true' }} js: ${{ steps.filter.outputs.js || 'true' }} products-frontend: ${{ steps.filter.outputs.products-frontend || 'true' }} + general: ${{ steps.filter.outputs.general || 'true' }} + devex: ${{ steps.filter.outputs.devex || 'true' }} + test-rules: ${{ steps.filter.outputs.test-rules || 'true' }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -73,6 +82,43 @@ jobs: - 'products/**' - '.semgrep/rules/security/prefer-codegen-api.yaml' - '.github/workflows/ci-security.yaml' + # semgrep-general scans everything its own --exclude flags don't drop, + # so these negations must stay one-for-one with those flags below. + # Dropping a flag there without dropping it here is the dangerous + # direction: the files come into scan scope while the job stops being + # triggered for them. + general: + - '**' + - '!cli/**' + - '!common/**' + - '!ee/**' + - '!frontend/**' + - '!livestream/**' + - '!nodejs/**' + - '!posthog/**' + - '!products/**' + - '!rust/**' + - '!.semgrep/**' + - '!docs/**' + - '!services/**' + # Mirrors the directory list semgrep-devex passes to semgrep. Its + # rules span python/typescript/generic, so this can't narrow by + # extension. + devex: + - 'bin/**' + - 'common/**' + - 'ee/**' + - 'frontend/**' + - 'packages/**' + - 'posthog/**' + - 'products/**' + - '.semgrep/rules/devex/**' + - '.github/workflows/ci-security.yaml' + # `semgrep --test .semgrep/` only exercises the rules against their + # own committed fixtures, so nothing outside .semgrep/ can change it. + test-rules: + - '.semgrep/**' + - '.github/workflows/ci-security.yaml' semgrep-python: needs: changes @@ -274,6 +320,8 @@ jobs: # scans GitHub Actions and other repo-wide config semgrep-general: + needs: changes + if: needs.changes.outputs.general == 'true' runs-on: ubuntu-latest timeout-minutes: 20 @@ -292,7 +340,8 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - # exclude all directories already scanned by other jobs + # exclude all directories already scanned by other jobs. + # Keep this list one-for-one with the `general` filter in the changes job. - name: Run Semgrep run: | docker run --rm -v "${{ github.workspace }}:/src" -w /src \ @@ -332,6 +381,8 @@ jobs: # - ERROR-severity rules fail the job on any finding (regression guards). # See `.semgrep/rules/devex/README.md`. semgrep-devex: + needs: changes + if: needs.changes.outputs.devex == 'true' runs-on: ubuntu-latest timeout-minutes: 20 @@ -383,6 +434,8 @@ jobs: bin/ common/ ee/ frontend/ packages/ posthog/ products/ semgrep-test-rules: + needs: changes + if: needs.changes.outputs.test-rules == 'true' runs-on: ubuntu-latest timeout-minutes: 10 From 2d255119b376d0cf444976ffc628399bb52292af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n-Otero?= Date: Tue, 28 Jul 2026 10:14:41 -0400 Subject: [PATCH 2/3] perf(ci): tighten the semgrep filter comments The comments added earlier in this PR were verbose, and the truncation NOTE contradicted the pre-existing lines it was appended to: those said the `|| 'true'` default makes scans run unconditionally, then the new text immediately qualified that it only covers a missing output. Merged into one statement, 9 lines to 6, with every fact intact. The `general` comment now also records why the filter is shaped as '**' plus negations rather than positive includes: a new top-level directory then fails open and still triggers the scan. That rationale was in the PR description but had never made it into the file, which is the one place a future editor would look before restructuring the filter. The one-for-one warning about semgrep-general's --exclude flags is preserved, including which direction of drift is the dangerous one. Comments only. Verified by parsing the workflow, re-parsing the `filters:` literal block (its comments are part of the string the action receives), and deep-comparing against the previous commit: byte-identical. Separately re-asserted that the `general` negations are still one-for-one with the job's --exclude flags. Generated-By: PostHog Code Task-Id: 7bad9f9c-04b8-40b0-b44c-e967a3f8c731 --- .github/workflows/ci-security.yaml | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci-security.yaml b/.github/workflows/ci-security.yaml index 063bb73e8200..4d3f130696c6 100644 --- a/.github/workflows/ci-security.yaml +++ b/.github/workflows/ci-security.yaml @@ -16,15 +16,12 @@ env: jobs: # Job to decide which semgrep scans need to run. # See .github/actions/paths-filter/README.md for filter semantics - # NOTE: With token auth, paths-filter uses pulls.listFiles which caps at - # 3000 changed files. PRs exceeding that will silently truncate, but the - # outputs default to 'true' so scans run unconditionally (safe fallback). - # That default only covers an absent output (filter step skipped or failed). - # The include-only filters below also truncate toward 'true', since any - # returned page will contain a file matching them. `general` is the - # exception: it includes '**' and is decided purely by its excludes, so a - # truncated >3000-file PR whose returned page sits entirely inside those - # excluded trees resolves to 'false'. + # NOTE: with token auth, paths-filter uses pulls.listFiles, which caps at 3000 + # changed files and truncates silently past that. The `|| 'true'` defaults below + # only cover a missing output (filter step skipped or failed). Include-only + # filters truncate toward 'true' anyway, since any returned page holds a file + # matching them. `general` is decided purely by its excludes, so it can truncate + # to 'false' when the returned page sits entirely inside the excluded trees. changes: runs-on: ubuntu-latest timeout-minutes: 5 @@ -82,11 +79,11 @@ jobs: - 'products/**' - '.semgrep/rules/security/prefer-codegen-api.yaml' - '.github/workflows/ci-security.yaml' - # semgrep-general scans everything its own --exclude flags don't drop, - # so these negations must stay one-for-one with those flags below. - # Dropping a flag there without dropping it here is the dangerous - # direction: the files come into scan scope while the job stops being - # triggered for them. + # '**' plus negations, not positive includes, so a new top-level + # directory fails open and still triggers the scan. Keep the + # negations one-for-one with semgrep-general's --exclude flags: + # dropping an --exclude without dropping its negation here leaves + # the files in scan scope with nothing left to trigger the job. general: - '**' - '!cli/**' @@ -101,9 +98,8 @@ jobs: - '!.semgrep/**' - '!docs/**' - '!services/**' - # Mirrors the directory list semgrep-devex passes to semgrep. Its - # rules span python/typescript/generic, so this can't narrow by - # extension. + # Mirrors the directory list semgrep-devex passes to semgrep. Its rules + # span python/typescript/generic, so this can't narrow by extension. devex: - 'bin/**' - 'common/**' From 38c779d3d56f7e3e80d3dc4204f4599bce42ebd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Tue, 28 Jul 2026 11:04:48 -0400 Subject: [PATCH 3/3] fix(ci): harden semgrep path filters --- .github/workflows/ci-security.yaml | 60 +++++++++--------------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci-security.yaml b/.github/workflows/ci-security.yaml index 4d3f130696c6..383b605c83fd 100644 --- a/.github/workflows/ci-security.yaml +++ b/.github/workflows/ci-security.yaml @@ -14,14 +14,9 @@ env: SEMGREP_IMAGE: semgrep/semgrep:1.167.0@sha256:06938c1f365d3f67b8cedd8bc117607ae64253f88a0e768e9da9408548927dd6 jobs: - # Job to decide which semgrep scans need to run. + # Job to decide which path-scoped semgrep scans need to run. # See .github/actions/paths-filter/README.md for filter semantics - # NOTE: with token auth, paths-filter uses pulls.listFiles, which caps at 3000 - # changed files and truncates silently past that. The `|| 'true'` defaults below - # only cover a missing output (filter step skipped or failed). Include-only - # filters truncate toward 'true' anyway, since any returned page holds a file - # matching them. `general` is decided purely by its excludes, so it can truncate - # to 'false' when the returned page sits entirely inside the excluded trees. + # Pushes and PRs above GitHub's 3000-file API limit run every scan. changes: runs-on: ubuntu-latest timeout-minutes: 5 @@ -31,21 +26,24 @@ jobs: contents: read pull-requests: read outputs: - python: ${{ steps.filter.outputs.python || 'true' }} - go: ${{ steps.filter.outputs.go || 'true' }} - rust: ${{ steps.filter.outputs.rust || 'true' }} - js: ${{ steps.filter.outputs.js || 'true' }} - products-frontend: ${{ steps.filter.outputs.products-frontend || 'true' }} - general: ${{ steps.filter.outputs.general || 'true' }} - devex: ${{ steps.filter.outputs.devex || 'true' }} - test-rules: ${{ steps.filter.outputs.test-rules || 'true' }} + python: ${{ steps.oversized.outputs.force_all || steps.filter.outputs.python || 'true' }} + go: ${{ steps.oversized.outputs.force_all || steps.filter.outputs.go || 'true' }} + rust: ${{ steps.oversized.outputs.force_all || steps.filter.outputs.rust || 'true' }} + js: ${{ steps.oversized.outputs.force_all || steps.filter.outputs.js || 'true' }} + products-frontend: ${{ steps.oversized.outputs.force_all || steps.filter.outputs.products-frontend || 'true' }} + devex: ${{ steps.oversized.outputs.force_all || steps.filter.outputs.devex || 'true' }} + test-rules: ${{ steps.oversized.outputs.force_all || steps.filter.outputs.test-rules || 'true' }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: clean: false + - name: Force all scans for oversized pull requests + id: oversized + if: github.event_name == 'pull_request' && github.event.pull_request.changed_files > 3000 + run: echo "force_all=true" >> "$GITHUB_OUTPUT" - uses: ./.github/actions/paths-filter id: filter - if: github.event_name != 'push' # Run all scans on master push + if: github.event_name != 'push' && steps.oversized.outputs.force_all != 'true' with: filters: | python: @@ -79,27 +77,7 @@ jobs: - 'products/**' - '.semgrep/rules/security/prefer-codegen-api.yaml' - '.github/workflows/ci-security.yaml' - # '**' plus negations, not positive includes, so a new top-level - # directory fails open and still triggers the scan. Keep the - # negations one-for-one with semgrep-general's --exclude flags: - # dropping an --exclude without dropping its negation here leaves - # the files in scan scope with nothing left to trigger the job. - general: - - '**' - - '!cli/**' - - '!common/**' - - '!ee/**' - - '!frontend/**' - - '!livestream/**' - - '!nodejs/**' - - '!posthog/**' - - '!products/**' - - '!rust/**' - - '!.semgrep/**' - - '!docs/**' - - '!services/**' - # Mirrors the directory list semgrep-devex passes to semgrep. Its rules - # span python/typescript/generic, so this can't narrow by extension. + # Keep these paths aligned with semgrep-devex's scan targets. devex: - 'bin/**' - 'common/**' @@ -110,8 +88,7 @@ jobs: - 'products/**' - '.semgrep/rules/devex/**' - '.github/workflows/ci-security.yaml' - # `semgrep --test .semgrep/` only exercises the rules against their - # own committed fixtures, so nothing outside .semgrep/ can change it. + # Rule tests only read fixtures under .semgrep/. test-rules: - '.semgrep/**' - '.github/workflows/ci-security.yaml' @@ -316,8 +293,6 @@ jobs: # scans GitHub Actions and other repo-wide config semgrep-general: - needs: changes - if: needs.changes.outputs.general == 'true' runs-on: ubuntu-latest timeout-minutes: 20 @@ -336,8 +311,7 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - # exclude all directories already scanned by other jobs. - # Keep this list one-for-one with the `general` filter in the changes job. + # exclude all directories already scanned by other jobs - name: Run Semgrep run: | docker run --rm -v "${{ github.workspace }}:/src" -w /src \