From 8160dc94e8d91e69d8cf0c31b9d284f901efeb39 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 24 Jul 2026 07:34:51 +0200 Subject: [PATCH 1/6] simplify: make rules a plain function Its only caller, compiledRuleSet, is already memoised behind sync.OnceValue, so the inner OnceValue wrapper added a layer of indirection without saving any work. --- rules.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rules.go b/rules.go index 93998c5..1518b90 100644 --- a/rules.go +++ b/rules.go @@ -95,7 +95,7 @@ func contextual(vendor, body string) string { // the regex-compiled form actually used at scan time. // //nolint:funlen // single-source-of-truth for the ruleset -var rules = sync.OnceValue(func() []rule { +func rules() []rule { return []rule{ { // aws-access-key-id. Prefix list mirrors the gitleaks @@ -1991,7 +1991,7 @@ var rules = sync.OnceValue(func() []rule { keywords: []string{"secret-live-", "secret-test-"}, }, } -}) +} // compiledRule is the runtime form of a [rule]: its keywords are // folded into a [kwMask] over the catalogue's shared keyword index, From 9e5cdf58a2e560adf1a2a269699503a078c99cef Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 24 Jul 2026 07:35:50 +0200 Subject: [PATCH 2/6] simplify: reuse kwMask.orIn in scanSerial, derive AC pattern cap from mask size The two hand-unrolled OR blocks duplicated kwMask.orIn, which the compiler already inlines to the same code (verified with -gcflags=-m and A/B benchmarks). The 320-pattern cap is now computed from the kwMask array length so it can't silently drift if the mask grows. --- aho.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/aho.go b/aho.go index b3c0e2c..da625a1 100644 --- a/aho.go +++ b/aho.go @@ -73,7 +73,7 @@ type acAutomaton struct { // buildAhoCorasick compiles patterns into an automaton. Patterns // must be lower-cased ASCII. func buildAhoCorasick(patterns []string) *acAutomaton { - if len(patterns) > 320 { + if len(patterns) > len(kwMask{})*64 { panic("portcullis: too many AC patterns for kwMask") } @@ -227,24 +227,14 @@ func (a *acAutomaton) scanSerial(text string) (mask kwMask) { off := raw &^ acceptBit i++ if raw&acceptBit != 0 { - ap := &accept[off>>stateShift] - mask[0] |= ap[0] - mask[1] |= ap[1] - mask[2] |= ap[2] - mask[3] |= ap[3] - mask[4] |= ap[4] + mask.orIn(&accept[off>>stateShift]) } for i < n && off != 0 { raw = next[off+uint32(text[i])] off = raw &^ acceptBit i++ if raw&acceptBit != 0 { - ap := &accept[off>>stateShift] - mask[0] |= ap[0] - mask[1] |= ap[1] - mask[2] |= ap[2] - mask[3] |= ap[3] - mask[4] |= ap[4] + mask.orIn(&accept[off>>stateShift]) } } } From 7aef84b8cad536680adff615ba29ea47c27ec42e Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 24 Jul 2026 07:36:15 +0200 Subject: [PATCH 3/6] simplify: factor shared match-sort comparator into sortMatches dedupOverlapping and mergeOverlapping used the same inline comparator; a named helper documents the ordering contract once. --- portcullis.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/portcullis.go b/portcullis.go index 5c8c0f6..16df10c 100644 --- a/portcullis.go +++ b/portcullis.go @@ -67,6 +67,19 @@ func findMatches(text string) []Match { return matches } +// sortMatches orders matches by Start ascending, breaking ties by +// End descending so that at equal starts the longest span comes +// first — the order both [dedupOverlapping] and [mergeOverlapping] +// rely on for their single greedy pass. +func sortMatches(matches []Match) { + slices.SortFunc(matches, func(a, b Match) int { + if a.Start != b.Start { + return a.Start - b.Start + } + return b.End - a.End + }) +} + // dedupOverlapping collapses overlapping matches to one per underlying // span, keeping the longest. After sorting by Start asc, End desc, a // greedy walk drops anything contained in the last kept match, and a @@ -80,12 +93,7 @@ func dedupOverlapping(matches []Match) []Match { if len(matches) < 2 { return matches } - slices.SortFunc(matches, func(a, b Match) int { - if a.Start != b.Start { - return a.Start - b.Start - } - return b.End - a.End - }) + sortMatches(matches) out := matches[:0] for _, m := range matches { if len(out) == 0 || m.Start >= out[len(out)-1].End { @@ -186,12 +194,7 @@ func mergeOverlapping(matches []Match) []Match { if len(matches) < 2 { return matches } - slices.SortFunc(matches, func(a, b Match) int { - if a.Start != b.Start { - return a.Start - b.Start - } - return b.End - a.End - }) + sortMatches(matches) out := matches[:0] for _, m := range matches { if len(out) == 0 || m.Start >= out[len(out)-1].End { From 4441721d4ce42fefbab2203ee2be1a45a5f0af8f Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 24 Jul 2026 07:36:33 +0200 Subject: [PATCH 4/6] simplify: factor shared AC pre-filter prologue into scanPrelude findMatches and Contains duplicated the empty-input check, rule-set lookup, and keyword-mask early-out; one helper now owns that logic. --- portcullis.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/portcullis.go b/portcullis.go index 16df10c..de962db 100644 --- a/portcullis.go +++ b/portcullis.go @@ -36,13 +36,26 @@ func Find(text string) []Match { return dedupOverlapping(findMatches(text)) } -func findMatches(text string) []Match { +// scanPrelude runs the shared pre-filter for [Find] and [Contains]: +// the Aho–Corasick pass over text. ok is false when no rule could +// possibly match — empty input, or no keyword hit and no +// always-run rules — letting callers return without touching any +// regex. +func scanPrelude(text string) (rs *ruleSet, found kwMask, ok bool) { if text == "" { - return nil + return nil, kwMask{}, false } - rs := compiledRuleSet() - found := rs.ac.scan(text) + rs = compiledRuleSet() + found = rs.ac.scan(text) if found.empty() && !rs.hasAlwaysRun { + return nil, kwMask{}, false + } + return rs, found, true +} + +func findMatches(text string) []Match { + rs, found, ok := scanPrelude(text) + if !ok { return nil } var matches []Match @@ -112,12 +125,8 @@ func dedupOverlapping(matches []Match) []Match { // Contains reports whether text matches any built-in secret rule. // It is safe for concurrent use. func Contains(text string) bool { - if text == "" { - return false - } - rs := compiledRuleSet() - found := rs.ac.scan(text) - if found.empty() && !rs.hasAlwaysRun { + rs, found, ok := scanPrelude(text) + if !ok { return false } for i := range rs.rules { From 8a848457fe68b1b2a4ff78446e9028dcaf2975b8 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 24 Jul 2026 07:36:59 +0200 Subject: [PATCH 5/6] perf: compute line/col with one cursor per file in portcullis-scan lineCol rescanned data from offset 0 for every match, making output formatting O(len(file) x matches). Matches are emitted in ascending Start order, so a single incremental cursor gives identical line and column numbers in one pass. --- cmd/portcullis-scan/main.go | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/cmd/portcullis-scan/main.go b/cmd/portcullis-scan/main.go index cc56007..230911d 100644 --- a/cmd/portcullis-scan/main.go +++ b/cmd/portcullis-scan/main.go @@ -328,9 +328,18 @@ func scanFileBytes(path, root string, maxSize int64, scanBinary bool) ([]byte, e display = rel } var b strings.Builder + // Matches arrive in ascending Start order, so line/column are + // tracked with one cursor instead of rescanning data from the + // start for every match. + line, lastNL, pos := 1, -1, 0 for _, m := range matches { - line, col := lineCol(data, m.Start) - fmt.Fprintf(&b, "%s:%d:%d: %s\n", display, line, col, sanitizeValue(m.Value)) + for ; pos < m.Start; pos++ { + if data[pos] == '\n' { + line++ + lastNL = pos + } + } + fmt.Fprintf(&b, "%s:%d:%d: %s\n", display, line, pos-lastNL, sanitizeValue(m.Value)) } return []byte(b.String()), nil } @@ -386,23 +395,6 @@ func readIfScannable(path string, maxSize int64, scanBinary bool) (data []byte, return append(sniff, rest...), true, nil } -// lineCol returns the 1-based line and column for offset within data. -// Column counts bytes since the last newline. -func lineCol(data []byte, offset int) (line, col int) { - if offset > len(data) { - offset = len(data) - } - line = 1 - last := -1 - for i := range offset { - if data[i] == '\n' { - line++ - last = i - } - } - return line, offset - last -} - // sanitizeValue collapses CR / LF in v so a multi-line match (e.g. a // PEM block) stays on a single output line. func sanitizeValue(v string) string { From 65ad239c45c5d2f31e4eca8bd68ca983c1175ea8 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Fri, 24 Jul 2026 07:50:09 +0200 Subject: [PATCH 6/6] address review: fix stale docs, derive test cap from kwMask, cover line/col cursor - kwMask doc and the overflow test no longer hardcode 320; both now track len(kwMask)*64 so they can't drift from buildAhoCorasick. - scanPrelude doc mentions Redact, which also reaches it via findMatches. - New table-driven scanFileBytes test locks down the incremental line/column cursor: match at offset zero, two matches on one line, CRLF endings, and empty files. --- aho.go | 7 ++-- aho_test.go | 5 ++- cmd/portcullis-scan/main_test.go | 58 ++++++++++++++++++++++++++++++++ portcullis.go | 8 ++--- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/aho.go b/aho.go index da625a1..3d271c2 100644 --- a/aho.go +++ b/aho.go @@ -3,9 +3,10 @@ package portcullis // kwMask is a fixed-size bitset over keyword indices, used to record // which patterns occurred in a scanned input and which keywords each // rule subscribes to. Storing it as a small array keeps every test -// branch-free; the cap of 320 indices accommodates today's catalogue -// of ~285 unique keywords with limited remaining headroom for future -// rules (an overflow trips a deterministic panic in [buildAhoCorasick]). +// branch-free; the cap of len(kwMask)*64 indices (currently 320) +// accommodates today's catalogue of ~285 unique keywords with limited +// remaining headroom for future rules (an overflow trips a +// deterministic panic in [buildAhoCorasick]). type kwMask [5]uint64 func (m *kwMask) empty() bool { return m[0]|m[1]|m[2]|m[3]|m[4] == 0 } diff --git a/aho_test.go b/aho_test.go index a0fd913..5be9ffd 100644 --- a/aho_test.go +++ b/aho_test.go @@ -102,12 +102,11 @@ func TestAhoCorasickPropagatesAllKwMaskWords(t *testing.T) { } // TestAhoCorasickPanicOnTooManyPatterns verifies that buildAhoCorasick -// panics when given more than 320 patterns, which would overflow the -// kwMask bitset. +// panics when given more patterns than the kwMask bitset can index. func TestAhoCorasickPanicOnTooManyPatterns(t *testing.T) { t.Parallel() - patterns := make([]string, 321) + patterns := make([]string, len(kwMask{})*64+1) for i := range patterns { // Each pattern must be unique to avoid trie conflicts; encode // the index as a 3-letter base-26 string. diff --git a/cmd/portcullis-scan/main_test.go b/cmd/portcullis-scan/main_test.go index 5b19dc8..f4fa4ec 100644 --- a/cmd/portcullis-scan/main_test.go +++ b/cmd/portcullis-scan/main_test.go @@ -41,6 +41,64 @@ func TestRunPrintsEachSecretFound(t *testing.T) { assert.Equal(t, 3, strings.Count(out, "\n")) } +// TestScanFileBytesLineColTracking locks down the incremental +// line/column cursor: matches at offset zero, several matches on one +// line, CRLF line endings, and empty files. +func TestScanFileBytesLineColTracking(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + content string + want []string + }{ + { + name: "match at offset zero", + content: githubPAT + "\n", + want: []string{"f:1:1: " + githubPAT}, + }, + { + name: "two matches on the same line", + content: githubPAT + " " + githubPAT + "\n", + want: []string{ + "f:1:1: " + githubPAT, + "f:1:42: " + githubPAT, + }, + }, + { + name: "crlf line endings", + content: "clean line\r\n" + githubPAT + "\r\nkey: " + githubPAT + "\r\n", + want: []string{ + "f:2:1: " + githubPAT, + "f:3:6: " + githubPAT, + }, + }, + { + name: "empty file", + content: "", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + path := filepath.Join(root, "f") + require.NoError(t, os.WriteFile(path, []byte(tc.content), 0o644)) + + buf, err := scanFileBytes(path, root, defaultMaxSize, false) + + require.NoError(t, err) + var wantOut string + if len(tc.want) > 0 { + wantOut = strings.Join(tc.want, "\n") + "\n" + } + assert.Equal(t, wantOut, string(buf)) + }) + } +} + func TestRunReturnsZeroWhenNoSecrets(t *testing.T) { t.Parallel() diff --git a/portcullis.go b/portcullis.go index de962db..15b924a 100644 --- a/portcullis.go +++ b/portcullis.go @@ -36,10 +36,10 @@ func Find(text string) []Match { return dedupOverlapping(findMatches(text)) } -// scanPrelude runs the shared pre-filter for [Find] and [Contains]: -// the Aho–Corasick pass over text. ok is false when no rule could -// possibly match — empty input, or no keyword hit and no -// always-run rules — letting callers return without touching any +// scanPrelude runs the shared pre-filter behind [Find], [Redact], +// and [Contains]: the Aho–Corasick pass over text. ok is false when +// no rule could possibly match — empty input, or no keyword hit and +// no always-run rules — letting callers return without touching any // regex. func scanPrelude(text string) (rs *ruleSet, found kwMask, ok bool) { if text == "" {