diff --git a/aho.go b/aho.go index b3c0e2c..3d271c2 100644 --- a/aho.go +++ b/aho.go @@ -3,9 +3,10 @@ package portcullis // kwMask is a fixed-size bitset over keyword indices, used to record // which patterns occurred in a scanned input and which keywords each // rule subscribes to. Storing it as a small array keeps every test -// branch-free; the cap of 320 indices accommodates today's catalogue -// of ~285 unique keywords with limited remaining headroom for future -// rules (an overflow trips a deterministic panic in [buildAhoCorasick]). +// branch-free; the cap of len(kwMask)*64 indices (currently 320) +// accommodates today's catalogue of ~285 unique keywords with limited +// remaining headroom for future rules (an overflow trips a +// deterministic panic in [buildAhoCorasick]). type kwMask [5]uint64 func (m *kwMask) empty() bool { return m[0]|m[1]|m[2]|m[3]|m[4] == 0 } @@ -73,7 +74,7 @@ type acAutomaton struct { // buildAhoCorasick compiles patterns into an automaton. Patterns // must be lower-cased ASCII. func buildAhoCorasick(patterns []string) *acAutomaton { - if len(patterns) > 320 { + if len(patterns) > len(kwMask{})*64 { panic("portcullis: too many AC patterns for kwMask") } @@ -227,24 +228,14 @@ func (a *acAutomaton) scanSerial(text string) (mask kwMask) { off := raw &^ acceptBit i++ if raw&acceptBit != 0 { - ap := &accept[off>>stateShift] - mask[0] |= ap[0] - mask[1] |= ap[1] - mask[2] |= ap[2] - mask[3] |= ap[3] - mask[4] |= ap[4] + mask.orIn(&accept[off>>stateShift]) } for i < n && off != 0 { raw = next[off+uint32(text[i])] off = raw &^ acceptBit i++ if raw&acceptBit != 0 { - ap := &accept[off>>stateShift] - mask[0] |= ap[0] - mask[1] |= ap[1] - mask[2] |= ap[2] - mask[3] |= ap[3] - mask[4] |= ap[4] + mask.orIn(&accept[off>>stateShift]) } } } diff --git a/aho_test.go b/aho_test.go index a0fd913..5be9ffd 100644 --- a/aho_test.go +++ b/aho_test.go @@ -102,12 +102,11 @@ func TestAhoCorasickPropagatesAllKwMaskWords(t *testing.T) { } // TestAhoCorasickPanicOnTooManyPatterns verifies that buildAhoCorasick -// panics when given more than 320 patterns, which would overflow the -// kwMask bitset. +// panics when given more patterns than the kwMask bitset can index. func TestAhoCorasickPanicOnTooManyPatterns(t *testing.T) { t.Parallel() - patterns := make([]string, 321) + patterns := make([]string, len(kwMask{})*64+1) for i := range patterns { // Each pattern must be unique to avoid trie conflicts; encode // the index as a 3-letter base-26 string. diff --git a/cmd/portcullis-scan/main.go b/cmd/portcullis-scan/main.go index cc56007..230911d 100644 --- a/cmd/portcullis-scan/main.go +++ b/cmd/portcullis-scan/main.go @@ -328,9 +328,18 @@ func scanFileBytes(path, root string, maxSize int64, scanBinary bool) ([]byte, e display = rel } var b strings.Builder + // Matches arrive in ascending Start order, so line/column are + // tracked with one cursor instead of rescanning data from the + // start for every match. + line, lastNL, pos := 1, -1, 0 for _, m := range matches { - line, col := lineCol(data, m.Start) - fmt.Fprintf(&b, "%s:%d:%d: %s\n", display, line, col, sanitizeValue(m.Value)) + for ; pos < m.Start; pos++ { + if data[pos] == '\n' { + line++ + lastNL = pos + } + } + fmt.Fprintf(&b, "%s:%d:%d: %s\n", display, line, pos-lastNL, sanitizeValue(m.Value)) } return []byte(b.String()), nil } @@ -386,23 +395,6 @@ func readIfScannable(path string, maxSize int64, scanBinary bool) (data []byte, return append(sniff, rest...), true, nil } -// lineCol returns the 1-based line and column for offset within data. -// Column counts bytes since the last newline. -func lineCol(data []byte, offset int) (line, col int) { - if offset > len(data) { - offset = len(data) - } - line = 1 - last := -1 - for i := range offset { - if data[i] == '\n' { - line++ - last = i - } - } - return line, offset - last -} - // sanitizeValue collapses CR / LF in v so a multi-line match (e.g. a // PEM block) stays on a single output line. func sanitizeValue(v string) string { diff --git a/cmd/portcullis-scan/main_test.go b/cmd/portcullis-scan/main_test.go index 5b19dc8..f4fa4ec 100644 --- a/cmd/portcullis-scan/main_test.go +++ b/cmd/portcullis-scan/main_test.go @@ -41,6 +41,64 @@ func TestRunPrintsEachSecretFound(t *testing.T) { assert.Equal(t, 3, strings.Count(out, "\n")) } +// TestScanFileBytesLineColTracking locks down the incremental +// line/column cursor: matches at offset zero, several matches on one +// line, CRLF line endings, and empty files. +func TestScanFileBytesLineColTracking(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + content string + want []string + }{ + { + name: "match at offset zero", + content: githubPAT + "\n", + want: []string{"f:1:1: " + githubPAT}, + }, + { + name: "two matches on the same line", + content: githubPAT + " " + githubPAT + "\n", + want: []string{ + "f:1:1: " + githubPAT, + "f:1:42: " + githubPAT, + }, + }, + { + name: "crlf line endings", + content: "clean line\r\n" + githubPAT + "\r\nkey: " + githubPAT + "\r\n", + want: []string{ + "f:2:1: " + githubPAT, + "f:3:6: " + githubPAT, + }, + }, + { + name: "empty file", + content: "", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + root := t.TempDir() + path := filepath.Join(root, "f") + require.NoError(t, os.WriteFile(path, []byte(tc.content), 0o644)) + + buf, err := scanFileBytes(path, root, defaultMaxSize, false) + + require.NoError(t, err) + var wantOut string + if len(tc.want) > 0 { + wantOut = strings.Join(tc.want, "\n") + "\n" + } + assert.Equal(t, wantOut, string(buf)) + }) + } +} + func TestRunReturnsZeroWhenNoSecrets(t *testing.T) { t.Parallel() diff --git a/portcullis.go b/portcullis.go index 5c8c0f6..15b924a 100644 --- a/portcullis.go +++ b/portcullis.go @@ -36,13 +36,26 @@ func Find(text string) []Match { return dedupOverlapping(findMatches(text)) } -func findMatches(text string) []Match { +// scanPrelude runs the shared pre-filter behind [Find], [Redact], +// and [Contains]: the Aho–Corasick pass over text. ok is false when +// no rule could possibly match — empty input, or no keyword hit and +// no always-run rules — letting callers return without touching any +// regex. +func scanPrelude(text string) (rs *ruleSet, found kwMask, ok bool) { if text == "" { - return nil + return nil, kwMask{}, false } - rs := compiledRuleSet() - found := rs.ac.scan(text) + rs = compiledRuleSet() + found = rs.ac.scan(text) if found.empty() && !rs.hasAlwaysRun { + return nil, kwMask{}, false + } + return rs, found, true +} + +func findMatches(text string) []Match { + rs, found, ok := scanPrelude(text) + if !ok { return nil } var matches []Match @@ -67,6 +80,19 @@ func findMatches(text string) []Match { return matches } +// sortMatches orders matches by Start ascending, breaking ties by +// End descending so that at equal starts the longest span comes +// first — the order both [dedupOverlapping] and [mergeOverlapping] +// rely on for their single greedy pass. +func sortMatches(matches []Match) { + slices.SortFunc(matches, func(a, b Match) int { + if a.Start != b.Start { + return a.Start - b.Start + } + return b.End - a.End + }) +} + // dedupOverlapping collapses overlapping matches to one per underlying // span, keeping the longest. After sorting by Start asc, End desc, a // greedy walk drops anything contained in the last kept match, and a @@ -80,12 +106,7 @@ func dedupOverlapping(matches []Match) []Match { if len(matches) < 2 { return matches } - slices.SortFunc(matches, func(a, b Match) int { - if a.Start != b.Start { - return a.Start - b.Start - } - return b.End - a.End - }) + sortMatches(matches) out := matches[:0] for _, m := range matches { if len(out) == 0 || m.Start >= out[len(out)-1].End { @@ -104,12 +125,8 @@ func dedupOverlapping(matches []Match) []Match { // Contains reports whether text matches any built-in secret rule. // It is safe for concurrent use. func Contains(text string) bool { - if text == "" { - return false - } - rs := compiledRuleSet() - found := rs.ac.scan(text) - if found.empty() && !rs.hasAlwaysRun { + rs, found, ok := scanPrelude(text) + if !ok { return false } for i := range rs.rules { @@ -186,12 +203,7 @@ func mergeOverlapping(matches []Match) []Match { if len(matches) < 2 { return matches } - slices.SortFunc(matches, func(a, b Match) int { - if a.Start != b.Start { - return a.Start - b.Start - } - return b.End - a.End - }) + sortMatches(matches) out := matches[:0] for _, m := range matches { if len(out) == 0 || m.Start >= out[len(out)-1].End { diff --git a/rules.go b/rules.go index 93998c5..1518b90 100644 --- a/rules.go +++ b/rules.go @@ -95,7 +95,7 @@ func contextual(vendor, body string) string { // the regex-compiled form actually used at scan time. // //nolint:funlen // single-source-of-truth for the ruleset -var rules = sync.OnceValue(func() []rule { +func rules() []rule { return []rule{ { // aws-access-key-id. Prefix list mirrors the gitleaks @@ -1991,7 +1991,7 @@ var rules = sync.OnceValue(func() []rule { keywords: []string{"secret-live-", "secret-test-"}, }, } -}) +} // compiledRule is the runtime form of a [rule]: its keywords are // folded into a [kwMask] over the catalogue's shared keyword index,