From a7f942848931a542c8672c8b4040a10315fc1ebe Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 1 Oct 2026 11:08:04 +0100 Subject: [PATCH] Handle SPDX declarations cut off by the byte limit Keep an overlong expression only when one rule match covers it; otherwise report the tag as a clue and drop matches inside it. --- README.md | 4 ++ spdx.go | 97 ++++++++++++++++++++++-------- spdx_boundary_api_test.go | 123 ++++++++++++++++++++++++++++++++++++++ spdx_test.go | 11 ++-- 4 files changed, 205 insertions(+), 30 deletions(-) create mode 100644 spdx_boundary_api_test.go diff --git a/README.md b/README.md index 124d39e..aa76b3f 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,10 @@ including lines with a shared source-comment prefix, remain detections. Demoted matches stay visible in the file report but do not contribute to the repository expression totals. +An `SPDX-License-Identifier` expression longer than 1,024 bytes is parsed only +when a single rule match covers all of it. Otherwise the tag is reported as an +`spdx-license-identifier` clue, and matches inside the expression are dropped. + The default `project` scope skips hidden, dependency, build, cache, and test-data directories. Use `-scope all` for dependency-license scans. An explicit `-skip` list applies in either scope. Both scopes exclude `.git`. diff --git a/spdx.go b/spdx.go index d5ce14b..dc5585d 100644 --- a/spdx.go +++ b/spdx.go @@ -124,6 +124,7 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool { coverage := spdxMatchSpans(result) next, coveredEnd := 0, 0 var declarations []spdxDeclaration + var cutOff []spdxSpan pending := make(map[string]*Detection) for offset := first; offset < len(input); { anchor := indexSPDXAnchor(input, offset) @@ -135,11 +136,24 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool { offset = anchor + 1 continue } - start, end := spdxExpressionSpan(input, tagEnd) + start, end, complete := spdxExpressionSpan(input, tagEnd) + if !complete { + start, end, _ = spdxExpressionSpanWithin(input, tagEnd, len(input)) + } offset = max(end, tagEnd) if start >= end { continue } + for next < len(coverage) && coverage[next].start <= start { + coveredEnd = max(coveredEnd, coverage[next].end) + next++ + } + covered := coveredEnd >= end + if !complete && !covered { + // Component matches would state part of the expression as the whole. + cutOff = append(cutOff, spdxSpan{anchor, end}) + continue + } expression, identifiers, scanCodeIDs := m.engine.spdx.normalizeExpression(input[start:end]) if expression == "" { continue @@ -149,11 +163,7 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool { Start: anchor, End: end, }) - for next < len(coverage) && coverage[next].start <= start { - coveredEnd = max(coveredEnd, coverage[next].end) - next++ - } - if coveredEnd >= end { + if covered { declarations = append(declarations, spdxDeclaration{anchor: anchor, start: start, end: end, covered: true}) continue } @@ -173,14 +183,12 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool { detection.Matches = append(detection.Matches, match) } changed := removeMatches(result, func(match Match) bool { - index := sort.Search(len(declarations), func(i int) bool { return declarations[i].end >= match.End }) - if index == len(declarations) { - return false - } - declaration := declarations[index] - return match.Start >= declaration.anchor && - (!declaration.covered || match.Start > declaration.start || match.End < declaration.end) + return supersededBySPDX(match, declarations, cutOff) }) + for _, span := range cutOff { + result.Clues = append(result.Clues, m.spdxCutOffClue(input, span)) + changed = true + } if len(pending) == 0 { return changed } @@ -197,6 +205,36 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool { return true } +// spdxCutOffClue marks a tag whose expression exceeds the parse limit and is +// not covered by a rule match. +func (m *Matcher) spdxCutOffClue(input []byte, span spdxSpan) Match { + clue := Match{RuleID: spdxTagRuleID, Kind: KindClue, Method: SpdxID, Start: span.start, End: span.end} + if m.matchedText { + clue.Matched = slices.Clone(input[span.start:span.end]) + } + return clue +} + +// supersededBySPDX reports whether a declaration or cut-off tag replaces match. +func supersededBySPDX(match Match, declarations []spdxDeclaration, cutOff []spdxSpan) bool { + if spanContains(cutOff, match.Start) { + return true + } + index := sort.Search(len(declarations), func(i int) bool { return declarations[i].end >= match.End }) + if index == len(declarations) { + return false + } + declaration := declarations[index] + return match.Start >= declaration.anchor && + (!declaration.covered || match.Start > declaration.start || match.End < declaration.end) +} + +// spanContains reports whether offset falls inside one of the sorted spans. +func spanContains(spans []spdxSpan, offset int) bool { + index := sort.Search(len(spans), func(i int) bool { return spans[i].end > offset }) + return index < len(spans) && offset >= spans[index].start +} + // Earlier declarations cannot remove a match extending over a later expression. func spdxMatchSpans(result *Result) []spdxSpan { var spans []spdxSpan @@ -261,26 +299,33 @@ func spdxTagEnd(input []byte, anchor int) int { // spdxExpressionSpan returns the trimmed byte range of the expression that // follows a tag colon. The expression ends at end-of-line, a closing block -// comment marker, or maxSPDXExpressionBytes. -func spdxExpressionSpan(input []byte, from int) (int, int) { - limit := min(len(input), from+maxSPDXExpressionBytes) +// comment marker, or EOF. The boolean is false when maxSPDXExpressionBytes +// cuts it off. +func spdxExpressionSpan(input []byte, from int) (int, int, bool) { + return spdxExpressionSpanWithin(input, from, maxSPDXExpressionBytes) +} + +func spdxExpressionSpanWithin(input []byte, from, maxBytes int) (int, int, bool) { + limit := min(len(input), from+maxBytes) end := limit - for offset := from; offset < limit; offset++ { + complete := limit == len(input) +scan: + for offset := from; offset <= limit && offset < len(input); offset++ { switch input[offset] { case '\n', '\r': - end = offset + end, complete = offset, true + break scan case '*': - if offset+1 < limit && input[offset+1] == '/' { - end = offset + if offset+1 < len(input) && input[offset+1] == '/' { + end, complete = offset, true + break scan } case '-': - if offset+2 < limit && input[offset+1] == '-' && input[offset+2] == '>' { - end = offset + if offset+2 < len(input) && input[offset+1] == '-' && input[offset+2] == '>' { + end, complete = offset, true + break scan } } - if end != limit { - break - } } start := from for start < end && input[start] == ' ' || start < end && input[start] == '\t' { @@ -289,7 +334,7 @@ func spdxExpressionSpan(input []byte, from int) (int, int) { for end > start && (input[end-1] == ' ' || input[end-1] == '\t') { end-- } - return start, end + return start, end, complete } // normalizeExpression parses raw SPDX expression bytes and returns the diff --git a/spdx_boundary_api_test.go b/spdx_boundary_api_test.go new file mode 100644 index 0000000..45b2111 --- /dev/null +++ b/spdx_boundary_api_test.go @@ -0,0 +1,123 @@ +package licenses_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/git-pkgs/licenses" +) + +func TestSPDXExpressionBoundary(t *testing.T) { + t.Parallel() + matcher, err := licenses.New() + if err != nil { + t.Fatal(err) + } + const limit = 1024 + for _, size := range []int{limit - 1, limit, limit + 1} { + for _, ending := range []string{"", "\n", "\r\n", "*/", "-->"} { + t.Run(fmt.Sprintf("%d/%q", size, ending), func(t *testing.T) { + const suffix = "AND LicenseRef-Example" + expression := "MIT" + strings.Repeat(" ", size-len("MIT")-len(suffix)) + suffix + input := "SPDX-License-Identifier:" + expression + ending + result, err := matcher.Match(context.Background(), []byte(input)) + if err != nil { + t.Fatal(err) + } + if size > limit { + requireSPDXCutOffClue(t, result, len(input)-len(ending)) + return + } + const want = "MIT AND LicenseRef-scancode-unknown-spdx" + if len(result.SPDXDeclarations) != 1 || result.SPDXDeclarations[0].Expression != want || + result.SPDXDeclarations[0].End != len(input)-len(ending) { + t.Fatalf("declarations = %+v", result.SPDXDeclarations) + } + if len(result.Detections) != 1 || result.Detections[0].Expression != want { + t.Fatalf("detections = %+v", result.Detections) + } + }) + } + } +} + +func TestSPDXExpressionLimitDropsComponentMatches(t *testing.T) { + t.Parallel() + matcher, err := licenses.New() + if err != nil { + t.Fatal(err) + } + input := "SPDX-License-Identifier: MIT" + strings.Repeat(" ", 1020) + " AND LicenseRef-Example\n" + result, err := matcher.Match(context.Background(), []byte(input)) + if err != nil { + t.Fatal(err) + } + requireSPDXCutOffClue(t, result, len(input)-1) +} + +func TestSPDXExpressionLimitKeepsCoveringMatch(t *testing.T) { + t.Parallel() + matcher, err := licenses.New() + if err != nil { + t.Fatal(err) + } + input := "SPDX-License-Identifier: MIT" + strings.Repeat(" ", 1020) + " OR Apache-2.0\n" + result, err := matcher.Match(context.Background(), []byte(input)) + if err != nil { + t.Fatal(err) + } + const want = "MIT OR Apache-2.0" + if len(result.SPDXDeclarations) != 1 || result.SPDXDeclarations[0].Expression != want || + result.SPDXDeclarations[0].End != len(input)-1 { + t.Fatalf("declarations = %+v", result.SPDXDeclarations) + } + if len(result.Detections) != 1 || result.Detections[0].Expression != want || len(result.Clues) != 0 { + t.Fatalf("result = %+v", result) + } +} + +func requireSPDXCutOffClue(t *testing.T, result licenses.Result, end int) { + t.Helper() + if len(result.Detections) != 0 || len(result.SPDXDeclarations) != 0 { + t.Fatalf("cut-off expression produced licensing: %+v", result) + } + if len(result.Clues) != 1 || result.Clues[0].RuleID != "spdx-license-identifier" || + result.Clues[0].Kind != licenses.KindClue || result.Clues[0].Start != 0 || result.Clues[0].End != end { + t.Fatalf("clues = %+v", result.Clues) + } +} + +func TestScanRepositorySPDXExpressionLimit(t *testing.T) { + t.Parallel() + matcher, err := licenses.New() + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + cutOff := "// SPDX-License-Identifier: MIT" + strings.Repeat(" ", 1020) + " AND LicenseRef-Example\n" + source := "// SPDX-License-Identifier: Apache-2.0\n" + cutOff + // Source maps escape newlines, so a header tag runs past the limit. + sourceMap := `{"sourcesContent":["// SPDX-License-Identifier: MIT\n` + strings.Repeat(`export const x = 1;\n`, 80) + `"]}` + for name, content := range map[string]string{"main.go": source, "main.js.map": sourceMap} { + if err := os.WriteFile(filepath.Join(root, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + report, err := licenses.ScanRepository(context.Background(), matcher, root, licenses.DefaultScanOptions()) + if err != nil { + t.Fatal(err) + } + if len(report.Errors) != 0 { + t.Fatalf("errors = %+v", report.Errors) + } + if len(report.Expressions) != 1 || report.Expressions[0].Expression != "Apache-2.0" { + t.Fatalf("expressions = %+v", report.Expressions) + } + if report.Summary.FilesWithClues != 2 { + t.Fatalf("summary = %+v", report.Summary) + } +} diff --git a/spdx_test.go b/spdx_test.go index 91501ba..b334a8f 100644 --- a/spdx_test.go +++ b/spdx_test.go @@ -341,7 +341,10 @@ func TestSPDXExpressionSpan(t *testing.T) { {input: "", want: ""}, } for _, test := range tests { - start, end := spdxExpressionSpan([]byte(test.input), 0) + start, end, complete := spdxExpressionSpan([]byte(test.input), 0) + if !complete { + t.Fatalf("span(%q) was truncated", test.input) + } if got := test.input[start:end]; got != test.want { t.Errorf("span(%q) = %q, want %q", test.input, got, test.want) } @@ -355,9 +358,9 @@ func TestSPDXExpressionSpanBounded(t *testing.T) { for i := range input { input[i] = 'A' } - _, end := spdxExpressionSpan(input, 0) - if end != maxSPDXExpressionBytes { - t.Fatalf("end = %d, want %d", end, maxSPDXExpressionBytes) + _, end, complete := spdxExpressionSpan(input, 0) + if end != maxSPDXExpressionBytes || complete { + t.Fatalf("end = %d, complete = %v", end, complete) } }