Reject SPDX declarations cut off by the byte limit

README.md

@@ -1,3 +1,3 @@
 # licenses
 
 Go library and command for matching license text and scanning repositories
@@ -69,6 +69,10 @@
 Demoted matches stay visible in the file report but do not contribute to the
 repository expression totals.
 
+An `SPDX-License-Identifier` expression longer than 1,024 bytes is parsed only
+when a single rule match covers all of it. Otherwise the tag is reported as an
+`spdx-license-identifier` clue, and matches inside the expression are dropped.
+
 The default `project` scope skips hidden, dependency, build, cache, and
 test-data directories. Use `-scope all` for dependency-license scans. An
 explicit `-skip` list applies in either scope. Both scopes exclude `.git`.

spdx.go

@@ -124,6 +124,7 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool {
 	coverage := spdxMatchSpans(result)
 	next, coveredEnd := 0, 0
 	var declarations []spdxDeclaration
+	var cutOff []spdxSpan
 	pending := make(map[string]*Detection)
 	for offset := first; offset < len(input); {
 		anchor := indexSPDXAnchor(input, offset)
@@ -135,11 +136,24 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool {
 			offset = anchor + 1
 			continue
 		}
-		start, end := spdxExpressionSpan(input, tagEnd)
+		start, end, complete := spdxExpressionSpan(input, tagEnd)
+		if !complete {
+			start, end, _ = spdxExpressionSpanWithin(input, tagEnd, len(input))
+		}
 		offset = max(end, tagEnd)
 		if start >= end {
 			continue
 		}
+		for next < len(coverage) && coverage[next].start <= start {
+			coveredEnd = max(coveredEnd, coverage[next].end)
+			next++
+		}
+		covered := coveredEnd >= end
+		if !complete && !covered {
+			// Component matches would state part of the expression as the whole.
+			cutOff = append(cutOff, spdxSpan{anchor, end})
+			continue
+		}
 		expression, identifiers, scanCodeIDs := m.engine.spdx.normalizeExpression(input[start:end])
 		if expression == "" {
 			continue
@@ -149,11 +163,7 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool {
 			Start:      anchor,
 			End:        end,
 		})
-		for next < len(coverage) && coverage[next].start <= start {
-			coveredEnd = max(coveredEnd, coverage[next].end)
-			next++
-		}
-		if coveredEnd >= end {
+		if covered {
 			declarations = append(declarations, spdxDeclaration{anchor: anchor, start: start, end: end, covered: true})
 			continue
 		}
@@ -173,14 +183,12 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool {
 		detection.Matches = append(detection.Matches, match)
 	}
 	changed := removeMatches(result, func(match Match) bool {
-		index := sort.Search(len(declarations), func(i int) bool { return declarations[i].end >= match.End })
-		if index == len(declarations) {
-			return false
-		}
-		declaration := declarations[index]
-		return match.Start >= declaration.anchor &&
-			(!declaration.covered || match.Start > declaration.start || match.End < declaration.end)
+		return supersededBySPDX(match, declarations, cutOff)
 	})
+	for _, span := range cutOff {
+		result.Clues = append(result.Clues, m.spdxCutOffClue(input, span))
+		changed = true
+	}
 	if len(pending) == 0 {
 		return changed
 	}
@@ -197,6 +205,36 @@ func (m *Matcher) matchSPDXTags(input []byte, result *Result) bool {
 	return true
 }
 
+// spdxCutOffClue marks a tag whose expression exceeds the parse limit and is
+// not covered by a rule match.
+func (m *Matcher) spdxCutOffClue(input []byte, span spdxSpan) Match {
+	clue := Match{RuleID: spdxTagRuleID, Kind: KindClue, Method: SpdxID, Start: span.start, End: span.end}
+	if m.matchedText {
+		clue.Matched = slices.Clone(input[span.start:span.end])
+	}
+	return clue
+}
+
+// supersededBySPDX reports whether a declaration or cut-off tag replaces match.
+func supersededBySPDX(match Match, declarations []spdxDeclaration, cutOff []spdxSpan) bool {
+	if spanContains(cutOff, match.Start) {
+		return true
+	}
+	index := sort.Search(len(declarations), func(i int) bool { return declarations[i].end >= match.End })
+	if index == len(declarations) {
+		return false
+	}
+	declaration := declarations[index]
+	return match.Start >= declaration.anchor &&
+		(!declaration.covered || match.Start > declaration.start || match.End < declaration.end)
+}
+
+// spanContains reports whether offset falls inside one of the sorted spans.
+func spanContains(spans []spdxSpan, offset int) bool {
+	index := sort.Search(len(spans), func(i int) bool { return spans[i].end > offset })
+	return index < len(spans) && offset >= spans[index].start
+}
+
 // Earlier declarations cannot remove a match extending over a later expression.
 func spdxMatchSpans(result *Result) []spdxSpan {
 	var spans []spdxSpan
@@ -261,26 +299,33 @@ func spdxTagEnd(input []byte, anchor int) int {
 
 // spdxExpressionSpan returns the trimmed byte range of the expression that
 // follows a tag colon. The expression ends at end-of-line, a closing block
-// comment marker, or maxSPDXExpressionBytes.
-func spdxExpressionSpan(input []byte, from int) (int, int) {
-	limit := min(len(input), from+maxSPDXExpressionBytes)
+// comment marker, or EOF. The boolean is false when maxSPDXExpressionBytes
+// cuts it off.
+func spdxExpressionSpan(input []byte, from int) (int, int, bool) {
+	return spdxExpressionSpanWithin(input, from, maxSPDXExpressionBytes)
+}
+
+func spdxExpressionSpanWithin(input []byte, from, maxBytes int) (int, int, bool) {
+	limit := min(len(input), from+maxBytes)
 	end := limit
-	for offset := from; offset < limit; offset++ {
+	complete := limit == len(input)
+scan:
+	for offset := from; offset <= limit && offset < len(input); offset++ {
 		switch input[offset] {
 		case '\n', '\r':
-			end = offset
+			end, complete = offset, true
+			break scan
 		case '*':
-			if offset+1 < limit && input[offset+1] == '/' {
-				end = offset
+			if offset+1 < len(input) && input[offset+1] == '/' {
+				end, complete = offset, true
+				break scan
 			}
 		case '-':
-			if offset+2 < limit && input[offset+1] == '-' && input[offset+2] == '>' {
-				end = offset
+			if offset+2 < len(input) && input[offset+1] == '-' && input[offset+2] == '>' {
+				end, complete = offset, true
+				break scan
 			}
 		}
-		if end != limit {
-			break
-		}
 	}
 	start := from
 	for start < end && input[start] == ' ' || start < end && input[start] == '\t' {
@@ -289,7 +334,7 @@ func spdxExpressionSpan(input []byte, from int) (int, int) {
 	for end > start && (input[end-1] == ' ' || input[end-1] == '\t') {
 		end--
 	}
-	return start, end
+	return start, end, complete
 }
 
 // normalizeExpression parses raw SPDX expression bytes and returns the

spdx_boundary_api_test.go

@@ -0,0 +1,123 @@
+package licenses_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/git-pkgs/licenses"
+)
+
+func TestSPDXExpressionBoundary(t *testing.T) {
+	t.Parallel()
+	matcher, err := licenses.New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	const limit = 1024
+	for _, size := range []int{limit - 1, limit, limit + 1} {
+		for _, ending := range []string{"", "\n", "\r\n", "*/", "-->"} {
+			t.Run(fmt.Sprintf("%d/%q", size, ending), func(t *testing.T) {
+				const suffix = "AND LicenseRef-Example"
+				expression := "MIT" + strings.Repeat(" ", size-len("MIT")-len(suffix)) + suffix
+				input := "SPDX-License-Identifier:" + expression + ending
+				result, err := matcher.Match(context.Background(), []byte(input))
+				if err != nil {
+					t.Fatal(err)
+				}
+				if size > limit {
+					requireSPDXCutOffClue(t, result, len(input)-len(ending))
+					return
+				}
+				const want = "MIT AND LicenseRef-scancode-unknown-spdx"
+				if len(result.SPDXDeclarations) != 1 || result.SPDXDeclarations[0].Expression != want ||
+					result.SPDXDeclarations[0].End != len(input)-len(ending) {
+					t.Fatalf("declarations = %+v", result.SPDXDeclarations)
+				}
+				if len(result.Detections) != 1 || result.Detections[0].Expression != want {
+					t.Fatalf("detections = %+v", result.Detections)
+				}
+			})
+		}
+	}
+}
+
+func TestSPDXExpressionLimitDropsComponentMatches(t *testing.T) {
+	t.Parallel()
+	matcher, err := licenses.New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	input := "SPDX-License-Identifier: MIT" + strings.Repeat(" ", 1020) + " AND LicenseRef-Example\n"
+	result, err := matcher.Match(context.Background(), []byte(input))
+	if err != nil {
+		t.Fatal(err)
+	}
+	requireSPDXCutOffClue(t, result, len(input)-1)
+}
+
+func TestSPDXExpressionLimitKeepsCoveringMatch(t *testing.T) {
+	t.Parallel()
+	matcher, err := licenses.New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	input := "SPDX-License-Identifier: MIT" + strings.Repeat(" ", 1020) + " OR Apache-2.0\n"
+	result, err := matcher.Match(context.Background(), []byte(input))
+	if err != nil {
+		t.Fatal(err)
+	}
+	const want = "MIT OR Apache-2.0"
+	if len(result.SPDXDeclarations) != 1 || result.SPDXDeclarations[0].Expression != want ||
+		result.SPDXDeclarations[0].End != len(input)-1 {
+		t.Fatalf("declarations = %+v", result.SPDXDeclarations)
+	}
+	if len(result.Detections) != 1 || result.Detections[0].Expression != want || len(result.Clues) != 0 {
+		t.Fatalf("result = %+v", result)
+	}
+}
+
+func requireSPDXCutOffClue(t *testing.T, result licenses.Result, end int) {
+	t.Helper()
+	if len(result.Detections) != 0 || len(result.SPDXDeclarations) != 0 {
+		t.Fatalf("cut-off expression produced licensing: %+v", result)
+	}
+	if len(result.Clues) != 1 || result.Clues[0].RuleID != "spdx-license-identifier" ||
+		result.Clues[0].Kind != licenses.KindClue || result.Clues[0].Start != 0 || result.Clues[0].End != end {
+		t.Fatalf("clues = %+v", result.Clues)
+	}
+}
+
+func TestScanRepositorySPDXExpressionLimit(t *testing.T) {
+	t.Parallel()
+	matcher, err := licenses.New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	root := t.TempDir()
+	cutOff := "// SPDX-License-Identifier: MIT" + strings.Repeat(" ", 1020) + " AND LicenseRef-Example\n"
+	source := "// SPDX-License-Identifier: Apache-2.0\n" + cutOff
+	// Source maps escape newlines, so a header tag runs past the limit.
+	sourceMap := `{"sourcesContent":["// SPDX-License-Identifier: MIT\n` + strings.Repeat(`export const x = 1;\n`, 80) + `"]}`
+	for name, content := range map[string]string{"main.go": source, "main.js.map": sourceMap} {
+		if err := os.WriteFile(filepath.Join(root, name), []byte(content), 0600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	report, err := licenses.ScanRepository(context.Background(), matcher, root, licenses.DefaultScanOptions())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(report.Errors) != 0 {
+		t.Fatalf("errors = %+v", report.Errors)
+	}
+	if len(report.Expressions) != 1 || report.Expressions[0].Expression != "Apache-2.0" {
+		t.Fatalf("expressions = %+v", report.Expressions)
+	}
+	if report.Summary.FilesWithClues != 2 {
+		t.Fatalf("summary = %+v", report.Summary)
+	}
+}

spdx_test.go

@@ -1,3 +1,3 @@
 package licenses
 
 import (
@@ -341,7 +341,10 @@
 		{input: "", want: ""},
 	}
 	for _, test := range tests {
-		start, end := spdxExpressionSpan([]byte(test.input), 0)
+		start, end, complete := spdxExpressionSpan([]byte(test.input), 0)
+		if !complete {
+			t.Fatalf("span(%q) was truncated", test.input)
+		}
 		if got := test.input[start:end]; got != test.want {
 			t.Errorf("span(%q) = %q, want %q", test.input, got, test.want)
 		}
@@ -355,9 +358,9 @@
 	for i := range input {
 		input[i] = 'A'
 	}
-	_, end := spdxExpressionSpan(input, 0)
-	if end != maxSPDXExpressionBytes {
-		t.Fatalf("end = %d, want %d", end, maxSPDXExpressionBytes)
+	_, end, complete := spdxExpressionSpan(input, 0)
+	if end != maxSPDXExpressionBytes || complete {
+		t.Fatalf("end = %d, complete = %v", end, complete)
 	}
 }