Skip to content

perf(license): remove quadratic rescans on long repetitive attribution lists #1452

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mstykow merged 1 commit into from
Oct 11, 2026
Merged

perf(license): remove quadratic rescans on long repetitive attribution lists #1452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 132 additions & 29 deletions src/license_detection/match_refine/merge.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,74 @@ fn combine_matches(a: &LicenseMatch, b: &LicenseMatch) -> LicenseMatch {
merged
}

const NONE: usize = usize::MAX;

/// Matches in merge order, with O(1) removal so merging a long run of
/// matches for one rule stays linear instead of shifting the tail per removal.
struct MatchList {
items: Vec<LicenseMatch>,
next: Vec<usize>,
prev: Vec<usize>,
head: usize,
}

impl MatchList {
fn new(items: Vec<LicenseMatch>) -> Self {
let len = items.len();
Self {
next: (0..len)
.map(|i| if i + 1 < len { i + 1 } else { NONE })
.collect(),
prev: (0..len)
.map(|i| if i == 0 { NONE } else { i - 1 })
.collect(),
items,
head: 0,
}
}

/// Unlinks `index` and returns its successor.
fn remove(&mut self, index: usize) -> usize {
let (prev, next) = (self.prev[index], self.next[index]);
if prev == NONE {
self.head = next;
} else {
self.next[prev] = next;
}
if next != NONE {
self.prev[next] = prev;
}
next
}

/// Removes the anchor at `index` and returns the next anchor. Removing the
/// first element also skips the new first element.
fn remove_anchor(&mut self, index: usize) -> Option<usize> {
let was_first = self.prev[index] == NONE;
let next = self.remove(index);
let anchor = if was_first && next != NONE {
self.next[next]
} else {
next
};
Some(anchor).filter(|&n| n != NONE)
}

fn into_ordered(self) -> Vec<LicenseMatch> {
let Self {
items, next, head, ..
} = self;
let mut slots: Vec<Option<LicenseMatch>> = items.into_iter().map(Some).collect();
let mut ordered = Vec::new();
let mut index = head;
while index != NONE {
ordered.extend(slots[index].take());
index = next[index];
}
ordered
}
}

/// Merge overlapping and adjacent matches for the same rule.
///
/// Based on Python: `merge_matches()` (match.py:869-1068)
Expand Down Expand Up @@ -140,30 +208,31 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec<LicenseMatch>
let rule_length = rule_matches[0].rule_length;
let max_rule_side_dist = (rule_length / 2).clamp(1, MAX_DIST);

let mut rule_matches: Vec<LicenseMatch> =
rule_matches.iter().map(|m| (*m).clone()).collect();
let mut i = 0;
let mut list = MatchList::new(rule_matches.iter().map(|m| (*m).clone()).collect());
let mut anchor = Some(list.head);

while i < rule_matches.len().saturating_sub(1) {
let mut j = i + 1;
while let Some(i) = anchor
&& list.next[i] != NONE
{
let mut j = list.next[i];

while j < rule_matches.len() {
let current = &rule_matches[i];
let next = &rule_matches[j];
while j != NONE {
let current = &list.items[i];
let next = &list.items[j];
let same_coordinate_kind =
current.has_rule_alignment() == next.has_rule_alignment();
let both_rule_aligned = current.has_rule_alignment() && next.has_rule_alignment();

if !same_coordinate_kind {
j += 1;
j = list.next[j];
continue;
}

if !both_rule_aligned
&& current.matcher == MatcherKind::SpdxId
&& next.matcher == MatcherKind::SpdxId
{
j += 1;
j = list.next[j];
continue;
}

Expand All @@ -178,7 +247,7 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec<LicenseMatch>
if current.qspan_set() == next.qspan_set()
&& (!both_rule_aligned || current.ispan_set() == next.ispan_set())
{
rule_matches.remove(j);
j = list.remove(j);
continue;
}

Expand All @@ -189,22 +258,20 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec<LicenseMatch>
let current_mag = current.qspan_magnitude();
let next_mag = next.qspan_magnitude();
if current_mag <= next_mag {
rule_matches.remove(j);
j = list.remove(j);
continue;
} else {
rule_matches.remove(i);
i = i.saturating_sub(1);
anchor = list.remove_anchor(i);
break;
}
}

if current.qcontains(next) {
rule_matches.remove(j);
j = list.remove(j);
continue;
}
if next.qcontains(current) {
rule_matches.remove(i);
i = i.saturating_sub(1);
anchor = list.remove_anchor(i);
break;
}

Expand All @@ -213,8 +280,8 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec<LicenseMatch>
if !both_rule_aligned
|| combined.query_span().len() == combined.effective_ispan().len()
{
rule_matches[i] = combined;
rule_matches.remove(j);
list.items[i] = combined;
j = list.remove(j);
continue;
}
}
Expand All @@ -223,16 +290,15 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec<LicenseMatch>
if !both_rule_aligned
|| combined.query_span().len() == combined.effective_ispan().len()
{
rule_matches[j] = combined;
rule_matches.remove(i);
i = i.saturating_sub(1);
list.items[j] = combined;
anchor = list.remove_anchor(i);
break;
}
}

if next.is_after(current) {
rule_matches[i] = combine_matches(current, next);
rule_matches.remove(j);
list.items[i] = combine_matches(current, next);
j = list.remove(j);
continue;
}

Expand All @@ -251,19 +317,21 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec<LicenseMatch>
if qoverlap > 0 {
let ioverlap = current.ispan_overlap(next);
if qoverlap == ioverlap {
rule_matches[i] = combine_matches(current, next);
rule_matches.remove(j);
list.items[i] = combine_matches(current, next);
j = list.remove(j);
continue;
}
}
}
}

j += 1;
j = list.next[j];
}
if anchor == Some(i) {
anchor = Some(list.next[i]).filter(|&n| n != NONE);
}
i += 1;
}
merged.extend(rule_matches);
merged.extend(list.into_ordered());
}

merged
Expand Down Expand Up @@ -388,6 +456,41 @@ mod tests {
}
}

fn list_of(len: usize) -> MatchList {
MatchList::new(
(1..=len)
.map(|line| create_test_match("#1", line, line, MatchScore::MAX, 100.0, 100))
.collect(),
)
}

fn ordered_lines(list: MatchList) -> Vec<usize> {
list.into_ordered().iter().map(|m| m.start_token).collect()
}

#[test]
fn test_match_list_removes_in_place_and_keeps_order() {
let mut list = list_of(5);
assert_eq!(list.remove(2), 3);
assert_eq!(list.remove(4), NONE);
assert_eq!(ordered_lines(list), vec![1, 2, 4]);
}

#[test]
fn test_match_list_removing_the_first_anchor_skips_the_new_first() {
let mut list = list_of(4);
assert_eq!(list.remove_anchor(0), Some(2));
assert_eq!(ordered_lines(list), vec![2, 3, 4]);
}

#[test]
fn test_match_list_removing_a_later_anchor_continues_with_its_successor() {
let mut list = list_of(4);
assert_eq!(list.remove_anchor(1), Some(2));
assert_eq!(list.remove_anchor(3), None);
assert_eq!(ordered_lines(list), vec![1, 3]);
}

fn create_test_match(
rule_identifier: &str,
start_line: usize,
Expand Down
52 changes: 49 additions & 3 deletions src/license_detection/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,9 +304,47 @@ fn filter_redundant_same_expression_seq_containers(
.collect()
}

/// Fully-matched exact matches ordered by start position, so a wrapper only
/// examines the exact matches near its own span.
struct ExactMatchIndex {
entries: Vec<(usize, usize)>,
max_extent: usize,
}

impl ExactMatchIndex {
fn new(candidate_contained_matches: &[LicenseMatch]) -> Self {
let mut entries = Vec::new();
let mut max_extent = 0;
for (index, m) in candidate_contained_matches.iter().enumerate() {
if m.matcher == aho_match::MATCH_AHO && has_full_match_coverage(m) {
let (start, end) = m.qspan_bounds();
entries.push((start, index));
max_extent = max_extent.max(end - start);
}
}
entries.sort_unstable();
Self {
entries,
max_extent,
}
}

/// Indices, in original order, of the exact matches that may overlap `start..end`.
fn near(&self, start: usize, end: usize) -> Vec<usize> {
let first = self
.entries
.partition_point(|&(s, _)| s < start.saturating_sub(self.max_extent));
let last = self.entries.partition_point(|&(s, _)| s < end);
let mut indices: Vec<usize> = self.entries[first..last].iter().map(|&(_, i)| i).collect();
indices.sort_unstable();
indices
}
}

fn is_redundant_low_coverage_composite_seq_wrapper(
container: &LicenseMatch,
candidate_contained_matches: &[LicenseMatch],
exact_matches: &ExactMatchIndex,
) -> bool {
// Provenant-specific tuning (no ScanCode equivalent): only very-low-coverage seq
// wrappers are eligible for redundant-composite dropping. A seq match at or above
Expand All @@ -318,8 +356,11 @@ fn is_redundant_low_coverage_composite_seq_wrapper(

let container_qspan_set = container.qspan_set();

let children: Vec<&LicenseMatch> = candidate_contained_matches
.iter()
let (container_start, container_end) = container.qspan_bounds();
let children: Vec<&LicenseMatch> = exact_matches
.near(container_start, container_end)
.into_iter()
.map(|index| &candidate_contained_matches[index])
.filter(|m| {
m.matcher == aho_match::MATCH_AHO
&& has_full_match_coverage(m)
Expand Down Expand Up @@ -374,10 +415,15 @@ fn filter_redundant_low_coverage_composite_seq_wrappers(
seq_matches: Vec<LicenseMatch>,
candidate_contained_matches: &[LicenseMatch],
) -> Vec<LicenseMatch> {
let exact_matches = ExactMatchIndex::new(candidate_contained_matches);
seq_matches
.into_iter()
.filter(|m| {
!is_redundant_low_coverage_composite_seq_wrapper(m, candidate_contained_matches)
!is_redundant_low_coverage_composite_seq_wrapper(
m,
candidate_contained_matches,
&exact_matches,
)
})
.collect()
}
Expand Down
Loading
Loading