diff --git a/src/license_detection/match_refine/merge.rs b/src/license_detection/match_refine/merge.rs index 9b8fe791b..e0811c6c3 100644 --- a/src/license_detection/match_refine/merge.rs +++ b/src/license_detection/match_refine/merge.rs @@ -91,6 +91,74 @@ fn combine_matches(a: &LicenseMatch, b: &LicenseMatch) -> LicenseMatch { merged } +const NONE: usize = usize::MAX; + +/// Matches in merge order, with O(1) removal so merging a long run of +/// matches for one rule stays linear instead of shifting the tail per removal. +struct MatchList { + items: Vec, + next: Vec, + prev: Vec, + head: usize, +} + +impl MatchList { + fn new(items: Vec) -> Self { + let len = items.len(); + Self { + next: (0..len) + .map(|i| if i + 1 < len { i + 1 } else { NONE }) + .collect(), + prev: (0..len) + .map(|i| if i == 0 { NONE } else { i - 1 }) + .collect(), + items, + head: 0, + } + } + + /// Unlinks `index` and returns its successor. + fn remove(&mut self, index: usize) -> usize { + let (prev, next) = (self.prev[index], self.next[index]); + if prev == NONE { + self.head = next; + } else { + self.next[prev] = next; + } + if next != NONE { + self.prev[next] = prev; + } + next + } + + /// Removes the anchor at `index` and returns the next anchor. Removing the + /// first element also skips the new first element. + fn remove_anchor(&mut self, index: usize) -> Option { + let was_first = self.prev[index] == NONE; + let next = self.remove(index); + let anchor = if was_first && next != NONE { + self.next[next] + } else { + next + }; + Some(anchor).filter(|&n| n != NONE) + } + + fn into_ordered(self) -> Vec { + let Self { + items, next, head, .. + } = self; + let mut slots: Vec> = items.into_iter().map(Some).collect(); + let mut ordered = Vec::new(); + let mut index = head; + while index != NONE { + ordered.extend(slots[index].take()); + index = next[index]; + } + ordered + } +} + /// Merge overlapping and adjacent matches for the same rule. /// /// Based on Python: `merge_matches()` (match.py:869-1068) @@ -140,22 +208,23 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec let rule_length = rule_matches[0].rule_length; let max_rule_side_dist = (rule_length / 2).clamp(1, MAX_DIST); - let mut rule_matches: Vec = - rule_matches.iter().map(|m| (*m).clone()).collect(); - let mut i = 0; + let mut list = MatchList::new(rule_matches.iter().map(|m| (*m).clone()).collect()); + let mut anchor = Some(list.head); - while i < rule_matches.len().saturating_sub(1) { - let mut j = i + 1; + while let Some(i) = anchor + && list.next[i] != NONE + { + let mut j = list.next[i]; - while j < rule_matches.len() { - let current = &rule_matches[i]; - let next = &rule_matches[j]; + while j != NONE { + let current = &list.items[i]; + let next = &list.items[j]; let same_coordinate_kind = current.has_rule_alignment() == next.has_rule_alignment(); let both_rule_aligned = current.has_rule_alignment() && next.has_rule_alignment(); if !same_coordinate_kind { - j += 1; + j = list.next[j]; continue; } @@ -163,7 +232,7 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec && current.matcher == MatcherKind::SpdxId && next.matcher == MatcherKind::SpdxId { - j += 1; + j = list.next[j]; continue; } @@ -178,7 +247,7 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec if current.qspan_set() == next.qspan_set() && (!both_rule_aligned || current.ispan_set() == next.ispan_set()) { - rule_matches.remove(j); + j = list.remove(j); continue; } @@ -189,22 +258,20 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec let current_mag = current.qspan_magnitude(); let next_mag = next.qspan_magnitude(); if current_mag <= next_mag { - rule_matches.remove(j); + j = list.remove(j); continue; } else { - rule_matches.remove(i); - i = i.saturating_sub(1); + anchor = list.remove_anchor(i); break; } } if current.qcontains(next) { - rule_matches.remove(j); + j = list.remove(j); continue; } if next.qcontains(current) { - rule_matches.remove(i); - i = i.saturating_sub(1); + anchor = list.remove_anchor(i); break; } @@ -213,8 +280,8 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec if !both_rule_aligned || combined.query_span().len() == combined.effective_ispan().len() { - rule_matches[i] = combined; - rule_matches.remove(j); + list.items[i] = combined; + j = list.remove(j); continue; } } @@ -223,16 +290,15 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec if !both_rule_aligned || combined.query_span().len() == combined.effective_ispan().len() { - rule_matches[j] = combined; - rule_matches.remove(i); - i = i.saturating_sub(1); + list.items[j] = combined; + anchor = list.remove_anchor(i); break; } } if next.is_after(current) { - rule_matches[i] = combine_matches(current, next); - rule_matches.remove(j); + list.items[i] = combine_matches(current, next); + j = list.remove(j); continue; } @@ -251,19 +317,21 @@ pub fn merge_overlapping_matches(matches: &[LicenseMatch]) -> Vec if qoverlap > 0 { let ioverlap = current.ispan_overlap(next); if qoverlap == ioverlap { - rule_matches[i] = combine_matches(current, next); - rule_matches.remove(j); + list.items[i] = combine_matches(current, next); + j = list.remove(j); continue; } } } } - j += 1; + j = list.next[j]; + } + if anchor == Some(i) { + anchor = Some(list.next[i]).filter(|&n| n != NONE); } - i += 1; } - merged.extend(rule_matches); + merged.extend(list.into_ordered()); } merged @@ -388,6 +456,41 @@ mod tests { } } + fn list_of(len: usize) -> MatchList { + MatchList::new( + (1..=len) + .map(|line| create_test_match("#1", line, line, MatchScore::MAX, 100.0, 100)) + .collect(), + ) + } + + fn ordered_lines(list: MatchList) -> Vec { + list.into_ordered().iter().map(|m| m.start_token).collect() + } + + #[test] + fn test_match_list_removes_in_place_and_keeps_order() { + let mut list = list_of(5); + assert_eq!(list.remove(2), 3); + assert_eq!(list.remove(4), NONE); + assert_eq!(ordered_lines(list), vec![1, 2, 4]); + } + + #[test] + fn test_match_list_removing_the_first_anchor_skips_the_new_first() { + let mut list = list_of(4); + assert_eq!(list.remove_anchor(0), Some(2)); + assert_eq!(ordered_lines(list), vec![2, 3, 4]); + } + + #[test] + fn test_match_list_removing_a_later_anchor_continues_with_its_successor() { + let mut list = list_of(4); + assert_eq!(list.remove_anchor(1), Some(2)); + assert_eq!(list.remove_anchor(3), None); + assert_eq!(ordered_lines(list), vec![1, 3]); + } + fn create_test_match( rule_identifier: &str, start_line: usize, diff --git a/src/license_detection/mod.rs b/src/license_detection/mod.rs index 8a241b015..d89e7b71a 100644 --- a/src/license_detection/mod.rs +++ b/src/license_detection/mod.rs @@ -304,9 +304,47 @@ fn filter_redundant_same_expression_seq_containers( .collect() } +/// Fully-matched exact matches ordered by start position, so a wrapper only +/// examines the exact matches near its own span. +struct ExactMatchIndex { + entries: Vec<(usize, usize)>, + max_extent: usize, +} + +impl ExactMatchIndex { + fn new(candidate_contained_matches: &[LicenseMatch]) -> Self { + let mut entries = Vec::new(); + let mut max_extent = 0; + for (index, m) in candidate_contained_matches.iter().enumerate() { + if m.matcher == aho_match::MATCH_AHO && has_full_match_coverage(m) { + let (start, end) = m.qspan_bounds(); + entries.push((start, index)); + max_extent = max_extent.max(end - start); + } + } + entries.sort_unstable(); + Self { + entries, + max_extent, + } + } + + /// Indices, in original order, of the exact matches that may overlap `start..end`. + fn near(&self, start: usize, end: usize) -> Vec { + let first = self + .entries + .partition_point(|&(s, _)| s < start.saturating_sub(self.max_extent)); + let last = self.entries.partition_point(|&(s, _)| s < end); + let mut indices: Vec = self.entries[first..last].iter().map(|&(_, i)| i).collect(); + indices.sort_unstable(); + indices + } +} + fn is_redundant_low_coverage_composite_seq_wrapper( container: &LicenseMatch, candidate_contained_matches: &[LicenseMatch], + exact_matches: &ExactMatchIndex, ) -> bool { // Provenant-specific tuning (no ScanCode equivalent): only very-low-coverage seq // wrappers are eligible for redundant-composite dropping. A seq match at or above @@ -318,8 +356,11 @@ fn is_redundant_low_coverage_composite_seq_wrapper( let container_qspan_set = container.qspan_set(); - let children: Vec<&LicenseMatch> = candidate_contained_matches - .iter() + let (container_start, container_end) = container.qspan_bounds(); + let children: Vec<&LicenseMatch> = exact_matches + .near(container_start, container_end) + .into_iter() + .map(|index| &candidate_contained_matches[index]) .filter(|m| { m.matcher == aho_match::MATCH_AHO && has_full_match_coverage(m) @@ -374,10 +415,15 @@ fn filter_redundant_low_coverage_composite_seq_wrappers( seq_matches: Vec, candidate_contained_matches: &[LicenseMatch], ) -> Vec { + let exact_matches = ExactMatchIndex::new(candidate_contained_matches); seq_matches .into_iter() .filter(|m| { - !is_redundant_low_coverage_composite_seq_wrapper(m, candidate_contained_matches) + !is_redundant_low_coverage_composite_seq_wrapper( + m, + candidate_contained_matches, + &exact_matches, + ) }) .collect() } diff --git a/src/license_detection/seq_match/matching.rs b/src/license_detection/seq_match/matching.rs index 3511f5f09..e60b556b8 100644 --- a/src/license_detection/seq_match/matching.rs +++ b/src/license_detection/seq_match/matching.rs @@ -21,21 +21,135 @@ use crate::license_detection::query::QueryRun; use crate::models::LineNumber; use crate::models::MatchScore; use bit_set::BitSet; -use rustc_hash::FxHashMap; -use std::collections::HashMap; +use rustc_hash::{FxHashMap, FxHashSet}; use super::MATCH_SEQ; use super::candidates::Candidate; +type HighPostings<'a> = FxHashMap; +type TokenOccurrences = FxHashMap>; + +/// Sorted matchable legalese positions per query token, so a search can visit +/// only the positions whose token occurs in the rule range being matched. +fn token_occurrences( + query_tokens: &[TokenId], + matchables: &BitSet, + len_legalese: usize, +) -> TokenOccurrences { + let mut occurrences = TokenOccurrences::default(); + for (i, &token) in query_tokens.iter().enumerate() { + if token.as_usize() < len_legalese && matchables.contains(i) { + occurrences.entry(token).or_default().push(i); + } + } + occurrences +} + +const MAX_SPARSE_RULE_TOKENS: usize = 8; + +/// Ascending merge of the occurrence lists of a few tokens. +struct MergedPositions<'a> { + lists: Vec<&'a [usize]>, +} + +impl Iterator for MergedPositions<'_> { + type Item = usize; + + fn next(&mut self) -> Option { + let list = self + .lists + .iter_mut() + .filter(|list| !list.is_empty()) + .min_by_key(|list| list[0])?; + let position = list[0]; + *list = &list[1..]; + Some(position) + } +} + +/// The query positions in `query_lo..query_hi` that can contribute to a match +/// against `rule_lo..rule_hi`, or `None` when a dense scan is cheaper. A dense +/// scan visits every position; any other position leaves the rolling run map +/// empty, which a gap between returned positions reproduces. +fn sparse_positions<'a>( + context: &MatchSearchContext<'a>, + query_lo: usize, + query_hi: usize, + rule_lo: usize, + rule_hi: usize, +) -> Option> { + let mut rule_set: Vec = Vec::new(); + for &token in &context.rule_tokens[rule_lo..rule_hi] { + if context.high_postings.contains_key(&token) && !rule_set.contains(&token) { + if rule_set.len() == MAX_SPARSE_RULE_TOKENS { + return None; + } + rule_set.push(token); + } + } + + let budget = (query_hi - query_lo) / 4; + let mut total = 0; + let mut lists = Vec::with_capacity(rule_set.len()); + for token in rule_set { + let Some(all) = context.occurrences.get(&token) else { + continue; + }; + let start = all.partition_point(|&p| p < query_lo); + let end = all.partition_point(|&p| p < query_hi); + total += end - start; + if total > budget { + return None; + } + lists.push(&all[start..end]); + } + Some(MergedPositions { lists }) +} + struct MatchSearchContext<'a> { query_tokens: &'a [TokenId], rule_tokens: &'a [TokenId], - high_postings: &'a HashMap>, + high_postings: &'a HighPostings<'a>, len_legalese: usize, matchables: &'a BitSet, + run_potential: &'a [usize], + occurrences: &'a TokenOccurrences, cancel: Option<&'a CancelToken>, } +/// `potential[i]` bounds the longest diagonal run that can end at or after `i`: +/// each position must be matchable, legalese and in the rule's postings, and +/// each adjacent query pair must also be adjacent somewhere in the rule. +fn run_potential( + query_tokens: &[TokenId], + rule_tokens: &[TokenId], + matchables: &BitSet, + high_postings: &HighPostings<'_>, + len_legalese: usize, +) -> Vec { + let rule_pairs: FxHashSet<(TokenId, TokenId)> = + rule_tokens.windows(2).map(|w| (w[0], w[1])).collect(); + let mut potential = vec![0usize; query_tokens.len() + 1]; + let mut run = 0usize; + for (i, &token) in query_tokens.iter().enumerate() { + let active = token.as_usize() < len_legalese + && matchables.contains(i) + && high_postings.contains_key(&token); + run = if !active { + 0 + } else if i > 0 && run > 0 && rule_pairs.contains(&(query_tokens[i - 1], token)) { + run + 1 + } else { + 1 + }; + potential[i] = run; + } + for i in (0..query_tokens.len()).rev() { + potential[i] = potential[i].max(potential[i + 1]); + } + potential +} + /// Find the longest matching block between query and rule token sequences. /// /// Uses dynamic programming to find the longest contiguous matching subsequence. @@ -78,11 +192,27 @@ fn find_longest_match_impl( let mut j2len: FxHashMap = FxHashMap::default(); let mut new_j2len: FxHashMap = FxHashMap::default(); - for (offset, i) in (query_lo..query_hi).enumerate() { + let visited: Box + '_> = + match sparse_positions(context, query_lo, query_hi, rule_lo, rule_hi) { + Some(positions) => Box::new(positions), + None => Box::new(query_lo..query_hi), + }; + let mut prev_i = None; + + for (offset, i) in visited.enumerate() { if offset.is_multiple_of(256) { crate::license_detection::ensure_not_cancelled(context.cancel)?; } + if prev_i.is_none_or(|prev| prev + 1 != i) { + j2len.clear(); + } + prev_i = Some(i); + + if best_size >= (rule_hi - rule_lo).min(context.run_potential[i]) { + break; + } + new_j2len.clear(); let cur_a = context.query_tokens[i]; @@ -90,7 +220,7 @@ fn find_longest_match_impl( && context.matchables.contains(i) && let Some(positions) = context.high_postings.get(&cur_a) { - for &j in positions { + for &j in *positions { if j < rule_lo { continue; } @@ -267,6 +397,7 @@ pub(crate) fn seq_match_with_candidates_and_cancel( // `qstart <= max_matchable` is an O(1) equivalent of // `matchables.iter().any(|p| p >= qstart)`, avoiding an O(N) scan per round. let max_matchable = matchables.iter().max(); + let occurrences = token_occurrences(query_tokens, &matchables, len_legalese); for (candidate_index, candidate) in candidates.iter().enumerate() { if candidate_index.is_multiple_of(8) { @@ -275,24 +406,33 @@ pub(crate) fn seq_match_with_candidates_and_cancel( let rid = candidate.rid; let rule_tokens = index.rule_tokens(rid); - let high_postings: HashMap> = index + let high_postings: HighPostings<'_> = index .high_postings_by_rid .get(&rid) .map(|hp| { hp.iter() .filter(|(tid, _)| candidate.high_set_intersection.contains(&tid.raw())) - .map(|(&tid, postings)| (tid, postings.clone())) + .map(|(&tid, postings)| (tid, postings.as_slice())) .collect() }) .unwrap_or_default(); if let Some(rule_tokens) = rule_tokens { + let run_potential = run_potential( + query_tokens, + rule_tokens, + &matchables, + &high_postings, + len_legalese, + ); let context = MatchSearchContext { query_tokens, rule_tokens, high_postings: &high_postings, len_legalese, matchables: &matchables, + run_potential: &run_potential, + occurrences: &occurrences, cancel, }; @@ -405,12 +545,20 @@ pub(crate) fn seq_match_with_candidates_and_cancel( mod tests { use super::*; use crate::license_detection::index::dictionary::{TokenId, tid}; + use std::collections::HashMap; use std::ops::Range; fn tids(values: &[u16]) -> Vec { values.iter().copied().map(tid).collect() } + fn borrowed_postings(postings: &HashMap>) -> super::HighPostings<'_> { + postings + .iter() + .map(|(&tid, v)| (tid, v.as_slice())) + .collect() + } + fn find_longest_match_for_test( query_tokens: &[TokenId], rule_tokens: &[TokenId], @@ -420,12 +568,23 @@ mod tests { len_legalese: usize, matchables: &BitSet, ) -> (usize, usize, usize) { + let high_postings = borrowed_postings(high_postings); + let occurrences = token_occurrences(query_tokens, matchables, len_legalese); + let potential = run_potential( + query_tokens, + rule_tokens, + matchables, + &high_postings, + len_legalese, + ); let context = MatchSearchContext { query_tokens, rule_tokens, - high_postings, + high_postings: &high_postings, len_legalese, matchables, + run_potential: &potential, + occurrences: &occurrences, cancel: None, }; find_longest_match_impl( @@ -446,18 +605,150 @@ mod tests { len_legalese: usize, matchables: &BitSet, ) -> Vec<(usize, usize, usize)> { + let high_postings = borrowed_postings(high_postings); + let occurrences = token_occurrences(query_tokens, matchables, len_legalese); + let potential = run_potential( + query_tokens, + rule_tokens, + matchables, + &high_postings, + len_legalese, + ); let context = MatchSearchContext { query_tokens, rule_tokens, - high_postings, + high_postings: &high_postings, len_legalese, matchables, + run_potential: &potential, + occurrences: &occurrences, cancel: None, }; match_blocks_impl(&context, query_range.start, query_range.end) .expect("Sequence block matching without cancel should not time out") } + /// Exhaustive longest-match search with none of the production shortcuts. + fn reference_longest_match( + query: &[TokenId], + rule: &[TokenId], + query_range: Range, + rule_range: Range, + len_legalese: usize, + matchables: &BitSet, + ) -> (usize, usize, usize) { + let active = |i: usize, j: usize| { + query[i] == rule[j] && query[i].as_usize() < len_legalese && matchables.contains(i) + }; + let mut run = vec![vec![0usize; rule.len()]; query.len()]; + let (mut best_i, mut best_j, mut best_size) = (query_range.start, rule_range.start, 0); + for i in query_range.clone() { + for j in rule_range.clone() { + if !active(i, j) { + continue; + } + let previous = if i > query_range.start && j > rule_range.start { + run[i - 1][j - 1] + } else { + 0 + }; + run[i][j] = previous + 1; + if run[i][j] > best_size { + best_size = run[i][j]; + (best_i, best_j) = (i + 1 - best_size, j + 1 - best_size); + } + } + } + if best_size > 0 { + while best_i > query_range.start + && best_j > rule_range.start + && query[best_i - 1] == rule[best_j - 1] + && matchables.contains(best_i - 1) + { + best_i -= 1; + best_j -= 1; + best_size += 1; + } + while best_i + best_size < query_range.end + && best_j + best_size < rule_range.end + && query[best_i + best_size] == rule[best_j + best_size] + && matchables.contains(best_i + best_size) + { + best_size += 1; + } + } + (best_i, best_j, best_size) + } + + #[test] + fn test_sparse_and_dense_scans_match_an_exhaustive_search() { + // Appending more than `MAX_SPARSE_RULE_TOKENS` tokens that never occur in + // the query forces the dense scan without changing any match. + let len_legalese = 60; + let dummies: Vec = (50..51 + MAX_SPARSE_RULE_TOKENS as u16).collect(); + let mut state = 0x2545_f491_4f6c_dd1d_u64; + let mut next = |bound: usize| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state % bound as u64) as usize + }; + + for _ in 0..300 { + let query: Vec = (0..300 + next(200)).map(|_| next(40) as u16).collect(); + let rule: Vec = (0..4 + next(5)).map(|_| next(24) as u16).collect(); + let mut dense_rule = rule.clone(); + dense_rule.extend(&dummies); + + let query_lo = next(query.len() / 2); + let query_range = query_lo..query_lo + 1 + next(query.len() - query_lo); + let rule_lo = next(rule.len() / 2); + let rule_range = rule_lo..rule_lo + 1 + next(rule.len() - rule_lo); + + let query_tokens = tids(&query); + let matchables: BitSet = (0..query.len()).filter(|_| next(8) != 0).collect(); + let search = |rule: &[u16], rule_range: Range| { + let rule_tokens = tids(rule); + let mut postings: HashMap> = HashMap::new(); + for (j, token) in rule_tokens.iter().enumerate() { + if token.as_usize() < len_legalese { + postings.entry(*token).or_default().push(j); + } + } + find_longest_match_for_test( + &query_tokens, + &rule_tokens, + query_range.clone(), + rule_range, + &postings, + len_legalese, + &matchables, + ) + }; + let reference = |rule_range: Range| { + reference_longest_match( + &query_tokens, + &tids(&rule), + query_range.clone(), + rule_range, + len_legalese, + &matchables, + ) + }; + + assert_eq!( + search(&rule, rule_range.clone()), + reference(rule_range.clone()), + "sparse: query {query:?} {query_range:?} rule {rule:?} {rule_range:?}" + ); + assert_eq!( + search(&dense_rule, 0..dense_rule.len()), + reference(0..rule.len()), + "dense: query {query:?} {query_range:?} rule {rule:?}" + ); + } + } + #[test] fn test_find_longest_match_basic() { let query_tokens = tids(&[0, 1, 2, 3]);