From 2858855d75b18ac15265083de89e55947a88ca96 Mon Sep 17 00:00:00 2001 From: Dwight Gunning Date: Thu, 1 Oct 2026 17:34:29 -0400 Subject: [PATCH] fix(grep): document="EX-10.1" selects exactly EX-10.1; literal matches keep source offsets (5qqy, 6th2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 5qqy: Filing._attachment_matches tested document.upper() in doc_type, a substring match, so "EX-10.1" also selected EX-10.10..EX-10.19. Replaced by _select_attachments, which sees the whole list: an exact filename wins, then an exact document type, and the old substring match applies only when nothing matched exactly, so "EX-10" still selects every EX-10 exhibit. PR #1380 added only an exact-filename check ahead of the substring test, which leaves the type collision in place. SeeQC's S-1 (0001213900-26-073222): "agreement" in EX-10.1 is 133 matches; the old filter returned 378, adding EX-10.10/11/12. 6th2: the literal branch of _grep_text found positions in text.lower() and sliced the original text with them. "İ".lower() is two characters, so each one before a match shifted it by one. Literal search now runs re.escape(pattern) with IGNORECASE over the original text. It keeps the old scan's overlapping matches ("aa" in "aaa" is two) by searching again from m.start() + 1 rather than using finditer; regex mode is unchanged. Co-Authored-By: Claude Opus 5.5 --- changelog.d/5qqy.fixed.md | 1 + changelog.d/6th2.fixed.md | 1 + docs/guides/full-text-search.md | 5 +- edgar/_filings.py | 37 ++++--- edgar/search/grep.py | 65 ++++++------- .../search/test_grep_document_and_offsets.py | 97 +++++++++++++++++++ 6 files changed, 156 insertions(+), 50 deletions(-) create mode 100644 changelog.d/5qqy.fixed.md create mode 100644 changelog.d/6th2.fixed.md create mode 100644 tests/search/test_grep_document_and_offsets.py diff --git a/changelog.d/5qqy.fixed.md b/changelog.d/5qqy.fixed.md new file mode 100644 index 000000000..a544f2fbb --- /dev/null +++ b/changelog.d/5qqy.fixed.md @@ -0,0 +1 @@ +**`Filing.grep(document="EX-10.1")` also searched EX-10.10 through EX-10.19.** In SeeQC's S-1 it returned 378 matches for "agreement" where EX-10.1 holds 133. An exact filename or document type now wins, and a substring match applies only when nothing matches exactly, so `document="EX-10"` still selects every EX-10 exhibit. (bead edgartools-5qqy) diff --git a/changelog.d/6th2.fixed.md b/changelog.d/6th2.fixed.md new file mode 100644 index 000000000..f6ae892b0 --- /dev/null +++ b/changelog.d/6th2.fixed.md @@ -0,0 +1 @@ +**A literal `grep` returned shifted matches after a character whose lowercase is longer.** Positions were found in the lowercased text and applied to the original, so four "İ" before "going concern" returned "g concern dou". Literal matches are now found in the original text, case-insensitively. (bead edgartools-6th2) diff --git a/docs/guides/full-text-search.md b/docs/guides/full-text-search.md index 97c88ffd3..50cb06656 100644 --- a/docs/guides/full-text-search.md +++ b/docs/guides/full-text-search.md @@ -189,8 +189,11 @@ m.context # Surrounding text (~100 chars each side) # Only the primary filing document filing.grep("risk factor", document="primary") -# Only a specific exhibit +# Only a specific exhibit (EX-10.1 exactly, not EX-10.10) filing.grep("intellectual property", document="EX-10.1") + +# Every EX-10 exhibit: a value that matches no type exactly matches by substring +filing.grep("intellectual property", document="EX-10") ``` #### Regex Support diff --git a/edgar/_filings.py b/edgar/_filings.py index 10f8d75ec..bb45f2d32 100644 --- a/edgar/_filings.py +++ b/edgar/_filings.py @@ -2191,7 +2191,10 @@ def grep(self, pattern: str, *, regex: bool = False, document: Optional[str] = N pattern: Text to search for (exact match, case-insensitive) regex: If True, treat pattern as a regular expression document: Narrow search to a specific document. Use "primary" for - the main filing document, or a document type like "EX-10.1" + the main filing document, a document type like "EX-10.1" + (exactly that exhibit, not EX-10.10), or a filename. A value + that matches no type or filename exactly, like "EX-10", + selects every document whose type or filename contains it. Returns: GrepResult containing GrepMatch objects with location and context @@ -2211,9 +2214,7 @@ def grep(self, pattern: str, *, regex: bool = False, document: Optional[str] = N except Exception: attachments = [] - for attachment in attachments: - if document and not self._attachment_matches(attachment, document): - continue + for attachment in self._select_attachments(attachments, document): if attachment.empty or attachment.is_binary(): continue @@ -2237,14 +2238,26 @@ def grep(self, pattern: str, *, regex: bool = False, document: Optional[str] = N return GrepResult(pattern, all_matches) @staticmethod - def _attachment_matches(attachment, document: str) -> bool: - """Whether `attachment` satisfies grep()'s `document` filter.""" - if document.lower() == "primary": - return attachment.sequence_number == "1" - doc_type = (attachment.document_type or "").upper() - if document.upper() in doc_type: - return True - return document.lower() in (attachment.document or "").lower() + def _select_attachments(attachments, document: Optional[str]) -> list: + """The attachments grep()'s `document` filter selects. + + An exact filename wins, then an exact document type, and only when + neither matches anything does a substring match apply, so "EX-10.1" + selects EX-10.1 and not EX-10.10 through EX-10.19, while "EX-10" still + selects every EX-10 exhibit. + """ + attachments = list(attachments) + if not document: + return attachments + wanted = document.lower() + if wanted == "primary": + return [a for a in attachments if a.sequence_number == "1"] + for field in ("document", "document_type"): + exact = [a for a in attachments if (getattr(a, field) or "").lower() == wanted] + if exact: + return exact + return [a for a in attachments + if wanted in (a.document_type or "").lower() or wanted in (a.document or "").lower()] @staticmethod def _attachment_location(attachment) -> str: diff --git a/edgar/search/grep.py b/edgar/search/grep.py index 7a8798177..7a2f09084 100644 --- a/edgar/search/grep.py +++ b/edgar/search/grep.py @@ -13,7 +13,7 @@ import re from dataclasses import dataclass -from typing import List, Optional +from typing import Iterator, List, Optional from edgar.richtools import repr_rich @@ -104,6 +104,13 @@ def __repr_html__(self): return repr_rich(self.__rich__()) +def _overlapping(compiled: re.Pattern, text: str) -> Iterator[re.Match]: + pos = 0 + while (m := compiled.search(text, pos)) is not None: + yield m + pos = m.start() + 1 + + def _grep_text(text: str, pattern: str, location: str, regex: bool = False, context_chars: int = 100) -> List[GrepMatch]: """Core grep function: search text for pattern, return matches with context. @@ -113,47 +120,31 @@ def _grep_text(text: str, pattern: str, location: str, if not text or not pattern: return [] - matches = [] - if regex: try: compiled = re.compile(pattern, re.IGNORECASE) except re.error: return [] - for m in compiled.finditer(text): - start = max(0, m.start() - context_chars) - end = min(len(text), m.end() + context_chars) - context = text[start:end].strip() - if start > 0: - context = "..." + context - if end < len(text): - context = context + "..." - matches.append(GrepMatch( - location=location, - match=m.group(), - context=context, - )) + found = compiled.finditer(text) else: - # Case-insensitive substring search - text_lower = text.lower() - pattern_lower = pattern.lower() - start_pos = 0 - while True: - pos = text_lower.find(pattern_lower, start_pos) - if pos == -1: - break - ctx_start = max(0, pos - context_chars) - ctx_end = min(len(text), pos + len(pattern) + context_chars) - context = text[ctx_start:ctx_end].strip() - if ctx_start > 0: - context = "..." + context - if ctx_end < len(text): - context = context + "..." - matches.append(GrepMatch( - location=location, - match=text[pos:pos + len(pattern)], - context=context, - )) - start_pos = pos + 1 + # Searched in the original text, not text.lower(): lowercasing can change + # length ("İ" becomes two characters) and shift every later position. + # Overlapping matches are kept, as the substring scan this replaced did. + found = _overlapping(re.compile(re.escape(pattern), re.IGNORECASE), text) + + matches = [] + for m in found: + start = max(0, m.start() - context_chars) + end = min(len(text), m.end() + context_chars) + context = text[start:end].strip() + if start > 0: + context = "..." + context + if end < len(text): + context = context + "..." + matches.append(GrepMatch( + location=location, + match=m.group(), + context=context, + )) return matches diff --git a/tests/search/test_grep_document_and_offsets.py b/tests/search/test_grep_document_and_offsets.py new file mode 100644 index 000000000..1f9b57b46 --- /dev/null +++ b/tests/search/test_grep_document_and_offsets.py @@ -0,0 +1,97 @@ +"""Filing.grep document selection and literal-match offsets. + +edgartools-5qqy: document="EX-10.1" was a substring test against the document +type, so it also searched EX-10.10 through EX-10.19. + +edgartools-6th2: a literal grep found positions in text.lower() and sliced the +original text with them. Lowercasing "İ" gives two characters, so every match +after one came back shifted. +""" +from collections import Counter +from types import SimpleNamespace +from unittest.mock import PropertyMock, patch + +import pytest + +from edgar import Filing +from edgar.search.grep import _grep_text + + +def _attachment(seq, document_type, document, text=""): + return SimpleNamespace(sequence_number=seq, document_type=document_type, document=document, + empty=False, is_binary=lambda: False, text=lambda: text) + + +ATTACHMENTS = [ + _attachment("1", "S-1", "ea0288470-s1.htm", "primary agreement"), + _attachment("2", "EX-10.1", "ex10-1.htm", "stockholder support agreement"), + _attachment("3", "EX-10.10", "ex10-10.htm", "employment agreement"), + _attachment("4", "EX-10.11", "ex10-11.htm", "employment agreement"), + _attachment("5", "EX-21.1", "ex21-1.htm", "subsidiaries"), +] + + +def _selected(document): + return [a.sequence_number for a in Filing._select_attachments(ATTACHMENTS, document)] + + +@pytest.mark.fast +class TestDocumentSelection: + + def test_an_exact_type_selects_only_that_exhibit(self): + assert _selected("EX-10.1") == ["2"] + assert _selected("ex-10.1") == ["2"] + + def test_an_exact_filename_wins(self): + assert _selected("ex10-10.htm") == ["3"] + + def test_a_partial_name_still_selects_by_substring(self): + assert _selected("EX-10") == ["2", "3", "4"] + assert _selected("ex21") == ["5"] + + def test_primary_and_no_filter(self): + assert _selected("primary") == ["1"] + assert _selected(None) == ["1", "2", "3", "4", "5"] + + def test_grep_reports_matches_from_the_selected_exhibit_only(self): + filing = Filing(cik=1, company="Test", form="S-1", filing_date="2026-06-01", + accession_no="0000000001-26-000001") + with patch.object(Filing, "attachments", new_callable=PropertyMock, return_value=ATTACHMENTS): + result = filing.grep("agreement", document="EX-10.1") + assert [m.location for m in result] == ["EX-10.1"] + + +@pytest.mark.fast +class TestLiteralOffsets: + + def test_text_whose_lowercase_is_longer_does_not_shift_the_match(self): + text = "İİİİ Istanbul text says going concern doubt here" + assert "İ".lower() != "İ" and len("İ".lower()) == 2 # the premise + + [m] = _grep_text(text, "going concern", "p") + + assert m.match == "going concern" + assert m.context == text + + def test_the_match_keeps_the_source_casing(self): + [m] = _grep_text("Substantial Doubt about Going Concern", "going concern", "p") + assert m.match == "Going Concern" + + def test_overlapping_matches_are_still_found(self): + assert [m.match for m in _grep_text("aaa", "aa", "p")] == ["aa", "aa"] + + def test_regex_metacharacters_in_a_literal_are_literal(self): + assert [m.match for m in _grep_text("rate of 5.0% (fixed)", "5.0% (fixed)", "p")] == ["5.0% (fixed)"] + assert _grep_text("rate of 510% fixed", "5.0%", "p") == [] + + +@pytest.mark.network +def test_seeqc_s1_ex_10_1_excludes_the_employment_agreements(): + """SeeQC's S-1 (0001213900-26-073222) has EX-10.1, a Stockholder Support Agreement, + and EX-10.10 to EX-10.12, employment agreements. "agreement" in EX-10.1 is 133 + matches; the substring filter added 245 more from the other three.""" + from edgar import find + result = find("0001213900-26-073222").grep("agreement", document="EX-10.1") + + assert Counter(m.location for m in result) == {"EX-10.1": 133} + assert "STOCKHOLDER SUPPORT AGREEMENT" in result[0].context