fix(documents): collapse whitespace runs inside a text node (#1370)

changelog.d/1370.fixed.md

@@ -0,0 +1 @@
+- **Hard-wrapped paragraph text no longer keeps its line breaks** — `HTMLParser` now collapses whitespace runs inside a text node to one space, as a browser does, while `<br>` and `<pre>` still break lines. Item 7 of Google's FY2004 10-K had about 500 mid-sentence newlines. (#1370)

edgar/documents/parser.py

@@ -3,13 +3,15 @@
 """
 
 import logging
+import re
 import time
 from typing import List, Optional, Union
 
 import lxml.html
 from lxml import etree
 from lxml.html import HtmlElement
 
+from edgar.core import is_probably_html
 from edgar.documents.config import ParserConfig
 from edgar.documents.document import Document, DocumentMetadata
 from edgar.documents.exceptions import DocumentTooLargeError, HTMLParsingError, InvalidConfigurationError
@@ -27,6 +29,14 @@
 
 logger = logging.getLogger(__name__)
 
+# Filing.html() wraps a plain-text primary document in exactly this. Its line
+# breaks are the document's structure ("Item  8.01  Other Events" on its own
+# line), so it is parsed as preformatted text rather than collapsed as a browser
+# collapses HTML whitespace (gh-1370).
+_PLAIN_TEXT_WRAPPER = '<html><body><div>'
+_PREFORMATTED_WRAPPER = '<html><body><div style="white-space: pre-wrap">'
+_COMMENT = re.compile(r'<!--.*?-->', re.DOTALL)
+
 
 class HTMLParser:
     """
@@ -133,6 +143,14 @@ def parse(self, html: Union[str, bytes]) -> Document:
             # Store original HTML BEFORE preprocessing (needed for TOC analysis)
             original_html = html
 
+            if html.startswith(_PLAIN_TEXT_WRAPPER):
+                html = _PREFORMATTED_WRAPPER + html[len(_PLAIN_TEXT_WRAPPER):]
+            elif not is_probably_html(html) and _COMMENT.sub('', html).strip():
+                # A raw .txt primary document handed straight to the parser
+                # (Filing.parsed_items on a 1999 8-K) is plain text too. Input with
+                # nothing but comments is not a document; it still fails to parse.
+                html = f"{_PREFORMATTED_WRAPPER}{html}</div></body></html>"
+
             # Preprocessing (will remove ix:hidden for rendering)
             html = self.preprocessor.process(html)
 

edgar/documents/strategies/document_builder.py

@@ -2,6 +2,7 @@
 Document builder that converts parsed HTML tree into document nodes.
 """
 
+import re
 from typing import Any, Dict, Optional
 
 from lxml.html import HtmlElement
@@ -24,6 +25,10 @@
 from edgar.documents.table_nodes import Cell, Row, TableNode
 from edgar.documents.types import ParseContext, SemanticType, Style
 
+# HTML's "ASCII whitespace". Deliberately excludes \xa0: a non-breaking space is content.
+_ASCII_WHITESPACE_RUN = re.compile(r'[ \t\n\r\f]+')
+_PRE_WHITE_SPACE = re.compile(r'white-space\s*:\s*(pre|break-spaces)', re.IGNORECASE)
+
 
 class DocumentBuilder:
     """
@@ -104,18 +109,34 @@ def build(self, tree: HtmlElement) -> DocumentNode:
         return root
 
     @staticmethod
-    def _collapse_edges(text: str) -> str:
+    def _is_preformatted(element: Optional[HtmlElement]) -> bool:
+        """True when element sits in a <pre> or a white-space: pre* box, where newlines are content."""
+        while element is not None:
+            tag = element.tag
+            if isinstance(tag, str):
+                if tag.lower() == 'pre' or _PRE_WHITE_SPACE.search(element.get('style') or ''):
+                    return True
+            element = element.getparent()
+        return False
+
+    def _collapse_edges(self, text: str, context: Optional[HtmlElement] = None) -> str:
         """Collapse a text node's edge whitespace to a single space instead of deleting it.
 
         lxml puts the whitespace that separates a word from an adjacent inline element on
         the edge of element.text / element.tail. Stripping it destroys the word boundary
         ('per <font>$1,000</font>' -> 'per$1,000') and nothing downstream can tell that
         apart from a genuine mid-word split. Same rule the preprocessor follows: collapse,
         never delete. Whitespace-only nodes still return '' — those carry no word.
+
+        Runs of whitespace inside the node collapse to one space too, as a browser renders
+        them: older filings hard-wrap <P> text at ~150 columns (gh-1370). `context` is the
+        element whose content the text is; inside <pre> the inner whitespace is kept.
         """
         stripped = text.strip()
         if not stripped:
             return ''
+        if not self._is_preformatted(context):
+            stripped = _ASCII_WHITESPACE_RUN.sub(' ', stripped)
         return (' ' if text[:1].isspace() else '') + stripped + (' ' if text[-1:].isspace() else '')
 
     def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]:
@@ -139,7 +160,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
                     parent.add_child(text_node)
                 else:
                     if element.tail.strip():
-                        text_node = TextNode(content=self._collapse_edges(element.tail))
+                        text_node = TextNode(content=self._collapse_edges(element.tail, element.getparent()))
                         parent.add_child(text_node)
             return None
 
@@ -187,7 +208,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
                                 node.add_child(text_node)
                         else:
                             if element.text.strip():
-                                text_node = TextNode(content=self._collapse_edges(element.text))
+                                text_node = TextNode(content=self._collapse_edges(element.text, element))
                                 node.add_child(text_node)
 
                     # Process child elements
@@ -201,7 +222,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
                             parent.add_child(text_node)
                         else:
                             if element.tail.strip():
-                                text_node = TextNode(content=self._collapse_edges(element.tail))
+                                text_node = TextNode(content=self._collapse_edges(element.tail, element.getparent()))
                                 parent.add_child(text_node)
                             elif element.tail.isspace():
                                 # Even if tail is just whitespace, preserve the spacing info
@@ -216,7 +237,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
                             parent.add_child(text_node)
                         else:
                             if element.tail.strip():
-                                text_node = TextNode(content=self._collapse_edges(element.tail))
+                                text_node = TextNode(content=self._collapse_edges(element.tail, element.getparent()))
                                 parent.add_child(text_node)
                             elif element.tail.isspace():
                                 # Even if tail is just whitespace, preserve the spacing info
@@ -234,7 +255,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
                         parent.add_child(text_node)
                     else:
                         if element.tail.strip():
-                            text_node = TextNode(content=self._collapse_edges(element.tail))
+                            text_node = TextNode(content=self._collapse_edges(element.tail, element.getparent()))
                             parent.add_child(text_node)
 
             # Exit XBRL context
@@ -591,6 +612,7 @@ def _get_element_text(self, element: HtmlElement) -> str:
         """Get text content from element."""
         inline = element.tag.lower() in self.INLINE_ELEMENTS
         text_parts = []
+        block_breaks: set = set()  # indexes of parts that separate block children
 
         # Get element's direct text
         if element.text:
@@ -619,10 +641,21 @@ def _get_element_text(self, element: HtmlElement) -> str:
                 # their trailing text is still part of the document. (gh-898)
                 if child.tail:
                     if inline:
+                        if child.tail.isspace() and child.tag.lower() not in self.INLINE_ELEMENTS:
+                            # The line break between two block children of an inline
+                            # element (<font><p>..</p>\n<p>..</p></font>) is the only
+                            # thing separating their paragraphs; it is not a run to
+                            # collapse (Merck's 2005 CORRESP letter lost all 67 lines).
+                            block_breaks.add(len(text_parts))
                         text_parts.append(child.tail)
                     elif child.tail.strip():
                         text_parts.append(child.tail.strip())
 
+        # Collapse inner whitespace runs as a browser does; edges keep one space (gh-1370)
+        if not self.config.preserve_whitespace and not self._is_preformatted(element):
+            text_parts = [part if i in block_breaks else _ASCII_WHITESPACE_RUN.sub(' ', part)
+                          for i, part in enumerate(text_parts)]
+
         # A single preserved inline part is returned as-is; otherwise space-join
         # so word boundaries survive (the preprocessor strips whitespace adjacent
         # to tags, and this re-separates the parts).

edgar/offerings/prospectus/_424b_cover.py

@@ -244,6 +244,7 @@ def _clean_agent_name(name: str) -> str:
 # Defined-abbreviation pattern: '<Full Name> ("ABBR")'.
 _ABBREV_DEF_RE = re.compile(
     r"([A-Z][A-Za-z0-9 ,.&'-]{3,45}?)\s*\(\s*[\"“]([A-Za-z&.']{2,10})[\"”]\s*\)")
+_SENTENCE_TAIL_RE = re.compile(r"[A-Z][a-z]+\.\s+(?=[A-Z])")
 
 
 def _resolve_abbreviation(name: str, text: str) -> str:
@@ -255,7 +256,14 @@ def _resolve_abbreviation(name: str, text: str) -> str:
     """
     for m in _ABBREV_DEF_RE.finditer(text):
         if m.group(2).strip() == name:
-            return m.group(1).strip()
+            full = m.group(1).strip()
+            # The capture starts at the leftmost capital, so it can open on the
+            # end of the previous sentence: "... per Note. BofA Securities, Inc."
+            # A single capitalised word and a full stop is that sentence's end;
+            # "J.P. Morgan" and "Goldman Sachs & Co. LLC" do not open that way.
+            while (lead := _SENTENCE_TAIL_RE.match(full)) is not None:
+                full = full[lead.end():]
+            return full
     return name
 
 

tests/documents/test_html_parser_regressions.py

@@ -156,6 +156,68 @@ def test_unspaced_edges_still_stay_glued(self):
         assert 'RegulationS-T(the rule)' in text
 
 
+class TestHardWrappedTextNode:
+    """gh-1370: whitespace runs INSIDE a text node collapse to one space.
+
+    Older EDGAR HTML is hard-wrapped at ~150 columns inside <P>. Google's FY2004
+    10-K (0001193125-05-065298) yielded ~500 mid-sentence newlines in Item 7.
+    """
+
+    def test_newline_inside_a_paragraph_collapses(self):
+        html = ('<html><body><p>Our stock has been quoted since August 19,\n2004. '
+                'Prior to that time there was no market.</p></body></html>')
+        text = parse_html(html, ParserConfig(detect_sections=False)).text()
+        assert text.strip() == ('Our stock has been quoted since August 19, 2004. '
+                                'Prior to that time there was no market.')
+
+    def test_newline_inside_an_inline_font_collapses(self):
+        html = ('<html><body><p><font>net revenues\n   increased by 118%</font> '
+                'in 2004.</p></body></html>')
+        text = parse_html(html).text()
+        assert 'net revenues increased by 118% in 2004.' in text
+
+    def test_br_is_still_a_line_break(self):
+        html = '<html><body><p>Mountain View,<br>California</p></body></html>'
+        text = parse_html(html).text()
+        assert 'Mountain View,\nCalifornia' in text
+
+    def test_pre_keeps_its_newlines(self):
+        html = '<html><body><pre>Line one\nLine two</pre></body></html>'
+        text = parse_html(html).text()
+        assert 'Line one\nLine two' in text
+
+    def test_wrapped_plain_text_document_keeps_its_lines(self):
+        """Filing.html() wraps a .txt primary document in <html><body><div>.
+
+        Its lines are its structure: GMAC's 2005 8-K (0000040729-05-000026) puts
+        "Item  8.01  Other Events" on its own line, and collapsing it into the
+        paragraph above lost the item.
+        """
+        html = ('<html><body><div>registered public accounting firm, Deloitte & Touche LLP.\n\n'
+                'Item  8.01  Other Events\n\nIn order to analyze</div></body></html>')
+        text = parse_html(html).text()
+        assert '\nItem 8.01 Other Events\n' in text  # the preprocessor squeezes spaces, as on main
+
+    def test_raw_plain_text_keeps_its_lines(self):
+        """A raw .txt document handed straight to the parser (Filing.parsed_items)."""
+        text = parse_html('ITEM 7. Financial Statements and Exhibits\n\n'
+                          'ITEM 8. Not applicable.').text()
+        assert 'Exhibits\n' in text and '\nITEM 8. Not applicable.' in text
+
+    def test_paragraphs_inside_an_inline_wrapper_stay_separate(self):
+        """<font> around block <p>s: the break between them is not a run to collapse.
+
+        Merck's 2005 CORRESP letter (0000065873-05-000060) wraps every paragraph
+        in one <font SIZE="2">, and collapsing joined all 67 lines into one.
+        """
+        html = ('<html><body><font SIZE="2"><p>ALLTEL CORPORATION</p>\n'
+                '<p>One Allied Drive, Little Rock,\nArkansas 72202</p>\n'
+                '<p>June 22, 2005</p></font></body></html>')
+        lines = [line.strip() for line in parse_html(html).text().splitlines() if line.strip()]
+        assert lines == ['ALLTEL CORPORATION', 'One Allied Drive, Little Rock, Arkansas 72202',
+                         'June 22, 2005']
+
+
 class TestSpacerElementWordGluing:
     """A third copy of the same bug: whitespace-only spacer elements.
 

tests/issues/regression/test_dt1f1_era_item_titles.py

@@ -143,7 +143,7 @@ def test_both_era_sections_are_detected(doc_1999):
     # header cells sat in a column with one substantial entry per several rows,
     # which the scorer read as spacing. ZERO tokens are lost and the number
     # sequence is untouched -- the 547 added characters are those headers.
-    assert len(sections["part_iii_item_14"].text()) == 16610
+    assert len(sections["part_iii_item_14"].text()) == 16609  # -1: a whitespace run inside a text node collapses (gh-1370)
     assert sections["part_iii_item_14"].text().startswith("Item 14: Exhibits")
 
     # The neighbours are unchanged — this adds sections, it does not re-cut the
@@ -190,8 +190,8 @@ def test_the_lookups_resolve_without_the_legacy_parser():
     assert len(report["Item 4"]) == 177
     assert len(report["4"]) == 177  # the short spelling resolves too
     # Re-pinned for edgartools-y0ri; see the note in the test above.
-    assert len(report["Item 14"]) == 16610
-    assert len(report["14"]) == 16610
+    assert len(report["Item 14"]) == 16609  # -1: a whitespace run inside a text node collapses (gh-1370)
+    assert len(report["14"]) == 16609  # -1: a whitespace run inside a text node collapses (gh-1370)
 
     items = report.items
     assert "Item 4" in items

tests/issues/regression/test_dt1f1_wrapped_item_headers.py

@@ -132,15 +132,15 @@ def test_20f_wrapped_headers_resolve_without_the_legacy_parser():
     report = _without_legacy(TwentyF)(filing)
 
     # 'Item 6. Directors, Senior Management and\nEmployees'
-    assert len(report["Item 6"]) == 42395  # table cells no longer fused (edgartools-wzgu)
+    assert len(report["Item 6"]) == 42393  # wzgu; -2 for whitespace runs collapsed inside a text node (gh-1370)
     assert report["Item 6"].split("\n", 1)[0].startswith("Item 6.")
 
     # 'Item 11. Quantitative and Qualitative Disclosures\nAbout Market Risk.'
     assert len(report["Item 11"]) == 268
     assert report["Item 11"].split("\n", 1)[0].startswith("Item 11.")
 
     # And the same answers on the real class, legacy fallback and all.
-    assert len(TwentyF(filing)["Item 6"]) == 42395  # table cells no longer fused (edgartools-wzgu)
+    assert len(TwentyF(filing)["Item 6"]) == 42393  # wzgu; -2 for gh-1370
     assert len(TwentyF(filing)["Item 11"]) == 268