` still break lines. Item 7 of Google's FY2004 10-K had about 500 mid-sentence newlines. (#1370)
diff --git a/edgar/documents/parser.py b/edgar/documents/parser.py
index d0ff8e06d..6e92dcdf2 100644
--- a/edgar/documents/parser.py
+++ b/edgar/documents/parser.py
@@ -3,6 +3,7 @@
"""
import logging
+import re
import time
from typing import List, Optional, Union
@@ -10,6 +11,7 @@
from lxml import etree
from lxml.html import HtmlElement
+from edgar.core import is_probably_html
from edgar.documents.config import ParserConfig
from edgar.documents.document import Document, DocumentMetadata
from edgar.documents.exceptions import DocumentTooLargeError, HTMLParsingError, InvalidConfigurationError
@@ -27,6 +29,14 @@
logger = logging.getLogger(__name__)
+# Filing.html() wraps a plain-text primary document in exactly this. Its line
+# breaks are the document's structure ("Item 8.01 Other Events" on its own
+# line), so it is parsed as preformatted text rather than collapsed as a browser
+# collapses HTML whitespace (gh-1370).
+_PLAIN_TEXT_WRAPPER = ''
+_PREFORMATTED_WRAPPER = ''
+_COMMENT = re.compile(r'', re.DOTALL)
+
class HTMLParser:
"""
@@ -133,6 +143,14 @@ def parse(self, html: Union[str, bytes]) -> Document:
# Store original HTML BEFORE preprocessing (needed for TOC analysis)
original_html = html
+ if html.startswith(_PLAIN_TEXT_WRAPPER):
+ html = _PREFORMATTED_WRAPPER + html[len(_PLAIN_TEXT_WRAPPER):]
+ elif not is_probably_html(html) and _COMMENT.sub('', html).strip():
+ # A raw .txt primary document handed straight to the parser
+ # (Filing.parsed_items on a 1999 8-K) is plain text too. Input with
+ # nothing but comments is not a document; it still fails to parse.
+ html = f"{_PREFORMATTED_WRAPPER}{html}"
+
# Preprocessing (will remove ix:hidden for rendering)
html = self.preprocessor.process(html)
diff --git a/edgar/documents/strategies/document_builder.py b/edgar/documents/strategies/document_builder.py
index dff654a89..a0df78e01 100644
--- a/edgar/documents/strategies/document_builder.py
+++ b/edgar/documents/strategies/document_builder.py
@@ -2,6 +2,7 @@
Document builder that converts parsed HTML tree into document nodes.
"""
+import re
from typing import Any, Dict, Optional
from lxml.html import HtmlElement
@@ -24,6 +25,10 @@
from edgar.documents.table_nodes import Cell, Row, TableNode
from edgar.documents.types import ParseContext, SemanticType, Style
+# HTML's "ASCII whitespace". Deliberately excludes \xa0: a non-breaking space is content.
+_ASCII_WHITESPACE_RUN = re.compile(r'[ \t\n\r\f]+')
+_PRE_WHITE_SPACE = re.compile(r'white-space\s*:\s*(pre|break-spaces)', re.IGNORECASE)
+
class DocumentBuilder:
"""
@@ -104,7 +109,17 @@ def build(self, tree: HtmlElement) -> DocumentNode:
return root
@staticmethod
- def _collapse_edges(text: str) -> str:
+ def _is_preformatted(element: Optional[HtmlElement]) -> bool:
+ """True when element sits in a or a white-space: pre* box, where newlines are content."""
+ while element is not None:
+ tag = element.tag
+ if isinstance(tag, str):
+ if tag.lower() == 'pre' or _PRE_WHITE_SPACE.search(element.get('style') or ''):
+ return True
+ element = element.getparent()
+ return False
+
+ def _collapse_edges(self, text: str, context: Optional[HtmlElement] = None) -> str:
"""Collapse a text node's edge whitespace to a single space instead of deleting it.
lxml puts the whitespace that separates a word from an adjacent inline element on
@@ -112,10 +127,16 @@ def _collapse_edges(text: str) -> str:
('per $1,000' -> 'per$1,000') and nothing downstream can tell that
apart from a genuine mid-word split. Same rule the preprocessor follows: collapse,
never delete. Whitespace-only nodes still return '' — those carry no word.
+
+ Runs of whitespace inside the node collapse to one space too, as a browser renders
+ them: older filings hard-wrap text at ~150 columns (gh-1370). `context` is the
+ element whose content the text is; inside
the inner whitespace is kept.
"""
stripped = text.strip()
if not stripped:
return ''
+ if not self._is_preformatted(context):
+ stripped = _ASCII_WHITESPACE_RUN.sub(' ', stripped)
return (' ' if text[:1].isspace() else '') + stripped + (' ' if text[-1:].isspace() else '')
def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]:
@@ -139,7 +160,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
parent.add_child(text_node)
else:
if element.tail.strip():
- text_node = TextNode(content=self._collapse_edges(element.tail))
+ text_node = TextNode(content=self._collapse_edges(element.tail, element.getparent()))
parent.add_child(text_node)
return None
@@ -187,7 +208,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
node.add_child(text_node)
else:
if element.text.strip():
- text_node = TextNode(content=self._collapse_edges(element.text))
+ text_node = TextNode(content=self._collapse_edges(element.text, element))
node.add_child(text_node)
# Process child elements
@@ -201,7 +222,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
parent.add_child(text_node)
else:
if element.tail.strip():
- text_node = TextNode(content=self._collapse_edges(element.tail))
+ text_node = TextNode(content=self._collapse_edges(element.tail, element.getparent()))
parent.add_child(text_node)
elif element.tail.isspace():
# Even if tail is just whitespace, preserve the spacing info
@@ -216,7 +237,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
parent.add_child(text_node)
else:
if element.tail.strip():
- text_node = TextNode(content=self._collapse_edges(element.tail))
+ text_node = TextNode(content=self._collapse_edges(element.tail, element.getparent()))
parent.add_child(text_node)
elif element.tail.isspace():
# Even if tail is just whitespace, preserve the spacing info
@@ -234,7 +255,7 @@ def _process_element(self, element: HtmlElement, parent: Node) -> Optional[Node]
parent.add_child(text_node)
else:
if element.tail.strip():
- text_node = TextNode(content=self._collapse_edges(element.tail))
+ text_node = TextNode(content=self._collapse_edges(element.tail, element.getparent()))
parent.add_child(text_node)
# Exit XBRL context
@@ -591,6 +612,7 @@ def _get_element_text(self, element: HtmlElement) -> str:
"""Get text content from element."""
inline = element.tag.lower() in self.INLINE_ELEMENTS
text_parts = []
+ block_breaks: set = set() # indexes of parts that separate block children
# Get element's direct text
if element.text:
@@ -619,10 +641,21 @@ def _get_element_text(self, element: HtmlElement) -> str:
# their trailing text is still part of the document. (gh-898)
if child.tail:
if inline:
+ if child.tail.isspace() and child.tag.lower() not in self.INLINE_ELEMENTS:
+ # The line break between two block children of an inline
+ # element (..
\n..
) is the only
+ # thing separating their paragraphs; it is not a run to
+ # collapse (Merck's 2005 CORRESP letter lost all 67 lines).
+ block_breaks.add(len(text_parts))
text_parts.append(child.tail)
elif child.tail.strip():
text_parts.append(child.tail.strip())
+ # Collapse inner whitespace runs as a browser does; edges keep one space (gh-1370)
+ if not self.config.preserve_whitespace and not self._is_preformatted(element):
+ text_parts = [part if i in block_breaks else _ASCII_WHITESPACE_RUN.sub(' ', part)
+ for i, part in enumerate(text_parts)]
+
# A single preserved inline part is returned as-is; otherwise space-join
# so word boundaries survive (the preprocessor strips whitespace adjacent
# to tags, and this re-separates the parts).
diff --git a/edgar/offerings/prospectus/_424b_cover.py b/edgar/offerings/prospectus/_424b_cover.py
index e36b2e705..f229c8c93 100644
--- a/edgar/offerings/prospectus/_424b_cover.py
+++ b/edgar/offerings/prospectus/_424b_cover.py
@@ -244,6 +244,7 @@ def _clean_agent_name(name: str) -> str:
# Defined-abbreviation pattern: ' ("ABBR")'.
_ABBREV_DEF_RE = re.compile(
r"([A-Z][A-Za-z0-9 ,.&'-]{3,45}?)\s*\(\s*[\"“]([A-Za-z&.']{2,10})[\"”]\s*\)")
+_SENTENCE_TAIL_RE = re.compile(r"[A-Z][a-z]+\.\s+(?=[A-Z])")
def _resolve_abbreviation(name: str, text: str) -> str:
@@ -255,7 +256,14 @@ def _resolve_abbreviation(name: str, text: str) -> str:
"""
for m in _ABBREV_DEF_RE.finditer(text):
if m.group(2).strip() == name:
- return m.group(1).strip()
+ full = m.group(1).strip()
+ # The capture starts at the leftmost capital, so it can open on the
+ # end of the previous sentence: "... per Note. BofA Securities, Inc."
+ # A single capitalised word and a full stop is that sentence's end;
+ # "J.P. Morgan" and "Goldman Sachs & Co. LLC" do not open that way.
+ while (lead := _SENTENCE_TAIL_RE.match(full)) is not None:
+ full = full[lead.end():]
+ return full
return name
diff --git a/tests/documents/test_html_parser_regressions.py b/tests/documents/test_html_parser_regressions.py
index 14a44ca60..8cfb77b00 100644
--- a/tests/documents/test_html_parser_regressions.py
+++ b/tests/documents/test_html_parser_regressions.py
@@ -156,6 +156,68 @@ def test_unspaced_edges_still_stay_glued(self):
assert 'RegulationS-T(the rule)' in text
+class TestHardWrappedTextNode:
+ """gh-1370: whitespace runs INSIDE a text node collapse to one space.
+
+ Older EDGAR HTML is hard-wrapped at ~150 columns inside . Google's FY2004
+ 10-K (0001193125-05-065298) yielded ~500 mid-sentence newlines in Item 7.
+ """
+
+ def test_newline_inside_a_paragraph_collapses(self):
+ html = ('
Our stock has been quoted since August 19,\n2004. '
+ 'Prior to that time there was no market.
')
+ text = parse_html(html, ParserConfig(detect_sections=False)).text()
+ assert text.strip() == ('Our stock has been quoted since August 19, 2004. '
+ 'Prior to that time there was no market.')
+
+ def test_newline_inside_an_inline_font_collapses(self):
+ html = ('net revenues\n increased by 118% '
+ 'in 2004.
')
+ text = parse_html(html).text()
+ assert 'net revenues increased by 118% in 2004.' in text
+
+ def test_br_is_still_a_line_break(self):
+ html = 'Mountain View,
California
'
+ text = parse_html(html).text()
+ assert 'Mountain View,\nCalifornia' in text
+
+ def test_pre_keeps_its_newlines(self):
+ html = 'Line one\nLine two
'
+ text = parse_html(html).text()
+ assert 'Line one\nLine two' in text
+
+ def test_wrapped_plain_text_document_keeps_its_lines(self):
+ """Filing.html() wraps a .txt primary document in .
+
+ Its lines are its structure: GMAC's 2005 8-K (0000040729-05-000026) puts
+ "Item 8.01 Other Events" on its own line, and collapsing it into the
+ paragraph above lost the item.
+ """
+ html = ('registered public accounting firm, Deloitte & Touche LLP.\n\n'
+ 'Item 8.01 Other Events\n\nIn order to analyze')
+ text = parse_html(html).text()
+ assert '\nItem 8.01 Other Events\n' in text # the preprocessor squeezes spaces, as on main
+
+ def test_raw_plain_text_keeps_its_lines(self):
+ """A raw .txt document handed straight to the parser (Filing.parsed_items)."""
+ text = parse_html('ITEM 7. Financial Statements and Exhibits\n\n'
+ 'ITEM 8. Not applicable.').text()
+ assert 'Exhibits\n' in text and '\nITEM 8. Not applicable.' in text
+
+ def test_paragraphs_inside_an_inline_wrapper_stay_separate(self):
+ """ around block s: the break between them is not a run to collapse.
+
+ Merck's 2005 CORRESP letter (0000065873-05-000060) wraps every paragraph
+ in one , and collapsing joined all 67 lines into one.
+ """
+ html = ('ALLTEL CORPORATION
\n'
+ 'One Allied Drive, Little Rock,\nArkansas 72202
\n'
+ 'June 22, 2005
')
+ lines = [line.strip() for line in parse_html(html).text().splitlines() if line.strip()]
+ assert lines == ['ALLTEL CORPORATION', 'One Allied Drive, Little Rock, Arkansas 72202',
+ 'June 22, 2005']
+
+
class TestSpacerElementWordGluing:
"""A third copy of the same bug: whitespace-only spacer elements.
diff --git a/tests/issues/regression/test_dt1f1_era_item_titles.py b/tests/issues/regression/test_dt1f1_era_item_titles.py
index f3ba1d373..34a20eec2 100644
--- a/tests/issues/regression/test_dt1f1_era_item_titles.py
+++ b/tests/issues/regression/test_dt1f1_era_item_titles.py
@@ -143,7 +143,7 @@ def test_both_era_sections_are_detected(doc_1999):
# header cells sat in a column with one substantial entry per several rows,
# which the scorer read as spacing. ZERO tokens are lost and the number
# sequence is untouched -- the 547 added characters are those headers.
- assert len(sections["part_iii_item_14"].text()) == 16610
+ assert len(sections["part_iii_item_14"].text()) == 16609 # -1: a whitespace run inside a text node collapses (gh-1370)
assert sections["part_iii_item_14"].text().startswith("Item 14: Exhibits")
# The neighbours are unchanged — this adds sections, it does not re-cut the
@@ -190,8 +190,8 @@ def test_the_lookups_resolve_without_the_legacy_parser():
assert len(report["Item 4"]) == 177
assert len(report["4"]) == 177 # the short spelling resolves too
# Re-pinned for edgartools-y0ri; see the note in the test above.
- assert len(report["Item 14"]) == 16610
- assert len(report["14"]) == 16610
+ assert len(report["Item 14"]) == 16609 # -1: a whitespace run inside a text node collapses (gh-1370)
+ assert len(report["14"]) == 16609 # -1: a whitespace run inside a text node collapses (gh-1370)
items = report.items
assert "Item 4" in items
diff --git a/tests/issues/regression/test_dt1f1_wrapped_item_headers.py b/tests/issues/regression/test_dt1f1_wrapped_item_headers.py
index 799ca4e28..4393e8361 100644
--- a/tests/issues/regression/test_dt1f1_wrapped_item_headers.py
+++ b/tests/issues/regression/test_dt1f1_wrapped_item_headers.py
@@ -132,7 +132,7 @@ def test_20f_wrapped_headers_resolve_without_the_legacy_parser():
report = _without_legacy(TwentyF)(filing)
# 'Item 6. Directors, Senior Management and\nEmployees'
- assert len(report["Item 6"]) == 42395 # table cells no longer fused (edgartools-wzgu)
+ assert len(report["Item 6"]) == 42393 # wzgu; -2 for whitespace runs collapsed inside a text node (gh-1370)
assert report["Item 6"].split("\n", 1)[0].startswith("Item 6.")
# 'Item 11. Quantitative and Qualitative Disclosures\nAbout Market Risk.'
@@ -140,7 +140,7 @@ def test_20f_wrapped_headers_resolve_without_the_legacy_parser():
assert report["Item 11"].split("\n", 1)[0].startswith("Item 11.")
# And the same answers on the real class, legacy fallback and all.
- assert len(TwentyF(filing)["Item 6"]) == 42395 # table cells no longer fused (edgartools-wzgu)
+ assert len(TwentyF(filing)["Item 6"]) == 42393 # wzgu; -2 for gh-1370
assert len(TwentyF(filing)["Item 11"]) == 268
diff --git a/tests/issues/regression/test_filing_text_baseline.py b/tests/issues/regression/test_filing_text_baseline.py
index 757c2f197..f60f7c728 100644
--- a/tests/issues/regression/test_filing_text_baseline.py
+++ b/tests/issues/regression/test_filing_text_baseline.py
@@ -168,6 +168,46 @@ def sha256(text):
# already ("nothing was repaired in the text itself"). The invariant is a
# property of an individual re-capture, not of this file.
+# Re-captured against b022ad29ac3965248c473b5790cf8e124887eac6: gh-1370: whitespace runs inside a text node collapse to one space as a browser renders them, so hard-wrapped paragraph text rejoins. Block-level breaks inside inline wrappers and plain-text documents keep their lines.
+# 4 of 5 filings moved; 1 byte-identical.
+# 0001193125-20-052640: 5782 -> 3795 lines, ~2142 changed
+# -(Exact
+# -Name of Registrant as Specified in Its Charter)
+# +(Exact Name of Registrant as Specified in Its Charter)
+# -Indicate by check mark if the registrant is a well-known seasoned issuer, as defined in Rule 405 of the Securities
+# -Act. Yes ☐ No ☒
+# -
+# -Indicate by check mark if the registrant is not required to
+# -file reports pursuant to Section 13 or Section 15(d) of the Act. Yes ☐ No ☒
+# 0000065873-05-000060: 68 -> 42 lines, ~21 changed
+# -One Allied Drive, Little Rock,
+# -Arkansas 72202
+# +One Allied Drive, Little Rock, Arkansas 72202
+# -Re: ALLTEL
+# -Corporation
+# +Re: ALLTEL Corporation
+# -The undersigned respectively requests
+# -that the effective date of the above-referenced Registration Statement be
+# 0000950137-05-004969: 71 -> 62 lines, ~8 changed
+# -Check the appropriate box below if the Form 8-K filing is intended to simultaneously satisfy the
+# -filing obligation of the registrant under any of the following provisions:
+# +Check the appropriate box below if the Form 8-K filing is intended to simultaneously satisfy the filing obligation of the registrant under any of the
+# -Item 5.02. Departure of Directors or Principal Officers; Election of Directors; Appointment of
+# -Principal Officers.
+# +Item 5.02. Departure of Directors or Principal Officers; Election of Directors; Appointment of Principal Officers.
+# -On April 26, 2005, the board of directors of Exelon Corporation elected Thomas J. Ridge to serve as
+# -a Class I director, effective May 2, 2005. Mr. Ridge, age 59, served as Secretary of the United
+# 0001481057-23-010389: 1482 -> 696 lines, ~533 changed
+# -The initial estimated value of the Notes as of the
+# -
+# -pricing date is expected to be between $907.80 and $957.80 per $1,000 in principal amount of Notes, which is less than the public offering
+# -
+# -price listed below. The actual value of your Notes at any time will reflect many factors and cannot be predicted with accuracy. See
+# -
+# -“Risk Factors” beginning on page PS-8 of this pricing supplement and “Structuring the Notes” on page PS-16 of
+# -
+# Verified mechanically: every changed line differs by whitespace only.
+
BASELINE = {
# Modern iXBRL 10-K
"0000320193-23-000106": (
@@ -211,13 +251,13 @@ def sha256(text):
# Re-captured for edgartools-wzgu (short columns kept): a numbered list laid
# out as a table regains its "1." .. "9." markers and seven rule lines
# resize; no token is lost -- 3,803 numbers become 3,812, the old in order.
- "4174954dcdec09bd337c289a74d72cbeeff222d7897f2a80d4a90e8fa27142df",
+ "857b48fd9e2cd67cd65a66aa01debab8a23600a10174782b98432b0890a86c6f",
),
# CORRESP — the shape where the two paths already agreed before the fix
"0000065873-05-000060": (
dict(form="CORRESP", filing_date="2005-11-03", company="MERCK & CO INC",
cik=65873, accession_no="0000065873-05-000060"),
- "733944a9b6e911a63321cd3d37c3a1408ad3ac215e9a93e4ed04f549e5c42b6e",
+ "c195843d15e1aef88638c220a0f5714553e0cd2361252615ff137932a4efecbf",
),
# 2005-era HTML 8-K
"0000950137-05-004969": (
@@ -231,13 +271,13 @@ def sha256(text):
# unchanged at 2,629 characters and
# "".join(before.split()) == "".join(after.split()) holds, so nothing but
# whitespace moved. This is the only filing of the five that changed.
- "37211411d05d2120eece22347d8c7cbfde46680d516751fbcce17f060dcb7b5e",
+ "d97e03eeb34ae623257b4d6f63ccffb96ef7168edb2895b127ee5d80412d979b",
),
# 424B2 structured note
"0001481057-23-010389": (
dict(form="424B2", filing_date="2023-12-13", company="BANK OF AMERICA CORP /DE/",
cik=70858, accession_no="0001481057-23-010389"),
- "32a49882a47542bf4e0d03af0dd389571a83c2d42452825b7cb138dedcb4b1c2",
+ "4ebe4fd0ceab8a79412156bd691542d1b13959897d8f028d3e2b4de481426ace",
),
}
diff --git a/tests/issues/regression/test_issue_2h4c.py b/tests/issues/regression/test_issue_2h4c.py
index 8e2ac5cfb..6f2b94bc8 100644
--- a/tests/issues/regression/test_issue_2h4c.py
+++ b/tests/issues/regression/test_issue_2h4c.py
@@ -113,6 +113,12 @@ def test_resolve_abbreviation_expands_defined_tag():
assert _resolve_abbreviation("BofAS", text) == "BofA Securities, Inc."
# Already a full name (not a defined tag) — unchanged.
assert _resolve_abbreviation("Barclays Capital Inc.", text) == "Barclays Capital Inc."
+ # The capture must not open on the end of the previous sentence (gh-1370 joined
+ # "per Note.\nBofA ..." onto one line), but a name's own periods stay.
+ joined = 'The underwriting discount is $0.10 per Note. BofA Securities, Inc. (“BofAS”), an affiliate.'
+ assert _resolve_abbreviation("BofAS", joined) == "BofA Securities, Inc."
+ jpm = 'Agent: J.P. Morgan Securities LLC (“JPMS”), an affiliate.'
+ assert _resolve_abbreviation("JPMS", jpm) == "J.P. Morgan Securities LLC"
class TestCoverAgentExtraction: