0.4.0 pre-release: quote_blocks without deepcopy; LegaldownDeprecationWarning

README.md

@@ -265,7 +265,12 @@ is the same function for files on disk, and the one `parse_template` takes as `r
 > `validate_document` is now `validate`, `serialize_document` is now `serialize` (text) or `save`
 > (file), and the importer callbacks `import_definitions=` and `import_attachment_definitions=` are
 > replaced by `resolve=`. They still work and raise a `DeprecationWarning`; they are deprecated
-> since 0.4.0 and will be removed in 0.5.0.
+> since 0.4.0 and will be removed in 0.5.0. Every deprecation warning legaldown raises is a
+> `legaldown.LegaldownDeprecationWarning` (a `DeprecationWarning` subclass), so one filter catches
+> them all: `warnings.simplefilter("error", legaldown.LegaldownDeprecationWarning)` in code, or
+> `filterwarnings = ["error::legaldown.LegaldownDeprecationWarning"]` in pytest's configuration
+> (Python's own `-W` option and `PYTHONWARNINGS` ignore a category from a package, as they read it
+> before the package can be imported).
 >
 > **Changed in 0.4.0:** `ValidationResult` is a plain value of two parts, `diagnostics` and `index`.
 > `errors`, `warnings` and `infos` are read-only lists taken from `diagnostics` (changing them
@@ -709,7 +714,7 @@ checks and what it does not.
 > **Deprecated:** `assemble(text, answers, load_file=)`, `template_questions` and `needed_questions`
 > are replaced by `Template` and `Form`. They still work, unchanged (but for `Question`, below, and
 > that a `load_file=` loader is never asked for a path that is absolute or leads out of the template's
-> directory, §2.3), and raise a `DeprecationWarning`; they are deprecated since 0.4.0 and will be removed in 0.5.0. Two things
+> directory, §2.3), and raise a `LegaldownDeprecationWarning`; they are deprecated since 0.4.0 and will be removed in 0.5.0. Two things
 > differ in the new API: it asks the questions of an included fragment where its `{{include:}}` is
 > (the functions ask them after the body), and it refuses a template that has Errors in the
 > validator's template rules (the functions assemble it). `legaldown assemble` follows the new API,

src/legaldown/__init__.py

@@ -17,6 +17,9 @@
 """
 from __future__ import annotations
 
+# The category of every deprecation warning legaldown raises
+from ._deprecation import LegaldownDeprecationWarning
+
 # Assembly (§15.7): a named capability (§17.6)
 from .assembly import AssemblyError, AssemblyResult, Question, assemble, needed_questions, template_questions
 from .files import LoadFile, file_loader
@@ -168,4 +171,6 @@
     # Deprecated importer callbacks
     "DefinitionsImporter",  # deprecated since 0.4.0, removed in 0.5.0: use ``validate(resolve=)``
     "AttachmentDefinitionsImporter",  # deprecated since 0.4.0, removed in 0.5.0: use ``validate(resolve=)``
+    # What each deprecated name above warns with when used: one category to filter
+    "LegaldownDeprecationWarning",
 ]

src/legaldown/_deprecation.py

@@ -0,0 +1,14 @@
+"""The warning category of legaldown's deprecations."""
+from __future__ import annotations
+
+
+class LegaldownDeprecationWarning(DeprecationWarning):
+    """A deprecated legaldown API was used: it is removed in the next minor
+    release. Every deprecation warning legaldown raises is of this category,
+    so one filter covers them all; a ``DeprecationWarning`` filter still
+    matches them too."""
+
+
+# Named where it is imported from, ``legaldown.LegaldownDeprecationWarning``
+# (a warnings filter written as a qualified name, a pickled warning).
+LegaldownDeprecationWarning.__module__ = "legaldown"

src/legaldown/assembly.py

@@ -39,6 +39,7 @@
 from functools import cache
 from typing import Any
 
+from ._deprecation import LegaldownDeprecationWarning
 from .definitions import list_fragments
 from .directives import PLACEHOLDER_TYPE_PARAMS, Directive, format_value, lex
 from .files import LoadFile, relative_path
@@ -1785,7 +1786,7 @@ def assemble(
     warnings.warn(
         "legaldown.assemble() is deprecated since 0.4.0 and will be removed in 0.5.0; "
         "use legaldown.load_template() or parse_template(), then .form(answers).assemble()",
-        DeprecationWarning,
+        LegaldownDeprecationWarning,
         stacklevel=2,
     )
     try:
@@ -1845,7 +1846,7 @@ def template_questions(
     warnings.warn(
         "legaldown.template_questions() is deprecated since 0.4.0 and will be removed in 0.5.0; "
         "use legaldown.load_template() or parse_template(), then .questions",
-        DeprecationWarning,
+        LegaldownDeprecationWarning,
         stacklevel=2,
     )
     return _questions(_read(template, load_file))
@@ -1869,7 +1870,7 @@ def needed_questions(
     warnings.warn(
         "legaldown.needed_questions() is deprecated since 0.4.0 and will be removed in 0.5.0; "
         "use legaldown.load_template() or parse_template(), then .form(answers).questions",
-        DeprecationWarning,
+        LegaldownDeprecationWarning,
         stacklevel=2,
     )
     t = _read(template, load_file)

src/legaldown/parser.py

@@ -9,7 +9,6 @@
 from __future__ import annotations
 
 import bisect
-import copy
 import os
 import re
 import warnings
@@ -21,6 +20,7 @@
 
 import yaml
 
+from ._deprecation import LegaldownDeprecationWarning
 from .definitions import DefinitionAnchor, block_fragments, find_definition_anchors
 from .directives import Directive, lex
 from .markdown import (
@@ -1357,14 +1357,20 @@ def quote_content(text: str, depth: int = 0) -> tuple[tuple[Block, ...], tuple[_
     how many list items and quotes the content is in (``_parse_list``).
     Cached, as the validator reads a quote's blocks more than once: the
     blocks are shared, not to be changed."""
+    return _read_quote_content(text, depth)
+
+
+def _read_quote_content(text: str, depth: int = 0) -> tuple[tuple[Block, ...], tuple[_BlockSpan, ...]]:
+    """``quote_content`` uncached: blocks of its own each time, which no
+    other caller holds."""
     layout = _Layout()
     blocks, _sections = _parse_body(text.split("\n"), layout, headings=False, depth=depth)
     return tuple(blocks), tuple(layout.preamble)
 
 
 def quote_blocks(block: Block, *, depth: int = 0) -> list[Block]:
     """The blocks the block quote *block* holds, as the validator reads them
-    (``quote_content``): each a fresh copy, free to change, since the
+    (``quote_content``): each block read afresh, free to change, since the
     validator keeps its own reading of a quote's content. A heading in one
     is a ``heading`` block, not a section, and no directive is lifted into
     block fields (§4.1). *depth*: how many list items and quotes *block* is
@@ -1376,7 +1382,9 @@ def quote_blocks(block: Block, *, depth: int = 0) -> list[Block]:
         raise ValueError(f"not a block quote: {block.kind}")
     if depth >= MAX_QUOTE_DEPTH:
         return [Block(kind="paragraph", text=block.text)] if block.text.strip() else []
-    return copy.deepcopy(list(quote_content(block.text, depth + 1)[0]))
+    # Read again rather than deep-copied from the cache: a copy recurses once
+    # per nesting level, and fails on a list nested near MAX_LIST_DEPTH.
+    return list(_read_quote_content(block.text, depth + 1)[0])
 
 
 def parse_item_content(text: str) -> list[Block]:
@@ -1469,7 +1477,7 @@ def parse_document(source: str, *, filename: str = "") -> Document:
     warnings.warn(
         "legaldown.parse_document() is deprecated since 0.4.0 and will be removed in 0.5.0; "
         "use legaldown.parse() for a string, or legaldown.load() for a file",
-        DeprecationWarning,
+        LegaldownDeprecationWarning,
         stacklevel=2,
     )
     return parse(source, filename=filename)

src/legaldown/serializer.py

@@ -13,6 +13,7 @@
 
 import yaml
 
+from ._deprecation import LegaldownDeprecationWarning
 from .directives import format_value
 from .files import write_atomically
 from .markdown import (
@@ -640,7 +641,7 @@ def serialize_document(document: Document) -> str:
     warnings.warn(
         "legaldown.serialize_document() is deprecated since 0.4.0 and will be removed in 0.5.0; "
         "use legaldown.serialize()",
-        DeprecationWarning,
+        LegaldownDeprecationWarning,
         stacklevel=2,
     )
     return serialize(document)

src/legaldown/validator/core.py

@@ -14,6 +14,7 @@
 from functools import cache, partial
 from typing import Any
 
+from .._deprecation import LegaldownDeprecationWarning
 from ..directives import (
     DIRECTIVE_PARAMS,
     KNOWN_DIRECTIVES,
@@ -636,7 +637,7 @@ def validate_document(
     warnings.warn(
         "legaldown.validate_document() is deprecated since 0.4.0 and will be removed in 0.5.0; "
         "use legaldown.validate()",
-        DeprecationWarning,
+        LegaldownDeprecationWarning,
         stacklevel=2,
     )
     resolve = _resolver(document, None, import_definitions, import_attachment_definitions, stacklevel=3)
@@ -659,7 +660,7 @@ def _resolver(
         warnings.warn(
             "import_definitions and import_attachment_definitions are deprecated since 0.4.0 "
             "and will be removed in 0.5.0; use validate(resolve=), which reads the files itself",
-            DeprecationWarning,
+            LegaldownDeprecationWarning,
             stacklevel=stacklevel,
         )
         return resolve

src/legaldown/validator/templates.py

@@ -300,7 +300,7 @@ def is_drafting_note(quote: Block) -> bool:
 
 def drafting_note_blocks(quote: Block, *, depth: int = 0) -> list[Block]:
     """The blocks of drafting note *quote* without its ``[!DRAFTING]`` marker
-    line (§15.6), as ``quote_blocks`` reads the note's content: fresh copies.
+    line (§15.6), as ``quote_blocks`` reads the note's content: fresh blocks.
     The marker starts the first paragraph or heading, and is cut from it,
     the block going when nothing else is in it; where it does not (a marker
     line indented as code), the note is what is written after its first

tests/test_deep_nesting.py

@@ -0,0 +1,147 @@
+"""Deep nesting up to the parser's limits, and past them: lists nested
+``MAX_LIST_DEPTH`` deep, quotes nested ``MAX_QUOTE_DEPTH`` deep, and the two
+in each other. Each public path reads such a document without a
+``RecursionError``: past the limits the parser reads the rest as text, so no
+document nests deeper than that, but a path that recurses more per level than
+the parser does can still run out of stack below them."""
+from __future__ import annotations
+
+import pytest
+
+from legaldown import Document, parse, parse_template, serialize, validate
+from legaldown.grammar import MAX_LIST_DEPTH, MAX_QUOTE_DEPTH
+from legaldown.parser import quote_content
+from legaldown.syntax import (
+    block_fragments,
+    body_layout,
+    code_content,
+    drafting_note_blocks,
+    find_markers,
+    is_drafting_note,
+    iter_document_directives,
+    quote_blocks,
+)
+
+_FRONT = "---\ntitle: T\nquestions:\n  extras:\n    type: boolean\n---\n\n# A {#a}\n\n"
+
+#: What the innermost level holds: placeholders, a choice, fenced code.
+_INNER = ['x {{placeholder: who}} {{choose: extras, true="a", false="b"}}', "", "```", "{{placeholder: c}}", "```"]
+
+
+def _nest(levels: str) -> str:
+    """The body nesting ``_INNER`` in *levels*, outermost first: ``L`` a list
+    item, ``Q`` a block quote, ``D`` a drafting note."""
+    lines = list(_INNER)
+    for kind in reversed(levels):
+        if kind == "L":
+            lines = ["- " + lines[0]] + [f"  {line}" if line else "" for line in lines[1:]]
+        else:
+            lines = [f"> {line}" if line else ">" for line in lines]
+            if kind == "D":
+                lines.insert(0, "> [!DRAFTING]")
+    return "\n".join(lines)
+
+
+_L, _Q = MAX_LIST_DEPTH, MAX_QUOTE_DEPTH
+SHAPES = {
+    "lists at the limit": "L" * _L,
+    "lists past the limit": "L" * (_L + 6),
+    "quotes at the limit": "Q" * _Q,
+    "quotes past the limit": "Q" * (_Q + 4),
+    "lists below the limit in a quote": "Q" + "L" * (_L - 2),
+    "lists past the limit in a quote": "Q" + "L" * (_L + 6),
+    "lists past the limit in quotes": "Q" * (_Q - 1) + "L" * (_L + 6),
+    "quotes past the limit in lists": "L" * (_L - 1) + "Q" * (_Q + 4),
+    "quotes in lists in quotes": "L" * (_L - 4) + "Q" * _Q + "L" * 10,
+    "quotes and lists alternating": "QL" * (_Q + 4),
+    "lists and quotes alternating": "LQ" * (_Q + 4),
+    "lists in a drafting note": "D" + "L" * (_L + 6),
+    "lists in a drafting note in lists": "L" * 10 + "D" + "L" * (_L - 4),
+    "drafting notes in lists": "DL" * (_Q + 4),
+}
+
+
+@pytest.fixture(params=list(SHAPES), scope="module")
+def source(request) -> str:
+    return _FRONT + _nest(SHAPES[request.param]) + "\n"
+
+
+def _quotes(blocks, depth: int, found: list) -> list:
+    for block in blocks:
+        if block.kind == "quote":
+            found.append((block, depth))
+        for item in block.items or []:
+            _quotes(item.blocks, depth + 1, found)
+    return found
+
+
+def test_the_reported_quote_holds_its_nested_lists_as_fresh_blocks():
+    body = "\n".join("> " + "  " * i + "- x" for i in range(62))
+    [quote] = parse(_FRONT + body + "\n").sections[0].blocks
+    blocks = quote_blocks(quote)
+    cached = list(quote_content(quote.text, 1)[0])
+    assert blocks == cached
+    assert all(fresh is not shared for fresh, shared in zip(blocks, cached, strict=True))
+    depth, block = 0, blocks[0]
+    while block.kind == "unordered_list":
+        depth += 1
+        block = block.items[0].blocks[-1]
+    assert depth == 62
+    blocks[0].items.clear()  # the caller may change them; the cache keeps its reading
+    assert quote_content(quote.text, 1)[0][0].items != []
+    assert quote_blocks(quote) == cached
+
+
+def test_a_drafting_note_holding_lists_past_the_limit_has_fresh_blocks():
+    [note] = parse(_FRONT + _nest("D" + "L" * (_L + 6)) + "\n").sections[0].blocks
+    assert is_drafting_note(note)
+    blocks = drafting_note_blocks(note)
+    assert [block.kind for block in blocks] == ["unordered_list"]
+    blocks[0].items.clear()
+    assert drafting_note_blocks(note)[0].items != []
+
+
+def test_parse_validate_and_serialize(source):
+    document = parse(source)
+    validate(document)
+    validate(parse(source), final=True)
+    text = serialize(document)
+    assert serialize(parse(text)) == text
+
+
+def test_the_dict_round_trip(source):
+    document = parse(source)
+    assert Document.from_dict(document.to_dict()) == document
+
+
+def test_where_the_parts_are(source):
+    document = parse(source)
+    assert document.layout() is not None
+    assert document.line_of(0, 0) == 10
+    assert body_layout(source.split("---\n\n", 1)[1]) is not None
+
+
+def test_what_the_text_holds(source):
+    document = parse(source)
+    assert list(iter_document_directives(document)) != []
+    find_markers(document)
+    for block in document.sections[0].blocks:
+        block_fragments(block)
+
+
+def test_each_quote_read_into_blocks(source):
+    pending = _quotes(parse(source).sections[0].blocks, 0, [])
+    while pending:
+        quote, depth = pending.pop()
+        blocks = drafting_note_blocks(quote, depth=depth) if is_drafting_note(quote) else quote_blocks(quote, depth=depth)
+        _quotes(blocks, depth + 1, pending)
+        for block in blocks:
+            if block.kind == "code":
+                code_content(block)
+
+
+@pytest.mark.parametrize("answers", [{"extras": True, "who": "Ann", "c": "C"}, {"extras": False}])
+def test_assembly(source, answers):
+    result = parse_template(source).form(answers).assemble()
+    if result.ok:
+        parse(result.output)