From ba5be9906ff41c3acb89ca0ca5cc6b55600df733 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Tue, 3 Feb 2026 00:19:23 +0800 Subject: [PATCH 1/3] chore: Show less error message --- src/sphinxnotes/data/render/datanodes.py | 65 ++++++++++++++---------- src/sphinxnotes/data/utils/__init__.py | 15 +++--- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/sphinxnotes/data/render/datanodes.py b/src/sphinxnotes/data/render/datanodes.py index 3dae467..8d4616e 100644 --- a/src/sphinxnotes/data/render/datanodes.py +++ b/src/sphinxnotes/data/render/datanodes.py @@ -71,25 +71,34 @@ def render(self, host: Host) -> None: assert not self.rendered self.rendered = True - report = Report( - 'Render Debug Report', 'DEBUG', source=self.source, line=self.line - ) + # Clear empty reports. + Reporter(self).clear_empty() + + dbg = Report('Render Report', 'DEBUG', source=self.source, line=self.line) + + def get_error_report() -> Report: + if self.template.debug: + # Reuse the debug report as possible. + dbg['type'] = 'ERROR' + return dbg + return Report('Render Report', 'ERROR', source=self.source, line=self.line) # 1. Prepare context for Jinja template. if isinstance(self.data, PendingData): - report.text('Raw data:') - report.code(pformat(self.data.raw), lang='python') - report.text('Schema:') - report.code(pformat(self.data.schema), lang='python') + dbg.text('Raw data:') + dbg.code(pformat(self.data.raw), lang='python') + dbg.text('Schema:') + dbg.code(pformat(self.data.schema), lang='python') for hook in self._raw_data_hooks: hook(self, self.data.raw) try: data = self.data = self.data.parse() - except ValueError: + except ValueError as e: + report = get_error_report() report.text('Failed to parse raw data:') - report.excption() + report.exception(e) self += report return else: @@ -98,45 +107,47 @@ def render(self, host: Host) -> None: for hook in self._parsed_data_hooks: hook(self, data) - report.text(f'Parsed data (type: {type(data)}):') - report.code(pformat(data), lang='python') - report.text('Extra context (only keys):') - report.code(pformat(list(self.extra.keys())), lang='python') - report.text(f'Template (phase: {self.template.phase}):') - report.code(self.template.text, lang='jinja') + dbg.text(f'Parsed data (type: {type(data)}):') + dbg.code(pformat(data), lang='python') + dbg.text('Extra context (only keys):') + dbg.code(pformat(list(self.extra.keys())), lang='python') + dbg.text(f'Template (phase: {self.template.phase}):') + dbg.code(self.template.text, lang='jinja') # 2. Render the template and data to markup text. try: markup = TemplateRenderer(self.template.text).render(data, extra=self.extra) - except Exception: + except Exception as e: + report = get_error_report() report.text('Failed to render Jinja template:') - report.excption() + report.exception(e) self += report return for hook in self._markup_text_hooks: markup = hook(self, markup) - report.text('Rendered markup text:') - report.code(markup, lang='rst') + dbg.text('Rendered markup text:') + dbg.code(markup, lang='rst') # 3. Render the markup text to doctree nodes. try: ns, msgs = MarkupRenderer(host).render(markup, inline=self.inline) - except Exception: + except Exception as e: + report = get_error_report() report.text( 'Failed to render markup text ' f'to {"inline " if self.inline else ""}nodes:' ) - report.excption() + report.exception(e) self += report return - report.text(f'Rendered nodes (inline: {self.inline}):') - report.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') + dbg.text(f'Rendered nodes (inline: {self.inline}):') + dbg.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') if msgs: - report.text('Systemd messages:') - [report.node(msg) for msg in msgs] + dbg.text('Systemd messages:') + [dbg.node(msg) for msg in msgs] # 4. Add rendered nodes to container. for hook in self._rendered_nodes_hooks: @@ -145,9 +156,7 @@ def render(self, host: Host) -> None: self += ns if self.template.debug: - self += report - - Reporter(self).clear_empty() + self += dbg return diff --git a/src/sphinxnotes/data/utils/__init__.py b/src/sphinxnotes/data/utils/__init__.py index 58f1530..62b4f35 100644 --- a/src/sphinxnotes/data/utils/__init__.py +++ b/src/sphinxnotes/data/utils/__init__.py @@ -97,15 +97,14 @@ def find_nearest_block_element(node: nodes.Node | None) -> nodes.Element | None: class Report(nodes.system_message): - type Level = Literal['DEBUG', 'INFO', 'WARNING', 'ERROR'] + type Type = Literal['DEBUG', 'INFO', 'WARNING', 'ERROR'] - level: Level title: str def __init__( - self, title: str, level: Level = 'DEBUG', *children, **attributes + self, title: str, typ: Type = 'DEBUG', *children, **attributes ) -> None: - super().__init__(title + ':', type=level, level=2, *children, **attributes) + super().__init__(title + ':', type=typ, level=2, *children, **attributes) self.title = title def empty(self) -> bool: @@ -143,12 +142,16 @@ def list(self, lines: Iterable[str]) -> None: self.node(bullet_list) - def excption(self) -> None: + def traceback(self) -> None: # https://pygments.org/docs/lexers/#pygments.lexers.python.PythonTracebackLexer self.code(traceback.format_exc(), lang='pytb') + def exception(self, e: Exception) -> None: + # https://pygments.org/docs/lexers/#pygments.lexers.python.PythonTracebackLexer + self.code(str(e), lang='pytb') + def is_error(self) -> bool: - return self.level == 'ERROR' + return self['type'] == 'ERROR' type Inliner = RstInliner | tuple[nodes.document, nodes.Element] From a51d3a0beaef9c775c1ee7395e6838def385d3aa Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Tue, 3 Feb 2026 00:37:08 +0800 Subject: [PATCH 2/3] refactor: Impl _ParsedHook with Transform --- src/sphinxnotes/data/render/extractx.py | 8 ++-- src/sphinxnotes/data/render/markup.py | 8 +++- src/sphinxnotes/data/render/pipeline.py | 53 ++++++------------------- src/sphinxnotes/data/render/render.py | 3 +- tests/test_data.py | 2 +- 5 files changed, 25 insertions(+), 49 deletions(-) diff --git a/src/sphinxnotes/data/render/extractx.py b/src/sphinxnotes/data/render/extractx.py index e05277b..9a2db65 100644 --- a/src/sphinxnotes/data/render/extractx.py +++ b/src/sphinxnotes/data/render/extractx.py @@ -40,7 +40,7 @@ def generate(self, host: TransformHost) -> Any: ... class ExtraContextRegistry: names: set[str] parsing: dict[str, ParsePhaseExtraContext] - parsed: dict[str, ParsePhaseExtraContext] + parsed: dict[str, TransformPhaseExtraContext] post_transform: dict[str, TransformPhaseExtraContext] global_: dict[str, GlobalExtraContxt] @@ -70,7 +70,7 @@ def add_parsing_phase_context( self.parsing['_' + name] = ctxgen def add_parsed_phase_context( - self, name: str, ctxgen: ParsePhaseExtraContext + self, name: str, ctxgen: TransformPhaseExtraContext ) -> None: self._name_dedup(name) self.parsed['_' + name] = ctxgen @@ -163,7 +163,7 @@ def on_parsing(self, host: ParseHost) -> None: for name, ctxgen in self.registry.parsing.items(): self._safegen(name, lambda: ctxgen.generate(host)) - def on_parsed(self, host: ParseHost) -> None: + def on_parsed(self, host: TransformHost) -> None: for name, ctxgen in self.registry.parsed.items(): self._safegen(name, lambda: ctxgen.generate(host)) @@ -177,7 +177,7 @@ def _safegen(self, name: str, gen: Callable[[], Any]): self.node.extra[name] = gen() except Exception: self.report.text(f'Failed to generate extra context "{name}":') - self.report.excption() + self.report.traceback() def setup(app: Sphinx): diff --git a/src/sphinxnotes/data/render/markup.py b/src/sphinxnotes/data/render/markup.py index 198ab16..06c5dec 100644 --- a/src/sphinxnotes/data/render/markup.py +++ b/src/sphinxnotes/data/render/markup.py @@ -43,9 +43,13 @@ def _render(self, text: str) -> list[Node]: elif isinstance(self.host, SphinxTransform): # TODO: dont create parser for every time if version_info[0] >= 9: - parser = self.host.app.registry.create_source_parser('rst', env=self.host.env, config=self.host.config) + parser = self.host.app.registry.create_source_parser( + 'rst', env=self.host.env, config=self.host.config + ) else: - parser = self.host.app.registry.create_source_parser(self.host.app, 'rst') + parser = self.host.app.registry.create_source_parser( + self.host.app, 'rst' + ) settings = self.host.document.settings doc = new_document('', settings=settings) parser.parse(text, doc) diff --git a/src/sphinxnotes/data/render/pipeline.py b/src/sphinxnotes/data/render/pipeline.py index 7d9ba50..10bc3f2 100644 --- a/src/sphinxnotes/data/render/pipeline.py +++ b/src/sphinxnotes/data/render/pipeline.py @@ -48,6 +48,7 @@ from docutils import nodes from sphinx.util import logging from sphinx.util.docutils import SphinxDirective, SphinxRole +from sphinx.transforms import SphinxTransform from sphinx.transforms.post_transforms import SphinxPostTransform, ReferencesResolver from .render import HostWrapper, Phase, Template, Host, ParseHost, TransformHost @@ -232,70 +233,42 @@ def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: return ns, msgs -class _ParsedHook(SphinxDirective, Pipeline): +class _ParsedHook(SphinxTransform, Pipeline): + # Before almost all others. + default_priority = 100 + @override def process_pending_node(self, n: pending_node) -> bool: - self.state.document.note_source(n.source, n.line) # type: ignore[arg-type] - - # Generate and save parsed extra context for later use. - ExtraContextGenerator(n).on_parsed(cast(ParseHost, self)) - + ExtraContextGenerator(n).on_parsed(cast(TransformHost, self)) return n.template.phase == Phase.Parsed @override - def run(self) -> list[nodes.Node]: - for pending in self.state.document.findall(pending_node): + def apply(self, **kwargs): + for pending in self.document.findall(pending_node): self.queue_pending_node(pending) - # Hook system_message method to let it report the - # correct line number. - # TODO: self.state.document.note_source(source, line) # type: ignore[arg-type] - # def fix_lineno(level, message, *children, **kwargs): - # kwargs['line'] = pending.line - # return orig_sysmsg(level, message, *children, **kwargs) - - # self.state_machine.reporter.system_message = fix_lineno - - ns = self.render_queue() - assert len(ns) == 0 - - return [] # nothing to return - - -def _insert_parsed_hook(app, docname, content): - # NOTE: content is a single element list, representing the content of the - # source file. - # - # .. seealso:: https://www.sphinx-doc.org/en/master/extdev/event_callbacks.html#event-source-read - # - # TODO: markdown? - # TODO: rst_prelog? - content[-1] = content[-1] + '\n\n.. data.parsed-hook::' + self.render_queue() class _ResolvingHook(SphinxPostTransform, Pipeline): - # After resolving pending_xref. - default_priority = (ReferencesResolver.default_priority or 10) + 5 + # After resolving pending_xref + default_priority = (ReferencesResolver.default_priority or 10) + 5 @override def process_pending_node(self, n: pending_node) -> bool: - # Generate and save post transform extra context for later use. ExtraContextGenerator(n).on_post_transform(cast(TransformHost, self)) - - return n.template.phase == Phase.PostTranform + return n.template.phase == Phase.Resolving @override def apply(self, **kwargs): for pending in self.document.findall(pending_node): self.queue_pending_node(pending) - ns = self.render_queue() assert len(ns) == 0 def setup(app: Sphinx) -> None: # Hook for Phase.Parsed. - app.add_directive('data.parsed-hook', _ParsedHook) - app.connect('source-read', _insert_parsed_hook) + app.add_transform(_ParsedHook) # Hook for Phase.Resolving. app.add_post_transform(_ResolvingHook) diff --git a/src/sphinxnotes/data/render/render.py b/src/sphinxnotes/data/render/render.py index e54254e..02472a5 100644 --- a/src/sphinxnotes/data/render/render.py +++ b/src/sphinxnotes/data/render/render.py @@ -10,8 +10,7 @@ class Phase(Enum): Parsing = 'parsing' Parsed = 'parsed' - PostTranform = 'post-transform' - # TODO: transform? + Resolving = 'resolving' @classmethod def default(cls) -> Phase: diff --git a/tests/test_data.py b/tests/test_data.py index f91b34e..477cd0f 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -6,8 +6,8 @@ from data.data import Field, REGISTRY -class TestFieldParser(unittest.TestCase): +class TestFieldParser(unittest.TestCase): # ========================== # Basic Types # ========================== From 75f45442a19210be991d665fb9c7625160e0fd4e Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Tue, 3 Feb 2026 01:38:55 +0800 Subject: [PATCH 3/3] feat: Phase.Resolving works now --- src/sphinxnotes/data/data.py | 2 +- src/sphinxnotes/data/render/datanodes.py | 111 +++++++++++++---------- src/sphinxnotes/data/render/pipeline.py | 17 +++- src/sphinxnotes/data/utils/ctxproxy.py | 4 +- 4 files changed, 78 insertions(+), 56 deletions(-) diff --git a/src/sphinxnotes/data/data.py b/src/sphinxnotes/data/data.py index b1c93e3..1eae6b3 100644 --- a/src/sphinxnotes/data/data.py +++ b/src/sphinxnotes/data/data.py @@ -451,7 +451,7 @@ def by_option_store_value_error(opt: ByOption) -> ValueError: @dataclass(frozen=True) -class Schema(object): +class Schema(Unpicklable): name: Field | None attrs: dict[str, Field] | Field content: Field | None diff --git a/src/sphinxnotes/data/render/datanodes.py b/src/sphinxnotes/data/render/datanodes.py index 8d4616e..d212fcd 100644 --- a/src/sphinxnotes/data/render/datanodes.py +++ b/src/sphinxnotes/data/render/datanodes.py @@ -10,7 +10,6 @@ from .template import TemplateRenderer from ..data import RawData, PendingData, ParsedData from ..utils import ( - Unpicklable, Report, Reporter, find_nearest_block_element, @@ -24,7 +23,7 @@ class Base(nodes.Element): ... -class pending_node(Base, Unpicklable): +class pending_node(Base): # The data to be rendered by Jinja template. data: PendingData | ParsedData | dict[str, Any] # The extra context for Jina template. @@ -35,6 +34,8 @@ class pending_node(Base, Unpicklable): inline: bool #: Whether the rendering pipeline is finished (failed is also finished). rendered: bool + #: The report of render pipepine. + report: Report def __init__( self, @@ -51,6 +52,9 @@ def __init__( self.template = tmpl self.inline = inline self.rendered = False + self.report = Report( + 'Render Report', 'DEBUG', source=self.source, line=self.line + ) # Init hook lists. self._raw_data_hooks = [] @@ -58,11 +62,48 @@ def __init__( self._markup_text_hooks = [] self._rendered_nodes_hooks = [] + def get_error_report(self) -> Report: + if self.template.debug: + # Reuse the render report as possible. + self.report['type'] = 'ERROR' + return self.report + return Report('Render Report', 'ERROR', source=self.source, line=self.line) + + def ensure_data_parsed(self) -> ParsedData | dict[str, Any] | None: + """ + Ensure self.data is parsed (instance of ParsedData | dict[str, Any]). + if no, parse it. + """ + if not isinstance(self.data, PendingData): + return self.data + + self.report.text('Raw data:') + self.report.code(pformat(self.data.raw), lang='python') + self.report.text('Schema:') + self.report.code(pformat(self.data.schema), lang='python') + + for hook in self._raw_data_hooks: + hook(self, self.data.raw) + + try: + data = self.data = self.data.parse() + except ValueError as e: + report = self.get_error_report() + report.text('Failed to parse raw data:') + report.exception(e) + self += report + return None + + for hook in self._parsed_data_hooks: + hook(self, data) + + return data + def render(self, host: Host) -> None: """ The core function for rendering data to docutils nodes. - 1. Schema.parse(RawData) -> ParsedData + 1. Schema.parse(RawData) -> ParsedData (self.parse_data) 2. TemplateRenderer.render(ParsedData) -> Markup Text (``str``) 3. MarkupRenderer.render(Markup Text) -> doctree Nodes (list[nodes.Node]) """ @@ -74,51 +115,22 @@ def render(self, host: Host) -> None: # Clear empty reports. Reporter(self).clear_empty() - dbg = Report('Render Report', 'DEBUG', source=self.source, line=self.line) - - def get_error_report() -> Report: - if self.template.debug: - # Reuse the debug report as possible. - dbg['type'] = 'ERROR' - return dbg - return Report('Render Report', 'ERROR', source=self.source, line=self.line) - # 1. Prepare context for Jinja template. - if isinstance(self.data, PendingData): - dbg.text('Raw data:') - dbg.code(pformat(self.data.raw), lang='python') - dbg.text('Schema:') - dbg.code(pformat(self.data.schema), lang='python') - - for hook in self._raw_data_hooks: - hook(self, self.data.raw) - - try: - data = self.data = self.data.parse() - except ValueError as e: - report = get_error_report() - report.text('Failed to parse raw data:') - report.exception(e) - self += report - return - else: - data = self.data + if (data := self.ensure_data_parsed()) is None: + return # parse failure - for hook in self._parsed_data_hooks: - hook(self, data) - - dbg.text(f'Parsed data (type: {type(data)}):') - dbg.code(pformat(data), lang='python') - dbg.text('Extra context (only keys):') - dbg.code(pformat(list(self.extra.keys())), lang='python') - dbg.text(f'Template (phase: {self.template.phase}):') - dbg.code(self.template.text, lang='jinja') + self.report.text(f'Parsed data (type: {type(data)}):') + self.report.code(pformat(data), lang='python') + self.report.text('Extra context (only keys):') + self.report.code(pformat(list(self.extra.keys())), lang='python') + self.report.text(f'Template (phase: {self.template.phase}):') + self.report.code(self.template.text, lang='jinja') # 2. Render the template and data to markup text. try: markup = TemplateRenderer(self.template.text).render(data, extra=self.extra) except Exception as e: - report = get_error_report() + report = self.get_error_report() report.text('Failed to render Jinja template:') report.exception(e) self += report @@ -127,14 +139,14 @@ def get_error_report() -> Report: for hook in self._markup_text_hooks: markup = hook(self, markup) - dbg.text('Rendered markup text:') - dbg.code(markup, lang='rst') + self.report.text('Rendered markup text:') + self.report.code(markup, lang='rst') # 3. Render the markup text to doctree nodes. try: ns, msgs = MarkupRenderer(host).render(markup, inline=self.inline) except Exception as e: - report = get_error_report() + report = self.get_error_report() report.text( 'Failed to render markup text ' f'to {"inline " if self.inline else ""}nodes:' @@ -143,20 +155,21 @@ def get_error_report() -> Report: self += report return - dbg.text(f'Rendered nodes (inline: {self.inline}):') - dbg.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') + self.report.text(f'Rendered nodes (inline: {self.inline}):') + self.report.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') if msgs: - dbg.text('Systemd messages:') - [dbg.node(msg) for msg in msgs] + self.report.text('Systemd messages:') + [self.report.node(msg) for msg in msgs] # 4. Add rendered nodes to container. for hook in self._rendered_nodes_hooks: hook(self, ns) + # TODO: set_source_info? self += ns if self.template.debug: - self += dbg + self += self.report return diff --git a/src/sphinxnotes/data/render/pipeline.py b/src/sphinxnotes/data/render/pipeline.py index 10bc3f2..c5b580f 100644 --- a/src/sphinxnotes/data/render/pipeline.py +++ b/src/sphinxnotes/data/render/pipeline.py @@ -73,7 +73,7 @@ def process_pending_node(self, n: pending_node) -> bool: """ You can add hooks to pending node here. - Return ``true`` if you want to render the pending node *immediately*, + Return ``true`` if you want to render the pending node *now*, otherwise it will be inserted to doctree directly andwaiting to later rendering """ @@ -123,7 +123,8 @@ def render_queue(self) -> list[pending_node]: while self._q: pending = self._q.pop() - if not self.process_pending_node(pending): + render_now = self.process_pending_node(pending) + if not render_now: ns.append(pending) continue @@ -246,12 +247,18 @@ def process_pending_node(self, n: pending_node) -> bool: def apply(self, **kwargs): for pending in self.document.findall(pending_node): self.queue_pending_node(pending) - self.render_queue() + + for n in self.render_queue(): + # NOTE: In the next Phase, doctrees will be pickled to disk. + # As :cls:`data.Schema` is **Unpicklable**, we should ensure + # ``pending_node.data`` is parsed, which means pending_node dropped + # the reference to Schema. + n.ensure_data_parsed() class _ResolvingHook(SphinxPostTransform, Pipeline): # After resolving pending_xref - default_priority = (ReferencesResolver.default_priority or 10) + 5 + default_priority = (ReferencesResolver.default_priority or 10) + 5 @override def process_pending_node(self, n: pending_node) -> bool: @@ -263,6 +270,8 @@ def apply(self, **kwargs): for pending in self.document.findall(pending_node): self.queue_pending_node(pending) ns = self.render_queue() + + # NOTE: Should no node left. assert len(ns) == 0 diff --git a/src/sphinxnotes/data/utils/ctxproxy.py b/src/sphinxnotes/data/utils/ctxproxy.py index f48c092..cd9816c 100644 --- a/src/sphinxnotes/data/utils/ctxproxy.py +++ b/src/sphinxnotes/data/utils/ctxproxy.py @@ -8,7 +8,6 @@ from sphinx.config import Config as SphinxConfig from ..utils import find_first_child -from ..utils import Unpicklable logger = logging.getLogger(__name__) @@ -22,8 +21,9 @@ def wrapped(self: Proxy) -> Any: return property(wrapped) +# FIXME: Unpicklable? @dataclass(frozen=True) -class Proxy(Unpicklable): +class Proxy: """ Proxy complex objects into context for convenient and secure access within Jinja templates.