feat: Phase.Resolving works now

src/sphinxnotes/data/data.py

@@ -451,7 +451,7 @@ def by_option_store_value_error(opt: ByOption) -> ValueError:
 
 
 @dataclass(frozen=True)
-class Schema(object):
+class Schema(Unpicklable):
     name: Field | None
     attrs: dict[str, Field] | Field
     content: Field | None

src/sphinxnotes/data/render/datanodes.py

@@ -10,7 +10,6 @@
 from .template import TemplateRenderer
 from ..data import RawData, PendingData, ParsedData
 from ..utils import (
-    Unpicklable,
     Report,
     Reporter,
     find_nearest_block_element,
@@ -24,7 +23,7 @@
 class Base(nodes.Element): ...
 
 
-class pending_node(Base, Unpicklable):
+class pending_node(Base):
     # The data to be rendered by Jinja template.
     data: PendingData | ParsedData | dict[str, Any]
     # The extra context for Jina template.
@@ -35,6 +34,8 @@ class pending_node(Base, Unpicklable):
     inline: bool
     #: Whether the rendering pipeline is finished (failed is also finished).
     rendered: bool
+    #: The report of render pipepine.
+    report: Report
 
     def __init__(
         self,
@@ -51,18 +52,58 @@ def __init__(
         self.template = tmpl
         self.inline = inline
         self.rendered = False
+        self.report = Report(
+            'Render Report', 'DEBUG', source=self.source, line=self.line
+        )
 
         # Init hook lists.
         self._raw_data_hooks = []
         self._parsed_data_hooks = []
         self._markup_text_hooks = []
         self._rendered_nodes_hooks = []
 
+    def get_error_report(self) -> Report:
+        if self.template.debug:
+            # Reuse the render report as possible.
+            self.report['type'] = 'ERROR'
+            return self.report
+        return Report('Render Report', 'ERROR', source=self.source, line=self.line)
+
+    def ensure_data_parsed(self) -> ParsedData | dict[str, Any] | None:
+        """
+        Ensure self.data is parsed (instance of ParsedData | dict[str, Any]).
+        if no, parse it.
+        """
+        if not isinstance(self.data, PendingData):
+            return self.data
+
+        self.report.text('Raw data:')
+        self.report.code(pformat(self.data.raw), lang='python')
+        self.report.text('Schema:')
+        self.report.code(pformat(self.data.schema), lang='python')
+
+        for hook in self._raw_data_hooks:
+            hook(self, self.data.raw)
+
+        try:
+            data = self.data = self.data.parse()
+        except ValueError as e:
+            report = self.get_error_report()
+            report.text('Failed to parse raw data:')
+            report.exception(e)
+            self += report
+            return None
+
+        for hook in self._parsed_data_hooks:
+            hook(self, data)
+
+        return data
+
     def render(self, host: Host) -> None:
         """
         The core function for rendering data to docutils nodes.
 
-        1. Schema.parse(RawData) -> ParsedData
+        1. Schema.parse(RawData) -> ParsedData (self.parse_data)
         2. TemplateRenderer.render(ParsedData) -> Markup Text (``str``)
         3. MarkupRenderer.render(Markup Text) -> doctree Nodes (list[nodes.Node])
         """
@@ -71,83 +112,64 @@ def render(self, host: Host) -> None:
         assert not self.rendered
         self.rendered = True
 
-        report = Report(
-            'Render Debug Report', 'DEBUG', source=self.source, line=self.line
-        )
+        # Clear empty reports.
+        Reporter(self).clear_empty()
 
         # 1. Prepare context for Jinja template.
-        if isinstance(self.data, PendingData):
-            report.text('Raw data:')
-            report.code(pformat(self.data.raw), lang='python')
-            report.text('Schema:')
-            report.code(pformat(self.data.schema), lang='python')
-
-            for hook in self._raw_data_hooks:
-                hook(self, self.data.raw)
-
-            try:
-                data = self.data = self.data.parse()
-            except ValueError:
-                report.text('Failed to parse raw data:')
-                report.excption()
-                self += report
-                return
-        else:
-            data = self.data
-
-        for hook in self._parsed_data_hooks:
-            hook(self, data)
+        if (data := self.ensure_data_parsed()) is None:
+            return  # parse failure
 
-        report.text(f'Parsed data (type: {type(data)}):')
-        report.code(pformat(data), lang='python')
-        report.text('Extra context (only keys):')
-        report.code(pformat(list(self.extra.keys())), lang='python')
-        report.text(f'Template (phase: {self.template.phase}):')
-        report.code(self.template.text, lang='jinja')
+        self.report.text(f'Parsed data (type: {type(data)}):')
+        self.report.code(pformat(data), lang='python')
+        self.report.text('Extra context (only keys):')
+        self.report.code(pformat(list(self.extra.keys())), lang='python')
+        self.report.text(f'Template (phase: {self.template.phase}):')
+        self.report.code(self.template.text, lang='jinja')
 
         # 2. Render the template and data to markup text.
         try:
             markup = TemplateRenderer(self.template.text).render(data, extra=self.extra)
-        except Exception:
+        except Exception as e:
+            report = self.get_error_report()
             report.text('Failed to render Jinja template:')
-            report.excption()
+            report.exception(e)
             self += report
             return
 
         for hook in self._markup_text_hooks:
             markup = hook(self, markup)
 
-        report.text('Rendered markup text:')
-        report.code(markup, lang='rst')
+        self.report.text('Rendered markup text:')
+        self.report.code(markup, lang='rst')
 
         # 3. Render the markup text to doctree nodes.
         try:
             ns, msgs = MarkupRenderer(host).render(markup, inline=self.inline)
-        except Exception:
+        except Exception as e:
+            report = self.get_error_report()
             report.text(
                 'Failed to render markup text '
                 f'to {"inline " if self.inline else ""}nodes:'
             )
-            report.excption()
+            report.exception(e)
             self += report
             return
 
-        report.text(f'Rendered nodes (inline: {self.inline}):')
-        report.code('\n\n'.join([n.pformat() for n in ns]), lang='xml')
+        self.report.text(f'Rendered nodes (inline: {self.inline}):')
+        self.report.code('\n\n'.join([n.pformat() for n in ns]), lang='xml')
         if msgs:
-            report.text('Systemd messages:')
-            [report.node(msg) for msg in msgs]
+            self.report.text('Systemd messages:')
+            [self.report.node(msg) for msg in msgs]
 
         # 4. Add rendered nodes to container.
         for hook in self._rendered_nodes_hooks:
             hook(self, ns)
+
         # TODO: set_source_info?
         self += ns
 
         if self.template.debug:
-            self += report
-
-        Reporter(self).clear_empty()
+            self += self.report
 
         return
 

src/sphinxnotes/data/render/extractx.py

@@ -40,7 +40,7 @@ def generate(self, host: TransformHost) -> Any: ...
 class ExtraContextRegistry:
     names: set[str]
     parsing: dict[str, ParsePhaseExtraContext]
-    parsed: dict[str, ParsePhaseExtraContext]
+    parsed: dict[str, TransformPhaseExtraContext]
     post_transform: dict[str, TransformPhaseExtraContext]
     global_: dict[str, GlobalExtraContxt]
 
@@ -70,7 +70,7 @@ def add_parsing_phase_context(
         self.parsing['_' + name] = ctxgen
 
     def add_parsed_phase_context(
-        self, name: str, ctxgen: ParsePhaseExtraContext
+        self, name: str, ctxgen: TransformPhaseExtraContext
     ) -> None:
         self._name_dedup(name)
         self.parsed['_' + name] = ctxgen
@@ -163,7 +163,7 @@ def on_parsing(self, host: ParseHost) -> None:
         for name, ctxgen in self.registry.parsing.items():
             self._safegen(name, lambda: ctxgen.generate(host))
 
-    def on_parsed(self, host: ParseHost) -> None:
+    def on_parsed(self, host: TransformHost) -> None:
         for name, ctxgen in self.registry.parsed.items():
             self._safegen(name, lambda: ctxgen.generate(host))
 
@@ -177,7 +177,7 @@ def _safegen(self, name: str, gen: Callable[[], Any]):
             self.node.extra[name] = gen()
         except Exception:
             self.report.text(f'Failed to generate extra context "{name}":')
-            self.report.excption()
+            self.report.traceback()
 
 
 def setup(app: Sphinx):

src/sphinxnotes/data/render/markup.py

@@ -43,9 +43,13 @@ def _render(self, text: str) -> list[Node]:
         elif isinstance(self.host, SphinxTransform):
             # TODO: dont create parser for every time
             if version_info[0] >= 9:
-                parser = self.host.app.registry.create_source_parser('rst', env=self.host.env, config=self.host.config)
+                parser = self.host.app.registry.create_source_parser(
+                    'rst', env=self.host.env, config=self.host.config
+                )
             else:
-                parser = self.host.app.registry.create_source_parser(self.host.app, 'rst')
+                parser = self.host.app.registry.create_source_parser(
+                    self.host.app, 'rst'
+                )
             settings = self.host.document.settings
             doc = new_document('<generated text>', settings=settings)
             parser.parse(text, doc)

src/sphinxnotes/data/render/pipeline.py

@@ -48,6 +48,7 @@
 from docutils import nodes
 from sphinx.util import logging
 from sphinx.util.docutils import SphinxDirective, SphinxRole
+from sphinx.transforms import SphinxTransform
 from sphinx.transforms.post_transforms import SphinxPostTransform, ReferencesResolver
 
 from .render import HostWrapper, Phase, Template, Host, ParseHost, TransformHost
@@ -72,7 +73,7 @@ def process_pending_node(self, n: pending_node) -> bool:
         """
         You can add hooks to pending node here.
 
-        Return ``true`` if you want to render the pending node *immediately*,
+        Return ``true`` if you want to render the pending node *now*,
         otherwise it will be inserted to doctree directly andwaiting to later
         rendering
         """
@@ -122,7 +123,8 @@ def render_queue(self) -> list[pending_node]:
         while self._q:
             pending = self._q.pop()
 
-            if not self.process_pending_node(pending):
+            render_now = self.process_pending_node(pending)
+            if not render_now:
                 ns.append(pending)
                 continue
 
@@ -232,70 +234,50 @@ def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]:
         return ns, msgs
 
 
-class _ParsedHook(SphinxDirective, Pipeline):
+class _ParsedHook(SphinxTransform, Pipeline):
+    # Before almost all others.
+    default_priority = 100
+
     @override
     def process_pending_node(self, n: pending_node) -> bool:
-        self.state.document.note_source(n.source, n.line)  # type: ignore[arg-type]
-
-        # Generate and save parsed extra context for later use.
-        ExtraContextGenerator(n).on_parsed(cast(ParseHost, self))
-
+        ExtraContextGenerator(n).on_parsed(cast(TransformHost, self))
         return n.template.phase == Phase.Parsed
 
     @override
-    def run(self) -> list[nodes.Node]:
-        for pending in self.state.document.findall(pending_node):
+    def apply(self, **kwargs):
+        for pending in self.document.findall(pending_node):
             self.queue_pending_node(pending)
-            # Hook system_message method to let it report the
-            # correct line number.
-            # TODO: self.state.document.note_source(source, line)  # type: ignore[arg-type]
-            # def fix_lineno(level, message, *children, **kwargs):
-            #     kwargs['line'] = pending.line
-            #     return orig_sysmsg(level, message, *children, **kwargs)
-
-            # self.state_machine.reporter.system_message = fix_lineno
-
-        ns = self.render_queue()
-        assert len(ns) == 0
-
-        return []  # nothing to return
-
 
-def _insert_parsed_hook(app, docname, content):
-    # NOTE: content is a single element list, representing the content of the
-    # source file.
-    #
-    # .. seealso:: https://www.sphinx-doc.org/en/master/extdev/event_callbacks.html#event-source-read
-    #
-    # TODO: markdown?
-    # TODO: rst_prelog?
-    content[-1] = content[-1] + '\n\n.. data.parsed-hook::'
+        for n in self.render_queue():
+            # NOTE: In the next Phase, doctrees will be pickled to disk.
+            # As :cls:`data.Schema` is **Unpicklable**, we should ensure
+            # ``pending_node.data`` is parsed, which means pending_node dropped
+            # the reference to Schema.
+            n.ensure_data_parsed()
 
 
 class _ResolvingHook(SphinxPostTransform, Pipeline):
-    # After resolving pending_xref.
+    # After resolving pending_xref
     default_priority = (ReferencesResolver.default_priority or 10) + 5
 
     @override
     def process_pending_node(self, n: pending_node) -> bool:
-        # Generate and save post transform extra context for later use.
         ExtraContextGenerator(n).on_post_transform(cast(TransformHost, self))
-
-        return n.template.phase == Phase.PostTranform
+        return n.template.phase == Phase.Resolving
 
     @override
     def apply(self, **kwargs):
         for pending in self.document.findall(pending_node):
             self.queue_pending_node(pending)
-
         ns = self.render_queue()
+
+        # NOTE: Should no node left.
         assert len(ns) == 0
 
 
 def setup(app: Sphinx) -> None:
     # Hook for Phase.Parsed.
-    app.add_directive('data.parsed-hook', _ParsedHook)
-    app.connect('source-read', _insert_parsed_hook)
+    app.add_transform(_ParsedHook)
 
     # Hook for Phase.Resolving.
     app.add_post_transform(_ResolvingHook)

src/sphinxnotes/data/render/render.py

@@ -10,8 +10,7 @@
 class Phase(Enum):
     Parsing = 'parsing'
     Parsed = 'parsed'
-    PostTranform = 'post-transform'
-    # TODO: transform?
+    Resolving = 'resolving'
 
     @classmethod
     def default(cls) -> Phase: