From 7b72bae5f0078fe81e4caff8179a06c162ec38ff Mon Sep 17 00:00:00 2001 From: Malcolm Nixon Date: Thu, 16 Jul 2026 00:44:25 -0400 Subject: [PATCH] Fix infinite mutual recursion for empty graph with explicit auto/hierarchical algorithm AutoLayoutAlgorithm.ApplyCore's empty-graph (count == 0) fast path unconditionally delegated to the internal HierarchicalLayoutAlgorithm instance. That instance's own registry resolves both 'auto' and 'hierarchical' back to this same AutoLayoutAlgorithm instance (needed so nested 'auto' scopes re-evaluate correctly). Since an empty graph can never contain a container, HierarchicalLayoutAlgorithm.LayoutScope always takes its flat fast path and calls straight back into this same method with the same empty graph whenever the graph or options declare Algorithm as 'auto' or 'hierarchical' - infinite mutual recursion that overflows the stack (confirmed by reproduction: process exit code 0xC00000FD, STATUS_STACK_OVERFLOW). Fix: resolve the cascaded effective algorithm id directly in the empty-graph fast path, and short-circuit straight to the default leaf (layered) whenever it is self-referential, before ever consulting _hierarchical. Every other id still routes through _hierarchical.ApplyCore exactly as before, preserving existing resolution (and resolution-error) behavior unchanged. Added regression tests proving an empty graph with Algorithm explicitly set to 'auto' or 'hierarchical' resolves without crashing. --- .../AutoLayoutAlgorithm.cs | 27 ++++++++++-- .../AutoLayoutAlgorithmTests.cs | 43 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/DemaConsulting.Rendering.Layout/AutoLayoutAlgorithm.cs b/src/DemaConsulting.Rendering.Layout/AutoLayoutAlgorithm.cs index 04dffc4..75d1c91 100644 --- a/src/DemaConsulting.Rendering.Layout/AutoLayoutAlgorithm.cs +++ b/src/DemaConsulting.Rendering.Layout/AutoLayoutAlgorithm.cs @@ -166,9 +166,30 @@ protected internal override LayoutTree ApplyCore(LayoutGraph graph, LayoutOption var count = nodes.Count; if (count == 0) { - // No top-level nodes: nothing to route or split. Hierarchical is a pure pass-through to its - // default leaf (layered) when no container is present, so this is equivalent to routing an - // empty graph through any bundled algorithm. + // No top-level nodes: nothing to route or split. Resolve the cascaded effective algorithm + // id ourselves before delegating anywhere: this engine's own registry resolves both "auto" + // (this engine's own AlgorithmId) and "hierarchical" (HierarchicalLayoutAlgorithm.AlgorithmId) + // back to this very instance (see the constructor remarks - needed so a nested container + // scope re-evaluates "auto" correctly), and an empty graph can never contain a container, so + // HierarchicalLayoutAlgorithm.LayoutScope would always take its flat fast path and call + // straight back into this same method with the same empty graph whenever the effective id is + // one of those two - an infinite mutual recursion that overflows the stack (a fault that + // cannot be caught, confirmed by reproduction). Rewriting the id in the options snapshot + // passed down cannot + // mask this: a graph's own explicit Algorithm override always wins over whatever is passed + // in as options (see PropertyHolder.OverlayOnto), so the trap would still fire on the very + // next call if the graph itself carries the override directly. Guard by short-circuiting + // straight to the default leaf (layered) here, before _hierarchical is ever consulted, + // whenever the effective id is self-referential; every other id - including a genuinely + // unknown one - still routes through _hierarchical.ApplyCore exactly as before, preserving + // its existing resolution behavior (and resolution-error behavior) unchanged. + var effectiveEmpty = graph.OverlayOnto(options); + var algorithmId = effectiveEmpty.Get(CoreOptions.Algorithm); + if (algorithmId is AlgorithmId or HierarchicalLayoutAlgorithm.AlgorithmId) + { + return _layered.ApplyCore(graph, effectiveEmpty); + } + return _hierarchical.ApplyCore(graph, options); } diff --git a/test/DemaConsulting.Rendering.Layout.Tests/AutoLayoutAlgorithmTests.cs b/test/DemaConsulting.Rendering.Layout.Tests/AutoLayoutAlgorithmTests.cs index 14ef244..abe6c3a 100644 --- a/test/DemaConsulting.Rendering.Layout.Tests/AutoLayoutAlgorithmTests.cs +++ b/test/DemaConsulting.Rendering.Layout.Tests/AutoLayoutAlgorithmTests.cs @@ -60,6 +60,49 @@ public void Apply_EmptyGraph_ReturnsEmptyTree() Assert.Empty(tree.Nodes.OfType()); } + /// + /// Proves that an empty graph that explicitly declares as + /// "auto" (this engine's own ) resolves + /// without recursing: previously, delegating straight to the internal + /// instance for an empty graph would have that engine's + /// own registry resolve "auto" back to this very + /// instance, which would see the same empty graph and recurse forever - a stack overflow that + /// cannot be caught, confirmed by reproduction (process exit code 0xC00000FD, + /// STATUS_STACK_OVERFLOW) before this test's corresponding fix. + /// + [Fact] + public void Apply_EmptyGraph_ExplicitAutoAlgorithm_ReturnsEmptyTree() + { + // Arrange + var graph = new LayoutGraph(); + graph.Set(CoreOptions.Algorithm, "auto"); + + // Act + var tree = new AutoLayoutAlgorithm().Apply(graph); + + // Assert + Assert.Empty(tree.Nodes.OfType()); + } + + /// + /// Proves that an empty graph that explicitly declares as + /// "hierarchical" () also resolves + /// without recursing, for the same self-referential reason as the "auto" case above. + /// + [Fact] + public void Apply_EmptyGraph_ExplicitHierarchicalAlgorithm_ReturnsEmptyTree() + { + // Arrange + var graph = new LayoutGraph(); + graph.Set(CoreOptions.Algorithm, "hierarchical"); + + // Act + var tree = new AutoLayoutAlgorithm().Apply(graph); + + // Assert + Assert.Empty(tree.Nodes.OfType()); + } + /// /// Proves that a single fully-connected component routes entirely through the layered algorithm, /// taking the zero-copy fast path (byte-identical to invoking