Fix Indentation Of Nested Lists

lib/html2markdown/converter.ex

@@ -88,13 +88,13 @@ defmodule Html2Markdown.Converter do
     do: process_children_to_iolist(children, opts)
 
   defp process_node_to_iolist({"ul", _, children}, opts),
-    do: process_ul_list_to_iolist(children, opts)
+    do: process_ul_list_to_iolist(children, opts, 0)
 
   defp process_node_to_iolist({"ol", _, children}, opts),
-    do: process_ol_list_to_iolist(children, opts)
+    do: process_ol_list_to_iolist(children, opts, 0)
 
   defp process_node_to_iolist({"li", _, children}, opts),
-    do: ["- ", process_children_to_iolist(children, opts), "\n"]
+    do: [process_list_item_to_iolist({"li", [], children}, opts, 0), "\n"]
 
   defp process_node_to_iolist({"pre", _, [{"code", [{"class", classes}], children}]}, opts),
     do: process_code_block_to_iolist(classes, children, opts)
@@ -409,33 +409,74 @@ defmodule Html2Markdown.Converter do
     ["\n", result, "\n"]
   end
 
-  defp process_ul_list_to_iolist(children, opts) when is_list(children) do
+  defp process_ul_list_to_iolist(children, opts, depth) when is_list(children) do
     children
-    |> Enum.map(&process_list_item_to_iolist(&1, opts))
+    |> Enum.map(&process_list_item_to_iolist(&1, opts, depth))
     |> Enum.intersperse("\n")
   end
 
-  defp process_ol_list_to_iolist(children, opts) when is_list(children) do
+  defp process_ol_list_to_iolist(children, opts, depth) when is_list(children) do
     children
     |> Enum.with_index(1)
     |> Enum.map(fn {child, index} ->
-      process_ordered_list_item_to_iolist(child, index, opts)
+      process_ordered_list_item_to_iolist(child, index, opts, depth)
     end)
     |> Enum.intersperse("\n")
   end
 
-  defp process_list_item_to_iolist({"li", _, children}, opts),
-    do: ["- ", process_children_to_iolist(children, opts)]
+  defp process_list_item_to_iolist({"li", _, children}, opts, depth) do
+    indent = list_indent(depth)
+    {nested_lists, other_children} = split_nested_lists(children)
+    content = process_children_to_iolist(other_children, opts)
+    nested = process_nested_lists_to_iolist(nested_lists, opts, depth)
+
+    case {content, nested} do
+      {"", []} -> [indent, "-"]
+      {content, []} -> [indent, "- ", content]
+      {"", nested} -> [indent, "-", "\n", nested]
+      {content, nested} -> [indent, "- ", content, "\n", nested]
+    end
+  end
 
-  defp process_list_item_to_iolist(other, opts),
+  defp process_list_item_to_iolist(other, opts, _depth),
     do: process_node_to_iolist(other, opts)
 
-  defp process_ordered_list_item_to_iolist({"li", _, children}, index, opts),
-    do: [Integer.to_string(index), ". ", process_children_to_iolist(children, opts)]
+  defp process_ordered_list_item_to_iolist({"li", _, children}, index, opts, depth) do
+    indent = list_indent(depth)
+    marker = [Integer.to_string(index), ". "]
+    {nested_lists, other_children} = split_nested_lists(children)
+    content = process_children_to_iolist(other_children, opts)
+    nested = process_nested_lists_to_iolist(nested_lists, opts, depth)
+
+    case {content, nested} do
+      {"", []} -> [indent, Integer.to_string(index), "."]
+      {content, []} -> [indent, marker, content]
+      {"", nested} -> [indent, Integer.to_string(index), ".", "\n", nested]
+      {content, nested} -> [indent, marker, content, "\n", nested]
+    end
+  end
 
-  defp process_ordered_list_item_to_iolist(other, _index, opts),
+  defp process_ordered_list_item_to_iolist(other, _index, opts, _depth),
     do: process_node_to_iolist(other, opts)
 
+  defp list_indent(depth), do: String.duplicate("    ", depth)
+
+  defp split_nested_lists(children) do
+    Enum.split_with(children, fn
+      {tag, _, _} when tag in ["ul", "ol"] -> true
+      _ -> false
+    end)
+  end
+
+  defp process_nested_lists_to_iolist(nested_lists, opts, depth) do
+    nested_lists
+    |> Enum.map(fn
+      {"ul", _, children} -> process_ul_list_to_iolist(children, opts, depth + 1)
+      {"ol", _, children} -> process_ol_list_to_iolist(children, opts, depth + 1)
+    end)
+    |> Enum.intersperse("\n")
+  end
+
   # Context-aware processing for better spacing control
   defp process_children_with_context(children, opts, context) do
     final_context = determine_context(children, context)

lib/html2markdown/table_converter.ex

@@ -118,8 +118,7 @@ defmodule Html2Markdown.TableConverter do
     end)
   end
 
-  defp process_table_row({"tr", _attrs, cells}, column_count, opts)
-       when is_list(cells) and length(cells) > 0 do
+  defp process_table_row({"tr", _attrs, [_ | _] = cells}, column_count, opts) do
     case List.first(cells) do
       nil ->
         "|  |"

mix.exs

@@ -31,9 +31,9 @@ defmodule Html2Markdown.MixProject do
       {:floki, ">= 0.38.0"},
 
       # Dev & Test
-      {:ex_doc, "~> 0.38.4", only: :dev, runtime: false},
-      {:sobelow, "~> 0.14.0", only: [:dev, :test], runtime: false},
-      {:credo, "~> 1.7.12", only: [:dev, :test], runtime: false},
+      {:ex_doc, "~> 0.40.3", only: :dev, runtime: false},
+      {:sobelow, "~> 0.15.0", only: [:dev, :test], runtime: false},
+      {:credo, "~> 1.7.19", only: [:dev, :test], runtime: false},
       {:excoveralls, "~> 0.18.5", only: [:dev, :test], runtime: false},
       {:rambo, "~> 0.3.4", only: [:dev, :test], runtime: false}
     ]

mix.lock

@@ -1,16 +1,16 @@
 %{
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
-  "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"},
-  "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"},
-  "ex_doc": {:hex, :ex_doc, "0.38.4", "ab48dff7a8af84226bf23baddcdda329f467255d924380a0cf0cee97bb9a9ede", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "f7b62346408a83911c2580154e35613eb314e0278aeea72ed7fedef9c1f165b2"},
+  "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"},
+  "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"},
+  "ex_doc": {:hex, :ex_doc, "0.40.3", "4a972ffe64bc07dc605af487e98fc19b72a4185f55ca031b94c0552d6071c1d9", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2756e357742fecd9749b489b85d67c9ce99c465f2e75728d9e6dc8d704b973de"},
   "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"},
-  "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"},
-  "floki": {:hex, :floki, "0.38.0", "62b642386fa3f2f90713f6e231da0fa3256e41ef1089f83b6ceac7a3fd3abf33", [:mix], [], "hexpm", "a5943ee91e93fb2d635b612caf5508e36d37548e84928463ef9dd986f0d1abd9"},
-  "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
-  "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"},
+  "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"},
+  "floki": {:hex, :floki, "0.38.4", "10f98971e892aed2c2f1b3a0f928e488e3797e1c6dd3dfd98db40b14e9a78bcf", [:mix], [], "hexpm", "bdb34645eee8e79845c7edaca2d4099a52804ee4d4a3ecc683a69451f0244973"},
+  "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"},
+  "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"},
   "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"},
-  "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"},
+  "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"},
   "rambo": {:hex, :rambo, "0.3.4", "8962ac3bd1a633ee9d0e8b44373c7913e3ce3d875b4151dcd060886092d2dce7", [:mix], [], "hexpm", "0cc54ed089fbbc84b65f4b8a774224ebfe60e5c80186fafc7910b3e379ad58f1"},
-  "sobelow": {:hex, :sobelow, "0.14.0", "dd82aae8f72503f924fe9dd97ffe4ca694d2f17ec463dcfd365987c9752af6ee", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "7ecf91e298acfd9b24f5d761f19e8f6e6ac585b9387fb6301023f1f2cd5eed5f"},
+  "sobelow": {:hex, :sobelow, "0.15.0", "b067d7f8522a9d758fa89cb2bfcbab7ad72c45a0993cb958c989c6fd956fdd56", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "24a800e2d7fa8c3bd21561b6ad8ad4745ed726a09fd606598981d9048708da98"},
 }

test/html2markdown/converter_test.exs

@@ -181,11 +181,10 @@ defmodule Html2Markdown.ConverterTest do
            {"li", [], ["Phoenix Features"]}
          ]}
 
-      result = Converter.process_node(nested_ul, opts)
-      assert String.contains?(result, "- Elixir Features")
-      assert String.contains?(result, "- Pattern Matching")
-      assert String.contains?(result, "- Actor Model")
-      assert String.contains?(result, "- Phoenix Features")
+      expected =
+        "- Elixir Features\n    - Pattern Matching\n    - Actor Model\n- Phoenix Features"
+
+      assert Converter.process_node(nested_ul, opts) == expected
     end
   end
 
@@ -266,11 +265,10 @@ defmodule Html2Markdown.ConverterTest do
            {"dd", [], ["Rich, interactive UIs without JavaScript"]}
          ]}
 
-      result = Converter.process_node(dl, opts)
-      assert String.contains?(result, "**Phoenix**")
-      assert String.contains?(result, ": A web framework for Elixir")
-      assert String.contains?(result, "**LiveView**")
-      assert String.contains?(result, ": Rich, interactive UIs without JavaScript")
+      expected =
+        "\n**Phoenix**\n: A web framework for Elixir\n\n**LiveView**\n: Rich, interactive UIs without JavaScript\n"
+
+      assert Converter.process_node(dl, opts) == expected
     end
 
     test "handles multiple definitions per term" do

test/support/fixtures/elixir.md

@@ -4,9 +4,10 @@
 
 ## Key Features of Elixir
 
-- **Functional Programming**: Elixir is built on the principles of functional programming, which emphasizes - Immutability
-- Higher-order functions
-- Recursive algorithms
+- **Functional Programming**: Elixir is built on the principles of functional programming, which emphasizes
+    - Immutability
+    - Higher-order functions
+    - Recursive algorithms
 - Concurrency and Scalability: Elixir leverages the power of the Erlang VM to provide lightweight processes and efficient message passing, enabling massive concurrency and scalability.
 - Fault-Tolerance: With its actor-based concurrency model and support for supervisors, Elixir allows you to build fault-tolerant systems that can handle failures gracefully.
 - Metaprogramming: Elixir provides powerful metaprogramming capabilities through macros, allowing you to extend the language and write expressive and reusable code.
@@ -18,9 +19,10 @@
 1. Sentence-based Chunking: This approach splits the text into individual sentences using punctuation markers such as periods, question marks, and exclamation points. Each sentence becomes a separate chunk, allowing for fine-grained retrieval and generation.
 2. Paragraph-based Chunking: With this approach, the text is divided into paragraphs based on the presence of newline characters or specific paragraph delimiters. Paragraphs provide a coherent and self-contained unit of information suitable for RAG.
 3. Semantic Chunking: Semantic chunking involves analyzing the text and identifying meaningful semantic units or phrases. This can be achieved using techniques like named entity recognition, noun phrase extraction, or dependency parsing. Semantic chunks capture the core concepts and ideas within the text.
-4. Custom Chunking: Elixir provides the flexibility to define custom chunking rules based on specific requirements. For example, you can chunk text based on a certain number of words, specific delimiters, or regular expressions that match particular patterns. - By number of words
-- By specific delimiters
-- By regular expressions
+4. Custom Chunking: Elixir provides the flexibility to define custom chunking rules based on specific requirements. For example, you can chunk text based on a certain number of words, specific delimiters, or regular expressions that match particular patterns.
+    - By number of words
+    - By specific delimiters
+    - By regular expressions
 
 The choice of chunking approach depends on the nature of the text and the desired granularity of retrieval and generation. Elixir's powerful string manipulation and pattern matching capabilities make it easy to implement various chunking strategies efficiently.