Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion packages/markitdown/src/markitdown/_markitdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,33 @@ def _read_charset_sample(file_stream: BinaryIO) -> bytes:
return sample


# A fenced code block carries the document's own content, blank lines included:
# a notebook cell, or a <pre> block, that separates two definitions with two
# blank lines has to come back with two. Only a closed fence is treated as a
# block, so an unterminated one is still normalized.
_FENCED_CODE_BLOCK = re.compile(
r"^(?P<fence>`{3,}|~{3,}).*?^(?P=fence)[`~]*[ \t]*$",
re.MULTILINE | re.DOTALL,
)
_BLANK_LINE_RUN = re.compile(r"\n{3,}")


def _collapse_blank_lines(markdown: str) -> str:
"""Collapse runs of blank lines, leaving fenced code blocks untouched."""
collapsed: List[str] = []
position = 0

for block in _FENCED_CODE_BLOCK.finditer(markdown):
collapsed.append(
_BLANK_LINE_RUN.sub("\n\n", markdown[position : block.start()])
)
collapsed.append(block.group(0))
position = block.end()

collapsed.append(_BLANK_LINE_RUN.sub("\n\n", markdown[position:]))
return "".join(collapsed)


# Lower priority values are tried first.
PRIORITY_SPECIFIC_FILE_FORMAT = (
0.0 # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia
Expand Down Expand Up @@ -686,7 +713,7 @@ def _convert(
res.text_content = "\n".join(
[line.rstrip() for line in re.split(r"\r?\n", res.text_content)]
)
res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content)
res.text_content = _collapse_blank_lines(res.text_content)
return res

# If we got this far without success, report any exceptions
Expand Down
98 changes: 98 additions & 0 deletions packages/markitdown/tests/test_code_block_blank_lines.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/usr/bin/env python3 -m pytest
"""Blank lines inside a fenced code block are content, not layout.

``MarkItDown._convert`` collapses runs of blank lines as it normalizes a
converter's output. Applied inside a code fence that rewrites the document's
own code -- PEP 8's two blank lines between top-level definitions came back
as one.
"""

import io
import json
from typing import List

from markitdown import MarkItDown, StreamInfo

CODE = "def first():\n return 1\n\n\ndef second():\n return 2"


def _convert(stream: io.BytesIO, extension: str, **kwargs: object) -> str:
return (
MarkItDown()
.convert_stream(stream, stream_info=StreamInfo(extension=extension, **kwargs))
.markdown
)


def _notebook(source: str) -> io.BytesIO:
lines: List[str] = source.splitlines(keepends=True)
notebook = {
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": lines,
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5,
}
return io.BytesIO(json.dumps(notebook).encode("utf-8"))


def _html(body: str) -> io.BytesIO:
return io.BytesIO(f"<html><body>{body}</body></html>".encode("utf-8"))


def test_notebook_code_cell_keeps_its_blank_lines() -> None:
markdown = _convert(_notebook(CODE), ".ipynb")

assert CODE in markdown


def test_html_code_block_keeps_its_blank_lines() -> None:
markdown = _convert(
_html(f"<pre><code>{CODE}\n</code></pre>"),
".html",
mimetype="text/html",
charset="utf-8",
)

assert CODE in markdown


def test_blank_lines_between_paragraphs_are_still_collapsed() -> None:
markdown = _convert(
_html("<p>First</p><p>Second</p>"),
".html",
mimetype="text/html",
charset="utf-8",
)

assert markdown == "First\n\nSecond"


def test_blank_lines_around_a_code_block_are_still_collapsed() -> None:
markdown = _convert(
_html("<p>Before</p><pre><code>x = 1\n</code></pre><p>After</p>"),
".html",
mimetype="text/html",
charset="utf-8",
)

assert markdown == "Before\n\n```\nx = 1\n```\n\nAfter"


def test_unterminated_fence_is_not_treated_as_a_code_block() -> None:
"""Only a closed fence protects its content; an open one is normalized."""
markdown = _convert(
io.BytesIO("```\nx = 1\n\n\n\ny = 2\n".encode("utf-8")),
".md",
mimetype="text/markdown",
charset="utf-8",
)

assert markdown == "```\nx = 1\n\ny = 2\n"