Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions tests/python/test_build_package_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,18 +236,18 @@ def test_pointer_is_removed_without_leaving_a_double_blank_line(self):
)
assert strip_llms_txt_pointer(content) == "# Getting started\n\nBody text here.\n"

def test_description_and_editions_lines_are_kept(self):
def test_description_and_trailing_content_are_kept(self):
content = (
"# Heading\n"
"\n"
"> For the complete documentation index, see [llms.txt](https://doc.ibexa.co/en/5.0/llms.txt).\n"
"\n"
"Some page description.\n"
"\n"
"Editions: Content, Experience\n"
"See also: Some related page.\n"
)
assert strip_llms_txt_pointer(content) == (
"# Heading\n\nSome page description.\n\nEditions: Content, Experience\n"
"# Heading\n\nSome page description.\n\nSee also: Some related page.\n"
)

def test_nested_project_url_is_also_stripped(self):
Expand Down
42 changes: 3 additions & 39 deletions tests/python/test_page_metadata.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from llms_txt.llmstxt_preprocess import (
editions_from_frontmatter,
expand_macros,
inject_page_metadata,
)
Expand All @@ -13,36 +12,22 @@ def test_llms_txt_pointer_always_inserted_after_first_h1():
assert inject_page_metadata(content) == (
f"# Title\n\n{LLMS_TXT_LINE}\n\nBody text."
)
assert inject_page_metadata(content, description="", editions=[]) == (
assert inject_page_metadata(content, description="") == (
f"# Title\n\n{LLMS_TXT_LINE}\n\nBody text."
)


def test_editions_inserted_after_first_h1():
content = "# Title\n\nBody text."
assert inject_page_metadata(content, editions=["Experience"]) == (
f"# Title\n\n{LLMS_TXT_LINE}\n\nEditions: Experience\n\nBody text."
)


def test_description_inserted_after_first_h1():
content = "# Title\n\nBody text."
assert inject_page_metadata(content, description="Configure the Storefront.") == (
f"# Title\n\n{LLMS_TXT_LINE}\n\nConfigure the Storefront.\n\nBody text."
)


def test_description_comes_before_editions():
content = "# Title\n\nBody text."
assert inject_page_metadata(content, description="A description.", editions=["Experience"]) == (
f"# Title\n\n{LLMS_TXT_LINE}\n\nA description.\n\nEditions: Experience\n\nBody text."
)


def test_prepended_when_no_h1():
content = "Body text."
assert inject_page_metadata(content, description="A description.", editions=["Experience"]) == (
f"{LLMS_TXT_LINE}\n\nA description.\n\nEditions: Experience\n\nBody text."
assert inject_page_metadata(content, description="A description.") == (
f"{LLMS_TXT_LINE}\n\nA description.\n\nBody text."
)


Expand All @@ -57,27 +42,6 @@ def test_llms_txt_url_respects_nested_site_path():
assert result == f"# Title\n\n{nested_line}\n\nBody text."


def test_frontmatter_edition_string():
assert editions_from_frontmatter({"edition": "headless experience"}) == ["Headless", "Experience"]


def test_frontmatter_editions_list():
assert editions_from_frontmatter({"editions": ["headless", "lts-update"]}) == ["Headless", "LTS Update"]


def test_frontmatter_edition_and_editions_merged():
result = editions_from_frontmatter({"edition": "experience", "editions": ["headless"]})
assert result == ["Experience", "Headless"]


def test_unknown_edition_passes_through():
assert editions_from_frontmatter({"edition": "custom"}) == ["custom"]


def test_empty_frontmatter():
assert editions_from_frontmatter({}) == []


def test_expand_macros_substitutes_scalars():
variables = {"product_name": "Ibexa DXP", "product_name_cdp": "Ibexa CDP"}
assert expand_macros("Install [[= product_name_cdp =]] with [[= product_name =]].", variables) == (
Expand Down
34 changes: 3 additions & 31 deletions tests/python/test_preprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,11 @@ def test_release_note_tags_appended_to_heading():
html = (
'<h2>Connector v5.0.7<a class="headerlink" href="#c">&para;</a></h2>'
'<div class="release-note__tags">'
'<div class="pill pill--headless"></div>'
'<div class="pill pill--experience"></div>'
'<div class="pill pill--new-feature"></div>'
'<div class="pill pill--first-release"></div>'
"</div>"
)
assert "## Connector v5.0.7 (Headless, Experience)" in to_markdown(html)
assert "## Connector v5.0.7 (New feature, First release)" in to_markdown(html)


def test_admonition_becomes_blockquote():
Expand Down Expand Up @@ -106,34 +106,6 @@ def test_card_with_empty_title_raises():
to_markdown(html)


def test_inline_pill_becomes_parenthetical():
html = '<p>Feature<span class="pill--inline pill--experience"></span> is available.</p>'
assert "Feature (Experience) is available." in to_markdown(html)


def test_adjacent_inline_pills_merged():
# Structure from update_from_5.0: pills separated by a space in a heading.
html = (
'<h3 id="db">Database update '
'<span class="pill pill--inline pill--headless"></span> '
'<span class="pill pill--inline pill--experience"></span>'
'<a class="headerlink" href="#db">&para;</a></h3>'
)
result = to_markdown(html)
assert "### Database update (Headless, Experience)" in result
assert ") (" not in result


def test_three_adjacent_inline_pills_merged():
html = (
"<p>Feature"
'<span class="pill--inline pill--headless"></span> '
'<span class="pill--inline pill--experience"></span> '
'<span class="pill--inline pill--lts-update"></span> is available.</p>'
)
assert "Feature (Headless, Experience, LTS Update) is available." in to_markdown(html)


def test_ol_start_attribute_preserved():
# <ol start="N"> (a list interrupted by other content) keeps its numbering.
html = "<ol><li>a</li><li>b</li></ol><p>note</p><ol start='3'><li>c</li><li>d</li></ol>"
Expand Down
4 changes: 1 addition & 3 deletions tools/llms_txt/hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

from llms_txt.llmstxt_preprocess import (
absolutize_image_urls,
editions_from_frontmatter,
expand_macros,
inject_page_metadata,
renumber_ordered_lists,
Expand Down Expand Up @@ -66,7 +65,6 @@ def on_page_content(html: str, *, page: "Page", config: "MkDocsConfig", **kwargs
return

frontmatter = _read_frontmatter(page, config)
editions = editions_from_frontmatter(frontmatter)
description = expand_macros(str(frontmatter.get("description") or ""), config.get("extra") or {})
if "[[=" in description:
# Unresolved macros must not leak into the output.
Expand All @@ -75,7 +73,7 @@ def on_page_content(html: str, *, page: "Page", config: "MkDocsConfig", **kwargs
# userguide project published under /projects/userguide/), unlike
# a hardcoded root-relative "/llms.txt".
llms_txt_url = urljoin(llmstxt._base_url, "llms.txt")
content = inject_page_metadata(page_info.content, description, editions, llms_txt_url)
content = inject_page_metadata(page_info.content, description, llms_txt_url)
content = renumber_ordered_lists(content)
# Same base URL and page directory the plugin uses for making link hrefs absolute.
page_dir = PurePosixPath(page.file.dest_uri).parent.as_posix()
Expand Down
94 changes: 6 additions & 88 deletions tools/llms_txt/llmstxt_preprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
mkdocs-llmstxt plugin (configured in ``plugins.yml``) before the HTML is
converted to Markdown.
- Markdown post-processing helpers (``renumber_ordered_lists``,
``inject_edition_badges``, ``editions_from_frontmatter``) applied by
``hooks.py`` to the Markdown the plugin generated.
``inject_page_metadata``) applied by ``hooks.py`` to the Markdown the
plugin generated.

This package (``llms_txt``) is installed as a dependency by other Ibexa doc
sites, which each keep a thin root-level ``llmstxt_preprocess.py`` shim
Expand All @@ -33,30 +33,18 @@
"pill--first-release": "First release",
}

FRONTMATTER_EDITION_DISPLAY = {
"lts-update": "LTS Update",
"experience": "Experience",
"headless": "Headless",
}


def preprocess(soup: Soup, output: str) -> None:
"""
Preprocess HTML to improve markdown conversion.

Runs with autoclean disabled so we can control the order:
1. Expand tabbed sets with labels before autoclean removes tabbed-labels.
2. Run autoclean-equivalent cleanup.
3. Replace inline edition badge spans with readable text.
4. Remove release notes filter UI.
5. Convert card macros to markdown lists.

Note: frontmatter edition injection is handled in hooks.py on_page_content,
where page.file.src_path is available directly.
3. Remove release notes filter UI.
4. Convert card macros to markdown lists.
"""
_process_tabbed_sets(soup)
_autoclean(soup)
_process_inline_pills(soup)
_process_release_note_tags(soup)
_process_release_note_dates(soup)
_process_release_notes_filters(soup)
Expand Down Expand Up @@ -183,60 +171,6 @@ def _should_remove(tag) -> bool:
Soup(f"<pre{attr}>{html_module.escape(code_elem.get_text())}</pre>", "html.parser")
)

# ---------------------------------------------------------------------------
# Inline edition badge spans (from snippet includes)
# ---------------------------------------------------------------------------

def _pill_edition(node) -> str:
"""Return the edition name of an inline pill span, or '' if not one."""
if getattr(node, "name", None) != "span":
return ""
classes = node.get("class") or []
if "pill--inline" not in classes:
return ""
for pill_cls, edition_name in PILL_CLASS_TO_EDITION.items():
if pill_cls in classes:
return edition_name
return ""


def _process_inline_pills(soup: Soup) -> None:
"""Replace inline edition pill spans with readable text.

Consecutive pills (possibly separated by whitespace) are merged into a
single parenthetical, e.g. ' (Headless, Experience)' instead of
' (Headless) (Experience)'.
"""
for span in soup.find_all("span", class_="pill--inline"):
if span.parent is None: # already consumed as part of a previous run
continue
edition = _pill_edition(span)
if not edition:
continue

# Collect the run of pills that follow, skipping whitespace between them.
editions = [edition]
consumed = []
node = span.next_sibling
pending_whitespace = []
while node is not None:
if isinstance(node, NavigableString) and not node.strip():
pending_whitespace.append(node)
node = node.next_sibling
continue
next_edition = _pill_edition(node)
if not next_edition:
break
editions.append(next_edition)
consumed += pending_whitespace + [node]
pending_whitespace = []
node = node.next_sibling

for extra_node in consumed:
extra_node.extract()
span.replace_with(soup.new_string(f" ({', '.join(editions)})"))


def _process_release_note_tags(soup: Soup) -> None:
"""Append edition labels from release-note__tags divs to their preceding heading.

Expand Down Expand Up @@ -399,20 +333,6 @@ def _process_cards(soup: Soup) -> None:
# Markdown post-processing (applied by hooks.py to the generated Markdown)
# ---------------------------------------------------------------------------

def editions_from_frontmatter(frontmatter: dict) -> list:
"""Map ``edition``/``editions`` frontmatter values to display names."""

def _to_list(value):
if isinstance(value, list):
return value
if isinstance(value, str):
return value.split()
return []

all_editions = _to_list(frontmatter.get("edition")) + _to_list(frontmatter.get("editions") or [])
return [FRONTMATTER_EDITION_DISPLAY.get(e, e) for e in all_editions if e]


_MACRO_RE = re.compile(r"\[\[=\s*(\w+)\s*=\]\]")


Expand All @@ -431,9 +351,9 @@ def _substitute(match: re.Match) -> str:


def inject_page_metadata(
content: str, description: str = "", editions: list = (), llms_txt_url: str = "/llms.txt"
content: str, description: str = "", llms_txt_url: str = "/llms.txt"
) -> str:
"""Insert the llms.txt pointer, page description, and an 'Editions: X, Y' line after the first h1 heading.
"""Insert the llms.txt pointer and page description after the first h1 heading.

``llms_txt_url`` must be the absolute URL of *this site's own* llms.txt
(e.g. via ``urljoin(base_url, "llms.txt")``), not a hardcoded root-relative
Expand All @@ -443,8 +363,6 @@ def inject_page_metadata(
metadata_lines = ["", f"> For the complete documentation index, see [llms.txt]({llms_txt_url})."]
if description:
metadata_lines += ["", description]
if editions:
metadata_lines += ["", "Editions: " + ", ".join(editions)]

lines = content.split("\n")
for i, line in enumerate(lines):
Expand Down
Loading