Skip to content

fix: add missing KB parser routes for mdx/mkd/html/htm/csv #9921

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
lxfight wants to merge 1 commit into
base: master
Choose a base branch
from
+67 −1
Draft

fix: add missing KB parser routes for mdx/mkd/html/htm/csv #9921

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion astrbot/core/knowledge_base/parsers/util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,28 @@


async def select_parser(ext: str) -> BaseParser:
if ext in {".md", ".txt", ".markdown", ".rst", ".adoc", ".xlsx", ".docx", ".xls"}:
# Markdown variants that the MarkdownChunker whitelist in kb_helper
# already accepts; parse them as plain text without extra dependencies.
if ext in {".mdx", ".mkd"}:
from .text_parser import TextParser

return TextParser()
# Formats supported by markitdown-no-magika (verified converters:
# HtmlConverter, CsvConverter, PlainTextConverter; note it has no
# RTF converter, so .rtf stays unsupported).
if ext in {
".csv",
".html",
".htm",
".md",
".txt",
".markdown",
".rst",
".adoc",
".xlsx",
".docx",
".xls",
}:
from .markitdown_parser import MarkitdownParser

return MarkitdownParser()
Expand Down
45 changes: 45 additions & 0 deletions tests/test_kb_parser_routing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
from __future__ import annotations

import pytest

from astrbot.core.knowledge_base.parsers.markitdown_parser import MarkitdownParser
from astrbot.core.knowledge_base.parsers.text_parser import TextParser
from astrbot.core.knowledge_base.parsers.util import select_parser


@pytest.mark.parametrize("ext", [".mdx", ".mkd"])
@pytest.mark.asyncio
async def test_markdown_variants_use_text_parser(ext: str) -> None:
# These extensions are accepted by the MarkdownChunker whitelist in
# kb_helper but previously had no parser route, so uploads always
# failed with "暂时不支持的文件格式".
parser = await select_parser(ext)

assert isinstance(parser, TextParser)


@pytest.mark.parametrize("ext", [".html", ".htm", ".csv"])
@pytest.mark.asyncio
async def test_html_csv_use_markitdown_parser(ext: str) -> None:
parser = await select_parser(ext)

assert isinstance(parser, MarkitdownParser)


@pytest.mark.asyncio
async def test_markitdown_parses_html_and_csv() -> None:
html = b"<html><body><h1>Title</h1><p>Hello <b>world</b></p></body></html>"
result = await MarkitdownParser().parse(html, "a.html")
assert "Title" in result.text and "Hello **world**" in result.text

csv = b"name,age\nalice,1\n"
result = await MarkitdownParser().parse(csv, "a.csv")
assert "alice" in result.text


@pytest.mark.asyncio
async def test_rtf_still_unsupported() -> None:
# markitdown-no-magika 0.1.2 has no RTF converter, so .rtf must keep
# raising instead of being routed to a parser that cannot handle it.
with pytest.raises(ValueError, match="暂时不支持的文件格式"):
await select_parser(".rtf")
Loading