From c26c183f6caf76c0789ced17d6b61fd24574657f Mon Sep 17 00:00:00 2001 From: "LongZhenren(Zhibo Zhang)" Date: Fri, 2 Oct 2026 23:08:06 +0800 Subject: [PATCH] fix(index): skip indented comment lines in WebSource url files The emptiness check used the stripped line while the comment check used the raw line, so indented or tab-prefixed comments were collected as URLs. Strip each line once and apply both checks to the cleaned value. --- index/src/pixelrag_index/sources/web.py | 4 +-- tests/test_web_source_urls.py | 34 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 tests/test_web_source_urls.py diff --git a/index/src/pixelrag_index/sources/web.py b/index/src/pixelrag_index/sources/web.py index 87b9fd1..8a5b215 100644 --- a/index/src/pixelrag_index/sources/web.py +++ b/index/src/pixelrag_index/sources/web.py @@ -69,9 +69,9 @@ def __init__( if p.exists(): with open(p, encoding="utf-8") as f: self._urls = [ - line.strip() + cleaned for line in f - if line.strip() and not line.startswith("#") + if (cleaned := line.strip()) and not cleaned.startswith("#") ] else: raise FileNotFoundError(f"urls_file not found: {urls_file}") diff --git a/tests/test_web_source_urls.py b/tests/test_web_source_urls.py new file mode 100644 index 0000000..8a5b29a --- /dev/null +++ b/tests/test_web_source_urls.py @@ -0,0 +1,34 @@ +"""Regression tests for WebSource URL file parsing.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / "index" / "src")) + +from pixelrag_index.sources.web import WebSource + + +def _write(tmp_path, content): + p = tmp_path / "urls.txt" + p.write_text(content, encoding="utf-8") + return WebSource(urls_file=str(p)) + + +def test_plain_urls_are_kept(tmp_path): + src = _write(tmp_path, "https://a.com\nhttps://b.com\n") + assert src._urls == ["https://a.com", "https://b.com"] + + +def test_indented_comment_lines_are_skipped(tmp_path): + src = _write(tmp_path, "https://a.com\n # indented comment\n\thttps://b.com\n") + assert src._urls == ["https://a.com", "https://b.com"] + + +def test_blank_and_full_comment_lines_are_skipped(tmp_path): + src = _write(tmp_path, "# top comment\n\nhttps://a.com\n \n# another\n") + assert src._urls == ["https://a.com"] + + +def test_surrounding_whitespace_is_stripped(tmp_path): + src = _write(tmp_path, " https://a.com \n") + assert src._urls == ["https://a.com"]