diff --git a/index/src/pixelrag_index/sources/web.py b/index/src/pixelrag_index/sources/web.py index 87b9fd1..8a5b215 100644 --- a/index/src/pixelrag_index/sources/web.py +++ b/index/src/pixelrag_index/sources/web.py @@ -69,9 +69,9 @@ def __init__( if p.exists(): with open(p, encoding="utf-8") as f: self._urls = [ - line.strip() + cleaned for line in f - if line.strip() and not line.startswith("#") + if (cleaned := line.strip()) and not cleaned.startswith("#") ] else: raise FileNotFoundError(f"urls_file not found: {urls_file}") diff --git a/tests/test_web_source_urls.py b/tests/test_web_source_urls.py new file mode 100644 index 0000000..8a5b29a --- /dev/null +++ b/tests/test_web_source_urls.py @@ -0,0 +1,34 @@ +"""Regression tests for WebSource URL file parsing.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / "index" / "src")) + +from pixelrag_index.sources.web import WebSource + + +def _write(tmp_path, content): + p = tmp_path / "urls.txt" + p.write_text(content, encoding="utf-8") + return WebSource(urls_file=str(p)) + + +def test_plain_urls_are_kept(tmp_path): + src = _write(tmp_path, "https://a.com\nhttps://b.com\n") + assert src._urls == ["https://a.com", "https://b.com"] + + +def test_indented_comment_lines_are_skipped(tmp_path): + src = _write(tmp_path, "https://a.com\n # indented comment\n\thttps://b.com\n") + assert src._urls == ["https://a.com", "https://b.com"] + + +def test_blank_and_full_comment_lines_are_skipped(tmp_path): + src = _write(tmp_path, "# top comment\n\nhttps://a.com\n \n# another\n") + assert src._urls == ["https://a.com"] + + +def test_surrounding_whitespace_is_stripped(tmp_path): + src = _write(tmp_path, " https://a.com \n") + assert src._urls == ["https://a.com"]