diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b0e3b2..f07ced5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,25 @@ Format loosely follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +## [2.7.3] - 2026-09-24 + +Same code as 2.7.2. The 2.7.2 tag was made on a commit that never reached +`master` (a direct push was declined by the repository rules), so this +release is tagged on the merge of the release PR instead. + +## [2.7.2] - 2026-09-24 + +Point **Pages I scraped earlier** at the folder you have in mind. + +### Fixed +- **Pages I scraped earlier** only accepted the exact run folder + (`scrape//`). Pointing at `scrape/` or at `pages.jsonl` was + rejected with a message that did not say where it looked. It now takes the + run folder, a file inside it, or a parent folder such as `scrape/` (the + newest run with pages is used). Errors give the full path checked and what + was expected there, and an unreadable `pages.jsonl` is reported instead + of crashing. + ## [2.7.1] - 2026-09-24 Live feedback while scraping, a Back option that goes back, and a way to diff --git a/CITATION.cff b/CITATION.cff index e4072de..0ffa053 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -12,7 +12,7 @@ authors: orcid: 'https://orcid.org/0009-0000-4888-1941' affiliation: William Jessup University repository-code: 'https://github.com/ianktoo/data-forge' -version: 2.7.1 +version: 2.7.3 date-released: '2026-09-24' abstract: >- DataForge is an open-source pipeline for building synthetic diff --git a/README.md b/README.md index f175625..e15c1b0 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,13 @@ [![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue)](https://www.python.org/) [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) -**Current release: 2.7.1.** Scraping several URLs now shows each page as -it is fetched, Back works in every menu, and the Multiple URLs box finishes on -Enter at an empty line. New: build a dataset from pages you already scraped -with no AI, without fetching them again. 2.7.0 added keyboard URL review, a -language choice on multilingual sites, and scraping just the page you gave. -Tested on Linux, Windows and macOS with Python 3.11 to 3.14, with standalone -binaries for Windows, Linux and Apple Silicon Macs. See the -[changelog](CHANGELOG.md#271---2026-09-24). +**Current release: 2.7.3.** **Pages I scraped earlier** now accepts the +run folder, a file in it, or `scrape/` (the newest run is used), and says +exactly where it looked when nothing is found. 2.7.1 added live feedback +while scraping, a working Back, and building a dataset from pages you +already scraped. Tested on Linux, Windows and macOS with Python 3.11 to +3.14, with standalone binaries for Windows, Linux and Apple Silicon Macs. +See the [changelog](CHANGELOG.md#273---2026-09-24). Fine-tuning needs data, and good domain data is trapped in documentation sites, knowledge bases and public archives. Getting it out usually means @@ -150,7 +149,8 @@ text), `-f csv` to write only the tables, and `--json` before the command for a machine-readable summary. AI agents get the same through the MCP tool `scrape_page`. To turn a scrape folder into training data later, start **Build an AI training dataset** in `dataforge` and choose **Pages I scraped -earlier**: the pages are not fetched again and the run starts at processing. +earlier** (a run folder, or `scrape/` for the newest run): the pages are not +fetched again and the run starts at processing. To update or remove DataForge, run `dataforge update` or `dataforge uninstall` yourself, in your own terminal. Both close DataForge first, so nothing is diff --git a/pyproject.toml b/pyproject.toml index 9636f96..d9e51c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm-web-crawler" -version = "2.7.1" +version = "2.7.3" description = "LLM data collection and synthetic fine-tuning dataset pipeline" readme = "README.md" requires-python = ">=3.11" diff --git a/src/dataforge/cli/app.py b/src/dataforge/cli/app.py index 19e8854..e4a198d 100644 --- a/src/dataforge/cli/app.py +++ b/src/dataforge/cli/app.py @@ -2386,9 +2386,14 @@ async def _collect_scrape_folder(state: dict) -> list[str] | None: folder = await prompts.ask_scrape_dir(_latest_scrape_dir()) if folder is None: return None - pages = qs.load_scrape_dir(folder) + try: + pages = qs.load_scrape_dir(folder) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as e: + ui.error(f"Could not read the scrape in [dim]{folder}[/]: {e}") + return [] if not pages: - ui.error(f"No pages with text in [dim]{folder.resolve()}[/]") + ui.error(f"No pages with text in [dim]{folder}[/]. Every page there failed " + "or was empty; re-run the scrape or pick another folder.") return [] words = sum(p.word_count for p in pages) ui.info(f"Loaded {len(pages)} page{'s' if len(pages) != 1 else ''} ({words:,} words) " diff --git a/src/dataforge/cli/prompts.py b/src/dataforge/cli/prompts.py index ea47fe9..854f481 100644 --- a/src/dataforge/cli/prompts.py +++ b/src/dataforge/cli/prompts.py @@ -134,26 +134,31 @@ async def ask_input_method(allow_scrape_folder: bool = False) -> str | None: async def ask_scrape_dir(default: str = "") -> Path | None: - """Folder written by a no-AI scrape (pages.jsonl or page_NNN.md files).""" + """Folder written by a no-AI scrape, resolved by :func:`scrape.resolve_scrape_dir`. + + The run folder, a file in it, or a parent such as ``scrape/`` (newest run). + """ + from dataforge.scrape import resolve_scrape_dir + def _ok(v: str) -> bool | str: - d = Path(clean_input(v)).expanduser() - if not d.is_dir(): - return f"Folder not found: {clean_input(v)}" - if not ((d / "pages.jsonl").is_file() or any(d.glob("page_*.md"))): - return "No pages.jsonl or page_NNN.md files in that folder" + if not clean_input(v): + return "Enter a folder path" + try: + resolve_scrape_dir(clean_input(v)) + except ValueError as e: + return str(e) return True path = await questionary.path( "Scrape folder:", default=default, - only_directories=True, - instruction="(Enter for the latest; drag a folder here or paste its path)", + instruction="(Enter for the latest; a run folder, its pages.jsonl, or scrape/ for the newest run)", validate=_ok, **_q(), ).ask_async() if path is None: return None - return Path(clean_input(path)).expanduser() + return resolve_scrape_dir(clean_input(path)) async def ask_discovery_scope(n_urls: int) -> str | None: diff --git a/src/dataforge/scrape.py b/src/dataforge/scrape.py index 54e69be..9d5a721 100644 --- a/src/dataforge/scrape.py +++ b/src/dataforge/scrape.py @@ -176,6 +176,41 @@ def write_outputs(pages: list[ScrapedPage], out_dir: Path, # ── A scrape folder as dataset input ────────────────────────────────────────── +def _has_pages(d: Path) -> bool: + return (d / "pages.jsonl").is_file() or any(d.glob("page_*.md")) + + +def resolve_scrape_dir(path: str | Path) -> Path: + """The scrape folder ``path`` points at, absolute. + + Accepts the folder itself, a file inside it (``pages.jsonl``, a + ``page_NNN.md``), or a folder of scrape runs such as ``scrape/``, where + the newest run with pages is used. Raises ``ValueError`` with a message + that names the full location looked at and what was expected there. + """ + p = Path(path).expanduser() + full = p.resolve() + if not p.exists(): + return _fail(f"Not found: {full}") + if p.is_file(): + if p.name == "pages.jsonl" or (p.name.startswith("page_") and p.suffix == ".md"): + return full.parent + return _fail(f"{full} is a file. Point to the scrape folder that holds pages.jsonl " + "or page_NNN.md files.") + if _has_pages(p): + return full + runs = sorted((d for d in p.iterdir() if d.is_dir() and _has_pages(d)), reverse=True) + if runs: + return runs[0].resolve() + return _fail(f"No scraped pages in {full}. Expected pages.jsonl or page_NNN.md files " + "in this folder or in one of its subfolders (a `dataforge scrape` run " + "writes them to scrape// by default).") + + +def _fail(msg: str) -> Path: + raise ValueError(msg) + + def load_scrape_dir(folder: Path) -> list[ScrapedPage]: """Read back the pages a scrape saved in ``folder``. Only pages with text. diff --git a/tests/test_quick_scrape.py b/tests/test_quick_scrape.py index b1ea91a..9d4c791 100644 --- a/tests/test_quick_scrape.py +++ b/tests/test_quick_scrape.py @@ -186,3 +186,34 @@ async def test_imported_pages_go_through_processing(site, tmp_path, tmp_settings ctx = await ProcessorAgent(ctx).run() assert ctx.processed_chunk_ids + + +def test_resolve_scrape_dir_accepts_folder_file_or_parent(tmp_path): + old, new = tmp_path / "scrape" / "20260101-000000", tmp_path / "scrape" / "20260201-000000" + for d in (old, new): + d.mkdir(parents=True) + (d / "pages.jsonl").write_text("", encoding="utf-8") + (tmp_path / "scrape" / "20260301-000000").mkdir() # newer but empty: skipped + assert qs.resolve_scrape_dir(old) == old.resolve() + assert qs.resolve_scrape_dir(old / "pages.jsonl") == old.resolve() + assert qs.resolve_scrape_dir(tmp_path / "scrape") == new.resolve() + + +def test_resolve_scrape_dir_errors_name_the_full_location(tmp_path): + with pytest.raises(ValueError, match="Not found: .*missing"): + qs.resolve_scrape_dir(tmp_path / "missing") + with pytest.raises(ValueError, match=r"No scraped pages in .*pages\.jsonl or page_NNN\.md"): + qs.resolve_scrape_dir(tmp_path) + other = tmp_path / "notes.txt" + other.write_text("x", encoding="utf-8") + with pytest.raises(ValueError, match="is a file"): + qs.resolve_scrape_dir(other) + assert str(tmp_path.resolve()) in _err(tmp_path) + + +def _err(path) -> str: + try: + qs.resolve_scrape_dir(path) + except ValueError as e: + return str(e) + return "" diff --git a/uv.lock b/uv.lock index 3ff76f0..501747e 100644 --- a/uv.lock +++ b/uv.lock @@ -1559,7 +1559,7 @@ wheels = [ [[package]] name = "llm-web-crawler" -version = "2.7.1" +version = "2.7.3" source = { editable = "." } dependencies = [ { name = "aiofiles" },