Release 2.7.3: scrape folder path handling

CHANGELOG.md

@@ -5,6 +5,25 @@ Format loosely follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [2.7.3] - 2026-09-24
+
+Same code as 2.7.2. The 2.7.2 tag was made on a commit that never reached
+`master` (a direct push was declined by the repository rules), so this
+release is tagged on the merge of the release PR instead.
+
+## [2.7.2] - 2026-09-24
+
+Point **Pages I scraped earlier** at the folder you have in mind.
+
+### Fixed
+- **Pages I scraped earlier** only accepted the exact run folder
+  (`scrape/<date-time>/`). Pointing at `scrape/` or at `pages.jsonl` was
+  rejected with a message that did not say where it looked. It now takes the
+  run folder, a file inside it, or a parent folder such as `scrape/` (the
+  newest run with pages is used). Errors give the full path checked and what
+  was expected there, and an unreadable `pages.jsonl` is reported instead
+  of crashing.
+
 ## [2.7.1] - 2026-09-24
 
 Live feedback while scraping, a Back option that goes back, and a way to

CITATION.cff

@@ -12,7 +12,7 @@ authors:
     orcid: 'https://orcid.org/0009-0000-4888-1941'
     affiliation: William Jessup University
 repository-code: 'https://github.com/ianktoo/data-forge'
-version: 2.7.1
+version: 2.7.3
 date-released: '2026-09-24'
 abstract: >-
   DataForge is an open-source pipeline for building synthetic

README.md

@@ -6,14 +6,13 @@
 [![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13%20%7C%203.14-blue)](https://www.python.org/)
 [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
 
-**Current release: 2.7.1.** Scraping several URLs now shows each page as
-it is fetched, Back works in every menu, and the Multiple URLs box finishes on
-Enter at an empty line. New: build a dataset from pages you already scraped
-with no AI, without fetching them again. 2.7.0 added keyboard URL review, a
-language choice on multilingual sites, and scraping just the page you gave.
-Tested on Linux, Windows and macOS with Python 3.11 to 3.14, with standalone
-binaries for Windows, Linux and Apple Silicon Macs. See the
-[changelog](CHANGELOG.md#271---2026-09-24).
+**Current release: 2.7.3.** **Pages I scraped earlier** now accepts the
+run folder, a file in it, or `scrape/` (the newest run is used), and says
+exactly where it looked when nothing is found. 2.7.1 added live feedback
+while scraping, a working Back, and building a dataset from pages you
+already scraped. Tested on Linux, Windows and macOS with Python 3.11 to
+3.14, with standalone binaries for Windows, Linux and Apple Silicon Macs.
+See the [changelog](CHANGELOG.md#273---2026-09-24).
 
 Fine-tuning needs data, and good domain data is trapped in documentation
 sites, knowledge bases and public archives. Getting it out usually means
@@ -150,7 +149,8 @@ text), `-f csv` to write only the tables, and `--json` before the command for
 a machine-readable summary. AI agents get the same through the MCP tool
 `scrape_page`. To turn a scrape folder into training data later, start
 **Build an AI training dataset** in `dataforge` and choose **Pages I scraped
-earlier**: the pages are not fetched again and the run starts at processing.
+earlier** (a run folder, or `scrape/` for the newest run): the pages are not
+fetched again and the run starts at processing.
 
 To update or remove DataForge, run `dataforge update` or `dataforge uninstall`
 yourself, in your own terminal. Both close DataForge first, so nothing is

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llm-web-crawler"
-version = "2.7.1"
+version = "2.7.3"
 description = "LLM data collection and synthetic fine-tuning dataset pipeline"
 readme = "README.md"
 requires-python = ">=3.11"

src/dataforge/cli/app.py

@@ -2386,9 +2386,14 @@ async def _collect_scrape_folder(state: dict) -> list[str] | None:
     folder = await prompts.ask_scrape_dir(_latest_scrape_dir())
     if folder is None:
         return None
-    pages = qs.load_scrape_dir(folder)
+    try:
+        pages = qs.load_scrape_dir(folder)
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError) as e:
+        ui.error(f"Could not read the scrape in [dim]{folder}[/]: {e}")
+        return []
     if not pages:
-        ui.error(f"No pages with text in [dim]{folder.resolve()}[/]")
+        ui.error(f"No pages with text in [dim]{folder}[/]. Every page there failed "
+                 "or was empty; re-run the scrape or pick another folder.")
         return []
     words = sum(p.word_count for p in pages)
     ui.info(f"Loaded {len(pages)} page{'s' if len(pages) != 1 else ''} ({words:,} words) "

src/dataforge/cli/prompts.py

@@ -134,26 +134,31 @@ async def ask_input_method(allow_scrape_folder: bool = False) -> str | None:
 
 
 async def ask_scrape_dir(default: str = "") -> Path | None:
-    """Folder written by a no-AI scrape (pages.jsonl or page_NNN.md files)."""
+    """Folder written by a no-AI scrape, resolved by :func:`scrape.resolve_scrape_dir`.
+
+    The run folder, a file in it, or a parent such as ``scrape/`` (newest run).
+    """
+    from dataforge.scrape import resolve_scrape_dir
+
     def _ok(v: str) -> bool | str:
-        d = Path(clean_input(v)).expanduser()
-        if not d.is_dir():
-            return f"Folder not found: {clean_input(v)}"
-        if not ((d / "pages.jsonl").is_file() or any(d.glob("page_*.md"))):
-            return "No pages.jsonl or page_NNN.md files in that folder"
+        if not clean_input(v):
+            return "Enter a folder path"
+        try:
+            resolve_scrape_dir(clean_input(v))
+        except ValueError as e:
+            return str(e)
         return True
 
     path = await questionary.path(
         "Scrape folder:",
         default=default,
-        only_directories=True,
-        instruction="(Enter for the latest; drag a folder here or paste its path)",
+        instruction="(Enter for the latest; a run folder, its pages.jsonl, or scrape/ for the newest run)",
         validate=_ok,
         **_q(),
     ).ask_async()
     if path is None:
         return None
-    return Path(clean_input(path)).expanduser()
+    return resolve_scrape_dir(clean_input(path))
 
 
 async def ask_discovery_scope(n_urls: int) -> str | None:

src/dataforge/scrape.py

@@ -176,6 +176,41 @@ def write_outputs(pages: list[ScrapedPage], out_dir: Path,
 
 # ── A scrape folder as dataset input ──────────────────────────────────────────
 
+def _has_pages(d: Path) -> bool:
+    return (d / "pages.jsonl").is_file() or any(d.glob("page_*.md"))
+
+
+def resolve_scrape_dir(path: str | Path) -> Path:
+    """The scrape folder ``path`` points at, absolute.
+
+    Accepts the folder itself, a file inside it (``pages.jsonl``, a
+    ``page_NNN.md``), or a folder of scrape runs such as ``scrape/``, where
+    the newest run with pages is used. Raises ``ValueError`` with a message
+    that names the full location looked at and what was expected there.
+    """
+    p = Path(path).expanduser()
+    full = p.resolve()
+    if not p.exists():
+        return _fail(f"Not found: {full}")
+    if p.is_file():
+        if p.name == "pages.jsonl" or (p.name.startswith("page_") and p.suffix == ".md"):
+            return full.parent
+        return _fail(f"{full} is a file. Point to the scrape folder that holds pages.jsonl "
+                     "or page_NNN.md files.")
+    if _has_pages(p):
+        return full
+    runs = sorted((d for d in p.iterdir() if d.is_dir() and _has_pages(d)), reverse=True)
+    if runs:
+        return runs[0].resolve()
+    return _fail(f"No scraped pages in {full}. Expected pages.jsonl or page_NNN.md files "
+                 "in this folder or in one of its subfolders (a `dataforge scrape` run "
+                 "writes them to scrape/<date-time>/ by default).")
+
+
+def _fail(msg: str) -> Path:
+    raise ValueError(msg)
+
+
 def load_scrape_dir(folder: Path) -> list[ScrapedPage]:
     """Read back the pages a scrape saved in ``folder``. Only pages with text.
 

tests/test_quick_scrape.py

@@ -186,3 +186,34 @@ async def test_imported_pages_go_through_processing(site, tmp_path, tmp_settings
 
     ctx = await ProcessorAgent(ctx).run()
     assert ctx.processed_chunk_ids
+
+
+def test_resolve_scrape_dir_accepts_folder_file_or_parent(tmp_path):
+    old, new = tmp_path / "scrape" / "20260101-000000", tmp_path / "scrape" / "20260201-000000"
+    for d in (old, new):
+        d.mkdir(parents=True)
+        (d / "pages.jsonl").write_text("", encoding="utf-8")
+    (tmp_path / "scrape" / "20260301-000000").mkdir()  # newer but empty: skipped
+    assert qs.resolve_scrape_dir(old) == old.resolve()
+    assert qs.resolve_scrape_dir(old / "pages.jsonl") == old.resolve()
+    assert qs.resolve_scrape_dir(tmp_path / "scrape") == new.resolve()
+
+
+def test_resolve_scrape_dir_errors_name_the_full_location(tmp_path):
+    with pytest.raises(ValueError, match="Not found: .*missing"):
+        qs.resolve_scrape_dir(tmp_path / "missing")
+    with pytest.raises(ValueError, match=r"No scraped pages in .*pages\.jsonl or page_NNN\.md"):
+        qs.resolve_scrape_dir(tmp_path)
+    other = tmp_path / "notes.txt"
+    other.write_text("x", encoding="utf-8")
+    with pytest.raises(ValueError, match="is a file"):
+        qs.resolve_scrape_dir(other)
+    assert str(tmp_path.resolve()) in _err(tmp_path)
+
+
+def _err(path) -> str:
+    try:
+        qs.resolve_scrape_dir(path)
+    except ValueError as e:
+        return str(e)
+    return ""