From 1b055e2671664aff3a609f1de22a4ebe4fc59164 Mon Sep 17 00:00:00 2001 From: Duc-Tam Nguyen Date: Mon, 10 Aug 2026 13:48:27 +0700 Subject: [PATCH] Describe --max-pages as a queue budget, not a render count The cap is applied in enqueuePage against c.enqueued, so the budget is spent when a URL is queued rather than when it renders. A page that fails, that robots.txt disallows (checked after dequeue), or that turns out not to be HTML has already taken its slot, so a run can save fewer pages than asked for. The guide now also documents that pages discovered past the cap are still persisted to state.json, so raising the cap and rerunning continues instead of starting over. Follow-up to #76. --- CHANGELOG.md | 7 +++++-- README.md | 2 +- cli/clone.go | 2 +- clone/config.go | 12 ++++++++---- docs/content/guides/scoping-a-crawl.md | 12 +++++++++--- docs/content/reference/cli.md | 2 +- docs/content/reference/release-notes.md | 5 +++-- 7 files changed, 28 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfd55b7..281306c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,11 @@ All notable changes to kage are recorded here. The format follows - `--exclude` and `--scope-prefix` now match complete path prefixes and their descendants rather than arbitrary substrings. If you relied on the old `--exclude` behaviour, pass the full path prefix. -- `--max-pages` is documented as attempting at most N page renders; failed - renders count toward the cap. +- `--max-pages` is documented as a cap on how many page URLs are queued rather + than how many render. The budget is spent in `enqueuePage`, so a page that + fails to render, that `robots.txt` disallows, or that turns out not to be HTML + has already taken its slot by the time kage finds out, and a run can save + fewer pages than the number asked for. ### Deprecated diff --git a/README.md b/README.md index b998ccf..62dc069 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ The flags you'll actually reach for: | Flag | Default | Meaning | |------|---------|---------| | `-o, --out` | `$HOME/data/kage` | Output root; the mirror lands in `//` | -| `-p, --max-pages` | `0` | Attempt at most N page renders (0 = no limit); failed renders count toward the cap | +| `-p, --max-pages` | `0` | Queue at most N page URLs (0 = no limit); pages that fail, are disallowed, or are not HTML still count | | `-d, --max-depth` | `0` | How many links deep to follow (0 = no limit) | | `--scope-prefix` | | Only crawl this path and its descendants, not similar path names | | `--subdomains` | `false` | Treat subdomains of the seed host as in scope | diff --git a/cli/clone.go b/cli/clone.go index a329d9d..9bc12fe 100644 --- a/cli/clone.go +++ b/cli/clone.go @@ -69,7 +69,7 @@ func newCloneCmd() *cobra.Command { fs.IntVar(&f.workers, "workers", 4, "concurrent page render workers") fs.IntVar(&f.assetWorkers, "asset-workers", 8, "concurrent asset download workers") fs.IntVar(&f.browserPages, "browser-pages", 4, "Chrome page-pool size") - fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "attempt at most N page renders (0 = unlimited)") + fs.IntVarP(&f.maxPages, "max-pages", "p", 0, "queue at most N page URLs (0 = unlimited)") fs.IntVarP(&f.maxDepth, "max-depth", "d", 0, "link-follow depth cap (0 = unlimited)") fs.StringVar(&f.traversal, "traversal", "bfs", "frontier order (ignored; the crawl is always breadth-first)") _ = fs.MarkDeprecated("traversal", "the crawl is always breadth-first; this flag was never read") diff --git a/clone/config.go b/clone/config.go index 3ba256e..378aff3 100644 --- a/clone/config.go +++ b/clone/config.go @@ -28,10 +28,14 @@ type Config struct { OutDir string // output root; the mirror lands in // Reserved string // reserved dir name for assets and state (default "_kage") - Workers int // page render workers - AssetWorkers int // HTTP asset download workers - BrowserPages int // Chrome page-pool size - MaxPages int // attempt at most N page renders (0 = unlimited) + Workers int // page render workers + AssetWorkers int // HTTP asset download workers + BrowserPages int // Chrome page-pool size + // MaxPages caps how many page URLs are queued, not how many render (0 = + // unlimited). The budget is spent in enqueuePage, so a page that fails, + // that robots.txt disallows, or that turns out not to be HTML has already + // taken its slot by the time the crawl finds out. + MaxPages int MaxDepth int // BFS/DFS depth cap (0 = unlimited) Traversal string MaxAssetBytes int64 diff --git a/docs/content/guides/scoping-a-crawl.md b/docs/content/guides/scoping-a-crawl.md index 270c829..aa20d16 100644 --- a/docs/content/guides/scoping-a-crawl.md +++ b/docs/content/guides/scoping-a-crawl.md @@ -11,7 +11,7 @@ the crawl. ## Limit by count and depth ```bash -# Attempt at most 200 page renders +# Queue at most 200 page URLs kage clone example.com --max-pages 200 # Only follow links three hops from the seed @@ -19,8 +19,14 @@ kage clone example.com --max-depth 3 ``` `--max-depth 0` (the default) means unlimited depth; `--max-pages 0` means -unlimited attempts. Failed renders count toward the page cap. Combine the flags -to put a hard ceiling on a run. +unlimited pages. Combine the flags to put a hard ceiling on a run. + +The `--max-pages` budget is spent when a URL is queued, not when it renders, so +a run can save fewer pages than the number you asked for. A page that fails to +render, that `robots.txt` disallows, or that turns out not to be HTML has +already taken its slot by the time kage finds out. Pages discovered after the +budget runs out are still written to `state.json`, so raising the cap and +running again continues where the last run stopped rather than starting over. ## Limit by path diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md index 4bbd6bb..45220fc 100644 --- a/docs/content/reference/cli.md +++ b/docs/content/reference/cli.md @@ -35,7 +35,7 @@ images, and fonts, and writes a browsable mirror to `//`. | Flag | Default | Meaning | |------|---------|---------| -| `-p, --max-pages` | `0` | Attempt at most N page renders (0 = unlimited); failures count toward the cap | +| `-p, --max-pages` | `0` | Queue at most N page URLs (0 = unlimited); pages that fail, are disallowed by `robots.txt`, or turn out not to be HTML still count | | `-d, --max-depth` | `0` | Link-follow depth cap (0 = unlimited) | | `--scope-prefix` | | Only crawl the path prefix and its descendants, not similar path names | | `--subdomains` | `false` | Treat subdomains of the seed host as in scope | diff --git a/docs/content/reference/release-notes.md b/docs/content/reference/release-notes.md index e0f8068..0f7c122 100644 --- a/docs/content/reference/release-notes.md +++ b/docs/content/reference/release-notes.md @@ -10,8 +10,9 @@ The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github - **Crawl path controls use path boundaries.** `--exclude` and `--scope-prefix` match a path and its descendants without catching unrelated - names that merely contain the same text. `--max-pages` now accurately says - that failed render attempts count toward its cap. + names that merely contain the same text. `--max-pages` is now described + accurately as a cap on queued page URLs rather than on renders, so a page that + fails, that `robots.txt` disallows, or that is not HTML still spends its slot. - **Multi-page ZIM packs get a usable landing page.** Mirrors without a root `index.html` list pages by title instead of opening an arbitrary first page; single-page archives still open directly on their article ([#62](https://github.com/tamnd/kage/issues/62)).