From 307f0dcc00bc0a1aca8d743ce0672479160e84db Mon Sep 17 00:00:00 2001 From: Ilya Gusev Date: Tue, 11 Aug 2026 22:01:21 +0000 Subject: [PATCH] feat(fetch): add --max-chars flag to override the 50000-char content cap Co-Authored-By: Claude Fable 5 --- README.md | 1 + src/commands/search.rs | 2 ++ src/daemon.rs | 16 +++++++++++----- src/main.rs | 10 ++++++++-- tests/e2e/test_fetch.py | 15 +++++++++++++++ 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e021ccb..8ceea5f 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ keenable fetch https://example.com # Fetch page content keenable fetch https://example.com -p # Pretty output keenable fetch https://example.com --live # Fetch the live page (skip cache) keenable fetch https://example.com --prompt "List all pricing tiers" # LLM extraction instead of the full page +keenable fetch https://example.com --max-chars 200000 # Raise the 50000-char content cap ``` ### Authentication diff --git a/src/commands/search.rs b/src/commands/search.rs index a430392..f6fa695 100644 --- a/src/commands/search.rs +++ b/src/commands/search.rs @@ -384,6 +384,7 @@ pub async fn fetch( url: &str, live: bool, prompt: Option, + max_chars: Option, human: bool, api_key: Option<&str>, ) { @@ -392,6 +393,7 @@ pub async fn fetch( urls: Some(vec![url.to_string()]), live, prompt, + max_chars, ..Default::default() }; diff --git a/src/daemon.rs b/src/daemon.rs index 6063511..15d4e74 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -15,6 +15,9 @@ pub struct DaemonRequest { /// instruction's output instead of the full page. #[serde(default)] pub prompt: Option, + /// Fetch only: content-length cap; the API defaults to 50000 when unset. + #[serde(default)] + pub max_chars: Option, } impl DaemonRequest { @@ -27,18 +30,21 @@ impl DaemonRequest { /// Query params for GET /v1/fetch, shared by the daemon and the direct /// HTTP path so fetch params can't drift between them. None when `urls` /// is missing. - pub fn fetch_query(&self) -> Option> { - let mut query: Vec<(&str, &str)> = self + pub fn fetch_query(&self) -> Option> { + let mut query: Vec<(&str, String)> = self .urls .as_ref()? .iter() - .map(|u| ("url", u.as_str())) + .map(|u| ("url", u.clone())) .collect(); if self.live { - query.push(("live", "true")); + query.push(("live", "true".to_string())); } if let Some(p) = &self.prompt { - query.push(("prompt", p.as_str())); + query.push(("prompt", p.clone())); + } + if let Some(m) = self.max_chars { + query.push(("max_chars", m.to_string())); } Some(query) } diff --git a/src/main.rs b/src/main.rs index aac754b..13d7ca5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -178,7 +178,7 @@ enum Commands { /// Fetch page content as markdown (outputs YAML by default, use -p for pretty output) #[command( - after_help = "Works without login (free tier). Log in for higher rate limits.\n\nExamples:\n keenable fetch https://example.com YAML output\n keenable fetch https://example.com -p Pretty output\n keenable fetch https://example.com --live Fetch the live page (skip cache)\n keenable fetch https://example.com --prompt \"List all pricing tiers\" Extract with an LLM\n keenable fetch https://example.com --api-key keen_***_***** Use a specific API key" + after_help = "Works without login (free tier). Log in for higher rate limits.\n\nExamples:\n keenable fetch https://example.com YAML output\n keenable fetch https://example.com -p Pretty output\n keenable fetch https://example.com --live Fetch the live page (skip cache)\n keenable fetch https://example.com --prompt \"List all pricing tiers\" Extract with an LLM\n keenable fetch https://example.com --max-chars 200000 Raise the 50000-char content cap\n keenable fetch https://example.com --api-key keen_***_***** Use a specific API key" )] Fetch { /// URL to fetch @@ -193,6 +193,10 @@ enum Commands { #[arg(long)] prompt: Option, + /// Truncate content at this many characters (default: 50000) + #[arg(long = "max-chars", value_parser = clap::value_parser!(u64).range(1..))] + max_chars: Option, + /// Pretty-print output for humans instead of YAML #[arg(short = 'p', long = "pretty")] pretty: bool, @@ -330,10 +334,12 @@ async fn main() { url, live, prompt, + max_chars, pretty, api_key, } => { - commands::search::fetch(&url, live, prompt, pretty, api_key.as_deref()).await; + commands::search::fetch(&url, live, prompt, max_chars, pretty, api_key.as_deref()) + .await; } Commands::Feedback { query, diff --git a/tests/e2e/test_fetch.py b/tests/e2e/test_fetch.py index f62be34..d51dffe 100644 --- a/tests/e2e/test_fetch.py +++ b/tests/e2e/test_fetch.py @@ -39,6 +39,21 @@ def test_fetch_prompt(kn): assert "This domain is for use in illustrative examples" not in data["content"] +def test_fetch_max_chars(kn): + res = kn("fetch", "https://example.com", "--max-chars", "50") + assert res.code == 0 + data = res.yaml() + # The truncation disclaimer is the load-bearing assert: a stale daemon + # that drops `max_chars` returns the full page with exit code 0. + assert "truncated to stay below 50 characters" in data["content"] + + +def test_fetch_max_chars_rejects_zero(kn): + res = kn("fetch", "https://example.com", "--max-chars", "0") + assert res.code == 2 + assert "invalid value" in res.err + + def test_pretty_fetch(kn): res = kn("fetch", "https://example.com", "-p") assert res.code == 0