From 8077bcda25ce3e709fd97ce46f283c1c42127eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abraham=20P=C3=A9rez?= Date: Fri, 9 Oct 2026 17:54:12 -0600 Subject: [PATCH 1/3] perf: bound hosted MCP crawl time to stop 60s timeouts and cut GB-Hours The hosted MCP (web/mcp) was the real cost driver: ~20% of invocations hit the 60s wall because focused_crawl fetches up to 10 pages at a 10s per-page timeout (worst case 100s). On Hobby memory is fixed at 1 GB, so duration is the only lever. - mcp_server.focused_crawl/compare_strategies: thread a `timeout` param down to the strategy/compare calls (default 10s unchanged for local). - mcp_http: pass HOSTED_TIMEOUT=5s per page; lower caps MAX_PAGES 10->8 and MAX_COMPARE_PAGES 6->5. Worst case now ~8*5=40s, inside budget. - vercel.json: maxDuration landing 60->10, MCP 60->45. - tests: assert the hosted handlers forward the tighter timeout. Co-Authored-By: Claude Opus 4.8 --- src/bytecrawl/mcp_http.py | 21 ++++++++++++++++----- src/bytecrawl/mcp_server.py | 10 ++++++---- tests/test_mcp_http.py | 8 ++++++-- vercel.json | 4 ++-- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/bytecrawl/mcp_http.py b/src/bytecrawl/mcp_http.py index b51bcdb..1835703 100644 --- a/src/bytecrawl/mcp_http.py +++ b/src/bytecrawl/mcp_http.py @@ -38,11 +38,16 @@ from . import mcp_server as local from .security import assert_public_url -MAX_PAGES = 10 # hard cap per focused_crawl call (local allows 50) +MAX_PAGES = 8 # hard cap per focused_crawl call (local allows 50) # compare_strategies is three crawls that the rate limiter charges as one -# request, so it gets a smaller budget: 3 x 6 = 18 fetches, about two +# request, so it gets a smaller budget: 3 x 5 = 15 fetches, about two # focused_crawls' worth. Same bounded-cost rule, applied to a heavier call. -MAX_COMPARE_PAGES = 6 +MAX_COMPARE_PAGES = 5 +# Per-page HTTP timeout on the hosted server (local default is 10s). Kept +# tight so one slow/dead page can't eat the whole function budget: worst case +# is MAX_PAGES * HOSTED_TIMEOUT ~= 40s, inside the 45s maxDuration in +# vercel.json. This is what keeps crawls from hitting the wall and timing out. +HOSTED_TIMEOUT = 5 RATE_LIMIT = 20 # requests per window per client IP RATE_WINDOW = 60.0 # seconds @@ -176,7 +181,11 @@ def focused_crawl( ) -> dict: _guarded(url) return local.focused_crawl( - url, query=query, strategy=strategy, max_pages=min(max_pages, MAX_PAGES) + url, + query=query, + strategy=strategy, + max_pages=min(max_pages, MAX_PAGES), + timeout=HOSTED_TIMEOUT, ) @@ -190,7 +199,9 @@ def focused_crawl( ) def compare_strategies(url: str, query: str, max_pages: int = MAX_COMPARE_PAGES) -> dict: _guarded(url) - return local.compare_strategies(url, query=query, max_pages=min(max_pages, MAX_COMPARE_PAGES)) + return local.compare_strategies( + url, query=query, max_pages=min(max_pages, MAX_COMPARE_PAGES), timeout=HOSTED_TIMEOUT + ) @server.tool(description="Fetch a public JSON API endpoint.") diff --git a/src/bytecrawl/mcp_server.py b/src/bytecrawl/mcp_server.py index e7a349b..4daa83d 100644 --- a/src/bytecrawl/mcp_server.py +++ b/src/bytecrawl/mcp_server.py @@ -181,11 +181,13 @@ def list_links(url: str, raw: bool = False) -> dict: "(level by level). Returns pages ranked by relevance plus crawl stats." ) ) -def focused_crawl(url: str, query: str = "", strategy: str = "shark", max_pages: int = 20) -> dict: +def focused_crawl( + url: str, query: str = "", strategy: str = "shark", max_pages: int = 20, timeout: int = 10 +) -> dict: if strategy not in _STRATEGIES: raise ValueError(f"strategy must be one of {sorted(_STRATEGIES)}") max_pages = min(max_pages, 50) # keep agent calls bounded and polite - crawler = _STRATEGIES[strategy](query=query, delay=DELAY) + crawler = _STRATEGIES[strategy](query=query, delay=DELAY, timeout=timeout) result = crawler.crawl(url, max_pages=max_pages) # A walled, throttled or unreachable seed comes back as an empty crawl; raise # it as the same blocked/rate-limited/unreachable error a single fetch would @@ -212,10 +214,10 @@ def focused_crawl(url: str, query: str = "", strategy: str = "shark", max_pages: "the winner." ) ) -def compare_strategies(url: str, query: str, max_pages: int = 20) -> dict: +def compare_strategies(url: str, query: str, max_pages: int = 20, timeout: int = 10) -> dict: max_pages = min(max_pages, 50) # same bound as focused_crawl, per strategy with _seed_errors_as_tool_errors(): - out = compare(url, query, max_pages=max_pages, delay=DELAY) + out = compare(url, query, max_pages=max_pages, delay=DELAY, timeout=timeout) return {"url": url, "query": query, "max_pages": max_pages, **out} diff --git a/tests/test_mcp_http.py b/tests/test_mcp_http.py index 087019a..2b2e142 100644 --- a/tests/test_mcp_http.py +++ b/tests/test_mcp_http.py @@ -132,25 +132,29 @@ class TestCaps: def test_focused_crawl_clamped(self, monkeypatch): seen = {} - def spy(url, query="", strategy="shark", max_pages=0): + def spy(url, query="", strategy="shark", max_pages=0, timeout=0): seen["max_pages"] = max_pages + seen["timeout"] = timeout return {} monkeypatch.setattr(mcp_http.local, "focused_crawl", spy) mcp_http.focused_crawl("http://8.8.8.8/", max_pages=9999) assert seen["max_pages"] == mcp_http.MAX_PAGES + assert seen["timeout"] == mcp_http.HOSTED_TIMEOUT def test_compare_strategies_clamped_lower(self, monkeypatch): """Three crawls behind one rate-limited request, so a tighter cap.""" seen = {} - def spy(url, query="", max_pages=0): + def spy(url, query="", max_pages=0, timeout=0): seen["max_pages"] = max_pages + seen["timeout"] = timeout return {} monkeypatch.setattr(mcp_http.local, "compare_strategies", spy) mcp_http.compare_strategies("http://8.8.8.8/", query="x", max_pages=9999) assert seen["max_pages"] == mcp_http.MAX_COMPARE_PAGES + assert seen["timeout"] == mcp_http.HOSTED_TIMEOUT assert mcp_http.MAX_COMPARE_PAGES < mcp_http.MAX_PAGES diff --git a/vercel.json b/vercel.json index 8c44396..63c15af 100644 --- a/vercel.json +++ b/vercel.json @@ -6,7 +6,7 @@ "use": "@vercel/python", "config": { "includeFiles": ["web/landing/templates/**", "web/landing/static/**", "src/bytecrawl/**"], - "maxDuration": 60 + "maxDuration": 10 } }, { @@ -14,7 +14,7 @@ "use": "@vercel/python", "config": { "includeFiles": ["src/bytecrawl/**"], - "maxDuration": 60 + "maxDuration": 45 } } ], From fd985e8303446cac267bd65bff2ca1889b8b0e05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abraham=20P=C3=A9rez?= Date: Sat, 10 Oct 2026 09:01:42 -0600 Subject: [PATCH 2/3] perf: lower hosted MCP rate limit 20 -> 5 req/min per IP Cuts the volume a single client can push at the hosted /mcp endpoint, which is the main remaining cost driver. Best-effort (in-memory, per warm instance), so it throttles a hammering IP without being a hard cap. Co-Authored-By: Claude Opus 4.8 --- src/bytecrawl/mcp_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bytecrawl/mcp_http.py b/src/bytecrawl/mcp_http.py index 1835703..77d0897 100644 --- a/src/bytecrawl/mcp_http.py +++ b/src/bytecrawl/mcp_http.py @@ -48,7 +48,7 @@ # is MAX_PAGES * HOSTED_TIMEOUT ~= 40s, inside the 45s maxDuration in # vercel.json. This is what keeps crawls from hitting the wall and timing out. HOSTED_TIMEOUT = 5 -RATE_LIMIT = 20 # requests per window per client IP +RATE_LIMIT = 5 # requests per window per client IP (best-effort, per warm instance) RATE_WINDOW = 60.0 # seconds From 9696c86e4f611e602897e9dc8f22518f28ce99da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abraham=20P=C3=A9rez?= Date: Sat, 10 Oct 2026 09:05:17 -0600 Subject: [PATCH 3/3] fix: keep landing maxDuration at 60s (its analyzer crawls 20 pages) The landing /crawl demo fetches up to 20 pages and needs ~60s; 10s would kill it mid-crawl. Landing cost is negligible (~0.0038 GB-Hrs), so lowering it saves nothing while breaking a feature. Only the MCP is capped (45s). Co-Authored-By: Claude Opus 4.8 --- vercel.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vercel.json b/vercel.json index 63c15af..855be3b 100644 --- a/vercel.json +++ b/vercel.json @@ -6,7 +6,7 @@ "use": "@vercel/python", "config": { "includeFiles": ["web/landing/templates/**", "web/landing/static/**", "src/bytecrawl/**"], - "maxDuration": 10 + "maxDuration": 60 } }, {