diff --git a/src/bytecrawl/mcp_http.py b/src/bytecrawl/mcp_http.py index b51bcdb..77d0897 100644 --- a/src/bytecrawl/mcp_http.py +++ b/src/bytecrawl/mcp_http.py @@ -38,12 +38,17 @@ from . import mcp_server as local from .security import assert_public_url -MAX_PAGES = 10 # hard cap per focused_crawl call (local allows 50) +MAX_PAGES = 8 # hard cap per focused_crawl call (local allows 50) # compare_strategies is three crawls that the rate limiter charges as one -# request, so it gets a smaller budget: 3 x 6 = 18 fetches, about two +# request, so it gets a smaller budget: 3 x 5 = 15 fetches, about two # focused_crawls' worth. Same bounded-cost rule, applied to a heavier call. -MAX_COMPARE_PAGES = 6 -RATE_LIMIT = 20 # requests per window per client IP +MAX_COMPARE_PAGES = 5 +# Per-page HTTP timeout on the hosted server (local default is 10s). Kept +# tight so one slow/dead page can't eat the whole function budget: worst case +# is MAX_PAGES * HOSTED_TIMEOUT ~= 40s, inside the 45s maxDuration in +# vercel.json. This is what keeps crawls from hitting the wall and timing out. +HOSTED_TIMEOUT = 5 +RATE_LIMIT = 5 # requests per window per client IP (best-effort, per warm instance) RATE_WINDOW = 60.0 # seconds @@ -176,7 +181,11 @@ def focused_crawl( ) -> dict: _guarded(url) return local.focused_crawl( - url, query=query, strategy=strategy, max_pages=min(max_pages, MAX_PAGES) + url, + query=query, + strategy=strategy, + max_pages=min(max_pages, MAX_PAGES), + timeout=HOSTED_TIMEOUT, ) @@ -190,7 +199,9 @@ def focused_crawl( ) def compare_strategies(url: str, query: str, max_pages: int = MAX_COMPARE_PAGES) -> dict: _guarded(url) - return local.compare_strategies(url, query=query, max_pages=min(max_pages, MAX_COMPARE_PAGES)) + return local.compare_strategies( + url, query=query, max_pages=min(max_pages, MAX_COMPARE_PAGES), timeout=HOSTED_TIMEOUT + ) @server.tool(description="Fetch a public JSON API endpoint.") diff --git a/src/bytecrawl/mcp_server.py b/src/bytecrawl/mcp_server.py index e7a349b..4daa83d 100644 --- a/src/bytecrawl/mcp_server.py +++ b/src/bytecrawl/mcp_server.py @@ -181,11 +181,13 @@ def list_links(url: str, raw: bool = False) -> dict: "(level by level). Returns pages ranked by relevance plus crawl stats." ) ) -def focused_crawl(url: str, query: str = "", strategy: str = "shark", max_pages: int = 20) -> dict: +def focused_crawl( + url: str, query: str = "", strategy: str = "shark", max_pages: int = 20, timeout: int = 10 +) -> dict: if strategy not in _STRATEGIES: raise ValueError(f"strategy must be one of {sorted(_STRATEGIES)}") max_pages = min(max_pages, 50) # keep agent calls bounded and polite - crawler = _STRATEGIES[strategy](query=query, delay=DELAY) + crawler = _STRATEGIES[strategy](query=query, delay=DELAY, timeout=timeout) result = crawler.crawl(url, max_pages=max_pages) # A walled, throttled or unreachable seed comes back as an empty crawl; raise # it as the same blocked/rate-limited/unreachable error a single fetch would @@ -212,10 +214,10 @@ def focused_crawl(url: str, query: str = "", strategy: str = "shark", max_pages: "the winner." ) ) -def compare_strategies(url: str, query: str, max_pages: int = 20) -> dict: +def compare_strategies(url: str, query: str, max_pages: int = 20, timeout: int = 10) -> dict: max_pages = min(max_pages, 50) # same bound as focused_crawl, per strategy with _seed_errors_as_tool_errors(): - out = compare(url, query, max_pages=max_pages, delay=DELAY) + out = compare(url, query, max_pages=max_pages, delay=DELAY, timeout=timeout) return {"url": url, "query": query, "max_pages": max_pages, **out} diff --git a/tests/test_mcp_http.py b/tests/test_mcp_http.py index 087019a..2b2e142 100644 --- a/tests/test_mcp_http.py +++ b/tests/test_mcp_http.py @@ -132,25 +132,29 @@ class TestCaps: def test_focused_crawl_clamped(self, monkeypatch): seen = {} - def spy(url, query="", strategy="shark", max_pages=0): + def spy(url, query="", strategy="shark", max_pages=0, timeout=0): seen["max_pages"] = max_pages + seen["timeout"] = timeout return {} monkeypatch.setattr(mcp_http.local, "focused_crawl", spy) mcp_http.focused_crawl("http://8.8.8.8/", max_pages=9999) assert seen["max_pages"] == mcp_http.MAX_PAGES + assert seen["timeout"] == mcp_http.HOSTED_TIMEOUT def test_compare_strategies_clamped_lower(self, monkeypatch): """Three crawls behind one rate-limited request, so a tighter cap.""" seen = {} - def spy(url, query="", max_pages=0): + def spy(url, query="", max_pages=0, timeout=0): seen["max_pages"] = max_pages + seen["timeout"] = timeout return {} monkeypatch.setattr(mcp_http.local, "compare_strategies", spy) mcp_http.compare_strategies("http://8.8.8.8/", query="x", max_pages=9999) assert seen["max_pages"] == mcp_http.MAX_COMPARE_PAGES + assert seen["timeout"] == mcp_http.HOSTED_TIMEOUT assert mcp_http.MAX_COMPARE_PAGES < mcp_http.MAX_PAGES diff --git a/vercel.json b/vercel.json index 8c44396..855be3b 100644 --- a/vercel.json +++ b/vercel.json @@ -14,7 +14,7 @@ "use": "@vercel/python", "config": { "includeFiles": ["src/bytecrawl/**"], - "maxDuration": 60 + "maxDuration": 45 } } ],