perf: land the MCP cost fix (never merged in #18) + lower rate limit 20->5

src/bytecrawl/mcp_http.py

@@ -38,12 +38,17 @@
 from . import mcp_server as local
 from .security import assert_public_url
 
-MAX_PAGES = 10  # hard cap per focused_crawl call (local allows 50)
+MAX_PAGES = 8  # hard cap per focused_crawl call (local allows 50)
 # compare_strategies is three crawls that the rate limiter charges as one
-# request, so it gets a smaller budget: 3 x 6 = 18 fetches, about two
+# request, so it gets a smaller budget: 3 x 5 = 15 fetches, about two
 # focused_crawls' worth. Same bounded-cost rule, applied to a heavier call.
-MAX_COMPARE_PAGES = 6
-RATE_LIMIT = 20  # requests per window per client IP
+MAX_COMPARE_PAGES = 5
+# Per-page HTTP timeout on the hosted server (local default is 10s). Kept
+# tight so one slow/dead page can't eat the whole function budget: worst case
+# is MAX_PAGES * HOSTED_TIMEOUT ~= 40s, inside the 45s maxDuration in
+# vercel.json. This is what keeps crawls from hitting the wall and timing out.
+HOSTED_TIMEOUT = 5
+RATE_LIMIT = 5  # requests per window per client IP (best-effort, per warm instance)
 RATE_WINDOW = 60.0  # seconds
 
 
@@ -176,7 +181,11 @@ def focused_crawl(
 ) -> dict:
     _guarded(url)
     return local.focused_crawl(
-        url, query=query, strategy=strategy, max_pages=min(max_pages, MAX_PAGES)
+        url,
+        query=query,
+        strategy=strategy,
+        max_pages=min(max_pages, MAX_PAGES),
+        timeout=HOSTED_TIMEOUT,
     )
 
 
@@ -190,7 +199,9 @@ def focused_crawl(
 )
 def compare_strategies(url: str, query: str, max_pages: int = MAX_COMPARE_PAGES) -> dict:
     _guarded(url)
-    return local.compare_strategies(url, query=query, max_pages=min(max_pages, MAX_COMPARE_PAGES))
+    return local.compare_strategies(
+        url, query=query, max_pages=min(max_pages, MAX_COMPARE_PAGES), timeout=HOSTED_TIMEOUT
+    )
 
 
 @server.tool(description="Fetch a public JSON API endpoint.")

src/bytecrawl/mcp_server.py

@@ -181,11 +181,13 @@ def list_links(url: str, raw: bool = False) -> dict:
         "(level by level). Returns pages ranked by relevance plus crawl stats."
     )
 )
-def focused_crawl(url: str, query: str = "", strategy: str = "shark", max_pages: int = 20) -> dict:
+def focused_crawl(
+    url: str, query: str = "", strategy: str = "shark", max_pages: int = 20, timeout: int = 10
+) -> dict:
     if strategy not in _STRATEGIES:
         raise ValueError(f"strategy must be one of {sorted(_STRATEGIES)}")
     max_pages = min(max_pages, 50)  # keep agent calls bounded and polite
-    crawler = _STRATEGIES[strategy](query=query, delay=DELAY)
+    crawler = _STRATEGIES[strategy](query=query, delay=DELAY, timeout=timeout)
     result = crawler.crawl(url, max_pages=max_pages)
     # A walled, throttled or unreachable seed comes back as an empty crawl; raise
     # it as the same blocked/rate-limited/unreachable error a single fetch would
@@ -212,10 +214,10 @@ def focused_crawl(url: str, query: str = "", strategy: str = "shark", max_pages:
         "the winner."
     )
 )
-def compare_strategies(url: str, query: str, max_pages: int = 20) -> dict:
+def compare_strategies(url: str, query: str, max_pages: int = 20, timeout: int = 10) -> dict:
     max_pages = min(max_pages, 50)  # same bound as focused_crawl, per strategy
     with _seed_errors_as_tool_errors():
-        out = compare(url, query, max_pages=max_pages, delay=DELAY)
+        out = compare(url, query, max_pages=max_pages, delay=DELAY, timeout=timeout)
     return {"url": url, "query": query, "max_pages": max_pages, **out}
 
 

tests/test_mcp_http.py

@@ -132,25 +132,29 @@ class TestCaps:
     def test_focused_crawl_clamped(self, monkeypatch):
         seen = {}
 
-        def spy(url, query="", strategy="shark", max_pages=0):
+        def spy(url, query="", strategy="shark", max_pages=0, timeout=0):
             seen["max_pages"] = max_pages
+            seen["timeout"] = timeout
             return {}
 
         monkeypatch.setattr(mcp_http.local, "focused_crawl", spy)
         mcp_http.focused_crawl("http://8.8.8.8/", max_pages=9999)
         assert seen["max_pages"] == mcp_http.MAX_PAGES
+        assert seen["timeout"] == mcp_http.HOSTED_TIMEOUT
 
     def test_compare_strategies_clamped_lower(self, monkeypatch):
         """Three crawls behind one rate-limited request, so a tighter cap."""
         seen = {}
 
-        def spy(url, query="", max_pages=0):
+        def spy(url, query="", max_pages=0, timeout=0):
             seen["max_pages"] = max_pages
+            seen["timeout"] = timeout
             return {}
 
         monkeypatch.setattr(mcp_http.local, "compare_strategies", spy)
         mcp_http.compare_strategies("http://8.8.8.8/", query="x", max_pages=9999)
         assert seen["max_pages"] == mcp_http.MAX_COMPARE_PAGES
+        assert seen["timeout"] == mcp_http.HOSTED_TIMEOUT
         assert mcp_http.MAX_COMPARE_PAGES < mcp_http.MAX_PAGES
 
 

vercel.json

@@ -14,7 +14,7 @@
       "use": "@vercel/python",
       "config": {
         "includeFiles": ["src/bytecrawl/**"],
-        "maxDuration": 60
+        "maxDuration": 45
       }
     }
   ],