diff --git a/benchmarks/run_suite.py b/benchmarks/run_suite.py index 4f42835..a42618a 100644 --- a/benchmarks/run_suite.py +++ b/benchmarks/run_suite.py @@ -107,6 +107,8 @@ def run_one( poll_seconds=poll_seconds, timeout=timeout, ) + if run.get("status") == "cancelled": + raise CloudAPIError("run {} was cancelled".format(run_id)) if run.get("status") not in TERMINAL: raise CloudAPIError( "run {} never reached a terminal state (last status: {!r})".format( diff --git a/deedchain/adapters/browser_use_cloud.py b/deedchain/adapters/browser_use_cloud.py index a6b412f..66680c6 100644 --- a/deedchain/adapters/browser_use_cloud.py +++ b/deedchain/adapters/browser_use_cloud.py @@ -32,7 +32,7 @@ API = "https://api.browser-use.com/api/v4/runs" ENV_KEY = "BROWSER_USE_API_KEY" -# Terminal states the Cloud API reports. +# Terminal outcomes eligible for scoring. Cancellation ends polling but is not scored. TERMINAL = ("completed", "failed") @@ -102,7 +102,7 @@ def wait_for_run( run: Dict[str, Any] = {} while time.time() < deadline: run = get_run(run_id, k, base_url=base_url) - if run.get("status") in TERMINAL: + if run.get("status") in TERMINAL or run.get("status") == "cancelled": return run time.sleep(poll_seconds) return run diff --git a/tests/test_cloud_polling_http.py b/tests/test_cloud_polling_http.py new file mode 100644 index 0000000..d9a7be3 --- /dev/null +++ b/tests/test_cloud_polling_http.py @@ -0,0 +1,67 @@ +"""Cloud polling uses real urllib and a local synthetic HTTP server.""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from benchmarks import run_suite +from deedchain.adapters import browser_use_cloud +from deedchain.tasks import load_all_tasks + + +@pytest.fixture +def cloud_server(): + state = {"status": "cancelled", "requests": []} + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + state["requests"].append(("GET", self.path)) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"id": "local-run", "status": state["status"]}).encode()) + + def do_POST(self): + state["requests"].append(("POST", self.path)) + self.rfile.read(int(self.headers["Content-Length"])) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"id": "local-run"}') + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield "http://127.0.0.1:{}/runs".format(server.server_port), state + finally: + server.shutdown() + server.server_close() + thread.join() + + +@pytest.mark.parametrize("status", ["completed", "failed", "cancelled"]) +def test_terminal_run_returns_after_one_http_request(cloud_server, status): + url, state = cloud_server + state["status"] = status + result = browser_use_cloud.wait_for_run( + "local-run", key="local-test-only", base_url=url, timeout=0.2, poll_seconds=0.01 + ) + assert result["status"] == status + assert state["requests"] == [("GET", "/runs/local-run")] + + +def test_cancelled_benchmark_is_excluded_without_waiting(cloud_server): + url, state = cloud_server + task = next(iter(load_all_tasks().values())) + with pytest.raises(browser_use_cloud.CloudAPIError, match="was cancelled"): + run_suite.run_one( + task, "https://example.com", "local-test-only", + api_base=url, timeout=0.2, poll_seconds=0.01, + ) + assert state["requests"] == [("POST", "/runs"), ("GET", "/runs/local-run")]