diff --git a/.gitattributes b/.gitattributes index d33fe5dc4..e329a999b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,10 @@ # Historical test snapshots must retain their original bytes on Windows. /crates/bsk-cli/tests/fixtures/legacy-skills/*.md text eol=lf + +# Release-test fixtures are hashed byte-for-byte by fixture-manifest.json, so a +# CRLF checkout makes every manifest check fail. Keep text fixtures on LF and +# media fixtures untouched. +/evals/browser/skills/browserskill-release-test/assets/fixtures/*.html text eol=lf +/evals/browser/skills/browserskill-release-test/assets/fixtures/*.vtt text eol=lf +/evals/browser/skills/browserskill-release-test/assets/fixtures/*.wav -text +/evals/browser/skills/browserskill-release-test/assets/fixtures/*.mp4 -text diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 016ef385e..63dee6b1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,9 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } node scripts/check-crate-skill.mjs + - name: Run release-test audit helper tests + run: python evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py + - name: Run installer tests (Windows PowerShell 5.1) shell: powershell run: ./scripts/install-windows.test.ps1 diff --git a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py index 24aa9f151..49acad467 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py +++ b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py @@ -151,6 +151,35 @@ def parse(text): return None +def launcher(exe, platform=None): + """Return the argv prefix that actually starts ``exe``. + + Windows cannot exec a file without an executable extension, even when it + carries a shebang, so the script-based ``--bsk`` wrappers described in the + evidence contract fail there with WinError 193. Run Python wrappers with the + current interpreter instead; native executables and ``.cmd``/``.bat`` shims + are launched unchanged. ``platform`` exists so the decision can be tested + without mutating ``os.name``, which would also change how ``pathlib`` builds + paths. + """ + path = Path(exe) + platform = os.name if platform is None else platform + suffix = path.suffix.lower() + if platform != "nt" or suffix in {".exe", ".com", ".bat", ".cmd"}: + return [exe] + # CreateProcessW rejects `.py`/`.pyw` too, with or without a shebang. + if suffix in {".py", ".pyw"}: + return [sys.executable or "python", str(path)] + try: + with path.open("rb") as handle: + shebang = handle.readline(256).decode("utf-8", "replace").strip() + except OSError: + return [exe] + if shebang.startswith("#!") and "python" in shebang.lower(): + return [sys.executable or "python", str(path)] + return [exe] + + def discover(bsk, directory): directory = Path(directory) directory.mkdir(parents=True, exist_ok=True) @@ -165,9 +194,11 @@ def visit(parts): return seen.add(key) r = subprocess.run( - [bsk, *parts, "--help"], + [*launcher(bsk), *parts, "--help"], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=20, env=env, ) @@ -175,7 +206,7 @@ def visit(parts): raise ValueError("Help discovery failed: " + key + " " + r.stderr) text = r.stdout file = directory / ((key.replace(" ", "__") or "root") + ".txt") - file.write_text(text) + file.write_text(text, encoding="utf-8") documents.append({"command": key, "path": file.name, "sha256": digest(file)}) match = re.search(r"^Commands:\s*\n(.*?)(?=^\S|\Z)", text, re.M | re.S) children = ( @@ -187,9 +218,11 @@ def visit(parts): # exposes its subcommands when supported by the candidate binary. if not parts and "recipe" not in children: probe = subprocess.run( - [bsk, "recipe", "--help"], + [*launcher(bsk), "recipe", "--help"], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=20, env=env, ) @@ -582,7 +615,11 @@ def execute(args): else (subprocess.PIPE if args.interactive else subprocess.DEVNULL) ) p = subprocess.Popen( - [exe, *argv], stdout=stdout, stderr=stderr, stdin=input_handle, env=env + [*launcher(exe), *argv], + stdout=stdout, + stderr=stderr, + stdin=input_handle, + env=env, ) if args.interactive: @@ -619,7 +656,11 @@ def forward(): code = p.wait(timeout=args.timeout) except subprocess.TimeoutExpired: timed_out = True - p.send_signal(signal.SIGINT) + if os.name == "nt": + # Windows has no SIGINT for processes; send_signal raises ValueError. + p.terminate() + else: + p.send_signal(signal.SIGINT) try: code = p.wait(timeout=5) except subprocess.TimeoutExpired: @@ -634,8 +675,8 @@ def forward(): if started_ms is not None and finished_ms is not None else 0.0 ) - out_text = (folder / "stdout.txt").read_text(errors="replace") - err_text = (folder / "stderr.txt").read_text(errors="replace") + out_text = (folder / "stdout.txt").read_text(encoding="utf-8", errors="replace") + err_text = (folder / "stderr.txt").read_text(encoding="utf-8", errors="replace") parsed_json = parse(out_text.strip() or err_text.strip()) metrics = extract_metrics(cmd_name, code, parsed_json, elapsed_ms) meta.update( @@ -697,8 +738,8 @@ def read_record(root, record_id): raise ValueError("Changed raw " + kind) if (d / "stdin.txt").exists() and digest(d / "stdin.txt") != m.get("stdin_sha256"): raise ValueError("Changed protocol input") - out = (d / "stdout.txt").read_text(errors="replace") - err = (d / "stderr.txt").read_text(errors="replace") + out = (d / "stdout.txt").read_text(encoding="utf-8", errors="replace") + err = (d / "stderr.txt").read_text(encoding="utf-8", errors="replace") parsed_json = parse(out.strip() or err.strip()) if "elapsed_ms" not in m: s_ms = parse_iso_ms(m.get("started")) diff --git a/evals/browser/skills/browserskill-release-test/scripts/serve.py b/evals/browser/skills/browserskill-release-test/scripts/serve.py index afb3e4363..30f27bc4f 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/serve.py +++ b/evals/browser/skills/browserskill-release-test/scripts/serve.py @@ -145,7 +145,9 @@ def do_GET(self): body = ( "[" + ",".join( - events.read_text().splitlines() if events.exists() else [] + events.read_text(encoding="utf-8", errors="replace").splitlines() + if events.exists() + else [] ) + "]" ) diff --git a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py index 500adf078..0c200e72f 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py +++ b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Behavioral tests of the audit helper, not BrowserSkill product tests.""" -import contextlib, csv, importlib.util, io, json, tempfile, time, unittest +import contextlib, csv, importlib.util, io, json, sys, tempfile, time, unittest from pathlib import Path from types import SimpleNamespace as NS from unittest.mock import patch @@ -37,10 +37,10 @@ def setUp(self): (self.skill / "references").mkdir(parents=True) (self.skill / "assets/fixtures").mkdir(parents=True) (self.skill / "assets/fixtures/minimal.html").write_text( - "synthetic" + "synthetic", encoding="utf-8" ) self.binary = self.base / "bsk" - self.binary.write_text(FAKE) + self.binary.write_text(FAKE, encoding="utf-8") self.binary.chmod(0o755) self.patch = patch.object(audit, "SKILL", self.skill) self.patch.start() @@ -139,14 +139,16 @@ def test_raw_output_tampering_blocks(self): self.init() rid = self.execute() self.fill(rid) - (self.root / "commands" / rid / "stdout.txt").write_text('{"ok":false}') + (self.root / "commands" / rid / "stdout.txt").write_text( + '{"ok":false}', encoding="utf-8" + ) self.assertEqual(self.gate()[0], 1) def test_candidate_change_blocks(self): self.init() rid = self.execute() self.fill(rid) - self.binary.write_text(FAKE + "\n# changed\n") + self.binary.write_text(FAKE + "\n# changed\n", encoding="utf-8") self.assertEqual(self.gate()[0], 1) def test_deleted_scenario_blocks(self): @@ -248,7 +250,7 @@ def test_zero_exit_script_failure_cannot_be_hidden(self): "elif '--unsatisfied'", "elif '--eval-false' in args:print(json.dumps({'ok':False,'marker':'present'}))\nelif '--unsatisfied'", ) - self.binary.write_text(body) + self.binary.write_text(body, encoding="utf-8") self.init() good = self.execute() bad = self.execute(["evaluate", "--eval-false"], "CMD-evaluate") @@ -337,7 +339,7 @@ def test_discovery_keeps_navigate_and_distinct_alias_help(self): elif p==['navigate']:print('Usage: bsk navigate [URL]\\n\\nCommands:\\n back Back\\n\\nArguments:\\n [URL] Destination\\n\\nOptions:\\n --help Help') else:print('Usage: bsk '+' '.join(p)) """ - self.binary.write_text(body) + self.binary.write_text(body, encoding="utf-8") inv = audit.discover(str(self.binary), self.base / "help") self.assertEqual( set(inv["commands"]), {"help", "navigate", "navigate back", "navigate-back"} @@ -355,7 +357,9 @@ def test_discovery_keeps_navigate_and_distinct_alias_help(self): def test_server_routes_and_event_oracle(self): root = self.skill / "assets/fixtures" - (root / "plan-recipe-lab.html").write_text("plan-recipe-lab") + (root / "plan-recipe-lab.html").write_text( + "plan-recipe-lab", encoding="utf-8" + ) server = serve.make_server(root, 0) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -407,7 +411,7 @@ def test_hidden_recipe_discovery_and_plan_recipe_benchmark(self): else: print(json.dumps({'ok': True, 'value': 'synthetic'})) """ - self.binary.write_text(body) + self.binary.write_text(body, encoding="utf-8") inv = audit.discover(str(self.binary), self.base / "help") self.assertEqual( set(inv["commands"]), @@ -608,8 +612,12 @@ def test_hidden_recipe_discovery_and_plan_recipe_benchmark(self): def test_adversarial_server_routes_bfcache_cors_and_media_mimes(self): root = self.skill / "assets/fixtures" - (root / "arena-bfcache-detail.html").write_text("bfcache") - (root / "captions-zh.vtt").write_text("WEBVTT\n\n00:00.000 --> 00:01.000\n你好\n") + (root / "arena-bfcache-detail.html").write_text( + "bfcache", encoding="utf-8" + ) + (root / "captions-zh.vtt").write_text( + "WEBVTT\n\n00:00.000 --> 00:01.000\n你好\n", encoding="utf-8" + ) (root / "arena-audio.wav").write_bytes(b"RIFF\x24\x00\x00\x00WAVE") server = serve.make_server(root, 0) thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -753,15 +761,20 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): def test_wave_commands_range_assertions_and_long_admin_home(self): (self.skill / "assets/fixtures/arena-frame-outer.html").write_text( - '' + '', + encoding="utf-8", ) self.init() - rewritten = (self.root / "fixtures/arena-frame-outer.html").read_text() + rewritten = (self.root / "fixtures/arena-frame-outer.html").read_text( + encoding="utf-8" + ) self.assertIn("http://localhost:4567/a", rewritten) self.assertIn("http://127.0.0.1:4567/b", rewritten) rid = self.execute() d = self.root / "commands" / rid - (d / "stdout.txt").write_text('{"ok": true, "size": 42, "label": "9"}') + (d / "stdout.txt").write_text( + '{"ok": true, "size": 42, "label": "9"}', encoding="utf-8" + ) m = audit.load(d / "meta.json") m["stdout_sha256"] = audit.digest(d / "stdout.txt") audit.write(d / "meta.json", m) @@ -825,6 +838,10 @@ def test_wave_commands_range_assertions_and_long_admin_home(self): with patch.object(audit.sys, "platform", "win32"): win_home = audit.resolve_admin_home(deep_run) self.assertEqual(Path(win_home), (deep_run / "admin-runtime").resolve()) + if sys.platform == "win32": + # Windows daemons use named pipes, so the AF_UNIX 96-byte cap and its + # symlink workaround do not apply there. + return resolved = Path(audit.resolve_admin_home(deep_run)) try: self.assertLessEqual(len(str(resolved / "run" / "daemon.sock")), 96) @@ -833,6 +850,81 @@ def test_wave_commands_range_assertions_and_long_admin_home(self): if resolved.is_symlink(): resolved.unlink() + def test_timeout_records_metadata_instead_of_crashing(self): + self.binary.write_text( + "#!/usr/bin/env python3\n" + "import sys, time\n" + "if '--help' in sys.argv:\n" + " print('Usage: bsk [OPTIONS] \\n\\nCommands:\\n status Show status\\n\\nOptions:\\n --help Help' if sys.argv[1:] == ['--help'] else 'Usage: bsk status [OPTIONS]')\n" + "else:\n" + " time.sleep(30)\n", + encoding="utf-8", + ) + self.init() + with contextlib.redirect_stdout(io.StringIO()): + audit.execute( + NS( + run=str(self.root), + case="FLOW", + argv=["status"], + timeout=1, + lane="browser", + stdin_file=None, + interactive=False, + ) + ) + meta = audit.load(next((self.root / "commands").glob("*/meta.json"))) + self.assertTrue(meta["timed_out"]) + self.assertNotEqual(meta.get("exit_code"), 0) + + def test_launcher_runs_python_wrappers_through_the_interpreter(self): + wrapper = self.base / "bsk-wrapper" + wrapper.write_text("#!/usr/bin/env python3\nprint('ok')\n", encoding="utf-8") + self.assertEqual( + audit.launcher(wrapper, "nt"), [sys.executable or "python", str(wrapper)] + ) + self.assertEqual(audit.launcher(wrapper, "posix"), [wrapper]) + for suffix in (".exe", ".cmd", ".bat"): + native = self.base / ("bsk" + suffix) + native.write_bytes(b"MZ") + self.assertEqual(audit.launcher(native, "nt"), [native]) + for suffix in (".py", ".pyw"): + script = self.base / ("bsk" + suffix) + script.write_text("print('ok')\n", encoding="utf-8") + self.assertEqual( + audit.launcher(script, "nt"), [sys.executable or "python", str(script)] + ) + self.assertEqual(audit.launcher(script, "posix"), [script]) + plain = self.base / "bsk-script" + plain.write_text("#!/bin/sh\necho ok\n", encoding="utf-8") + self.assertEqual(audit.launcher(plain, "nt"), [plain]) + + def test_events_endpoint_serves_utf8_payloads(self): + root = self.skill / "assets/fixtures" + server = serve.make_server(root, 0) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_port}" + with urllib.request.urlopen( + urllib.request.Request( + url + "/event", + data=json.dumps( + {"type": "note", "value": "中文事件"}, ensure_ascii=False + ).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + ) as response: + response.read() + with urllib.request.urlopen(url + "/events") as response: + body = response.read().decode("utf-8") + self.assertIn("中文事件", body) + finally: + server.shutdown() + server.server_close() + thread.join() + def test_shipped_scenarios_and_fixture_manifest_integrity(self): import re as _re @@ -1012,7 +1104,7 @@ def test_nested_menu_vom_assertions_and_gate_contract(self): key = "vom" if cmd == "sync" else "text" print(json.dumps({key: vom})) """ - self.binary.write_text(nested_cli) + self.binary.write_text(nested_cli, encoding="utf-8") self.binary.chmod(0o755) real_skill = Path(__file__).resolve().parents[1] nested_scenario = next( @@ -1252,7 +1344,7 @@ def test_slotted_list_extract_gate_contract(self): "coverage": {"truncated": False, "dataset_complete": "unknown"} })) """ - self.binary.write_text(slotted_cli) + self.binary.write_text(slotted_cli, encoding="utf-8") self.binary.chmod(0o755) real_skill = Path(__file__).resolve().parents[1] slotted_scenario = next(