diff --git a/.gitattributes b/.gitattributes
index d33fe5dc4..e329a999b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,10 @@
# Historical test snapshots must retain their original bytes on Windows.
/crates/bsk-cli/tests/fixtures/legacy-skills/*.md text eol=lf
+
+# Release-test fixtures are hashed byte-for-byte by fixture-manifest.json, so a
+# CRLF checkout makes every manifest check fail. Keep text fixtures on LF and
+# media fixtures untouched.
+/evals/browser/skills/browserskill-release-test/assets/fixtures/*.html text eol=lf
+/evals/browser/skills/browserskill-release-test/assets/fixtures/*.vtt text eol=lf
+/evals/browser/skills/browserskill-release-test/assets/fixtures/*.wav -text
+/evals/browser/skills/browserskill-release-test/assets/fixtures/*.mp4 -text
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 016ef385e..63dee6b1a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -170,6 +170,9 @@ jobs:
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
node scripts/check-crate-skill.mjs
+ - name: Run release-test audit helper tests
+ run: python evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py
+
- name: Run installer tests (Windows PowerShell 5.1)
shell: powershell
run: ./scripts/install-windows.test.ps1
diff --git a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py
index 24aa9f151..49acad467 100644
--- a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py
+++ b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py
@@ -151,6 +151,35 @@ def parse(text):
return None
+def launcher(exe, platform=None):
+ """Return the argv prefix that actually starts ``exe``.
+
+ Windows cannot exec a file without an executable extension, even when it
+ carries a shebang, so the script-based ``--bsk`` wrappers described in the
+ evidence contract fail there with WinError 193. Run Python wrappers with the
+ current interpreter instead; native executables and ``.cmd``/``.bat`` shims
+ are launched unchanged. ``platform`` exists so the decision can be tested
+ without mutating ``os.name``, which would also change how ``pathlib`` builds
+ paths.
+ """
+ path = Path(exe)
+ platform = os.name if platform is None else platform
+ suffix = path.suffix.lower()
+ if platform != "nt" or suffix in {".exe", ".com", ".bat", ".cmd"}:
+ return [exe]
+ # CreateProcessW rejects `.py`/`.pyw` too, with or without a shebang.
+ if suffix in {".py", ".pyw"}:
+ return [sys.executable or "python", str(path)]
+ try:
+ with path.open("rb") as handle:
+ shebang = handle.readline(256).decode("utf-8", "replace").strip()
+ except OSError:
+ return [exe]
+ if shebang.startswith("#!") and "python" in shebang.lower():
+ return [sys.executable or "python", str(path)]
+ return [exe]
+
+
def discover(bsk, directory):
directory = Path(directory)
directory.mkdir(parents=True, exist_ok=True)
@@ -165,9 +194,11 @@ def visit(parts):
return
seen.add(key)
r = subprocess.run(
- [bsk, *parts, "--help"],
+ [*launcher(bsk), *parts, "--help"],
capture_output=True,
text=True,
+ encoding="utf-8",
+ errors="replace",
timeout=20,
env=env,
)
@@ -175,7 +206,7 @@ def visit(parts):
raise ValueError("Help discovery failed: " + key + " " + r.stderr)
text = r.stdout
file = directory / ((key.replace(" ", "__") or "root") + ".txt")
- file.write_text(text)
+ file.write_text(text, encoding="utf-8")
documents.append({"command": key, "path": file.name, "sha256": digest(file)})
match = re.search(r"^Commands:\s*\n(.*?)(?=^\S|\Z)", text, re.M | re.S)
children = (
@@ -187,9 +218,11 @@ def visit(parts):
# exposes its subcommands when supported by the candidate binary.
if not parts and "recipe" not in children:
probe = subprocess.run(
- [bsk, "recipe", "--help"],
+ [*launcher(bsk), "recipe", "--help"],
capture_output=True,
text=True,
+ encoding="utf-8",
+ errors="replace",
timeout=20,
env=env,
)
@@ -582,7 +615,11 @@ def execute(args):
else (subprocess.PIPE if args.interactive else subprocess.DEVNULL)
)
p = subprocess.Popen(
- [exe, *argv], stdout=stdout, stderr=stderr, stdin=input_handle, env=env
+ [*launcher(exe), *argv],
+ stdout=stdout,
+ stderr=stderr,
+ stdin=input_handle,
+ env=env,
)
if args.interactive:
@@ -619,7 +656,11 @@ def forward():
code = p.wait(timeout=args.timeout)
except subprocess.TimeoutExpired:
timed_out = True
- p.send_signal(signal.SIGINT)
+ if os.name == "nt":
+ # Windows has no SIGINT for processes; send_signal raises ValueError.
+ p.terminate()
+ else:
+ p.send_signal(signal.SIGINT)
try:
code = p.wait(timeout=5)
except subprocess.TimeoutExpired:
@@ -634,8 +675,8 @@ def forward():
if started_ms is not None and finished_ms is not None
else 0.0
)
- out_text = (folder / "stdout.txt").read_text(errors="replace")
- err_text = (folder / "stderr.txt").read_text(errors="replace")
+ out_text = (folder / "stdout.txt").read_text(encoding="utf-8", errors="replace")
+ err_text = (folder / "stderr.txt").read_text(encoding="utf-8", errors="replace")
parsed_json = parse(out_text.strip() or err_text.strip())
metrics = extract_metrics(cmd_name, code, parsed_json, elapsed_ms)
meta.update(
@@ -697,8 +738,8 @@ def read_record(root, record_id):
raise ValueError("Changed raw " + kind)
if (d / "stdin.txt").exists() and digest(d / "stdin.txt") != m.get("stdin_sha256"):
raise ValueError("Changed protocol input")
- out = (d / "stdout.txt").read_text(errors="replace")
- err = (d / "stderr.txt").read_text(errors="replace")
+ out = (d / "stdout.txt").read_text(encoding="utf-8", errors="replace")
+ err = (d / "stderr.txt").read_text(encoding="utf-8", errors="replace")
parsed_json = parse(out.strip() or err.strip())
if "elapsed_ms" not in m:
s_ms = parse_iso_ms(m.get("started"))
diff --git a/evals/browser/skills/browserskill-release-test/scripts/serve.py b/evals/browser/skills/browserskill-release-test/scripts/serve.py
index afb3e4363..30f27bc4f 100644
--- a/evals/browser/skills/browserskill-release-test/scripts/serve.py
+++ b/evals/browser/skills/browserskill-release-test/scripts/serve.py
@@ -145,7 +145,9 @@ def do_GET(self):
body = (
"["
+ ",".join(
- events.read_text().splitlines() if events.exists() else []
+ events.read_text(encoding="utf-8", errors="replace").splitlines()
+ if events.exists()
+ else []
)
+ "]"
)
diff --git a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py
index 500adf078..0c200e72f 100644
--- a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py
+++ b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
"""Behavioral tests of the audit helper, not BrowserSkill product tests."""
-import contextlib, csv, importlib.util, io, json, tempfile, time, unittest
+import contextlib, csv, importlib.util, io, json, sys, tempfile, time, unittest
from pathlib import Path
from types import SimpleNamespace as NS
from unittest.mock import patch
@@ -37,10 +37,10 @@ def setUp(self):
(self.skill / "references").mkdir(parents=True)
(self.skill / "assets/fixtures").mkdir(parents=True)
(self.skill / "assets/fixtures/minimal.html").write_text(
- "
synthetic"
+ "synthetic", encoding="utf-8"
)
self.binary = self.base / "bsk"
- self.binary.write_text(FAKE)
+ self.binary.write_text(FAKE, encoding="utf-8")
self.binary.chmod(0o755)
self.patch = patch.object(audit, "SKILL", self.skill)
self.patch.start()
@@ -139,14 +139,16 @@ def test_raw_output_tampering_blocks(self):
self.init()
rid = self.execute()
self.fill(rid)
- (self.root / "commands" / rid / "stdout.txt").write_text('{"ok":false}')
+ (self.root / "commands" / rid / "stdout.txt").write_text(
+ '{"ok":false}', encoding="utf-8"
+ )
self.assertEqual(self.gate()[0], 1)
def test_candidate_change_blocks(self):
self.init()
rid = self.execute()
self.fill(rid)
- self.binary.write_text(FAKE + "\n# changed\n")
+ self.binary.write_text(FAKE + "\n# changed\n", encoding="utf-8")
self.assertEqual(self.gate()[0], 1)
def test_deleted_scenario_blocks(self):
@@ -248,7 +250,7 @@ def test_zero_exit_script_failure_cannot_be_hidden(self):
"elif '--unsatisfied'",
"elif '--eval-false' in args:print(json.dumps({'ok':False,'marker':'present'}))\nelif '--unsatisfied'",
)
- self.binary.write_text(body)
+ self.binary.write_text(body, encoding="utf-8")
self.init()
good = self.execute()
bad = self.execute(["evaluate", "--eval-false"], "CMD-evaluate")
@@ -337,7 +339,7 @@ def test_discovery_keeps_navigate_and_distinct_alias_help(self):
elif p==['navigate']:print('Usage: bsk navigate [URL]\\n\\nCommands:\\n back Back\\n\\nArguments:\\n [URL] Destination\\n\\nOptions:\\n --help Help')
else:print('Usage: bsk '+' '.join(p))
"""
- self.binary.write_text(body)
+ self.binary.write_text(body, encoding="utf-8")
inv = audit.discover(str(self.binary), self.base / "help")
self.assertEqual(
set(inv["commands"]), {"help", "navigate", "navigate back", "navigate-back"}
@@ -355,7 +357,9 @@ def test_discovery_keeps_navigate_and_distinct_alias_help(self):
def test_server_routes_and_event_oracle(self):
root = self.skill / "assets/fixtures"
- (root / "plan-recipe-lab.html").write_text("plan-recipe-lab")
+ (root / "plan-recipe-lab.html").write_text(
+ "plan-recipe-lab", encoding="utf-8"
+ )
server = serve.make_server(root, 0)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
@@ -407,7 +411,7 @@ def test_hidden_recipe_discovery_and_plan_recipe_benchmark(self):
else:
print(json.dumps({'ok': True, 'value': 'synthetic'}))
"""
- self.binary.write_text(body)
+ self.binary.write_text(body, encoding="utf-8")
inv = audit.discover(str(self.binary), self.base / "help")
self.assertEqual(
set(inv["commands"]),
@@ -608,8 +612,12 @@ def test_hidden_recipe_discovery_and_plan_recipe_benchmark(self):
def test_adversarial_server_routes_bfcache_cors_and_media_mimes(self):
root = self.skill / "assets/fixtures"
- (root / "arena-bfcache-detail.html").write_text("bfcache")
- (root / "captions-zh.vtt").write_text("WEBVTT\n\n00:00.000 --> 00:01.000\n你好\n")
+ (root / "arena-bfcache-detail.html").write_text(
+ "bfcache", encoding="utf-8"
+ )
+ (root / "captions-zh.vtt").write_text(
+ "WEBVTT\n\n00:00.000 --> 00:01.000\n你好\n", encoding="utf-8"
+ )
(root / "arena-audio.wav").write_bytes(b"RIFF\x24\x00\x00\x00WAVE")
server = serve.make_server(root, 0)
thread = threading.Thread(target=server.serve_forever, daemon=True)
@@ -753,15 +761,20 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):
def test_wave_commands_range_assertions_and_long_admin_home(self):
(self.skill / "assets/fixtures/arena-frame-outer.html").write_text(
- ''
+ '',
+ encoding="utf-8",
)
self.init()
- rewritten = (self.root / "fixtures/arena-frame-outer.html").read_text()
+ rewritten = (self.root / "fixtures/arena-frame-outer.html").read_text(
+ encoding="utf-8"
+ )
self.assertIn("http://localhost:4567/a", rewritten)
self.assertIn("http://127.0.0.1:4567/b", rewritten)
rid = self.execute()
d = self.root / "commands" / rid
- (d / "stdout.txt").write_text('{"ok": true, "size": 42, "label": "9"}')
+ (d / "stdout.txt").write_text(
+ '{"ok": true, "size": 42, "label": "9"}', encoding="utf-8"
+ )
m = audit.load(d / "meta.json")
m["stdout_sha256"] = audit.digest(d / "stdout.txt")
audit.write(d / "meta.json", m)
@@ -825,6 +838,10 @@ def test_wave_commands_range_assertions_and_long_admin_home(self):
with patch.object(audit.sys, "platform", "win32"):
win_home = audit.resolve_admin_home(deep_run)
self.assertEqual(Path(win_home), (deep_run / "admin-runtime").resolve())
+ if sys.platform == "win32":
+ # Windows daemons use named pipes, so the AF_UNIX 96-byte cap and its
+ # symlink workaround do not apply there.
+ return
resolved = Path(audit.resolve_admin_home(deep_run))
try:
self.assertLessEqual(len(str(resolved / "run" / "daemon.sock")), 96)
@@ -833,6 +850,81 @@ def test_wave_commands_range_assertions_and_long_admin_home(self):
if resolved.is_symlink():
resolved.unlink()
+ def test_timeout_records_metadata_instead_of_crashing(self):
+ self.binary.write_text(
+ "#!/usr/bin/env python3\n"
+ "import sys, time\n"
+ "if '--help' in sys.argv:\n"
+ " print('Usage: bsk [OPTIONS] \\n\\nCommands:\\n status Show status\\n\\nOptions:\\n --help Help' if sys.argv[1:] == ['--help'] else 'Usage: bsk status [OPTIONS]')\n"
+ "else:\n"
+ " time.sleep(30)\n",
+ encoding="utf-8",
+ )
+ self.init()
+ with contextlib.redirect_stdout(io.StringIO()):
+ audit.execute(
+ NS(
+ run=str(self.root),
+ case="FLOW",
+ argv=["status"],
+ timeout=1,
+ lane="browser",
+ stdin_file=None,
+ interactive=False,
+ )
+ )
+ meta = audit.load(next((self.root / "commands").glob("*/meta.json")))
+ self.assertTrue(meta["timed_out"])
+ self.assertNotEqual(meta.get("exit_code"), 0)
+
+ def test_launcher_runs_python_wrappers_through_the_interpreter(self):
+ wrapper = self.base / "bsk-wrapper"
+ wrapper.write_text("#!/usr/bin/env python3\nprint('ok')\n", encoding="utf-8")
+ self.assertEqual(
+ audit.launcher(wrapper, "nt"), [sys.executable or "python", str(wrapper)]
+ )
+ self.assertEqual(audit.launcher(wrapper, "posix"), [wrapper])
+ for suffix in (".exe", ".cmd", ".bat"):
+ native = self.base / ("bsk" + suffix)
+ native.write_bytes(b"MZ")
+ self.assertEqual(audit.launcher(native, "nt"), [native])
+ for suffix in (".py", ".pyw"):
+ script = self.base / ("bsk" + suffix)
+ script.write_text("print('ok')\n", encoding="utf-8")
+ self.assertEqual(
+ audit.launcher(script, "nt"), [sys.executable or "python", str(script)]
+ )
+ self.assertEqual(audit.launcher(script, "posix"), [script])
+ plain = self.base / "bsk-script"
+ plain.write_text("#!/bin/sh\necho ok\n", encoding="utf-8")
+ self.assertEqual(audit.launcher(plain, "nt"), [plain])
+
+ def test_events_endpoint_serves_utf8_payloads(self):
+ root = self.skill / "assets/fixtures"
+ server = serve.make_server(root, 0)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ url = f"http://127.0.0.1:{server.server_port}"
+ with urllib.request.urlopen(
+ urllib.request.Request(
+ url + "/event",
+ data=json.dumps(
+ {"type": "note", "value": "中文事件"}, ensure_ascii=False
+ ).encode("utf-8"),
+ headers={"Content-Type": "application/json"},
+ method="POST",
+ )
+ ) as response:
+ response.read()
+ with urllib.request.urlopen(url + "/events") as response:
+ body = response.read().decode("utf-8")
+ self.assertIn("中文事件", body)
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join()
+
def test_shipped_scenarios_and_fixture_manifest_integrity(self):
import re as _re
@@ -1012,7 +1104,7 @@ def test_nested_menu_vom_assertions_and_gate_contract(self):
key = "vom" if cmd == "sync" else "text"
print(json.dumps({key: vom}))
"""
- self.binary.write_text(nested_cli)
+ self.binary.write_text(nested_cli, encoding="utf-8")
self.binary.chmod(0o755)
real_skill = Path(__file__).resolve().parents[1]
nested_scenario = next(
@@ -1252,7 +1344,7 @@ def test_slotted_list_extract_gate_contract(self):
"coverage": {"truncated": False, "dataset_complete": "unknown"}
}))
"""
- self.binary.write_text(slotted_cli)
+ self.binary.write_text(slotted_cli, encoding="utf-8")
self.binary.chmod(0o755)
real_skill = Path(__file__).resolve().parents[1]
slotted_scenario = next(