Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -1,2 +1,10 @@
# Historical test snapshots must retain their original bytes on Windows.
/crates/bsk-cli/tests/fixtures/legacy-skills/*.md text eol=lf

# Release-test fixtures are hashed byte-for-byte by fixture-manifest.json, so a
# CRLF checkout makes every manifest check fail. Keep text fixtures on LF and
# media fixtures untouched.
/evals/browser/skills/browserskill-release-test/assets/fixtures/*.html text eol=lf
/evals/browser/skills/browserskill-release-test/assets/fixtures/*.vtt text eol=lf
/evals/browser/skills/browserskill-release-test/assets/fixtures/*.wav -text
/evals/browser/skills/browserskill-release-test/assets/fixtures/*.mp4 -text
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,9 @@ jobs:
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
node scripts/check-crate-skill.mjs

- name: Run release-test audit helper tests
run: python evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py

- name: Run installer tests (Windows PowerShell 5.1)
shell: powershell
run: ./scripts/install-windows.test.ps1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,35 @@ def parse(text):
return None


def launcher(exe, platform=None):
"""Return the argv prefix that actually starts ``exe``.

Windows cannot exec a file without an executable extension, even when it
carries a shebang, so the script-based ``--bsk`` wrappers described in the
evidence contract fail there with WinError 193. Run Python wrappers with the
current interpreter instead; native executables and ``.cmd``/``.bat`` shims
are launched unchanged. ``platform`` exists so the decision can be tested
without mutating ``os.name``, which would also change how ``pathlib`` builds
paths.
"""
path = Path(exe)
platform = os.name if platform is None else platform
suffix = path.suffix.lower()
if platform != "nt" or suffix in {".exe", ".com", ".bat", ".cmd"}:
return [exe]
# CreateProcessW rejects `.py`/`.pyw` too, with or without a shebang.
if suffix in {".py", ".pyw"}:
return [sys.executable or "python", str(path)]
try:
with path.open("rb") as handle:
shebang = handle.readline(256).decode("utf-8", "replace").strip()
except OSError:
return [exe]
if shebang.startswith("#!") and "python" in shebang.lower():
return [sys.executable or "python", str(path)]
return [exe]


def discover(bsk, directory):
directory = Path(directory)
directory.mkdir(parents=True, exist_ok=True)
Expand All @@ -165,17 +194,19 @@ def visit(parts):
return
seen.add(key)
r = subprocess.run(
[bsk, *parts, "--help"],
[*launcher(bsk), *parts, "--help"],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=20,
env=env,
)
if r.returncode:
raise ValueError("Help discovery failed: " + key + " " + r.stderr)
text = r.stdout
file = directory / ((key.replace(" ", "__") or "root") + ".txt")
file.write_text(text)
file.write_text(text, encoding="utf-8")
documents.append({"command": key, "path": file.name, "sha256": digest(file)})
match = re.search(r"^Commands:\s*\n(.*?)(?=^\S|\Z)", text, re.M | re.S)
children = (
Expand All @@ -187,9 +218,11 @@ def visit(parts):
# exposes its subcommands when supported by the candidate binary.
if not parts and "recipe" not in children:
probe = subprocess.run(
[bsk, "recipe", "--help"],
[*launcher(bsk), "recipe", "--help"],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=20,
env=env,
)
Expand Down Expand Up @@ -582,7 +615,11 @@ def execute(args):
else (subprocess.PIPE if args.interactive else subprocess.DEVNULL)
)
p = subprocess.Popen(
[exe, *argv], stdout=stdout, stderr=stderr, stdin=input_handle, env=env
[*launcher(exe), *argv],
stdout=stdout,
stderr=stderr,
stdin=input_handle,
env=env,
)
if args.interactive:

Expand Down Expand Up @@ -619,7 +656,11 @@ def forward():
code = p.wait(timeout=args.timeout)
except subprocess.TimeoutExpired:
timed_out = True
p.send_signal(signal.SIGINT)
if os.name == "nt":
# Windows has no SIGINT for processes; send_signal raises ValueError.
p.terminate()
else:
p.send_signal(signal.SIGINT)
try:
code = p.wait(timeout=5)
except subprocess.TimeoutExpired:
Expand All @@ -634,8 +675,8 @@ def forward():
if started_ms is not None and finished_ms is not None
else 0.0
)
out_text = (folder / "stdout.txt").read_text(errors="replace")
err_text = (folder / "stderr.txt").read_text(errors="replace")
out_text = (folder / "stdout.txt").read_text(encoding="utf-8", errors="replace")
err_text = (folder / "stderr.txt").read_text(encoding="utf-8", errors="replace")
parsed_json = parse(out_text.strip() or err_text.strip())
metrics = extract_metrics(cmd_name, code, parsed_json, elapsed_ms)
meta.update(
Expand Down Expand Up @@ -697,8 +738,8 @@ def read_record(root, record_id):
raise ValueError("Changed raw " + kind)
if (d / "stdin.txt").exists() and digest(d / "stdin.txt") != m.get("stdin_sha256"):
raise ValueError("Changed protocol input")
out = (d / "stdout.txt").read_text(errors="replace")
err = (d / "stderr.txt").read_text(errors="replace")
out = (d / "stdout.txt").read_text(encoding="utf-8", errors="replace")
err = (d / "stderr.txt").read_text(encoding="utf-8", errors="replace")
parsed_json = parse(out.strip() or err.strip())
if "elapsed_ms" not in m:
s_ms = parse_iso_ms(m.get("started"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,9 @@ def do_GET(self):
body = (
"["
+ ",".join(
events.read_text().splitlines() if events.exists() else []
events.read_text(encoding="utf-8", errors="replace").splitlines()
if events.exists()
else []
)
+ "]"
)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
"""Behavioral tests of the audit helper, not BrowserSkill product tests."""

import contextlib, csv, importlib.util, io, json, tempfile, time, unittest
import contextlib, csv, importlib.util, io, json, sys, tempfile, time, unittest
from pathlib import Path
from types import SimpleNamespace as NS
from unittest.mock import patch
Expand Down Expand Up @@ -37,10 +37,10 @@ def setUp(self):
(self.skill / "references").mkdir(parents=True)
(self.skill / "assets/fixtures").mkdir(parents=True)
(self.skill / "assets/fixtures/minimal.html").write_text(
"<title>synthetic</title>"
"<title>synthetic</title>", encoding="utf-8"
)
self.binary = self.base / "bsk"
self.binary.write_text(FAKE)
self.binary.write_text(FAKE, encoding="utf-8")
self.binary.chmod(0o755)
self.patch = patch.object(audit, "SKILL", self.skill)
self.patch.start()
Expand Down Expand Up @@ -139,14 +139,16 @@ def test_raw_output_tampering_blocks(self):
self.init()
rid = self.execute()
self.fill(rid)
(self.root / "commands" / rid / "stdout.txt").write_text('{"ok":false}')
(self.root / "commands" / rid / "stdout.txt").write_text(
'{"ok":false}', encoding="utf-8"
)
self.assertEqual(self.gate()[0], 1)

def test_candidate_change_blocks(self):
self.init()
rid = self.execute()
self.fill(rid)
self.binary.write_text(FAKE + "\n# changed\n")
self.binary.write_text(FAKE + "\n# changed\n", encoding="utf-8")
self.assertEqual(self.gate()[0], 1)

def test_deleted_scenario_blocks(self):
Expand Down Expand Up @@ -248,7 +250,7 @@ def test_zero_exit_script_failure_cannot_be_hidden(self):
"elif '--unsatisfied'",
"elif '--eval-false' in args:print(json.dumps({'ok':False,'marker':'present'}))\nelif '--unsatisfied'",
)
self.binary.write_text(body)
self.binary.write_text(body, encoding="utf-8")
self.init()
good = self.execute()
bad = self.execute(["evaluate", "--eval-false"], "CMD-evaluate")
Expand Down Expand Up @@ -337,7 +339,7 @@ def test_discovery_keeps_navigate_and_distinct_alias_help(self):
elif p==['navigate']:print('Usage: bsk navigate [URL]\\n\\nCommands:\\n back Back\\n\\nArguments:\\n [URL] Destination\\n\\nOptions:\\n --help Help')
else:print('Usage: bsk '+' '.join(p))
"""
self.binary.write_text(body)
self.binary.write_text(body, encoding="utf-8")
inv = audit.discover(str(self.binary), self.base / "help")
self.assertEqual(
set(inv["commands"]), {"help", "navigate", "navigate back", "navigate-back"}
Expand All @@ -355,7 +357,9 @@ def test_discovery_keeps_navigate_and_distinct_alias_help(self):

def test_server_routes_and_event_oracle(self):
root = self.skill / "assets/fixtures"
(root / "plan-recipe-lab.html").write_text("<title>plan-recipe-lab</title>")
(root / "plan-recipe-lab.html").write_text(
"<title>plan-recipe-lab</title>", encoding="utf-8"
)
server = serve.make_server(root, 0)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
Expand Down Expand Up @@ -407,7 +411,7 @@ def test_hidden_recipe_discovery_and_plan_recipe_benchmark(self):
else:
print(json.dumps({'ok': True, 'value': 'synthetic'}))
"""
self.binary.write_text(body)
self.binary.write_text(body, encoding="utf-8")
inv = audit.discover(str(self.binary), self.base / "help")
self.assertEqual(
set(inv["commands"]),
Expand Down Expand Up @@ -608,8 +612,12 @@ def test_hidden_recipe_discovery_and_plan_recipe_benchmark(self):

def test_adversarial_server_routes_bfcache_cors_and_media_mimes(self):
root = self.skill / "assets/fixtures"
(root / "arena-bfcache-detail.html").write_text("<title>bfcache</title>")
(root / "captions-zh.vtt").write_text("WEBVTT\n\n00:00.000 --> 00:01.000\n你好\n")
(root / "arena-bfcache-detail.html").write_text(
"<title>bfcache</title>", encoding="utf-8"
)
(root / "captions-zh.vtt").write_text(
"WEBVTT\n\n00:00.000 --> 00:01.000\n你好\n", encoding="utf-8"
)
(root / "arena-audio.wav").write_bytes(b"RIFF\x24\x00\x00\x00WAVE")
server = serve.make_server(root, 0)
thread = threading.Thread(target=server.serve_forever, daemon=True)
Expand Down Expand Up @@ -753,15 +761,20 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):

def test_wave_commands_range_assertions_and_long_admin_home(self):
(self.skill / "assets/fixtures/arena-frame-outer.html").write_text(
'<iframe src="http://localhost:4187/a"></iframe><iframe src="http://127.0.0.1:4187/b"></iframe>'
'<iframe src="http://localhost:4187/a"></iframe><iframe src="http://127.0.0.1:4187/b"></iframe>',
encoding="utf-8",
)
self.init()
rewritten = (self.root / "fixtures/arena-frame-outer.html").read_text()
rewritten = (self.root / "fixtures/arena-frame-outer.html").read_text(
encoding="utf-8"
)
self.assertIn("http://localhost:4567/a", rewritten)
self.assertIn("http://127.0.0.1:4567/b", rewritten)
rid = self.execute()
d = self.root / "commands" / rid
(d / "stdout.txt").write_text('{"ok": true, "size": 42, "label": "9"}')
(d / "stdout.txt").write_text(
'{"ok": true, "size": 42, "label": "9"}', encoding="utf-8"
)
m = audit.load(d / "meta.json")
m["stdout_sha256"] = audit.digest(d / "stdout.txt")
audit.write(d / "meta.json", m)
Expand Down Expand Up @@ -825,6 +838,10 @@ def test_wave_commands_range_assertions_and_long_admin_home(self):
with patch.object(audit.sys, "platform", "win32"):
win_home = audit.resolve_admin_home(deep_run)
self.assertEqual(Path(win_home), (deep_run / "admin-runtime").resolve())
if sys.platform == "win32":
# Windows daemons use named pipes, so the AF_UNIX 96-byte cap and its
# symlink workaround do not apply there.
return
resolved = Path(audit.resolve_admin_home(deep_run))
try:
self.assertLessEqual(len(str(resolved / "run" / "daemon.sock")), 96)
Expand All @@ -833,6 +850,81 @@ def test_wave_commands_range_assertions_and_long_admin_home(self):
if resolved.is_symlink():
resolved.unlink()

def test_timeout_records_metadata_instead_of_crashing(self):
self.binary.write_text(
"#!/usr/bin/env python3\n"
"import sys, time\n"
"if '--help' in sys.argv:\n"
" print('Usage: bsk [OPTIONS] <COMMAND>\\n\\nCommands:\\n status Show status\\n\\nOptions:\\n --help Help' if sys.argv[1:] == ['--help'] else 'Usage: bsk status [OPTIONS]')\n"
"else:\n"
" time.sleep(30)\n",
encoding="utf-8",
)
self.init()
with contextlib.redirect_stdout(io.StringIO()):
audit.execute(
NS(
run=str(self.root),
case="FLOW",
argv=["status"],
timeout=1,
lane="browser",
stdin_file=None,
interactive=False,
)
)
meta = audit.load(next((self.root / "commands").glob("*/meta.json")))
self.assertTrue(meta["timed_out"])
self.assertNotEqual(meta.get("exit_code"), 0)

def test_launcher_runs_python_wrappers_through_the_interpreter(self):
wrapper = self.base / "bsk-wrapper"
wrapper.write_text("#!/usr/bin/env python3\nprint('ok')\n", encoding="utf-8")
self.assertEqual(
audit.launcher(wrapper, "nt"), [sys.executable or "python", str(wrapper)]
)
self.assertEqual(audit.launcher(wrapper, "posix"), [wrapper])
for suffix in (".exe", ".cmd", ".bat"):
native = self.base / ("bsk" + suffix)
native.write_bytes(b"MZ")
self.assertEqual(audit.launcher(native, "nt"), [native])
for suffix in (".py", ".pyw"):
script = self.base / ("bsk" + suffix)
script.write_text("print('ok')\n", encoding="utf-8")
self.assertEqual(
audit.launcher(script, "nt"), [sys.executable or "python", str(script)]
)
self.assertEqual(audit.launcher(script, "posix"), [script])
plain = self.base / "bsk-script"
plain.write_text("#!/bin/sh\necho ok\n", encoding="utf-8")
self.assertEqual(audit.launcher(plain, "nt"), [plain])

def test_events_endpoint_serves_utf8_payloads(self):
root = self.skill / "assets/fixtures"
server = serve.make_server(root, 0)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
url = f"http://127.0.0.1:{server.server_port}"
with urllib.request.urlopen(
urllib.request.Request(
url + "/event",
data=json.dumps(
{"type": "note", "value": "中文事件"}, ensure_ascii=False
).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
) as response:
response.read()
with urllib.request.urlopen(url + "/events") as response:
body = response.read().decode("utf-8")
self.assertIn("中文事件", body)
finally:
server.shutdown()
server.server_close()
thread.join()

def test_shipped_scenarios_and_fixture_manifest_integrity(self):
import re as _re

Expand Down Expand Up @@ -1012,7 +1104,7 @@ def test_nested_menu_vom_assertions_and_gate_contract(self):
key = "vom" if cmd == "sync" else "text"
print(json.dumps({key: vom}))
"""
self.binary.write_text(nested_cli)
self.binary.write_text(nested_cli, encoding="utf-8")
self.binary.chmod(0o755)
real_skill = Path(__file__).resolve().parents[1]
nested_scenario = next(
Expand Down Expand Up @@ -1252,7 +1344,7 @@ def test_slotted_list_extract_gate_contract(self):
"coverage": {"truncated": False, "dataset_complete": "unknown"}
}))
"""
self.binary.write_text(slotted_cli)
self.binary.write_text(slotted_cli, encoding="utf-8")
self.binary.chmod(0o755)
real_skill = Path(__file__).resolve().parents[1]
slotted_scenario = next(
Expand Down
Loading