From 1d0b802e063d58520302221a8f0aeb17dc9e4f3d Mon Sep 17 00:00:00 2001 From: CuSO41108 <1737268347@qq.com> Date: Wed, 18 Nov 2026 01:33:56 +0800 Subject: [PATCH 1/4] fix(evals): keep release-test fixture bytes stable on Windows checkouts fixture-manifest.json hashes every release-test fixture byte-for-byte, so a CRLF checkout (the Git for Windows default with core.autocrlf=true) makes every manifest check fail. Follow the existing legacy-skills precedent: pin text fixtures to LF and leave media fixtures untouched. --- .gitattributes | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitattributes b/.gitattributes index d33fe5dc4..e329a999b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,10 @@ # Historical test snapshots must retain their original bytes on Windows. /crates/bsk-cli/tests/fixtures/legacy-skills/*.md text eol=lf + +# Release-test fixtures are hashed byte-for-byte by fixture-manifest.json, so a +# CRLF checkout makes every manifest check fail. Keep text fixtures on LF and +# media fixtures untouched. +/evals/browser/skills/browserskill-release-test/assets/fixtures/*.html text eol=lf +/evals/browser/skills/browserskill-release-test/assets/fixtures/*.vtt text eol=lf +/evals/browser/skills/browserskill-release-test/assets/fixtures/*.wav -text +/evals/browser/skills/browserskill-release-test/assets/fixtures/*.mp4 -text From c1f4dc54e2b18939f98fff7b0e16544802f61adf Mon Sep 17 00:00:00 2001 From: CuSO41108 <1737268347@qq.com> Date: Wed, 18 Nov 2026 01:35:01 +0800 Subject: [PATCH 2/4] fix(evals): decode audit helper text as UTF-8 instead of the locale codec serve.py wrote server-events.jsonl as UTF-8 but read it back with the platform default encoding, so a non-ASCII event aborted the request handler on any non-UTF-8 locale (GBK on zh-CN Windows) and silently mojibaked elsewhere. Pin every remaining text read/write in the helper, including CLI help capture and stdout/stderr evidence, to UTF-8, and cover the /events round trip. --- .../scripts/release_audit.py | 14 +++++++---- .../scripts/serve.py | 4 +++- .../scripts/test_release_audit.py | 23 +++++++++++++++++++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py index 24aa9f151..d3936bf76 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py +++ b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py @@ -168,6 +168,8 @@ def visit(parts): [bsk, *parts, "--help"], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=20, env=env, ) @@ -175,7 +177,7 @@ def visit(parts): raise ValueError("Help discovery failed: " + key + " " + r.stderr) text = r.stdout file = directory / ((key.replace(" ", "__") or "root") + ".txt") - file.write_text(text) + file.write_text(text, encoding="utf-8") documents.append({"command": key, "path": file.name, "sha256": digest(file)}) match = re.search(r"^Commands:\s*\n(.*?)(?=^\S|\Z)", text, re.M | re.S) children = ( @@ -190,6 +192,8 @@ def visit(parts): [bsk, "recipe", "--help"], capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=20, env=env, ) @@ -634,8 +638,8 @@ def forward(): if started_ms is not None and finished_ms is not None else 0.0 ) - out_text = (folder / "stdout.txt").read_text(errors="replace") - err_text = (folder / "stderr.txt").read_text(errors="replace") + out_text = (folder / "stdout.txt").read_text(encoding="utf-8", errors="replace") + err_text = (folder / "stderr.txt").read_text(encoding="utf-8", errors="replace") parsed_json = parse(out_text.strip() or err_text.strip()) metrics = extract_metrics(cmd_name, code, parsed_json, elapsed_ms) meta.update( @@ -697,8 +701,8 @@ def read_record(root, record_id): raise ValueError("Changed raw " + kind) if (d / "stdin.txt").exists() and digest(d / "stdin.txt") != m.get("stdin_sha256"): raise ValueError("Changed protocol input") - out = (d / "stdout.txt").read_text(errors="replace") - err = (d / "stderr.txt").read_text(errors="replace") + out = (d / "stdout.txt").read_text(encoding="utf-8", errors="replace") + err = (d / "stderr.txt").read_text(encoding="utf-8", errors="replace") parsed_json = parse(out.strip() or err.strip()) if "elapsed_ms" not in m: s_ms = parse_iso_ms(m.get("started")) diff --git a/evals/browser/skills/browserskill-release-test/scripts/serve.py b/evals/browser/skills/browserskill-release-test/scripts/serve.py index afb3e4363..30f27bc4f 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/serve.py +++ b/evals/browser/skills/browserskill-release-test/scripts/serve.py @@ -145,7 +145,9 @@ def do_GET(self): body = ( "[" + ",".join( - events.read_text().splitlines() if events.exists() else [] + events.read_text(encoding="utf-8", errors="replace").splitlines() + if events.exists() + else [] ) + "]" ) diff --git a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py index 500adf078..fd55515ff 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py +++ b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py @@ -833,6 +833,29 @@ def test_wave_commands_range_assertions_and_long_admin_home(self): if resolved.is_symlink(): resolved.unlink() + def test_events_endpoint_serves_utf8_payloads(self): + root = self.skill / "assets/fixtures" + server = serve.make_server(root, 0) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.server_port}" + urllib.request.urlopen( + urllib.request.Request( + url + "/event", + data=json.dumps( + {"type": "note", "value": "中文事件"}, ensure_ascii=False + ).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + ).read() + body = urllib.request.urlopen(url + "/events").read().decode("utf-8") + self.assertIn("中文事件", body) + finally: + server.shutdown() + server.server_close() + def test_shipped_scenarios_and_fixture_manifest_integrity(self): import re as _re From 0414dc7938691344e4d7cb54471ce3602b6d2d47 Mon Sep 17 00:00:00 2001 From: CuSO41108 <1737268347@qq.com> Date: Wed, 18 Nov 2026 01:36:13 +0800 Subject: [PATCH 3/4] fix(evals): launch script-based --bsk wrappers on Windows The evidence contract allows --bsk to point at a wrapper, but Windows cannot exec a file without an executable extension even when it carries a shebang, so discover() and execute() failed with WinError 193 and the audit tests could not run there. Prefix Python wrappers with the current interpreter, leave native executables and .cmd/.bat shims alone, and skip the AF_UNIX-only admin-home assertion on platforms that use named pipes. --- .../scripts/release_audit.py | 35 +++++++++++++++++-- .../scripts/test_release_audit.py | 21 ++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py index d3936bf76..ab373bad3 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py +++ b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py @@ -151,6 +151,31 @@ def parse(text): return None +def launcher(exe, platform=None): + """Return the argv prefix that actually starts ``exe``. + + Windows cannot exec a file without an executable extension, even when it + carries a shebang, so the script-based ``--bsk`` wrappers described in the + evidence contract fail there with WinError 193. Run Python wrappers with the + current interpreter instead; native executables and ``.cmd``/``.bat`` shims + are launched unchanged. ``platform`` exists so the decision can be tested + without mutating ``os.name``, which would also change how ``pathlib`` builds + paths. + """ + path = Path(exe) + platform = os.name if platform is None else platform + if platform != "nt" or path.suffix.lower() in {".exe", ".com", ".bat", ".cmd"}: + return [exe] + try: + with path.open("rb") as handle: + shebang = handle.readline(256).decode("utf-8", "replace").strip() + except OSError: + return [exe] + if shebang.startswith("#!") and "python" in shebang.lower(): + return [sys.executable or "python", str(path)] + return [exe] + + def discover(bsk, directory): directory = Path(directory) directory.mkdir(parents=True, exist_ok=True) @@ -165,7 +190,7 @@ def visit(parts): return seen.add(key) r = subprocess.run( - [bsk, *parts, "--help"], + [*launcher(bsk), *parts, "--help"], capture_output=True, text=True, encoding="utf-8", @@ -189,7 +214,7 @@ def visit(parts): # exposes its subcommands when supported by the candidate binary. if not parts and "recipe" not in children: probe = subprocess.run( - [bsk, "recipe", "--help"], + [*launcher(bsk), "recipe", "--help"], capture_output=True, text=True, encoding="utf-8", @@ -586,7 +611,11 @@ def execute(args): else (subprocess.PIPE if args.interactive else subprocess.DEVNULL) ) p = subprocess.Popen( - [exe, *argv], stdout=stdout, stderr=stderr, stdin=input_handle, env=env + [*launcher(exe), *argv], + stdout=stdout, + stderr=stderr, + stdin=input_handle, + env=env, ) if args.interactive: diff --git a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py index fd55515ff..96a4ca5c5 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py +++ b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Behavioral tests of the audit helper, not BrowserSkill product tests.""" -import contextlib, csv, importlib.util, io, json, tempfile, time, unittest +import contextlib, csv, importlib.util, io, json, sys, tempfile, time, unittest from pathlib import Path from types import SimpleNamespace as NS from unittest.mock import patch @@ -825,6 +825,10 @@ def test_wave_commands_range_assertions_and_long_admin_home(self): with patch.object(audit.sys, "platform", "win32"): win_home = audit.resolve_admin_home(deep_run) self.assertEqual(Path(win_home), (deep_run / "admin-runtime").resolve()) + if sys.platform == "win32": + # Windows daemons use named pipes, so the AF_UNIX 96-byte cap and its + # symlink workaround do not apply there. + return resolved = Path(audit.resolve_admin_home(deep_run)) try: self.assertLessEqual(len(str(resolved / "run" / "daemon.sock")), 96) @@ -833,6 +837,21 @@ def test_wave_commands_range_assertions_and_long_admin_home(self): if resolved.is_symlink(): resolved.unlink() + def test_launcher_runs_python_wrappers_through_the_interpreter(self): + wrapper = self.base / "bsk-wrapper" + wrapper.write_text("#!/usr/bin/env python3\nprint('ok')\n", encoding="utf-8") + self.assertEqual( + audit.launcher(wrapper, "nt"), [sys.executable or "python", str(wrapper)] + ) + self.assertEqual(audit.launcher(wrapper, "posix"), [wrapper]) + for suffix in (".exe", ".cmd", ".bat"): + native = self.base / ("bsk" + suffix) + native.write_bytes(b"MZ") + self.assertEqual(audit.launcher(native, "nt"), [native]) + plain = self.base / "bsk-script" + plain.write_text("#!/bin/sh\necho ok\n", encoding="utf-8") + self.assertEqual(audit.launcher(plain, "nt"), [plain]) + def test_events_endpoint_serves_utf8_payloads(self): root = self.skill / "assets/fixtures" server = serve.make_server(root, 0) From f7579cb843fae8aab798bec655430f3a077336a8 Mon Sep 17 00:00:00 2001 From: CuSO41108 <1737268347@qq.com> Date: Wed, 18 Nov 2026 12:39:58 +0800 Subject: [PATCH 4/4] fix(evals): close the Windows gaps found in review - Encode every helper text write explicitly; the captions fixture used the locale codec and failed on cp1252 Windows. - Run the audit helper tests in the existing windows-daemon CI job so fixture and encoding regressions surface on a real windows-latest checkout. - Treat shebang-less .py/.pyw --bsk wrappers as interpreter scripts, since CreateProcessW rejects them as well. - Terminate instead of send_signal(SIGINT) on Windows timeouts, where the latter raises ValueError before meta.json is written. - Match the other server tests: context-managed responses and thread.join(). --- .github/workflows/ci.yml | 3 + .../scripts/release_audit.py | 12 ++- .../scripts/test_release_audit.py | 86 +++++++++++++++---- 3 files changed, 81 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 016ef385e..63dee6b1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,9 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } node scripts/check-crate-skill.mjs + - name: Run release-test audit helper tests + run: python evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py + - name: Run installer tests (Windows PowerShell 5.1) shell: powershell run: ./scripts/install-windows.test.ps1 diff --git a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py index ab373bad3..49acad467 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/release_audit.py +++ b/evals/browser/skills/browserskill-release-test/scripts/release_audit.py @@ -164,8 +164,12 @@ def launcher(exe, platform=None): """ path = Path(exe) platform = os.name if platform is None else platform - if platform != "nt" or path.suffix.lower() in {".exe", ".com", ".bat", ".cmd"}: + suffix = path.suffix.lower() + if platform != "nt" or suffix in {".exe", ".com", ".bat", ".cmd"}: return [exe] + # CreateProcessW rejects `.py`/`.pyw` too, with or without a shebang. + if suffix in {".py", ".pyw"}: + return [sys.executable or "python", str(path)] try: with path.open("rb") as handle: shebang = handle.readline(256).decode("utf-8", "replace").strip() @@ -652,7 +656,11 @@ def forward(): code = p.wait(timeout=args.timeout) except subprocess.TimeoutExpired: timed_out = True - p.send_signal(signal.SIGINT) + if os.name == "nt": + # Windows has no SIGINT for processes; send_signal raises ValueError. + p.terminate() + else: + p.send_signal(signal.SIGINT) try: code = p.wait(timeout=5) except subprocess.TimeoutExpired: diff --git a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py index 96a4ca5c5..0c200e72f 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py +++ b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py @@ -37,10 +37,10 @@ def setUp(self): (self.skill / "references").mkdir(parents=True) (self.skill / "assets/fixtures").mkdir(parents=True) (self.skill / "assets/fixtures/minimal.html").write_text( - "