From d0160018f299557d6ebf3319614aa9a169cc46ee Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:26:34 +0800 Subject: [PATCH] feat(validation): select bounded native Node test names Bind optional native TAP test-name patterns to canonical argv, target identity and source provenance. Gate filtered runs behind a new Runner capability at planning and Job admission. Exclude synthetic Node file-wrapper passes from durable executed-test counts and add regression coverage. --- .../src/project_validation_tests.rs | 49 +++++++ crates/webcodex-core/src/runner_protocol.rs | 31 ++-- .../webcodex-core/src/runner_protocol/job.rs | 66 ++++++++- .../webcodex-core/src/validation_evidence.rs | 46 +++++- .../src/validation_evidence_tests.rs | 38 +++++ .../webcodex-core/src/validation_identity.rs | 19 ++- crates/webcodex-runner-config/src/lib.rs | 1 + .../src/job_updates/start.rs | 21 +++ .../webcodex-runner-registry/src/requests.rs | 10 ++ .../src/tests/project_dependency_policy.rs | 18 +++ .../src/tests/reconciliation.rs | 58 ++++++++ .../webcodex_runner/transport/registration.rs | 1 + .../src/webcodex_runner/validation/project.rs | 5 +- .../validation/project_tests.rs | 39 +++++- .../src/tests/input_schemas.rs | 10 +- .../src/tool_definition/testing.rs | 2 +- .../webcodex-validation/src/adapters/mod.rs | 35 +++-- crates/webcodex-validation/src/lib.rs | 5 +- crates/webcodex-validation/src/recipe.rs | 65 ++++++++- docs/MCP.md | 2 +- docs/MCP.zh-CN.md | 4 +- .../node-native-tap-project-validation.md | 4 +- src/tool_runtime/tests/validation_handoff.rs | 15 +- .../validation_handoff/project_validation.rs | 132 ++++++++++++++++++ src/tool_runtime/validation.rs | 26 ++++ src/tool_runtime/validation_profile/mod.rs | 4 +- 26 files changed, 647 insertions(+), 59 deletions(-) diff --git a/crates/webcodex-core/src/project_validation_tests.rs b/crates/webcodex-core/src/project_validation_tests.rs index 1f7af218a..c8b7b6830 100644 --- a/crates/webcodex-core/src/project_validation_tests.rs +++ b/crates/webcodex-core/src/project_validation_tests.rs @@ -96,6 +96,55 @@ fn project_node_check_step_identity_capability_and_metadata_are_closed() { assert_ne!(node, ruff); } +#[test] +fn node_native_name_filter_identity_and_canonical_argv_are_closed() { + use crate::runner_protocol::{RunnerCapabilities, ShellJobValidationStep}; + use crate::validation_identity::{ + structured_validation_target_identity, ToolValidationIdentityKind, + }; + let legacy: RunnerCapabilities = + serde_json::from_value(serde_json::json!({"project_validation_node_tap_v1":true})).unwrap(); + assert!(!legacy.project_validation_node_name_filter_v1); + let id = |filter: Option<&str>| { + structured_validation_target_identity( + ToolValidationIdentityKind::NodeNativeTest, + &serde_json::json!({"cwd":".","filter":filter}), + ) + .unwrap() + }; + let old = structured_validation_target_identity( + ToolValidationIdentityKind::NodeNativeTest, + &serde_json::json!({"cwd":"."}), + ) + .unwrap(); + assert_eq!(old, id(None)); + assert_eq!(old, id(Some(""))); + assert_ne!(old, id(Some("alpha"))); + assert_ne!(id(Some("alpha")), id(Some("beta"))); + let step = |filter: Option<&str>| { + let mut args = vec!["--test".to_string(), "--test-reporter=tap".to_string()]; + if let Some(filter) = filter { + args.push(format!("--test-name-pattern={filter}")); + } + ShellJobValidationStep { + name: "test".into(), + program: "node".into(), + args, + env: vec![], + } + }; + assert!(step(None).is_canonical()); + assert!(step(Some("alpha|beta")).is_canonical()); + assert!(step(Some(" alpha ")).is_canonical()); + assert_ne!(id(Some("alpha")), id(Some(" alpha "))); + for bad in ["", "\n", "-oops", &"x".repeat(201)] { + assert!(!step(Some(bad)).is_canonical(), "{bad:?}"); + } + let mut malformed = step(Some("alpha")); + malformed.args.push("--eval=process.exit(0)".into()); + assert!(!malformed.is_canonical()); +} + #[test] fn only_canonical_project_cargo_steps_accept_sorted_workspace_exclusions() { use crate::runner_protocol::ShellJobValidationStep; diff --git a/crates/webcodex-core/src/runner_protocol.rs b/crates/webcodex-core/src/runner_protocol.rs index 4e4d39071..91ccc37bc 100644 --- a/crates/webcodex-core/src/runner_protocol.rs +++ b/crates/webcodex-core/src/runner_protocol.rs @@ -29,18 +29,18 @@ mod tests; pub use job::{ normalize_cargo_excluded_packages, normalize_cargo_packages, normalize_cargo_value, normalize_go_packages, normalize_go_test_filter, normalize_go_test_packages, - normalize_pytest_filter, normalize_rust_test_filter, valid_rust_test_filter, - RunnerJobLogRequest, RunnerJobLogResponse, RunnerJobResult, RunnerJobStatusRequest, - RunnerJobStatusResponse, RunnerJobStopRequest, RunnerJobStopResponse, RunnerJobUpdateRequest, - RunnerJobUpdateResponse, RunnerJobsListRequest, RunnerJobsListResponse, RunnerShellJobResult, - ShellJobActivity, ShellJobActivityPhase, ShellJobActivitySource, ShellJobActivityState, - ShellJobCodexMetadata, ShellJobContext, ShellJobInfo, ShellJobInventory, ShellJobLogSnapshot, - ShellJobOpRequest, ShellJobOpResponse, ShellJobSnapshot, ShellJobStreamSnapshot, - ShellJobStructuredExecutionMetadata, ShellJobTestCountEvidence, ShellJobValidationMetadata, - ShellJobValidationProgress, ShellJobValidationStep, CARGO_PACKAGE_MAX_ITEMS, - CARGO_TEST_MIN_TESTS_MAX, CARGO_VALUE_MAX_BYTES, GO_TEST_PACKAGE_MAX_BYTES, - GO_TEST_PACKAGE_MAX_ITEMS, JOB_INVENTORY_MAX_ACTIVE_JOBS, JOB_INVENTORY_MAX_JOBS, - JOB_INVENTORY_MAX_SERIALIZED_BYTES, JOB_INVENTORY_MAX_TERMINAL_JOBS, + normalize_node_native_test_filter, normalize_pytest_filter, normalize_rust_test_filter, + valid_rust_test_filter, RunnerJobLogRequest, RunnerJobLogResponse, RunnerJobResult, + RunnerJobStatusRequest, RunnerJobStatusResponse, RunnerJobStopRequest, RunnerJobStopResponse, + RunnerJobUpdateRequest, RunnerJobUpdateResponse, RunnerJobsListRequest, RunnerJobsListResponse, + RunnerShellJobResult, ShellJobActivity, ShellJobActivityPhase, ShellJobActivitySource, + ShellJobActivityState, ShellJobCodexMetadata, ShellJobContext, ShellJobInfo, ShellJobInventory, + ShellJobLogSnapshot, ShellJobOpRequest, ShellJobOpResponse, ShellJobSnapshot, + ShellJobStreamSnapshot, ShellJobStructuredExecutionMetadata, ShellJobTestCountEvidence, + ShellJobValidationMetadata, ShellJobValidationProgress, ShellJobValidationStep, + CARGO_PACKAGE_MAX_ITEMS, CARGO_TEST_MIN_TESTS_MAX, CARGO_VALUE_MAX_BYTES, + GO_TEST_PACKAGE_MAX_BYTES, GO_TEST_PACKAGE_MAX_ITEMS, JOB_INVENTORY_MAX_ACTIVE_JOBS, + JOB_INVENTORY_MAX_JOBS, JOB_INVENTORY_MAX_SERIALIZED_BYTES, JOB_INVENTORY_MAX_TERMINAL_JOBS, JOB_SNAPSHOT_STREAM_MAX_BYTES, JOB_TERMINAL_RETENTION_SECS, RUNNER_JOB_CONCURRENCY_MAX, RUNNER_JOB_CONCURRENCY_MIN, RUST_TEST_FILTER_MAX_BYTES, VALIDATION_ASSERTION_NAME_MAX_CHARS, }; @@ -695,6 +695,12 @@ runner_capabilities! { #[serde(default, skip_serializing_if = "is_false")] pub project_validation_node_tap_v1: bool = false; } + /// Node native TAP test-name pattern (separate rolling-upgrade capability). + ProjectValidationNodeNameFilter => RUNNER_CAPABILITY_PROJECT_VALIDATION_NODE_NAME_FILTER("project_validation_node_name_filter_v1"), + v2_baseline = false { + #[serde(default, skip_serializing_if = "is_false")] + pub project_validation_node_name_filter_v1: bool = false; + } /// The Runner understands the first-class model-facing `go_test` tool identity /// and its durable `ShellJobValidationMetadata` contract. This is deliberately /// separate from Go JSON parsing support: older Runners may advertise @@ -2945,6 +2951,7 @@ mod envelope_tests { project_validation_python_ruff_v1: false, project_validation_node_script_check_v1: false, project_validation_node_tap_v1: false, + project_validation_node_name_filter_v1: false, project_validation_rust_clippy_v1: false, project_validation_rust_clippy_offline_v1: false, structured_go_test_tool: true, diff --git a/crates/webcodex-core/src/runner_protocol/job.rs b/crates/webcodex-core/src/runner_protocol/job.rs index 4070197a1..608985ac5 100644 --- a/crates/webcodex-core/src/runner_protocol/job.rs +++ b/crates/webcodex-core/src/runner_protocol/job.rs @@ -238,11 +238,26 @@ impl ShellJobValidationStep { } /// Exact native Node test executor; manifest and Runner capability are separate fences. + /// Accepts the canonical two-arg unfiltered form or the three-arg form with a + /// validated `--test-name-pattern=` appended by the Runner. pub fn is_structured_node_tap_test(&self) -> bool { self.name == "test" && self.program == "node" && self.env.is_empty() - && self.args == ["--test", "--test-reporter=tap"] + && self.args.first().map(String::as_str) == Some("--test") + && self.args.get(1).map(String::as_str) == Some("--test-reporter=tap") + && match self.args.len() { + 2 => true, + 3 => self.args[2] + .strip_prefix("--test-name-pattern=") + .is_some_and(|pattern| { + matches!( + normalize_node_native_test_filter(pattern), + Ok(Some(normalized)) if normalized == pattern + ) + }), + _ => false, + } } /// Fixed local framework CLI and report placeholder, rewritten only by the @@ -663,6 +678,27 @@ pub fn normalize_go_test_filter(raw: &str) -> Result, &'static st Ok((!raw.is_empty()).then(|| raw.to_owned())) } +/// Bounded native Node `--test-name-pattern` JavaScript regexp. Preserve +/// meaningful whitespace; blank-only input means no filter. Empty +/// means no filter. Rejects control characters, option-shaped values (leading +/// `-`), and values exceeding the shared 200-byte bound. The Runner transports +/// this as one literal argv `--test-name-pattern=`, never as shell. +pub fn normalize_node_native_test_filter(raw: &str) -> Result, &'static str> { + if raw.len() > RUST_TEST_FILTER_MAX_BYTES { + return Err("Node test filter exceeds 200 bytes"); + } + if raw.chars().any(char::is_control) { + return Err("Node test filter contains control characters"); + } + if raw.trim().is_empty() { + return Ok(None); + } + if raw.trim_start().starts_with('-') { + return Err("Node test filter must not start with '-'"); + } + Ok(Some(raw.to_string())) +} + fn is_canonical_go_vet_args(args: &[&str]) -> bool { if args.len() < 2 || args[0] != "vet" { return false; @@ -1113,6 +1149,29 @@ impl ShellJobValidationMetadata { ) ) || provenance.request.scope.is_some() || provenance.request.dependency_policy.is_some() + || (self.adapter.starts_with("node:") + && self.adapter.ends_with(":test") + && { + let requested = provenance + .request + .test + .as_ref() + .and_then(|test| test.filter.as_deref()) + .map(normalize_node_native_test_filter) + .transpose(); + match (self.adapter.as_str(), requested) { + ("node:tap:test", Ok(filter)) => { + self.steps[0].args.get(2).and_then(|arg| { + arg.strip_prefix("--test-name-pattern=") + }) != filter.flatten().as_deref() + } + ("node:vitest:test" | "node:jest:test", Ok(Some(Some(_)))) => { + true + } + (_, Err(_)) => true, + _ => false, + } + }) || (provenance.request.action == crate::project_validation::ProjectValidationAction::Lint && self.steps[0].args != ["--run", "lint"]) @@ -1168,10 +1227,7 @@ impl ShellJobValidationMetadata { "node:tap:test" => { self.tool == "project_validate" && self.kind == "test" - && step.name == "test" - && step.program == "node" - && step.args == ["--test", "--test-reporter=tap"] - && step.env.is_empty() + && step.is_structured_node_tap_test() } "node:script:check" => { self.tool == "project_validate" diff --git a/crates/webcodex-core/src/validation_evidence.rs b/crates/webcodex-core/src/validation_evidence.rs index c24729d01..9fe85f08a 100644 --- a/crates/webcodex-core/src/validation_evidence.rs +++ b/crates/webcodex-core/src/validation_evidence.rs @@ -253,6 +253,11 @@ fn node_native_test_summary(stdout: &str) -> Option { // Its N counts top-level assertions, not all nested tests in # tests. let root_count = body.last()?.strip_prefix("1..")?.parse::().ok()?; let mut root_assertions = 0u64; + // Native Node may emit a synthetic passing file assertion after root 1..0. + // Require that exact adjacent named wrapper before excluding its count. + let mut pending_empty_file_wrapper = false; + let mut empty_file_wrapper_name: Option<&str> = None; + let mut synthetic_file_wrappers = 0u64; let fields = [ "# tests ", "# suites ", @@ -274,11 +279,31 @@ fn node_native_test_summary(stdout: &str) -> Option { .strip_prefix("# duration_ms ") .and_then(|value| value.parse::().ok()) .is_some_and(|value| value.is_finite() && value >= 0.0); - if line.starts_with("1..") - || line.starts_with("TAP version ") - || repeated_count - || repeated_duration - { + if line.starts_with("TAP version ") || repeated_count || repeated_duration { + return None; + } + if pending_empty_file_wrapper { + let name = line.strip_prefix("# Subtest: ")?; + if name.is_empty() { + return None; + } + empty_file_wrapper_name = Some(name); + pending_empty_file_wrapper = false; + continue; + } + if let Some(name) = empty_file_wrapper_name.take() { + root_assertions = root_assertions.checked_add(1)?; + if *line != format!("ok {root_assertions} - {name}") { + return None; + } + synthetic_file_wrappers = synthetic_file_wrappers.checked_add(1)?; + continue; + } + if *line == "1..0" { + pending_empty_file_wrapper = true; + continue; + } + if line.starts_with("1..") { return None; } let ordinal = line @@ -291,7 +316,10 @@ fn node_native_test_summary(stdout: &str) -> Option { } } } - if root_assertions != root_count { + if pending_empty_file_wrapper + || empty_file_wrapper_name.is_some() + || root_assertions != root_count + { return None; } // Accounting lines must themselves be top-level (unindented). @@ -319,8 +347,12 @@ fn node_native_test_summary(stdout: &str) -> Option { { return None; } + let real_passed = passed.checked_sub(synthetic_file_wrappers)?; + if synthetic_file_wrappers > 0 && real_passed == 0 && failed == 0 && skipped == 0 && todo == 0 { + return None; + } Some(CargoTestSummary { - passed: Some(passed), + passed: Some(real_passed), failed: Some(failed), ignored: skipped.checked_add(todo), }) diff --git a/crates/webcodex-core/src/validation_evidence_tests.rs b/crates/webcodex-core/src/validation_evidence_tests.rs index 64a20e95c..ba7c42b8f 100644 --- a/crates/webcodex-core/src/validation_evidence_tests.rs +++ b/crates/webcodex-core/src/validation_evidence_tests.rs @@ -842,6 +842,44 @@ fn node_native_tap_counts_only_complete_conservative_reporter_trailers() { ); } +#[test] +fn node_native_tap_filtered_file_wrappers_never_inflate_real_test_count() { + use crate::validation_evidence::parse_node_native_test_diagnostics; + // Native Node 24 emits a synthetic passing file-level assertion when a + // test file has no names matching --test-name-pattern. In a multi-file + // run those assertions must not be counted as executed tests. + let mixed = "TAP version 13\n# Subtest: alpha\nok 1 - alpha\n1..0\n# Subtest: b.test.js\nok 2 - b.test.js\n1..2\n# tests 2\n# suites 0\n# pass 2\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n# duration_ms 87.4\n"; + let summary = parse_node_native_test_diagnostics(mixed, false) + .test_summary + .expect("one matched Node test can be proven despite a nonmatching file"); + assert_eq!( + (summary.passed, summary.failed, summary.ignored), + (Some(1), Some(0), Some(0)) + ); + for forged in [ + mixed.replace("# Subtest: b.test.js", "# Subtest: other.test.js"), + mixed.replace("# Subtest: b.test.js\n", ""), + mixed.replace("1..0\n# Subtest:", "1..0\n# diagnostic: fake\n# Subtest:"), + mixed.replace("ok 2 - b.test.js", "not ok 2 - b.test.js"), + mixed.replace("ok 2 - b.test.js", "ok 2 - b.test.js # SKIP"), + mixed.replace("1..0", "1..0\n1..0"), + ] { + assert!( + parse_node_native_test_diagnostics(&forged, false) + .test_summary + .is_none(), + "malformed file wrapper must not prove any executed tests" + ); + } + let none = "TAP version 13\n1..0\n# Subtest: a.test.js\nok 1 - a.test.js\n1..0\n# Subtest: b.test.js\nok 2 - b.test.js\n1..2\n# tests 2\n# suites 0\n# pass 2\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n# duration_ms 80.0\n"; + assert!( + parse_node_native_test_diagnostics(none, false) + .test_summary + .is_none(), + "zero matching tests must not become a successful test-count receipt" + ); +} + #[test] fn node_native_tap_root_plan_and_footer_are_authoritative() { use crate::validation_evidence::parse_node_native_test_diagnostics; diff --git a/crates/webcodex-core/src/validation_identity.rs b/crates/webcodex-core/src/validation_identity.rs index b5466a426..7b1ee9c89 100644 --- a/crates/webcodex-core/src/validation_identity.rs +++ b/crates/webcodex-core/src/validation_identity.rs @@ -246,9 +246,22 @@ pub fn structured_validation_target_identity( } serde_json::json!({"tool": tool_name, "kind": "check", "cwd": cwd, "script": script, "engine": "node-native-run-v1"}) } - ToolValidationIdentityKind::NodeNativeTest - | ToolValidationIdentityKind::NodeVitestTest - | ToolValidationIdentityKind::NodeJestTest => { + ToolValidationIdentityKind::NodeNativeTest => { + let filter = match obj.get("filter") { + None | Some(Value::Null) => None, + Some(value) => { + crate::runner_protocol::normalize_node_native_test_filter(value.as_str()?) + .ok()? + } + }; + // Keep the existing unfiltered Node target identity unchanged. + let mut semantic = serde_json::json!({"tool":tool_name,"kind":"test","cwd":cwd}); + if let Some(filter) = filter { + semantic["filter"] = Value::String(filter); + } + semantic + } + ToolValidationIdentityKind::NodeVitestTest | ToolValidationIdentityKind::NodeJestTest => { serde_json::json!({"tool":tool_name,"kind":"test","cwd":cwd}) } ToolValidationIdentityKind::PythonPytest => { diff --git a/crates/webcodex-runner-config/src/lib.rs b/crates/webcodex-runner-config/src/lib.rs index d87365435..3e1ec06e5 100644 --- a/crates/webcodex-runner-config/src/lib.rs +++ b/crates/webcodex-runner-config/src/lib.rs @@ -308,6 +308,7 @@ pub fn generated_runner_config_toml(opts: &RunnerInitOptions) -> Result RunnerCapabili capabilities.set(RunnerCapabilityId::ProjectValidationPythonRuff, true); capabilities.set(RunnerCapabilityId::ProjectValidationNodeScriptCheck, true); capabilities.set(RunnerCapabilityId::ProjectValidationNodeTap, true); + capabilities.set(RunnerCapabilityId::ProjectValidationNodeNameFilter, true); capabilities.set(RunnerCapabilityId::ProjectValidationRustClippy, true); capabilities.set(RunnerCapabilityId::ProjectValidationRustClippyOffline, true); // This binary also understands the first-class go_test durable metadata diff --git a/crates/webcodex-runner/src/webcodex_runner/validation/project.rs b/crates/webcodex-runner/src/webcodex_runner/validation/project.rs index ea0740924..e303612c5 100644 --- a/crates/webcodex-runner/src/webcodex_runner/validation/project.rs +++ b/crates/webcodex-runner/src/webcodex_runner/validation/project.rs @@ -8,7 +8,8 @@ use webcodex_core::validation_identity::{ }; use webcodex_validation::{ detect_validation_recipe, resolve_node_native_project_check, resolve_node_native_project_lint, - resolve_node_native_project_test, validate_node_project_markers, RecipeId, SemanticCheck, + resolve_node_native_project_test_with_filter, validate_node_project_markers, RecipeId, + SemanticCheck, }; pub(crate) fn plan( @@ -105,7 +106,7 @@ pub(crate) fn plan( if request.action == ProjectValidationAction::Lint { resolve_node_native_project_lint(&root, request.cwd.as_deref()) } else if action == SemanticCheck::Test { - resolve_node_native_project_test(&root, request.cwd.as_deref()) + resolve_node_native_project_test_with_filter(&root, request.cwd.as_deref(), filter) } else { resolve_node_native_project_check(&root, request.cwd.as_deref()) } diff --git a/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs b/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs index de760b09b..368ba6515 100644 --- a/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs +++ b/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs @@ -1160,7 +1160,44 @@ fn project_validation_node_native_test_has_exact_plan_and_stale_source_fence() { filter: Some("selected".into()), ..Default::default() }); - assert!(project::plan(&policy, ®istry, &filtered).is_err()); + let selected = project::plan(&policy, ®istry, &filtered) + .expect("bounded native Node test-name selection must be planned") + .0; + assert_eq!( + selected.step.args, + [ + "--test", + "--test-reporter=tap", + "--test-name-pattern=selected" + ] + ); + assert!(selected.step.is_structured_node_tap_test()); + assert!(selected.provenance.is_valid()); + assert_ne!(selected.validation_target_id, plan.validation_target_id); + assert_ne!( + selected.provenance.invocation_digest, + plan.provenance.invocation_digest + ); + let mut explicit = filtered.clone(); + explicit.adapter = ProjectValidationAdapter::Node; + let explicit_plan = project::plan(&policy, ®istry, &explicit).unwrap().0; + assert_eq!(selected.step, explicit_plan.step); + assert_eq!( + selected.validation_target_id, + explicit_plan.validation_target_id + ); + // A name filter is native Node-only, never a Jest/Vitest/opaque script arg. + for body in ["jest", "vitest run", "node --test && echo other"] { + fs::write( + root.join("package.json"), + format!(r#"{{"scripts":{{"test":"{body}"}}}}"#), + ) + .unwrap(); + assert!( + project::plan(&policy, ®istry, &filtered).is_err(), + "script {body} must not inherit native Node filtering" + ); + } } #[test] diff --git a/crates/webcodex-tool-contracts/src/tests/input_schemas.rs b/crates/webcodex-tool-contracts/src/tests/input_schemas.rs index 0e36f9f75..4a287db5c 100644 --- a/crates/webcodex-tool-contracts/src/tests/input_schemas.rs +++ b/crates/webcodex-tool-contracts/src/tests/input_schemas.rs @@ -1561,7 +1561,10 @@ fn project_validation_node_contract_advertises_bounded_test_and_execution_effect "Node TAP test only for exact scripts.test=node --test", "fixed installed CLI", "project_validation_node_json_v1", - "not for Node", + "Node native TAP only", + "--test-name-pattern=", + "project_validation_node_name_filter_v1", + "not for Vitest/Jest or custom scripts", "Project scripts may modify files or access network", "No arbitrary executable, argv or shell", ] { @@ -1586,6 +1589,11 @@ fn project_validation_node_contract_advertises_bounded_test_and_execution_effect "test":{"require_tests":false, "min_tests":2}}); assert!(test_support::validate_schema_instance(&with_count, schema).is_ok()); assert!(ToolCall::from_tool_name("project_validate", with_count).is_ok()); + let with_native_name_filter = json!({"project":"demo", "adapter":"node", "action":"test", + "test":{"filter":"selected.*", "min_tests":1}}); + assert!(test_support::validate_schema_instance(&with_native_name_filter, schema).is_ok()); + assert!(ToolCall::from_tool_name("project_validate", with_native_name_filter).is_ok()); + for (field, value) in [ ("executable", json!("node")), ("argv", json!(["--test"])), diff --git a/crates/webcodex-tool-contracts/src/tool_definition/testing.rs b/crates/webcodex-tool-contracts/src/tool_definition/testing.rs index 16ccad0c5..d2fff02d7 100644 --- a/crates/webcodex-tool-contracts/src/tool_definition/testing.rs +++ b/crates/webcodex-tool-contracts/src/tool_definition/testing.rs @@ -131,7 +131,7 @@ pub(super) const DEFINITIONS: &[ToolDefinition] = &[ ) .with_composition_policy(super::ToolCompositionPolicy::Sequential) .with_host_orchestration_hint(super::ToolHostOrchestrationHint::sequential()), - "Project validation. Project scripts may modify files or access network. Rust: format_check/check/lint/test; Go: check/test. Rust lint: cargo clippy --all-targets -- -D clippy::correctness; cargo_clippy evidence, project_validation_rust_clippy_v1. Clippy missing: no install/fallback. Node >=22.3: check>typecheck>lint; lint selects only scripts.lint (check identity); node --run without pre/post. Node TAP test only for exact scripts.test=node --test or node --test --test-reporter=tap. Vitest/Jest: fixed installed CLI, project_validation_node_json_v1 JSON counts; other scripts unproven. Scope: packages 1..8/all_packages=true; Rust Cargo workspace proven, Go single module. Rust exclude_packages (all_packages): 1..8 exact Cargo members; unknown/all excluded reject; project_workspace_exclusions_v1 planning/admission; not Go/Node/Python/format. test.filter: Rust substring, Go -run, pytest -k; not for Node. require_tests/min_tests need proven counts. Python/Ruff preinstalled; Ruff needs pyproject.toml [tool.ruff] target-version, no extend/fixes/cache/install. locked: Cargo --locked/Go -mod=readonly; not Python/Node/format. dependency_resolution_policy=offline: Rust check/lint/test Cargo --offline; Clippy requires project_validation_rust_clippy_offline_v1; cache miss fails, no online fallback. build scripts/tests may use network; not OS sandbox. Go GO111MODULE=on/GOWORK=off. No arbitrary executable, argv or shell. Same Runner Job; recheck provenance at queue; observe pending, never retry.", ).with_discovery_keywords(&["project", "validation", "build", "tests", "lint", "verify", "项目", "验证", "构建", "测试", "检查"]) + "Project validation. Project scripts may modify files or access network. Rust: format_check/check/lint/test; Go: check/test. Rust lint: cargo clippy --all-targets -- -D clippy::correctness; needs project_validation_rust_clippy_v1; missing Clippy fails. Node >=22.3: check>typecheck>lint; lint selects only scripts.lint; node --run without pre/post. Node TAP test only for exact scripts.test=node --test or node --test --test-reporter=tap. Vitest/Jest: fixed installed CLI, project_validation_node_json_v1 JSON counts; other scripts unproven. Scope: packages 1..8 or all_packages=true; Rust workspace proven, Go single module. Rust exclude_packages: 1..8 exact members (all_packages only); unknown/all excluded reject; project_workspace_exclusions_v1 at planning/admission; not Go/Node/Python/format. test.filter: Rust substring, Go -run, pytest -k; Node native TAP only: bounded regex in one Runner-owned --test-name-pattern= argv; project_validation_node_name_filter_v1 required; not for Vitest/Jest or custom scripts. require_tests/min_tests need proven counts. Python pytest/Ruff preinstalled; Ruff needs [tool.ruff] target-version; no fixes/install. locked: Cargo --locked/Go -mod=readonly only. dependency_resolution_policy=offline: Rust Cargo --offline; Clippy needs project_validation_rust_clippy_offline_v1; cache miss fails. build scripts/tests may use network; not OS sandbox. Go GO111MODULE=on/GOWORK=off. No arbitrary executable, argv or shell. Same Runner Job; recheck provenance at queue; observe pending, never retry.", ).with_discovery_keywords(&["project", "validation", "build", "tests", "lint", "verify", "项目", "验证", "构建", "测试", "检查"]) .with_execution(super::ToolExecutionContract::new( super::ToolExecutionForm::StructuredValidation, super::ToolExecutionLifetime::Runner, diff --git a/crates/webcodex-validation/src/adapters/mod.rs b/crates/webcodex-validation/src/adapters/mod.rs index 21b994b4d..6cdd8d1cf 100644 --- a/crates/webcodex-validation/src/adapters/mod.rs +++ b/crates/webcodex-validation/src/adapters/mod.rs @@ -120,7 +120,9 @@ pub enum ReadOnlyValidationOperation { PythonRuffCheck, PythonRuffFormat, NodeScriptCheck, - NodeNativeTest, + /// Native Node built-in test runner. Carries an optional bounded JS regexp + /// filter that is appended as `--test-name-pattern=`. + NodeNativeTest(Option), } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -149,6 +151,11 @@ impl ReadOnlyValidationOperation { options.filter = webcodex_core::runner_protocol::normalize_pytest_filter(filter) .map_err(|_| "test_filter_unsupported")?; } + Self::NodeNativeTest(stored_filter) => { + *stored_filter = + webcodex_core::runner_protocol::normalize_node_native_test_filter(filter) + .map_err(|_| "test_filter_unsupported")?; + } _ => return Err("test_filter_unsupported"), } Ok(self) @@ -166,7 +173,7 @@ impl ReadOnlyValidationOperation { | Self::PythonRuffCheck | Self::PythonRuffFormat | Self::NodeScriptCheck - | Self::NodeNativeTest + | Self::NodeNativeTest(_) | Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => { return Err("dependency_policy_unsupported") } @@ -248,7 +255,7 @@ impl ReadOnlyValidationOperation { | Self::PythonRuffCheck | Self::PythonRuffFormat | Self::NodeScriptCheck - | Self::NodeNativeTest + | Self::NodeNativeTest(_) | Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => None, Self::Cargo(CargoReadOnlyValidationOperation::Check(options)) | Self::Cargo(CargoReadOnlyValidationOperation::Clippy(options)) => { @@ -273,14 +280,14 @@ impl ReadOnlyValidationOperation { | Self::PythonRuffCheck | Self::PythonRuffFormat | Self::NodeScriptCheck - | Self::NodeNativeTest + | Self::NodeNativeTest(_) | Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => false, } } pub fn compatibility_profile(&self) -> ValidationCompatibilityProfile { match self { - Self::NodeNativeTest => ValidationCompatibilityProfile { + Self::NodeNativeTest(_) => ValidationCompatibilityProfile { tool_identity: "node:tap:test", validation_identity: ToolValidationIdentityKind::NodeNativeTest, }, @@ -343,7 +350,7 @@ impl ReadOnlyValidationOperation { pub fn build_readonly_plan(&self) -> Result { let adapter = validation_adapter_for_tool(self.compatibility_profile().tool_identity) .ok_or_else(|| match self { - Self::NodeScriptCheck | Self::NodeNativeTest => { + Self::NodeScriptCheck | Self::NodeNativeTest(_) => { "Node requires Runner-owned package manifest resolution" } _ => "Ruff requires Runner-owned project manifest resolution", @@ -352,7 +359,7 @@ impl ReadOnlyValidationOperation { Self::PythonRuffCheck | Self::PythonRuffFormat | Self::NodeScriptCheck - | Self::NodeNativeTest => { + | Self::NodeNativeTest(_) => { unreachable!("project-local adapter has no static command") } Self::Python(options) => adapter.build_readonly_plan(ValidationCommandOptions { @@ -385,12 +392,16 @@ impl ReadOnlyValidationOperation { pub fn validation_target_id(&self, cwd: Option<&str>) -> Option { let profile = self.compatibility_profile(); let arguments = match self { - Self::PythonRuffCheck - | Self::PythonRuffFormat - | Self::NodeScriptCheck - | Self::NodeNativeTest => { + Self::PythonRuffCheck | Self::PythonRuffFormat | Self::NodeScriptCheck => { serde_json::json!({"cwd":cwd}) } + Self::NodeNativeTest(filter) => { + let mut arguments = serde_json::json!({"cwd": cwd}); + if let Some(filter) = filter { + arguments["filter"] = serde_json::json!(filter); + } + arguments + } Self::Python(options) => serde_json::json!({"cwd":cwd,"filter":options.filter}), Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => serde_json::json!({ "cwd": cwd, @@ -531,7 +542,7 @@ pub fn project_validation_operation( ("python", Format) => Ok(ReadOnlyValidationOperation::PythonRuffFormat), ("node", _) if packages.is_some() || all_packages => Err("validation_scope_unsupported"), ("node", Check) => Ok(ReadOnlyValidationOperation::NodeScriptCheck), - ("node", Test) => Ok(ReadOnlyValidationOperation::NodeNativeTest), + ("node", Test) => Ok(ReadOnlyValidationOperation::NodeNativeTest(None)), ("node", _) => Err("validation_action_unsupported"), _ => Err("validation_adapter_unavailable"), } diff --git a/crates/webcodex-validation/src/lib.rs b/crates/webcodex-validation/src/lib.rs index bcb004431..72358a4bb 100644 --- a/crates/webcodex-validation/src/lib.rs +++ b/crates/webcodex-validation/src/lib.rs @@ -40,8 +40,9 @@ pub use evidence::{ }; pub use recipe::{ detect_validation_recipe, resolve_node_native_project_check, resolve_node_native_project_lint, - resolve_node_native_project_test, resolve_node_project_build, - resolve_project_validation_recipe, resolve_project_validation_recipe_with_resolution_policy, + resolve_node_native_project_test, resolve_node_native_project_test_with_filter, + resolve_node_project_build, resolve_project_validation_recipe, + resolve_project_validation_recipe_with_resolution_policy, resolve_project_validation_recipe_with_scope_exclusions, resolve_validation_recipe, resolve_validation_recipe_with_packages, resolve_validation_recipe_with_project_policy, validate_node_project_markers, RecipeError, RecipeId, ResolvedValidationRecipe, SemanticCheck, diff --git a/crates/webcodex-validation/src/recipe.rs b/crates/webcodex-validation/src/recipe.rs index a8c06a68e..6899cb3cf 100644 --- a/crates/webcodex-validation/src/recipe.rs +++ b/crates/webcodex-validation/src/recipe.rs @@ -733,14 +733,62 @@ pub fn resolve_node_native_project_test( resolve_node_native_project_action(execution_root, cwd, SemanticCheck::Test, None) } +/// Like [`resolve_node_native_project_test`] but with an optional bounded +/// `--test-name-pattern` filter. The filter is normalized and appended as a +/// single literal argv; empty/None preserves the old two-arg canonical step. +pub fn resolve_node_native_project_test_with_filter( + execution_root: &Path, + cwd: Option<&str>, + filter: Option<&str>, +) -> Result { + resolve_node_native_project_action_with_filter( + execution_root, + cwd, + SemanticCheck::Test, + None, + filter, + ) +} + fn resolve_node_native_project_action( execution_root: &Path, cwd: Option<&str>, action: SemanticCheck, selected_script: Option<&'static str>, +) -> Result { + resolve_node_native_project_action_with_filter( + execution_root, + cwd, + action, + selected_script, + None, + ) +} + +fn resolve_node_native_project_action_with_filter( + execution_root: &Path, + cwd: Option<&str>, + action: SemanticCheck, + selected_script: Option<&'static str>, + test_filter: Option<&str>, ) -> Result { const MAX_NODE_MANIFEST_BYTES: u64 = 1024 * 1024; + // Normalize and validate the filter before any I/O so invalid filters fail fast. + let normalized_filter = if action == SemanticCheck::Test { + test_filter + .map(webcodex_core::runner_protocol::normalize_node_native_test_filter) + .transpose() + .map_err(|_| filter_unsupported())? + .flatten() + } else { + // Filters are only valid for Test; check/lint must reject them. + if test_filter.is_some() { + return Err(filter_unsupported()); + } + None + }; + validate_node_project_markers(execution_root, cwd)?; let resolved = resolve_project_recipe_root(execution_root, cwd, Some(RecipeId::Node)) .map_err(map_project_recipe_error)?; @@ -771,7 +819,7 @@ fn resolve_node_native_project_action( { return Err(manifest_invalid()); } - let step = if action == SemanticCheck::Test { + let mut step = if action == SemanticCheck::Test { let body = scripts .and_then(|scripts| scripts.get(selected)) .and_then(Value::as_str) @@ -795,6 +843,17 @@ fn resolve_node_native_project_action( env: Vec::new(), } }; + // Append the filter as a single literal argv for native Node test runs. + // vitest/jest framework steps do not support this filter path. + if let Some(ref filter) = normalized_filter { + if step.is_structured_node_tap_test() { + step.args.push(format!("--test-name-pattern={filter}")); + } else { + // Filter was requested but this script uses a framework that + // cannot accept it (vitest/jest). + return Err(filter_unsupported()); + } + } debug_assert!(step.is_canonical()); let mut hasher = Sha256::new(); let relative = canonical @@ -826,7 +885,7 @@ fn resolve_node_native_project_action( steps: vec![step], invocation_digest, manifest_digest, - test_filter: None, + test_filter: normalized_filter, }) } @@ -984,6 +1043,8 @@ fn normalize_test_filter( .map(Option::flatten) .map_err(|_| filter_unsupported()), _ => { + // Generic Node scripts cannot consume test-name selection. The + // native TAP resolver has a distinct, canonical filtered path. reject_filter(filter)?; Ok(None) } diff --git a/docs/MCP.md b/docs/MCP.md index 4432a901b..75bd53836 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -584,7 +584,7 @@ work_on_project For intentional Go source formatting, `project_format` accepts `adapter="go"` or `auto` with 1–8 explicit `.go` paths under one nearest unambiguous `go.mod` recipe. It requires `project:write`, `job:run`, `project_format_v1` and the additive `project_format_go_v1` Runner capability. Installed `gofmt` reads private source bytes on stdin with no arguments and returns candidates on stdout; the Runner owns guarded writes and durable `state_changed` evidence. No workspace traversal, `-w`/rewrite/simplification flags, goimports, module loading, installs or automatic retries are introduced. `go.mod` is a bounded UTF-8 scope/identity witness, not parsed module semantics; native stdin fragments are accepted and formatting is not compilation. Missing tooling fails closed. Pending or uncertain work must be observed by its original Job, not redispatched. See [bounded project formatting](implementation/project-format-lifecycle.md). -For ordinary bounded project validation, prefer `project_validate`; project-authored scripts and tests may modify files or use network. It accepts only a closed `format_check` / `check` / `lint` / `test` intent plus an optional `auto` / `rust` / `go` / `python` / `node` adapter hint; the Runner resolves the nearest unambiguous recipe on its own registered filesystem and then starts the existing structured validation Job. Rust maps to `cargo fmt -- --check`, `cargo check --all-targets`, `cargo clippy --all-targets -- -D clippy::correctness`, or `cargo test`; Go maps to `go vet ./...` or `go test -json ./...`. Go project validation runs in Runner-owned single-module mode with `GO111MODULE=on` and `GOWORK=off`, so ambient module mode or parent `go.work` selection cannot silently change the gateway's workspace semantics. Its validation target identity is domain-separated from ambient Go specialist evidence, so success under different workspace semantics cannot reconcile a gateway failure. The standalone `go_test` specialist keeps its existing environment behavior. Optional `scope` selects exactly one portable package intent: bounded `packages` (1..8 entries) narrows Rust check/lint/test through repeated Cargo `-p` selectors and Go check/test through project-relative package patterns, while `all_packages=true` selects the complete project unit. Rust all-packages may add `exclude_packages` with 1–8 exact Cargo workspace member names; unknown names and excluding every member fail before execution. Rust build and check/lint/test support exclusions, while Go/Node/Python and `format_check` reject them. The additive `project_workspace_exclusions_v1` capability gates planning and typed Job admission. Rust all-packages maps to Cargo `--workspace` only when the Runner proves the effective Cargo workspace root is exactly the registered Project root and binds the in-Project Cargo manifest graph into the existing re-plan fence; Go all-packages keeps the canonical `./...` single-module scope. Scoped formatting fails closed. Python uses existing configured/profile/PATH Python 3 for pytest test (`python -m pytest --color=no -rA`) and project-local Ruff check/format_check (explicit target-version, no fixes, cache or bytecode). Python scope and dependency policy fail closed. Node supports Runner-owned native script-based `check` through `node --run ` (see #995), plus a separate, opt-in native TAP `test` profile when `package.json` declares exactly `node --test` (or `node --test --test-reporter=tap`). The native test executes fixed Runner argv, parses only complete TAP v13 summaries, and requires the distinct `project_validation_node_tap_v1` capability. Direct `scripts.test` values `vitest`, `vitest run`, and `jest` also support native tests through the installed local framework CLI. The Runner adds fixed official JSON reporter flags and a fresh private per-Job output file, bounded to 1 MiB. Only a complete consistent file supplies executed counts; stdout/stderr never do. Reports are removed after aggregate evidence is retained. Missing/oversized/invalid reports, timeout and cancellation cannot prove counts. JSON admission requires `project_validation_node_json_v1`; compound/custom scripts return `structured_test_report_unavailable` and remain usable through ordinary recipes. Node scope and filters remain unsupported. See [Node native TAP validation](implementation/node-native-tap-project-validation.md). Python test planning and Job admission require `project_validation_python_pytest_v1`, while Ruff check/format_check require the separate `project_validation_python_ruff_v1` capability. Missing selected tooling is a definite not-started failure with no automatic installation or fallback. See [Python/pytest validation](implementation/python-pytest-project-validation.md) for environment selection, evidence and same-Job behavior. The request never carries arbitrary executable, argv, shell grammar, installation or an explicit source-mutation intent; project-authored scripts and tests can still cause effects. Existing `cargo_*` / `go_test` tools remain available for ecosystem-specific advanced options. `project_validate` requires the additive `project_validation_v1` Runner capability; explicit `packages` additionally require `project_validation_package_scope_v1`, while `all_packages=true` requires `project_all_packages_v1`; exclusions additionally require `project_workspace_exclusions_v1` at planning and Job admission. Go project-validation Job admission additionally requires `project_go_single_module_v1`, so a Server cannot hand a Go gateway plan to an older Runner that may inherit ambient workspace state. +For ordinary bounded project validation, prefer `project_validate`; project-authored scripts and tests may modify files or use network. It accepts only a closed `format_check` / `check` / `lint` / `test` intent plus an optional `auto` / `rust` / `go` / `python` / `node` adapter hint; the Runner resolves the nearest unambiguous recipe on its own registered filesystem and then starts the existing structured validation Job. Rust maps to `cargo fmt -- --check`, `cargo check --all-targets`, `cargo clippy --all-targets -- -D clippy::correctness`, or `cargo test`; Go maps to `go vet ./...` or `go test -json ./...`. Go project validation runs in Runner-owned single-module mode with `GO111MODULE=on` and `GOWORK=off`, so ambient module mode or parent `go.work` selection cannot silently change the gateway's workspace semantics. Its validation target identity is domain-separated from ambient Go specialist evidence, so success under different workspace semantics cannot reconcile a gateway failure. The standalone `go_test` specialist keeps its existing environment behavior. Optional `scope` selects exactly one portable package intent: bounded `packages` (1..8 entries) narrows Rust check/lint/test through repeated Cargo `-p` selectors and Go check/test through project-relative package patterns, while `all_packages=true` selects the complete project unit. Rust all-packages may add `exclude_packages` with 1–8 exact Cargo workspace member names; unknown names and excluding every member fail before execution. Rust build and check/lint/test support exclusions, while Go/Node/Python and `format_check` reject them. The additive `project_workspace_exclusions_v1` capability gates planning and typed Job admission. Rust all-packages maps to Cargo `--workspace` only when the Runner proves the effective Cargo workspace root is exactly the registered Project root and binds the in-Project Cargo manifest graph into the existing re-plan fence; Go all-packages keeps the canonical `./...` single-module scope. Scoped formatting fails closed. Python uses existing configured/profile/PATH Python 3 for pytest test (`python -m pytest --color=no -rA`) and project-local Ruff check/format_check (explicit target-version, no fixes, cache or bytecode). Python scope and dependency policy fail closed. Node supports Runner-owned native script-based `check` through `node --run ` (see #995), plus a separate, opt-in native TAP `test` profile when `package.json` declares exactly `node --test` (or `node --test --test-reporter=tap`). The native test executes fixed Runner argv, parses only complete TAP v13 summaries, and requires the distinct `project_validation_node_tap_v1` capability. Direct `scripts.test` values `vitest`, `vitest run`, and `jest` also support native tests through the installed local framework CLI. The Runner adds fixed official JSON reporter flags and a fresh private per-Job output file, bounded to 1 MiB. Only a complete consistent file supplies executed counts; stdout/stderr never do. Reports are removed after aggregate evidence is retained. Missing/oversized/invalid reports, timeout and cancellation cannot prove counts. JSON admission requires `project_validation_node_json_v1`; compound/custom scripts return `structured_test_report_unavailable` and remain usable through ordinary recipes. Node workspace/package scopes and general test framework filters remain unsupported. The native TAP adapter alone accepts bounded `test.filter` (one JS regular-expression source, at most 200 UTF-8 bytes, no control characters), which the Runner passes as one literal `--test-name-pattern=` argument. The separate `project_validation_node_name_filter_v1` capability gates planning and typed Job admission. Node may produce synthetic passing file assertions when a test-name filter matches nothing; these never prove real executed tests or satisfy `min_tests`. The unfiltered argv and validation target identity remain unchanged. See [Node native TAP validation](implementation/node-native-tap-project-validation.md). Python test planning and Job admission require `project_validation_python_pytest_v1`, while Ruff check/format_check require the separate `project_validation_python_ruff_v1` capability. Missing selected tooling is a definite not-started failure with no automatic installation or fallback. See [Python/pytest validation](implementation/python-pytest-project-validation.md) for environment selection, evidence and same-Job behavior. The request never carries arbitrary executable, argv, shell grammar, installation or an explicit source-mutation intent; project-authored scripts and tests can still cause effects. Existing `cargo_*` / `go_test` tools remain available for ecosystem-specific advanced options. `project_validate` requires the additive `project_validation_v1` Runner capability; explicit `packages` additionally require `project_validation_package_scope_v1`, while `all_packages=true` requires `project_all_packages_v1`; exclusions additionally require `project_workspace_exclusions_v1` at planning and Job admission. Go project-validation Job admission additionally requires `project_go_single_module_v1`, so a Server cannot hand a Go gateway plan to an older Runner that may inherit ambient workspace state. An empty `exclude_packages: []` is equivalent to omission, including with explicit `packages`, and requires no exclusion capability. Nonempty exclusions affect top-level Cargo workspace selection; an excluded member may still build as a dependency. They are not an execution-permission or dependency-graph filter. diff --git a/docs/MCP.zh-CN.md b/docs/MCP.zh-CN.md index 21a9c7459..33b966e71 100644 --- a/docs/MCP.zh-CN.md +++ b/docs/MCP.zh-CN.md @@ -456,7 +456,7 @@ work_on_project 需要有意修改 Go 源码格式时，`project_format` 接受 `adapter="go"`，或配合显式 `.go` 路径使用 `auto`；一次选择同一最近且无歧义 `go.mod` recipe 下的 1–8 个文件。需要 `project:write`、`job:run`、`project_format_v1` 及新增的 `project_format_go_v1` Runner capability。已安装的 `gofmt` 使用固定空参数，从 stdin 读取私有源内容、向 stdout 输出候选；受保护写入和持久 `state_changed` 证据仍由 Runner 管理。不增加工作区遍历、`-w`/重写/简化参数、goimports、模块加载、安装或自动重试。`go.mod` 仅作有界 UTF-8 范围与文件身份凭据，不解析模块语义；保留 gofmt stdin 接受代码片段的原生行为，格式化不等于编译验证。工具缺失时明确失败。pending 或不确定结果须观察原 Job，不得重新派发。详见[有界项目格式化](implementation/project-format-lifecycle.md)。 -普通的有界 Project validation 优先使用 `project_validate`；项目自定义脚本与测试仍可能修改文件或访问网络。它只接受封闭的 `format_check` / `check` / `lint` / `test` intent，以及可选的 `auto` / `rust` / `go` / `python` / `node` adapter hint；Runner 在自己注册的真实文件系统上解析最近且无歧义的 recipe，然后进入现有 structured validation Job。Rust 分别映射到 `cargo fmt -- --check`、`cargo check --all-targets`、`cargo clippy --all-targets -- -D clippy::correctness`、`cargo test`；Go 映射到 `go vet ./...` 或 `go test -json ./...`。Go project validation 由 Runner 固定为 single-module 模式（`GO111MODULE=on`、`GOWORK=off`），因此 ambient module mode 或父目录 `go.work` 选择不会静默改变 gateway 的 workspace 语义；其 validation target identity 与 ambient Go specialist evidence 做 domain separation，因此不同 workspace 语义下产生的成功不会消解 gateway failure。独立的 `go_test` specialist 保持现有环境语义。可选 `scope` 只允许二选一的 portable package intent：有界 `packages`（1..8 项）把 Rust check/lint/test 映射为重复 Cargo `-p` selector，把 Go check/test 映射为 project-relative package pattern；`all_packages=true` 则选择完整 project unit。Rust all-packages 可附加 `exclude_packages`（1..8 个精确 Cargo workspace member 名称）；未知名称或排除全部成员会在执行前拒绝。Rust build 与 check/lint/test 支持排除；Go/Node/Python 和 `format_check` 明确拒绝。新增能力 `project_workspace_exclusions_v1` 同时门控 planning 与 typed Job admission。Rust all-packages 只有在 Runner 证明 effective Cargo workspace root 与 registered Project root 完全一致后才映射为 Cargo `--workspace`，并把 Project 内 Cargo manifest graph 绑定到既有 re-plan fence；Go all-packages 保持 canonical `./...` single-module scope。带 scope 的格式检查会 fail closed。Python 使用 configured/profile/PATH 中已有的 Python 3 执行 pytest test（`python -m pytest --color=no -rA`），也支持项目本地 Ruff check/format_check（显式 target-version、禁用自动修复、缓存与字节码）；Python 的 scope 与 dependency policy 均 fail closed。Node 已支持 Runner 管理的脚本式 `check`（#995），并在 `package.json` 的 `scripts.test` 精确声明 `node --test`（或 `node --test --test-reporter=tap`）时支持独立的原生 TAP `test`。Node 测试由 Runner 使用固定 argv 直接启动，仅完整 TAP v13 汇总可证明测试数量，且需要独立能力 `project_validation_node_tap_v1`。复合/自定义 test 脚本、scope 和 filter 仍不支持；下面说明独立的原生 Vitest/Jest JSON profile。详见 [Node 原生 TAP 验证](implementation/node-native-tap-project-validation.md)。Python test 的 planning 与 Job 准入要求 `project_validation_python_pytest_v1`，Ruff check/format_check 则独立要求 `project_validation_python_ruff_v1`；所选工具缺失时明确 not-started，不自动安装或 fallback。环境、证据与同 Job 行为见 [Python/pytest validation](implementation/python-pytest-project-validation.md)。请求不会携带 arbitrary executable、argv、shell grammar、安装动作或显式修改源文件的意图；项目脚本和测试仍可能产生副作用；需要 ecosystem-specific 高级参数时继续使用现有 `cargo_*` / `go_test`。`project_validate` 依赖 additive `project_validation_v1` Runner capability；显式 `packages` 额外要求 `project_validation_package_scope_v1`，而 `all_packages=true` 要求 `project_all_packages_v1`；Go project-validation Job 准入还额外要求 `project_go_single_module_v1`，因此 Server 不会把 Go gateway plan 交给仍可能继承 ambient workspace 状态的旧 Runner。 +普通的有界 Project validation 优先使用 `project_validate`；项目自定义脚本与测试仍可能修改文件或访问网络。它只接受封闭的 `format_check` / `check` / `lint` / `test` intent，以及可选的 `auto` / `rust` / `go` / `python` / `node` adapter hint；Runner 在自己注册的真实文件系统上解析最近且无歧义的 recipe，然后进入现有 structured validation Job。Rust 分别映射到 `cargo fmt -- --check`、`cargo check --all-targets`、`cargo clippy --all-targets -- -D clippy::correctness`、`cargo test`；Go 映射到 `go vet ./...` 或 `go test -json ./...`。Go project validation 由 Runner 固定为 single-module 模式（`GO111MODULE=on`、`GOWORK=off`），因此 ambient module mode 或父目录 `go.work` 选择不会静默改变 gateway 的 workspace 语义；其 validation target identity 与 ambient Go specialist evidence 做 domain separation，因此不同 workspace 语义下产生的成功不会消解 gateway failure。独立的 `go_test` specialist 保持现有环境语义。可选 `scope` 只允许二选一的 portable package intent：有界 `packages`（1..8 项）把 Rust check/lint/test 映射为重复 Cargo `-p` selector，把 Go check/test 映射为 project-relative package pattern；`all_packages=true` 则选择完整 project unit。Rust all-packages 可附加 `exclude_packages`（1..8 个精确 Cargo workspace member 名称）；未知名称或排除全部成员会在执行前拒绝。Rust build 与 check/lint/test 支持排除；Go/Node/Python 和 `format_check` 明确拒绝。新增能力 `project_workspace_exclusions_v1` 同时门控 planning 与 typed Job admission。Rust all-packages 只有在 Runner 证明 effective Cargo workspace root 与 registered Project root 完全一致后才映射为 Cargo `--workspace`，并把 Project 内 Cargo manifest graph 绑定到既有 re-plan fence；Go all-packages 保持 canonical `./...` single-module scope。带 scope 的格式检查会 fail closed。Python 使用 configured/profile/PATH 中已有的 Python 3 执行 pytest test（`python -m pytest --color=no -rA`），也支持项目本地 Ruff check/format_check（显式 target-version、禁用自动修复、缓存与字节码）；Python 的 scope 与 dependency policy 均 fail closed。Node 已支持 Runner 管理的脚本式 `check`（#995），并在 `package.json` 的 `scripts.test` 精确声明 `node --test`（或 `node --test --test-reporter=tap`）时支持独立的原生 TAP `test`。Node 测试由 Runner 使用固定 argv 直接启动，仅完整 TAP v13 汇总可证明测试数量，且需要独立能力 `project_validation_node_tap_v1`。复合/自定义 test 脚本和 scope 仍不支持；仅原生 Node TAP 允许有界 `test.filter` 正则表达式，由 Runner 固定追加单个字面 argv `--test-name-pattern=`，并由独立能力 `project_validation_node_name_filter_v1` 在规划和 Job 准入时门控。Node 在过滤后零匹配时仍可能生成文件级虚假通过记录，不能据此证明实际执行或满足 `min_tests`；无过滤时保留旧 argv 和目标身份；下面说明独立的原生 Vitest/Jest JSON profile。详见 [Node 原生 TAP 验证](implementation/node-native-tap-project-validation.md)。Python test 的 planning 与 Job 准入要求 `project_validation_python_pytest_v1`，Ruff check/format_check 则独立要求 `project_validation_python_ruff_v1`；所选工具缺失时明确 not-started，不自动安装或 fallback。环境、证据与同 Job 行为见 [Python/pytest validation](implementation/python-pytest-project-validation.md)。请求不会携带 arbitrary executable、argv、shell grammar、安装动作或显式修改源文件的意图；项目脚本和测试仍可能产生副作用；需要 ecosystem-specific 高级参数时继续使用现有 `cargo_*` / `go_test`。`project_validate` 依赖 additive `project_validation_v1` Runner capability；显式 `packages` 额外要求 `project_validation_package_scope_v1`，而 `all_packages=true` 要求 `project_all_packages_v1`；Go project-validation Job 准入还额外要求 `project_go_single_module_v1`，因此 Server 不会把 Go gateway plan 交给仍可能继承 ambient workspace 状态的旧 Runner。 空 `exclude_packages: []` 等价于省略，配合显式 `packages` 时也成立，且不要求排除能力。非空排除仅影响 Cargo workspace 顶层选择；被排除的成员仍可能作为依赖构建。这不是执行权限或依赖图过滤。 @@ -467,7 +467,7 @@ Cargo all-packages provenance 是有界的 package-selection witness，并非完 Node 构建通过检测到的 npm/pnpm/yarn/bun 执行 `package.json` 中已有的 `build` script，复用 packageManager/lockfile 检测；缺失/冲突的 manager evidence 或缺失 script 均 fail closed。Node 拒绝 scope 与 dependency policy，并要求 `project_build_node_v1`；不会安装依赖或包管理器；Runner 覆盖 Corepack 网络/下载/auto-pin 配置，防止隐式引导包管理器。脚本及 lifecycle hooks 可能写文件或访问网络。成功只证明脚本完成；通过 `finish_coding_task(outputs=...)` 显式检查产物元数据，产物存在并不证明功能正确。 -Node `project_validate(action=test)` 还支持 `scripts.test` 精确为 `vitest`、`vitest run` 或 `jest` 的已安装本地框架。Runner 使用固定官方 JSON reporter 参数与每个 Job 独占的新建私有报告文件（上限 1 MiB）。只有完整且一致的报告才能证明已执行测试数；不会从 stdout/stderr 推测数量。聚合 evidence 保留后删除报告。缺失、超限、无效报告、超时与取消均不能证明数量。JSON Job admission 要求 `project_validation_node_json_v1`。复合或自定义脚本返回 `structured_test_report_unavailable`，仍可用普通 recipe 执行；Node scope/filter 仍不支持。 +Node `project_validate(action=test)` 还支持 `scripts.test` 精确为 `vitest`、`vitest run` 或 `jest` 的已安装本地框架。Runner 使用固定官方 JSON reporter 参数与每个 Job 独占的新建私有报告文件（上限 1 MiB）。只有完整且一致的报告才能证明已执行测试数；不会从 stdout/stderr 推测数量。聚合 evidence 保留后删除报告。缺失、超限、无效报告、超时与取消均不能证明数量。JSON Job admission 要求 `project_validation_node_json_v1`。复合或自定义脚本返回 `structured_test_report_unavailable`，仍可用普通 recipe 执行；Node package/workspace scope 和自定义 Jest/Vitest 等脚本的过滤器仍不支持；仅原生 TAP 有界 `test.filter` 可用。 两个 gateway 都可选接受 `dependency_policy: {"mode":"locked"}`。这是 portable 的依赖解析保证，而不是宣称不同生态的原生 flag 完全等价：Rust build/check/lint/test 映射为 Cargo `--locked`，Go build/vet/test 映射为 `-mod=readonly`。它要求 adapter 不得为了让本次操作成功而修复或改写项目级依赖选择状态，但**不**表示关闭 registry/module/toolchain 网络访问；#962 中的 Rust build/check/test 已新增可选的离线依赖解析策略。`project_validate(action="format_check")` 会拒绝该 policy，而不是静默忽略。携带 policy 的 planning 与 typed Job admission 都要求 additive `project_dependency_policy_v1` Runner capability。locked validation 使用独立的 durable validation target identity；省略 policy 的请求保持原有 argv 与 identity。 diff --git a/docs/implementation/node-native-tap-project-validation.md b/docs/implementation/node-native-tap-project-validation.md index afb66f609..c63953440 100644 --- a/docs/implementation/node-native-tap-project-validation.md +++ b/docs/implementation/node-native-tap-project-validation.md @@ -6,7 +6,7 @@ An additive `project_validate(action=test)` profile under [#962](https://github. - Runner resolves the canonical Project root and selects a regular, Project-local `package.json` via the existing Node manifest guard. - The `scripts.test` value must be **exactly** `node --test` or `node --test --test-reporter=tap`. The manifest is used as an explicit opt-in signal; its script is never executed by `npm` or `node --run`. -- The Runner-owned command is always `node --test --test-reporter=tap`, with no caller-controlled argv, package/workspace scope, filter, dependency policy or environment additions. +- The Runner-owned command is always `node --test --test-reporter=tap`, with no caller-selected program or arbitrary argv. Bounded native `test.filter` adds one literal `--test-name-pattern=` argv under its own capability; default argv is unchanged, with no package/workspace scope, dependency policy or environment additions. - The same configured Node interpreter is probed and executed. The existing bounded version probe requires Node 22.3+, strips `NODE_OPTIONS`, and does not install or download any tooling. - An independent, non-baseline Runner capability `project_validation_node_tap_v1` gates both planning of explicit Node requests and Job admission. Auto-detected Node jobs receive an admission check against the **actual** retained Node test adapter. - Provenance binds the manifest content, command identity and Project root into durable source fences. The native Node path repeats the source recheck after interpreter preparation and before process handoff. @@ -22,7 +22,7 @@ Node tests themselves execute Project-authored code and can have filesystem/netw ## Excluded - General npm/pnpm/yarn/bun scripts, Vitest, Jest, TypeScript compilation, arbitrary TAP reporters. -- Custom test filters or package/workspace scope. +- Arbitrary framework/script filters or package/workspace scope. Native TAP name selection has its own additive `project_validation_node_name_filter_v1` Runner capability. - Any implicit dependency installation, isolation guarantee, or externally interpreted diagnostics. ## Regression requirements diff --git a/src/tool_runtime/tests/validation_handoff.rs b/src/tool_runtime/tests/validation_handoff.rs index 9e680f2b2..6b687e563 100644 --- a/src/tool_runtime/tests/validation_handoff.rs +++ b/src/tool_runtime/tests/validation_handoff.rs @@ -101,10 +101,17 @@ pub(super) async fn complete_project_validation_plan_request( r#"{"scripts":{"test":"node --test"}}"#, ) .unwrap(); - webcodex_validation::resolve_node_native_project_test(root.path(), None) - .unwrap() - .steps - .remove(0) + webcodex_validation::resolve_node_native_project_test_with_filter( + root.path(), + None, + semantic + .test + .as_ref() + .and_then(|test| test.filter.as_deref()), + ) + .unwrap() + .steps + .remove(0) } else if backend == "python" && check != webcodex_validation::SemanticCheck::Test { // Mock Runner planning still resolves a real local manifest. The Server // fixture is not proof of a real Ruff execution. diff --git a/src/tool_runtime/tests/validation_handoff/project_validation.rs b/src/tool_runtime/tests/validation_handoff/project_validation.rs index 041b9325d..f52e75031 100644 --- a/src/tool_runtime/tests/validation_handoff/project_validation.rs +++ b/src/tool_runtime/tests/validation_handoff/project_validation.rs @@ -27,6 +27,7 @@ async fn setup(grace_ms: u64) -> ToolRuntime { project_validation_python_pytest_v1: true, project_validation_python_ruff_v1: true, project_validation_node_tap_v1: true, + project_validation_node_name_filter_v1: true, project_validation_node_json_v1: true, structured_go_test_json: true, structured_go_test_tool: true, @@ -555,6 +556,137 @@ fn call_with_test(test: ProjectValidationTestOptions) -> ToolCall { call } +#[tokio::test] +async fn auto_node_filter_preserves_canonical_job_and_real_test_count() { + let runtime = setup(1).await; + let task = tokio::spawn({ + let runtime = runtime.clone(); + async move { + runtime + .dispatch_with_auth( + call_with_test(ProjectValidationTestOptions { + filter: Some("alpha".into()), + min_tests: Some(1), + ..Default::default() + }), + Some(&auth_context(None, true)), + ) + .await + } + }); + let (request, job_id) = reply_plan(&runtime, "node", ProjectValidationAction::Test).await; + let validation = request + .job_context + .as_ref() + .unwrap() + .validation + .as_ref() + .unwrap(); + assert!(validation.is_valid()); + assert_eq!(validation.adapter, "node:tap:test"); + assert_eq!( + validation.steps[0].args, + ["--test", "--test-reporter=tap", "--test-name-pattern=alpha"] + ); + assert_eq!( + validation + .project_validation + .as_ref() + .unwrap() + .request + .test + .as_ref() + .and_then(|test| test.filter.as_deref()), + Some("alpha") + ); + let pending = task.await.unwrap(); + assert_eq!(pending.output["execution_state"], "pending"); + assert_eq!( + pending.output["continuation"]["arguments"]["items"][0]["job_id"], + job_id + ); + let native = "TAP version 13\n# Subtest: alpha\nok 1 - alpha\n1..0\n# Subtest: b.test.js\nok 2 - b.test.js\n1..2\n# tests 2\n# suites 0\n# pass 2\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n# duration_ms 87.4\n"; + runtime + .runner_registry + .update_job(cargo_test_update( + "project-validation", + &request.request_id, + &job_id, + "completed", + native, + "", + Some(0), + completed_progress(), + true, + )) + .await + .unwrap(); + let status = runtime.job_status_for_auth(job_id, false, None).await; + assert_eq!(status.output["validation"]["passed"], true, "{status:?}"); + assert_eq!(status.output["validation"]["tests_run_count"], 1); +} + +#[tokio::test] +async fn auto_node_filter_rejects_old_runner_after_backend_resolution() { + let runtime = runtime_with_agent_project("project-validation"); + register_agent( + &runtime, + "project-validation", + None, + RunnerCapabilities { + async_shell_jobs: true, + structured_validation_argv: true, + project_validation_v1: true, + project_validation_test_options_v1: true, + project_validation_node_tap_v1: true, + project_validation_node_name_filter_v1: false, + ..Default::default() + }, + ) + .await; + let task = tokio::spawn({ + let runtime = runtime.clone(); + async move { + runtime + .dispatch_with_auth( + call_with_test(ProjectValidationTestOptions { + filter: Some("alpha".into()), + ..Default::default() + }), + Some(&auth_context(None, true)), + ) + .await + } + }); + let planning = wait_for_patch_agent_request(&runtime, "project-validation").await; + assert_eq!(planning.kind, "plan_project_validation"); + // Answer the one planning request without polling for a Job: old Runners + // must fail closed before admission, not create a second request. + complete_project_validation_plan_request( + &runtime, + "project-validation", + planning, + "agent-proj", + "node", + ProjectValidationAction::Test, + ) + .await; + let result = task.await.unwrap(); + assert!(!result.success); + assert!( + result + .error + .as_deref() + .unwrap_or("") + .contains("capability_unavailable"), + "{result:?}" + ); + assert!(runtime.runner_registry.list_jobs(Some(10)).await.is_empty()); + assert!(probe_patch_agent_request(&runtime, "project-validation") + .await + .is_none()); +} + #[tokio::test] async fn project_validation_test_options_reject_before_any_plan_on_old_runner() { let runtime = runtime_with_agent_project("project-validation"); diff --git a/src/tool_runtime/validation.rs b/src/tool_runtime/validation.rs index fb794f75a..a00e7d3c9 100644 --- a/src/tool_runtime/validation.rs +++ b/src/tool_runtime/validation.rs @@ -528,6 +528,32 @@ impl ToolRuntime { json!({"execution_source":"project_validate", "execution_state":"not_started", "command_started":false, "command_completed":false, "failure_kind":code, "detected_backend":detected_backend})), Err(_) => return ToolResult::err("invalid Runner validation plan; upgrade Server and Runner together"), }; + // For auto-selected Node, the Runner owns backend resolution. Gate + // the expanded argv after planning; Job admission rechecks the bit. + if plan.provenance.backend == "node" + && request + .test + .as_ref() + .and_then(|test| test.filter.as_deref()) + .is_some_and(|filter| !filter.trim().is_empty()) + { + let feature = self + .runner_registry + .get_runner_feature_set(&resolved.config.client_id) + .await; + if !feature.is_ok_and(|set| { + set.supports( + webcodex_core::runner_protocol::RunnerCapabilityId::ProjectValidationNodeNameFilter + ) + }) { + return ToolResult::err_with_output( + "capability_unavailable: filtered native Node tests require a newer Runner", + json!({"execution_source":"project_validate", "execution_state":"not_started", + "command_started":false, "command_completed":false, + "failure_kind":"capability_unavailable"}), + ); + } + } let project_workspace = plan.provenance.backend == "rust" && request.scope.as_ref().is_some_and( webcodex_core::project_validation::ProjectValidationScope::selects_all_packages, diff --git a/src/tool_runtime/validation_profile/mod.rs b/src/tool_runtime/validation_profile/mod.rs index f3a1a6c74..a18e59d56 100644 --- a/src/tool_runtime/validation_profile/mod.rs +++ b/src/tool_runtime/validation_profile/mod.rs @@ -18,10 +18,10 @@ pub(crate) struct ValidationRuntimeProfile { pub(crate) fn runtime_profile(operation: &ReadOnlyValidationOperation) -> ValidationRuntimeProfile { match operation { - ReadOnlyValidationOperation::NodeNativeTest => ValidationRuntimeProfile { + ReadOnlyValidationOperation::NodeNativeTest(_) => ValidationRuntimeProfile { default_timeout_secs: DEFAULT_CARGO_TEST_TIMEOUT_SECS, force_agent_handoff: true, - invalid_argument_guidance: "Native Node TAP or recognized direct Vitest/Jest only; no compound scripts, filter or package scope.", + invalid_argument_guidance: "Native Node TAP or recognized direct Vitest/Jest only; Node TAP accepts a bounded test-name filter, not compound scripts or package scope.", }, ReadOnlyValidationOperation::NodeScriptCheck => ValidationRuntimeProfile { default_timeout_secs: DEFAULT_CARGO_CHECK_TIMEOUT_SECS,