diff --git a/docs/development/build-and-test/pull-requests.md b/docs/development/build-and-test/pull-requests.md index 39bb5abee..1a134708b 100644 --- a/docs/development/build-and-test/pull-requests.md +++ b/docs/development/build-and-test/pull-requests.md @@ -71,6 +71,26 @@ Select the current native build using `MXC_FFI_DIR` for Node and the managed prebuilt-native MSBuild properties for .NET. These are opt-in host tests, not substitutes for the PR build matrix. +### Local Windows capture diagnostics benchmark + +From `src`, run: + +```text +cargo test --release -p mxc-sdk --lib benchmark_high_cardinality_network_capture -- --ignored --nocapture +``` + +This opt-in benchmark exercises the production network extractor, bounded +aggregation, denial deduplication, v5 document construction, JSON writing, and +version-specific validation. It compares repeated signatures with changing +filter IDs and ephemeral local ports for connections to the same destination, +at 4,096, 100,000, and 1,000,000 events. It reports retained groups, output bytes, +post-TDH processing time, document construction/writing time, and average +version-dispatching validation versus direct v5 deserialization time. +It also checks denial and overflow accounting. Timings are measurements, not +CI pass/fail thresholds. This synthetic decoded-event workload excludes TDH +decoding, ETL I/O, and native trace stop/close, so it is not an end-to-end OS +capture or teardown measurement. + ## Azure Pipelines (optional on PRs, required on `main`) The ADO pipeline (`MXC-PR-Build`) is the Azure version of the PR pipeline. The official diff --git a/docs/schema.md b/docs/schema.md index 60342e922..5fe124e73 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -106,6 +106,8 @@ depends on the backend; see its guide. When `network`, `network.egress`, or Both ingress controls also default to `deny` when omitted. `hostLoopback` resolves independently of `ingress.default`, so host-loopback access must be requested explicitly. +A runtime proxy may omit `network.egress`: the effective egress policy is +still `deny` with no direct allow or deny rules. A ProcessContainer proxy requires `ingress.default: "allow"`. Identity-scoped proxies set a non-blank `allowedProxyPeer` and keep `hostLoopback: "deny"`; identity-less host proxies omit `allowedProxyPeer` and require diff --git a/src/mxc-sdk/src/backends/process_container/common/base_container_helpers.rs b/src/mxc-sdk/src/backends/process_container/common/base_container_helpers.rs index 1b60e3950..9b955d883 100644 --- a/src/mxc-sdk/src/backends/process_container/common/base_container_helpers.rs +++ b/src/mxc-sdk/src/backends/process_container/common/base_container_helpers.rs @@ -17,7 +17,7 @@ use crate::process_security_environment_spec::process_security_environment_layou }; use crate::process_container_common::network_policy_helpers::{ - add_default_network_capabilities, ensure_capability, + add_default_network_capabilities, ensure_capability, INTERNET_CLIENT_CAPABILITY, }; use crate::process_container_common::secenv::SecurityEnvironmentVersion; @@ -31,13 +31,14 @@ pub(super) fn build_psec_v1_security_environment_spec( request: &ExecutionRequest, version: SecurityEnvironmentVersion, supports_network_ingress: bool, + capture_network_decisions: bool, ) -> Vec { assert_eq!( version.major, 1, "build_psec_v1_security_environment_spec only supports PSEC major version 1" ); let mut builder = flatbuffers::FlatBufferBuilder::with_capacity(1024); - let mut capabilities = effective_capabilities(&request.policy); + let mut capabilities = effective_capabilities(request, capture_network_decisions); if request.policy.network_proxy.is_enabled() && unrestricted_host_loopback_allowed(&request.policy) { @@ -73,7 +74,11 @@ pub(super) fn build_psec_v1_security_environment_spec( builder.finished_data().to_vec() } -fn effective_capabilities(policy: &ContainerPolicy) -> Vec { +fn effective_capabilities( + request: &ExecutionRequest, + capture_network_decisions: bool, +) -> Vec { + let policy = &request.policy; let mut capabilities: Vec<_> = policy .capabilities .iter() @@ -81,6 +86,16 @@ fn effective_capabilities(policy: &ContainerPolicy) -> Vec { .cloned() .collect(); add_default_network_capabilities(policy, &mut capabilities); + if policy.capture_denials.is_some() + && capture_network_decisions + && policy.network_egress.is_some() + && !policy.network_proxy.is_enabled() + { + // Let outbound attempts pass the capability gate so Tessera's egress + // policy makes and records the decision. Proxy mode must remain direct- + // egress denied and therefore deliberately omits internetClient. + ensure_capability(&mut capabilities, INTERNET_CLIENT_CAPABILITY); + } capabilities } diff --git a/src/mxc-sdk/src/backends/process_container/common/base_container_helpers/tests.rs b/src/mxc-sdk/src/backends/process_container/common/base_container_helpers/tests.rs index 46259a23d..c1ebce1bd 100644 --- a/src/mxc-sdk/src/backends/process_container/common/base_container_helpers/tests.rs +++ b/src/mxc-sdk/src/backends/process_container/common/base_container_helpers/tests.rs @@ -53,8 +53,12 @@ fn runtime_proxy_request(proxy: &TcpListener) -> ExecutionRequest { fn runtime_proxy_uses_peer_and_capability_without_native_ingress() { let proxy = TcpListener::bind(("127.0.0.1", 0)).unwrap(); let request = runtime_proxy_request(&proxy); - let bytes = - build_psec_v1_security_environment_spec(&request, SecurityEnvironmentVersion::V1_1, true); + let bytes = build_psec_v1_security_environment_spec( + &request, + SecurityEnvironmentVersion::V1_1, + true, + false, + ); let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); let network = spec.network_policy().expect("network policy"); let proxy_url = format!("http://{}", proxy.local_addr().unwrap()); @@ -87,8 +91,12 @@ fn runtime_proxy_creates_native_security_environment() { } let proxy = TcpListener::bind(("127.0.0.1", 0)).unwrap(); let request = runtime_proxy_request(&proxy); - let bytes = - build_psec_v1_security_environment_spec(&request, SecurityEnvironmentVersion::V1_1, true); + let bytes = build_psec_v1_security_environment_spec( + &request, + SecurityEnvironmentVersion::V1_1, + true, + false, + ); let environment = create(&bytes, PROCESS_SECURITY_ENVIRONMENT_FLAG_NONE).unwrap_or_else(|error| { panic!("valid runtime proxy policy failed to create PSEC: {error}") @@ -107,8 +115,12 @@ fn identity_scoped_proxy_does_not_grant_host_loopback() { .unwrap() .host_loopback = NetworkAction::Deny; request.policy.allowed_proxy_peer = Some("Contoso.Proxy_12345".to_string()); - let bytes = - build_psec_v1_security_environment_spec(&request, SecurityEnvironmentVersion::V1_0, false); + let bytes = build_psec_v1_security_environment_spec( + &request, + SecurityEnvironmentVersion::V1_0, + false, + false, + ); let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); let network = spec.network_policy().unwrap(); diff --git a/src/mxc-sdk/src/backends/process_container/common/base_container_runner.rs b/src/mxc-sdk/src/backends/process_container/common/base_container_runner.rs index 4ad7ec097..7731780c5 100644 --- a/src/mxc-sdk/src/backends/process_container/common/base_container_runner.rs +++ b/src/mxc-sdk/src/backends/process_container/common/base_container_runner.rs @@ -13,8 +13,10 @@ use std::path::{Path, PathBuf}; use std::ptr; use std::sync::Arc; -use crate::learning_mode_core::DenialAnalyzer; -use crate::learning_mode_windows::{EtlDenialAnalyzer, LearningModeApi, LearningModeError}; +use crate::learning_mode_core::{capture_diagnostics::CaptureAnalysis, AnalyzeError}; +use crate::learning_mode_windows::{ + EtlDenialAnalyzer, LearningModeApi, LearningModeError, LearningModeTraceSources, +}; use windows::Win32::Foundation::{ CloseHandle, GetLastError, SetHandleInformation, ERROR_CALL_NOT_IMPLEMENTED, E_NOTIMPL, HANDLE, HANDLE_FLAG_INHERIT, WAIT_OBJECT_0, WAIT_TIMEOUT, @@ -59,7 +61,7 @@ use crate::process_container_common::base_container_helpers::{ }; use crate::process_container_common::capture_output::{ combine_capture_and_cleanup_results, combine_process_and_teardown_results, - remove_internal_capture_file, unique_denials_output_paths, write_denials_document, + remove_internal_capture_file, unique_denials_output_paths, write_capture_denials_document, write_stderr_line_best_effort, }; use crate::process_container_common::job_object::UiJobObject; @@ -79,6 +81,16 @@ use windows::Win32::System::Threading::{ ResumeThread, CREATE_NO_WINDOW, CREATE_SUSPENDED, CREATE_UNICODE_ENVIRONMENT, }; +trait CaptureDenialAnalyzer { + fn analyze_capture(&self, source_path: &Path) -> Result; +} + +impl CaptureDenialAnalyzer for EtlDenialAnalyzer { + fn analyze_capture(&self, source_path: &Path) -> Result { + EtlDenialAnalyzer::analyze_capture(self, source_path) + } +} + /// Build the environment block handed to the contained child. /// /// Honors the three states of [`ExecutionRequest::env`]: @@ -206,6 +218,10 @@ impl CaptureSessionOps for CaptureSession { } trait CaptureSessionFactory: Send + Sync { + fn trace_sources( + &self, + ) -> Result; + fn begin( &self, sandbox_specification: &[u8], @@ -216,6 +232,12 @@ trait CaptureSessionFactory: Send + Sync { struct RealCaptureSessionFactory; impl CaptureSessionFactory for RealCaptureSessionFactory { + fn trace_sources( + &self, + ) -> Result { + LearningModeApi::load().map(|api| api.trace_sources()) + } + fn begin( &self, sandbox_specification: &[u8], @@ -279,16 +301,6 @@ impl BaseContainerRunner { } } - #[cfg(test)] - fn build_process_security_environment_spec(request: &ExecutionRequest) -> Vec { - let version = Self::choose_min_required_psec_version_for_request(request, false); - build_psec_v1_security_environment_spec( - request, - version, - version >= SecurityEnvironmentVersion::V1_1, - ) - } - #[cfg(test)] fn with_capture_factory(capture_factory: Arc) -> Self { Self { @@ -298,6 +310,32 @@ impl BaseContainerRunner { } } + fn build_capture_aware_security_environment_spec( + &self, + request: &ExecutionRequest, + version: SecurityEnvironmentVersion, + supports_network_ingress: bool, + ) -> Result, ScriptResponse> { + let capture_network_decisions = match request.policy.capture_denials.as_ref() { + Some(_) => self + .capture_factory + .trace_sources() + .map_err(|error| { + ScriptResponse::error(&format!( + "captureDenials: failed to inspect learning-mode trace sources: {error}" + )) + })? + .includes_network(), + None => false, + }; + Ok(build_psec_v1_security_environment_spec( + request, + version, + supports_network_ingress, + capture_network_decisions, + )) + } + fn cleanup_capture_begin_failure(&mut self, logger: &mut Logger) { // CaptureSession owns and closes the PSEC environment. No legacy // identity/tracking state is created for this path. @@ -548,11 +586,12 @@ impl BaseContainerRunner { let supports_network_ingress = psec_version >= SecurityEnvironmentVersion::V1_1 && secenv::query_support(SecurityEnvironmentSupport::NetworkIngress); - let process_security_environment_spec = build_psec_v1_security_environment_spec( - request, - psec_version, - supports_network_ingress, - ); + let process_security_environment_spec = self + .build_capture_aware_security_environment_spec( + request, + psec_version, + supports_network_ingress, + )?; let _ = writeln!( logger, "process security environment spec built (PSEC {}.{}, {} bytes)", @@ -1635,21 +1674,21 @@ impl BaseContainerSandboxProcess { /// Decodes a sealed capture into the JSON denials document at `output_path`. fn decode_and_write_denials( - analyzer: &dyn DenialAnalyzer, + analyzer: &dyn CaptureDenialAnalyzer, etl_path: &std::path::Path, output_path: &std::path::Path, exit_code: i32, ) -> std::io::Result { - let analysis = analyzer.analyze(etl_path).map_err(|error| { + let analysis = analyzer.analyze_capture(etl_path).map_err(|error| { std::io::Error::other(format!( "captureDenials failed to decode denials ETL: {error}" )) })?; - write_denials_document(analysis, exit_code, output_path) + write_capture_denials_document(analysis, exit_code, output_path) } fn decode_write_and_finalize( - analyzer: &dyn DenialAnalyzer, + analyzer: &dyn CaptureDenialAnalyzer, etl_path: &Path, etl_directory: Option<&Path>, output_path: &Path, @@ -2100,14 +2139,17 @@ fn managed_capture_output_path_in( mod tests { use super::*; use crate::learning_mode_core::{ - AccessType, AnalysisResult, AnalyzeError, DenialsDocument, DeniedResource, ResourceType, + capture_diagnostics::CaptureVerboseLoggingSummary, AccessType, AnalysisResult, + AnalyzeError, DenialsDocument, DeniedResource, ResourceType, }; use crate::mxc_common::models::{ - BaseProcessUiConfig, ClipboardPolicy, ContainerPolicy, NetworkAction, NetworkCidr, - NetworkEgressPolicy, NetworkPeer, NetworkPort, NetworkProtocol, NetworkRule, ProxyAddress, - ProxyConfig, UiPolicy, + BaseProcessUiConfig, ClipboardPolicy, ContainerPolicy, ContainmentBackend, NetworkAction, + NetworkCidr, NetworkEgressPolicy, NetworkIngressPolicy, NetworkPeer, NetworkPort, + NetworkProtocol, NetworkRule, ProxyAddress, ProxyConfig, UiPolicy, }; + use crate::mxc_common::network_parser::{parse_network_policy, NetworkSections}; use crate::mxc_common::ui_policy::EffectiveUiRestrictions; + use crate::mxc_common::wire; use crate::process_container_common::job_object::to_job_object_uilimit_mask; use crate::process_security_environment_spec::process_security_environment_layout as psec_layout; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -2241,6 +2283,7 @@ mod tests { } struct FakeCaptureFactory { + trace_sources: LearningModeTraceSources, begin_error: Option<(&'static str, i32)>, finish_error: Option<(&'static str, i32)>, begin_calls: AtomicUsize, @@ -2248,6 +2291,13 @@ mod tests { } impl CaptureSessionFactory for FakeCaptureFactory { + fn trace_sources( + &self, + ) -> Result + { + Ok(self.trace_sources) + } + fn begin( &self, _sandbox_specification: &[u8], @@ -2269,6 +2319,7 @@ mod tests { fn fake_capture_factory() -> Arc { Arc::new(FakeCaptureFactory { + trace_sources: LearningModeTraceSources::AccessAndNetwork, begin_error: None, finish_error: None, begin_calls: AtomicUsize::new(0), @@ -2276,14 +2327,32 @@ mod tests { }) } + impl BaseContainerRunner { + fn build_process_security_environment_spec(request: &ExecutionRequest) -> Vec { + let version = Self::choose_min_required_psec_version_for_request(request, false); + Self::with_capture_factory(fake_capture_factory()) + .build_capture_aware_security_environment_spec( + request, + version, + version >= SecurityEnvironmentVersion::V1_1, + ) + .unwrap() + } + } + struct FakeAnalyzer { result: Result, } - impl DenialAnalyzer for FakeAnalyzer { - fn analyze(&self, _source_path: &Path) -> Result { + impl CaptureDenialAnalyzer for FakeAnalyzer { + fn analyze_capture(&self, _source_path: &Path) -> Result { match &self.result { - Ok(result) => Ok(result.clone()), + Ok(result) => Ok(CaptureAnalysis { + denials: result.denials.clone(), + denied_resources_truncated: result.denied_resources_truncated, + verbose_logging: result.verbose_logging.clone(), + network_verbose_logging: CaptureVerboseLoggingSummary::default(), + }), Err(message) => Err(AnalyzeError::Decode((*message).to_string())), } } @@ -2387,8 +2456,16 @@ mod tests { assert_eq!(metadata.exit_code, 7); assert_eq!(metadata.total_denials, 1); let document: DenialsDocument = - serde_json::from_slice(&std::fs::read(output_path).unwrap()).unwrap(); + serde_json::from_slice(&std::fs::read(&output_path).unwrap()).unwrap(); assert_eq!(document.denials.len(), 1); + let verbose_path = + crate::learning_mode_core::verbose_logging_sibling_path(&output_path).unwrap(); + let verbose: serde_json::Value = + serde_json::from_slice(&std::fs::read(verbose_path).unwrap()).unwrap(); + assert_eq!( + verbose["version"], + crate::learning_mode_core::capture_diagnostics::CAPTURE_VERBOSE_LOGGING_VERSION + ); } #[test] @@ -2646,6 +2723,7 @@ mod tests { #[test] fn capture_factory_injects_begin_failure() { let factory = Arc::new(FakeCaptureFactory { + trace_sources: LearningModeTraceSources::AccessAndNetwork, begin_error: Some(( "StartLearningModeTrace", windows::Win32::Foundation::E_FAIL.0, @@ -2672,6 +2750,7 @@ mod tests { #[test] fn capture_factory_injects_finish_failure_once() { let factory = Arc::new(FakeCaptureFactory { + trace_sources: LearningModeTraceSources::AccessAndNetwork, begin_error: None, finish_error: Some(( "StopLearningModeTrace", @@ -2910,6 +2989,174 @@ mod tests { ); } + #[test] + fn capture_denials_directional_default_deny_reaches_tessera_network_policy() { + let mut request = ExecutionRequest::default(); + request.policy.capture_denials = Some(Default::default()); + request.policy.network_egress = Some(NetworkEgressPolicy { + default: NetworkAction::Deny, + ..Default::default() + }); + + let bytes = BaseContainerRunner::build_process_security_environment_spec(&request); + let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); + let egress = spec + .network_policy() + .and_then(|policy| policy.egress()) + .expect("PSEC must carry the Tessera egress policy"); + + assert_eq!(spec.capabilities(), Some("internetClient")); + assert_eq!(egress.default_action(), psec_layout::FilterAction::deny); + } + + #[test] + fn capture_capability_requires_a_network_aware_trace() { + let mut request = ExecutionRequest::default(); + request.policy.capture_denials = Some(Default::default()); + request.policy.network_egress = Some(NetworkEgressPolicy { + default: NetworkAction::Deny, + ..Default::default() + }); + + for (trace_sources, capture_enabled, expected_capabilities) in [ + (LearningModeTraceSources::AccessOnly, true, None), + ( + LearningModeTraceSources::AccessAndNetwork, + true, + Some("internetClient"), + ), + (LearningModeTraceSources::AccessOnly, false, None), + (LearningModeTraceSources::AccessAndNetwork, false, None), + ] { + request.policy.capture_denials = capture_enabled.then(Default::default); + let factory = Arc::new(FakeCaptureFactory { + trace_sources, + begin_error: None, + finish_error: None, + begin_calls: AtomicUsize::new(0), + finish_calls: Arc::new(AtomicUsize::new(0)), + }); + let runner = BaseContainerRunner::with_capture_factory(factory); + let bytes = runner + .build_capture_aware_security_environment_spec( + &request, + SecurityEnvironmentVersion::V1_0, + false, + ) + .unwrap(); + let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); + + assert_eq!(spec.capabilities(), expected_capabilities); + let egress = spec.network_policy().unwrap().egress().unwrap(); + assert_eq!(egress.default_action(), psec_layout::FilterAction::deny); + assert!(egress.allow().is_none()); + assert!(egress.deny().is_none()); + } + } + + #[test] + fn capture_capability_requires_an_explicit_direct_egress_section() { + for (network, expected_capabilities) in [ + (None, None), + ( + Some(wire::Network { + egress: None, + ingress: Some(wire::NetworkIngress { + default: Some(wire::NetworkAction::Deny), + host_loopback: Some(wire::NetworkAction::Deny), + }), + }), + None, + ), + ( + Some(wire::Network { + egress: Some(wire::NetworkEgress { + default: Some(wire::NetworkAction::Deny), + allow: None, + deny: None, + }), + ingress: None, + }), + Some("internetClient"), + ), + ] { + let mut request = ExecutionRequest::default(); + request.policy.capture_denials = Some(Default::default()); + parse_network_policy( + &mut request.policy, + NetworkSections { + network, + runtime: None, + process_container: None, + }, + &ContainmentBackend::ProcessContainer, + ) + .unwrap(); + + let bytes = BaseContainerRunner::build_process_security_environment_spec(&request); + let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); + let egress = spec + .network_policy() + .and_then(|policy| policy.egress()) + .expect("PSEC must retain deny-default egress"); + + assert_eq!(spec.capabilities(), expected_capabilities); + assert_eq!(egress.default_action(), psec_layout::FilterAction::deny); + } + } + + #[test] + fn directional_default_deny_without_capture_does_not_add_internet_capability() { + let mut request = ExecutionRequest::default(); + request.policy.network_egress = Some(NetworkEgressPolicy { + default: NetworkAction::Deny, + ..Default::default() + }); + + let bytes = BaseContainerRunner::build_process_security_environment_spec(&request); + let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); + let egress = spec + .network_policy() + .and_then(|policy| policy.egress()) + .expect("PSEC must retain deny-default egress"); + + assert!(spec.capabilities().is_none()); + assert_eq!(egress.default_action(), psec_layout::FilterAction::deny); + } + + #[test] + fn capture_denials_proxy_preserves_proxy_capability_posture() { + let mut request = ExecutionRequest::default(); + request.policy.capture_denials = Some(Default::default()); + request.policy.runtime_network_proxy_specified = true; + request.policy.network_proxy = ProxyConfig { + address: Some(ProxyAddress::new("127.0.0.1".to_string(), 8080)), + }; + request.policy.network_egress = Some(NetworkEgressPolicy { + default: NetworkAction::Deny, + ..Default::default() + }); + request.policy.network_ingress = Some(NetworkIngressPolicy { + default: NetworkAction::Allow, + host_loopback: NetworkAction::Allow, + }); + + let bytes = BaseContainerRunner::build_process_security_environment_spec(&request); + let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); + + assert_eq!( + spec.capabilities(), + Some("privateNetworkClientServer,networkLoopback") + ); + assert!(spec + .capabilities() + .is_none_or(|capabilities| !capabilities.contains("internetClient"))); + assert!(spec + .network_policy() + .and_then(|network| network.proxy()) + .is_some()); + } + #[test] fn build_process_security_environment_spec_ignores_empty_capability() { let mut request = ExecutionRequest::default(); @@ -3180,7 +3427,7 @@ mod tests { let version = BaseContainerRunner::choose_min_required_psec_version_for_request(&request, true); assert_eq!(version, SecurityEnvironmentVersion::V1_0); - let bytes = build_psec_v1_security_environment_spec(&request, version, false); + let bytes = build_psec_v1_security_environment_spec(&request, version, false, false); let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); let network = spec.network_policy().unwrap(); assert_eq!(spec.version().minor(), 0); @@ -3220,6 +3467,7 @@ mod tests { &ExecutionRequest::default(), SecurityEnvironmentVersion { major: 2, minor: 0 }, false, + false, ); } @@ -3230,6 +3478,7 @@ mod tests { &request, SecurityEnvironmentVersion::V1_1, false, + false, ); let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); @@ -3250,6 +3499,7 @@ mod tests { &request, SecurityEnvironmentVersion::V1_0, false, + false, ); let spec = psec_layout::root_as_process_security_environment(&bytes).unwrap(); let network = spec.network_policy().expect("network policy"); diff --git a/src/mxc-sdk/src/backends/process_container/common/capture_output.rs b/src/mxc-sdk/src/backends/process_container/common/capture_output.rs index 5e5be87d3..f12e1df23 100644 --- a/src/mxc-sdk/src/backends/process_container/common/capture_output.rs +++ b/src/mxc-sdk/src/backends/process_container/common/capture_output.rs @@ -19,9 +19,10 @@ use std::path::{Path, PathBuf}; use crate::learning_mode_core::{ + capture_diagnostics::{write_capture_verbose_logging_document, CaptureAnalysis}, verbose_logging_sibling_path, write_document, write_paired_output_files, write_verbose_logging_document, AnalysisResult, DenialSummary, DenialsDocument, - DenialsOutputPointer, ExistingOutputPolicy, VerboseLoggingDocument, + DenialsOutputPointer, DeniedResource, ExistingOutputPolicy, VerboseLoggingDocument, }; use crate::mxc_common::models::CaptureDenialsOutput; @@ -61,22 +62,49 @@ pub fn write_denials_document( exit_code: i32, output_path: &Path, ) -> std::io::Result { - let verbose_logging_path = verbose_logging_output_path(output_path)?; - let summary = DenialSummary::new( + let verbose_logging_document = VerboseLoggingDocument::new(&analysis.verbose_logging); + write_denials_document_pair( + analysis.denials, + analysis.denied_resources_truncated, exit_code, - analysis.denials.len(), + output_path, + |writer| write_verbose_logging_document(writer, &verbose_logging_document), + ) +} + +/// Writes native capture analysis with the complete internal v5 diagnostics. +pub(crate) fn write_capture_denials_document( + analysis: CaptureAnalysis, + exit_code: i32, + output_path: &Path, +) -> std::io::Result { + let verbose_logging_document = analysis.verbose_document(); + write_denials_document_pair( + analysis.denials, analysis.denied_resources_truncated, - ); - let document = DenialsDocument::new(analysis.denials, summary); - let verbose_logging_document = VerboseLoggingDocument::new(&analysis.verbose_logging); + exit_code, + output_path, + |writer| write_capture_verbose_logging_document(writer, &verbose_logging_document), + ) +} +fn write_denials_document_pair( + denials: Vec, + denied_resources_truncated: bool, + exit_code: i32, + output_path: &Path, + write_verbose: impl FnOnce(&mut std::io::BufWriter) -> std::io::Result<()>, +) -> std::io::Result { + let verbose_logging_path = verbose_logging_output_path(output_path)?; + let summary = DenialSummary::new(exit_code, denials.len(), denied_resources_truncated); + let document = DenialsDocument::new(denials, summary); write_paired_output_files( "captureDenials", output_path, &verbose_logging_path, ExistingOutputPolicy::CreateNew, |writer| write_document(writer, &document), - |writer| write_verbose_logging_document(writer, &verbose_logging_document), + write_verbose, )?; let pointer = DenialsOutputPointer::new(output_path.to_string_lossy(), &document.summary); @@ -281,6 +309,11 @@ pub fn write_stderr_line_best_effort(message: std::fmt::Arguments<'_>) { #[cfg(test)] mod tests { use super::*; + use crate::learning_mode_core::capture_diagnostics::{ + CaptureVerboseLoggingOutcomeReason, CaptureVerboseLoggingProvider, + CaptureVerboseLoggingSignature, CaptureVerboseLoggingSummary, ConfigurationRecommendation, + NetworkDecisionReason, NetworkEndpoint, CAPTURE_VERBOSE_LOGGING_VERSION, + }; use crate::learning_mode_core::{ AccessType, DeniedResource, ResourceType, VerboseLoggingOutcomeReason, VerboseLoggingProvider, VerboseLoggingSignature, @@ -330,6 +363,74 @@ mod tests { assert!(verbose_logging_output_path(&output_path).unwrap().exists()); } + #[test] + fn native_capture_writes_version_five_reason_specific_guidance() { + let directory = tempfile::tempdir().expect("temp directory"); + let output_path = directory.path().join("denials.json"); + let mut network_verbose_logging = CaptureVerboseLoggingSummary::default(); + let mut retained_bytes = 0; + network_verbose_logging.record_with_byte_budget( + CaptureVerboseLoggingSignature { + provider: CaptureVerboseLoggingProvider::LearningModeNetworkDecision, + provider_guid: "{71237669-21C3-4101-BD2F-FF38945D725A}".to_string(), + event_id: 1, + event_name: Some("NetworkDecisionV1".to_string()), + reason: CaptureVerboseLoggingOutcomeReason::Actionable, + pid: 0, + access_type: Some(AccessType::Unknown), + resource_type: Some(ResourceType::Network), + network_decision_reason: Some(NetworkDecisionReason::DirectDefaultDeny), + configuration_recommendation: Some(ConfigurationRecommendation::AddEgressAllow), + network_endpoint: Some(NetworkEndpoint { + protocol: "tcp".to_string(), + remote_address: "203.0.113.10".to_string(), + remote_port: Some(443), + }), + properties: Vec::new(), + }, + &mut retained_bytes, + crate::learning_mode_core::MAX_VERBOSE_LOGGING_SIGNATURE_BYTES, + ); + let analysis = CaptureAnalysis { + denials: vec![DeniedResource { + resource: "tcp://203.0.113.10:443".to_string(), + resource_type: ResourceType::Network, + access_type: AccessType::Unknown, + pid: 0, + filetime: 99, + }], + denied_resources_truncated: false, + verbose_logging: Default::default(), + network_verbose_logging, + }; + + let metadata = write_capture_denials_document(analysis, 7, &output_path).unwrap(); + assert_eq!(metadata.kind, CaptureDenialsOutput::KIND); + assert_eq!(metadata.output_path, output_path.to_string_lossy()); + assert_eq!(metadata.exit_code, 7); + assert_eq!(metadata.total_denials, 1); + assert!(!metadata.denied_resources_truncated); + assert!(metadata.etl_path.is_none()); + + let verbose: serde_json::Value = serde_json::from_slice( + &std::fs::read(verbose_logging_output_path(&output_path).unwrap()).unwrap(), + ) + .unwrap(); + assert_eq!(verbose["version"], CAPTURE_VERBOSE_LOGGING_VERSION); + assert_eq!( + verbose["signatures"][0]["signature"]["networkDecisionReason"], + "directDefaultDeny" + ); + assert_eq!( + verbose["signatures"][0]["signature"]["configurationRecommendation"], + "addEgressAllow" + ); + assert_eq!( + verbose["signatures"][0]["signature"]["networkEndpoint"]["remoteAddress"], + "203.0.113.10" + ); + } + #[test] fn write_denials_document_writes_verbose_logging_aggregates() { let directory = tempfile::tempdir().expect("temp directory"); @@ -383,6 +484,26 @@ mod tests { assert_eq!(std::fs::read(verbose_logging_path).unwrap(), b"existing"); } + #[test] + fn native_verbose_logging_collision_leaves_actionable_output_absent() { + let directory = tempfile::tempdir().expect("temp directory"); + let output_path = directory.path().join("denials.json"); + let verbose_logging_path = verbose_logging_output_path(&output_path).unwrap(); + std::fs::write(&verbose_logging_path, b"existing").unwrap(); + let analysis = CaptureAnalysis { + denials: Vec::new(), + denied_resources_truncated: false, + verbose_logging: Default::default(), + network_verbose_logging: Default::default(), + }; + + write_capture_denials_document(analysis, 0, &output_path) + .expect_err("collision should fail"); + + assert!(!output_path.exists()); + assert_eq!(std::fs::read(verbose_logging_path).unwrap(), b"existing"); + } + #[test] fn second_staged_write_failure_leaves_no_final_output() { let directory = tempfile::tempdir().expect("temp directory"); diff --git a/src/mxc-sdk/src/core/learning_mode_windows/etl_decode.rs b/src/mxc-sdk/src/core/learning_mode_windows/etl_decode.rs index c18423307..33ef9add8 100644 --- a/src/mxc-sdk/src/core/learning_mode_windows/etl_decode.rs +++ b/src/mxc-sdk/src/core/learning_mode_windows/etl_decode.rs @@ -2689,6 +2689,111 @@ mod tests { ) } + #[test] + #[ignore = "manual benchmark: run with --release --ignored --nocapture"] + fn benchmark_high_cardinality_network_capture() { + use std::hint::black_box; + use std::time::Instant; + + use crate::learning_mode_core::capture_diagnostics::{ + parse_supported_verbose_document, write_capture_verbose_logging_document, + CaptureVerboseLoggingDocument, + }; + + // Exercise the production post-TDH path, not ETL I/O or OS trace teardown. + for event_count in [4_096usize, 100_000, MAX_PROCESSED_EVENTS] { + for distinct in [false, true] { + let mut event = network_event( + 777, + 600, + &[ + ("SourceDomain", "2"), + ("Reason", "100"), + ("FieldFlags", "63"), + ("ProviderGuid", "{2F8C6D14-3B7E-4A59-9C08-1D4E7A6B2F30}"), + ("SublayerGuid", "{7B1E9A2C-9D4F-4C8A-B321-5E6D2F8A1C44}"), + ("ApplicationId", r"\Device\HarddiskVolume3\app.exe"), + ("Protocol", "6"), + ("LocalAddress", "192.0.2.1"), + ("LocalPort", "49152"), + ("RemoteAddress", "203.0.113.10"), + ("RemotePort", "443"), + ], + ); + let mut accumulator = Accumulator::analyze(); + let started = Instant::now(); + for index in 0..event_count { + if distinct { + for (name, value) in &mut event.parts.props { + match name.as_str() { + // Descending keys also exercise front-of-vector insertion. + "FilterId" => *value = (10_000_000 - index).to_string(), + "LocalPort" => *value = (49_152 + index % 16_384).to_string(), + _ => {} + } + } + } + handle_decoded_event( + black_box(&event.parts), + event.pid, + event.filetime, + &mut accumulator, + ); + } + let analysis = accumulator.into_capture_analysis().unwrap(); + let decode_elapsed = started.elapsed(); + assert_eq!(analysis.denials.len(), 1); + assert_eq!( + analysis.network_verbose_logging.total_occurrences, + event_count as u64 + ); + let expected_groups = if distinct { + event_count.min(MAX_VERBOSE_LOGGING_GROUPS) + } else { + 1 + }; + assert_eq!( + analysis.network_verbose_logging.signatures.len(), + expected_groups + ); + assert_eq!( + analysis.network_verbose_logging.overflow_occurrences, + (if distinct { + event_count - expected_groups + } else { + 0 + }) as u64 + ); + + let started = Instant::now(); + let document = analysis.verbose_document(); + let mut bytes = Vec::new(); + write_capture_verbose_logging_document(&mut bytes, &document).unwrap(); + let write_elapsed = started.elapsed(); + + let started = Instant::now(); + for _ in 0..10 { + black_box(parse_supported_verbose_document(black_box(&bytes)).unwrap()); + } + let validate_elapsed = started.elapsed() / 10; + let started = Instant::now(); + for _ in 0..10 { + black_box( + serde_json::from_slice::(black_box(&bytes)) + .unwrap(), + ); + } + let typed_decode_elapsed = started.elapsed() / 10; + println!( + "events={event_count} distinct={distinct} groups={expected_groups} \ + bytes={} post_tdh={decode_elapsed:?} merge_write={write_elapsed:?} \ + versioned_validate={validate_elapsed:?} typed_decode={typed_decode_elapsed:?}", + bytes.len(), + ); + } + } + } + #[test] fn managed_network_events_route_actionable_and_verbose_outcomes() { let common = [ diff --git a/src/mxc-sdk/src/core/mxc_common/mod.rs b/src/mxc-sdk/src/core/mxc_common/mod.rs index abe02c2c2..1ae338a35 100644 --- a/src/mxc-sdk/src/core/mxc_common/mod.rs +++ b/src/mxc-sdk/src/core/mxc_common/mod.rs @@ -25,7 +25,7 @@ pub mod microvm_staging; pub mod models; pub mod mxc_error; pub mod network_blocks; -mod network_parser; +pub(crate) mod network_parser; pub mod policy_identity; pub use network_parser::host_is_canonical_loopback; pub mod proxy_env; diff --git a/src/mxc-sdk/src/core/mxc_common/network_parser.rs b/src/mxc-sdk/src/core/mxc_common/network_parser.rs index 12d1d9288..0935142de 100644 --- a/src/mxc-sdk/src/core/mxc_common/network_parser.rs +++ b/src/mxc-sdk/src/core/mxc_common/network_parser.rs @@ -66,12 +66,7 @@ fn convert_wire_proxy_at(url_str: &str, path: &str) -> Result) -> Result { - let egress = egress.unwrap_or(wire::NetworkEgress { - default: None, - allow: None, - deny: None, - }); +fn convert_egress(egress: wire::NetworkEgress) -> Result { Ok(NetworkEgressPolicy { default: egress.default.map(convert_action).unwrap_or_default(), allow: convert_rules(egress.allow.unwrap_or_default(), "network.egress.allow")?, @@ -119,14 +114,13 @@ pub(crate) fn parse_network_policy( // metadata is intentionally excluded because it may be supplied at exec. policy.network_mode_specified = has_directional_policy_fields(&network); - let egress = convert_egress(network.egress)?; - policy.network_egress = Some(egress); + policy.network_egress = network.egress.map(convert_egress).transpose()?; let ingress = convert_ingress(network.ingress); policy.network_ingress = Some(ingress); } None => { - policy.network_egress = Some(NetworkEgressPolicy::default()); + policy.network_egress = None; policy.network_ingress = Some(NetworkIngressPolicy::default()); } } @@ -171,9 +165,8 @@ fn convert_allowed_proxy_peer( fn validate_directional_proxy_policy(policy: &ContainerPolicy) -> Result<(), WxcError> { let Some(egress) = policy.network_egress.as_ref() else { - return Err(WxcError::ConfigParse( - "runtimeConfig.networkProxy requires an egress policy".to_string(), - )); + // Omitted egress is effective ruleless deny; preserve its authored absence. + return Ok(()); }; if egress.default != NetworkAction::Deny || !egress.allow.is_empty() || !egress.deny.is_empty() { @@ -471,6 +464,29 @@ mod proxy_policy_tests { ); } + #[test] + fn ingress_only_runtime_proxy_preserves_implicit_deny_egress() { + for containment in [ + ContainmentBackend::ProcessContainer, + ContainmentBackend::Lxc, + ContainmentBackend::Seatbelt, + ] { + let mut sections = directional_sections("http://127.0.0.1:8080"); + sections.network.as_mut().unwrap().egress = None; + let mut policy = ContainerPolicy::default(); + + parse_network_policy(&mut policy, sections, &containment).unwrap(); + + assert!(policy.network_egress.is_none()); + assert!(policy.network_mode_specified); + assert!(policy.runtime_network_proxy_specified); + assert!(policy.network_proxy.is_enabled()); + let ingress = policy.network_ingress.unwrap(); + assert_eq!(ingress.default, NetworkAction::Allow); + assert_eq!(ingress.host_loopback, NetworkAction::Allow); + } + } + #[test] fn reserved_loopback_peer_identity_is_rejected_case_insensitively() { let policy = ContainerPolicy { diff --git a/src/tools/wxc/src/audit.rs b/src/tools/wxc/src/audit.rs index 9ea638853..ca4ea9fcc 100644 --- a/src/tools/wxc/src/audit.rs +++ b/src/tools/wxc/src/audit.rs @@ -108,19 +108,21 @@ pub fn finalize( ) })?; validate_metadata(capture, &document)?; - let _: mxc_sdk::learning_mode_core::VerboseLoggingDocument = - serde_json::from_slice(&std::fs::read(&source_verbose_logging).map_err(|error| { - format!( - "failed to read captureDenials verbose logging output {}: {error}", - source_verbose_logging.display() - ) - })?) - .map_err(|error| { - format!( - "captureDenials verbose logging output {} is not valid JSON: {error}", - source_verbose_logging.display() - ) - })?; + let verbose_logging_bytes = std::fs::read(&source_verbose_logging).map_err(|error| { + format!( + "failed to read captureDenials verbose logging output {}: {error}", + source_verbose_logging.display() + ) + })?; + mxc_sdk::learning_mode_core::verbose_logging::validate_verbose_logging_document( + &verbose_logging_bytes, + ) + .map_err(|error| { + format!( + "captureDenials verbose logging output {} is not valid JSON: {error}", + source_verbose_logging.display() + ) + })?; let final_denials = context.log_dir.join("denials.json"); let final_verbose_logging = mxc_sdk::learning_mode_core::verbose_logging_sibling_path( @@ -372,6 +374,47 @@ mod tests { ); } + #[test] + fn finalize_accepts_version_five_verbose_logging() { + let directory = tempfile::tempdir().unwrap(); + let log_dir = directory.path().join("audit"); + std::fs::create_dir_all(&log_dir).unwrap(); + let source_denials = log_dir.join("denials.unique.json"); + let source_etl = log_dir.join("denials.unique.etl"); + let document = DenialsDocument::new(Vec::new(), DenialSummary::new(0, 0, false)); + std::fs::write(&source_denials, serde_json::to_vec(&document).unwrap()).unwrap(); + std::fs::write(&source_etl, b"etl").unwrap(); + let mut response = response_with_capture(&source_denials, &source_etl, 0); + let source_verbose_logging = + mxc_sdk::learning_mode_core::verbose_logging_sibling_path(&source_denials).unwrap(); + std::fs::write( + &source_verbose_logging, + br#"{ + "version": 5, + "signatures": [], + "summary": { + "totalOccurrences": 0, + "overflowOccurrences": 0, + "actionableOverflowOccurrences": 0, + "aggregateGroupsTruncated": false, + "processedEventsTruncated": false, + "actionableLimitReached": false + } + }"#, + ) + .unwrap(); + let context = AuditContext { + log_dir: log_dir.clone(), + config_path: None, + }; + + finalize(&mut response, &context, directory.path(), false).unwrap(); + + assert!(!source_verbose_logging.exists()); + let published = std::fs::read_to_string(log_dir.join("denials.verbose.json")).unwrap(); + assert!(published.contains("\"version\": 5")); + } + #[test] fn finalize_updates_metadata_before_postprocessing_failure() { let directory = tempfile::tempdir().unwrap();