From 4492e362294b5077f31646a9aad38c34c5555e2f Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Wed, 7 Oct 2026 17:08:20 -0700 Subject: [PATCH 01/17] Refuse AF_PACKET sockets in filtered LXC containers CAP_NET_RAW stays, because an explicit protocol: "icmp" allow needs a raw socket, and it also opens AF_PACKET, which reaches the interface below the hook the chains hang on. A syscall entering under numbering the filter was not built for kills the process rather than passing unchecked. AB#64150389 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9df5d37d-b3a2-487c-95df-ded398109b1d --- docs/backends/lxc/lxc-backend.md | 13 +- .../src/backends/lxc/common/lxc_bindings.rs | 124 +++++++++++++++++- .../src/backends/lxc/common/lxc_runner.rs | 5 +- ...xc_e2e_tests_e2e_lxc_network_capability.rs | 31 ++++- .../lxc_network_ga_egress_icmp_allowed.json | 2 +- .../scripts/run_lxc_network_ga_egress_test.sh | 10 +- 6 files changed, 163 insertions(+), 22 deletions(-) diff --git a/docs/backends/lxc/lxc-backend.md b/docs/backends/lxc/lxc-backend.md index d3ff17f04..f8239415f 100644 --- a/docs/backends/lxc/lxc-backend.md +++ b/docs/backends/lxc/lxc-backend.md @@ -335,10 +335,15 @@ The zone query should answer the zone you assigned. `CAP_NET_ADMIN` dropped from its bounding set whenever chains are installed, so it cannot. Default-deny closes external reachability for a container that does not deliberately tear it down, including services the workload itself starts. -- **Raw sockets bypass egress filtering.** `CAP_NET_RAW` is retained so that an - explicit `protocol: "icmp"` allow works. It also permits `AF_PACKET` sockets, - which write link-layer frames straight to the interface without traversing the - filter chain. +- **Only the command MXC attaches is held under the chains.** `CAP_NET_RAW` + stays, because an explicit `protocol: "icmp"` allow needs a raw socket. It + also permits `AF_PACKET` sockets, which write link-layer frames straight to + the interface without traversing the filter chain, and the attached command + carries a seccomp filter that fails `socket(AF_PACKET, ...)` with `EPERM` + whenever chains are installed. Container init and anything it starts are not + attached and are not filtered. A syscall entering under numbering the filter + was not built for, such as a 32-bit compat entry, kills the process rather + than passing unchecked. - **Policy is not in force while the container starts.** The chains are installed after the container has started and its address has settled, so container init and anything it starts run unfiltered in both directions for that interval. The diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index 93ff101a4..1197ac6cc 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -135,6 +135,14 @@ pub enum ContainerFirewall { Absent, } +/// Take away the two ways a workload can get around the chains MXC installed. +/// `CAP_NET_ADMIN` rewrites them and is dropped. Packets built on an +/// `AF_PACKET` socket go to the interface below the hook the chains hang on, +/// and a seccomp filter refuses that socket. +/// +/// `CAP_NET_RAW` stays. An `AF_INET` raw socket still goes through `OUTPUT`, +/// where the chains already filter it, and an explicit `protocol: "icmp"` +/// allow needs one. #[cfg(target_os = "linux")] fn confine_network_capabilities(command: &mut std::process::Command) { use std::os::unix::process::CommandExt; @@ -143,18 +151,122 @@ fn confine_network_capabilities(command: &mut std::process::Command) { const CAP_NET_ADMIN: libc::c_ulong = 12; // SAFETY: `pre_exec` runs between fork and exec, where only - // async-signal-safe work is permitted. `prctl` is a bare syscall and this - // closure allocates nothing and captures nothing. + // async-signal-safe work is permitted. `prctl` and `seccomp` are bare + // syscalls, the filter lives on the stack, and this closure allocates + // nothing and captures nothing. unsafe { command.pre_exec(|| { if libc::prctl(libc::PR_CAPBSET_DROP, CAP_NET_ADMIN, 0, 0, 0) != 0 { return Err(std::io::Error::last_os_error()); } - Ok(()) + refuse_packet_sockets() }); } } +/// Make `socket(AF_PACKET, ...)` fail with `EPERM` for this process and +/// everything it goes on to exec. A seccomp filter cannot be lifted once it is +/// installed, which is what holds the refusal against a workload trying to get +/// out of it. +/// +/// Needs `CAP_SYS_ADMIN`, which the runner already holds. Asking the kernel to +/// take the filter without it would mean setting `no_new_privs`, and that would +/// also strip the file capabilities a distribution's `ping` is shipped with. +#[cfg(target_os = "linux")] +fn refuse_packet_sockets() -> std::io::Result<()> { + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + compile_error!( + "refusing AF_PACKET needs this target's audit architecture and socket syscall number" + ); + + // Classic BPF opcodes, from linux/bpf_common.h: BPF_LD|BPF_W|BPF_ABS, + // BPF_JMP|BPF_JEQ|BPF_K, BPF_JMP|BPF_JGE|BPF_K, and BPF_RET|BPF_K. + const LOAD_WORD: u16 = 0x20; + const JUMP_IF_EQUAL: u16 = 0x15; + const JUMP_IF_AT_LEAST: u16 = 0x35; + const RETURN: u16 = 0x06; + + // Byte offsets into the `seccomp_data` the kernel presents, from + // linux/seccomp.h. `args` holds six 64-bit values; on a little-endian + // machine the low half of the first one sits at the lower address, and a + // socket domain is an `int` that fits there. + const SYSCALL_NUMBER: u32 = 0; + const ARCHITECTURE: u32 = 4; + const FIRST_ARGUMENT: u32 = 16; + + // x86-64 sets this bit in the syscall number to mean the x32 ABI, which + // numbers its calls differently. Neither supported architecture reaches + // this value any other way. + const FOREIGN_NUMBERING: u32 = 0x4000_0000; + + const REFUSE_WITH_EPERM: u32 = libc::SECCOMP_RET_ERRNO | (libc::EPERM as u32); + + // `seccomp` takes no flags here. Naming the argument keeps a bare zero out + // of a variadic call, where nothing would say what it meant. + const NO_FLAGS: libc::c_ulong = 0; + + // AUDIT_ARCH_* from linux/audit.h: the machine number in linux/elf-em.h + // with the 64-bit and little-endian flags set. The syscall numbers are from + // arch/x86/entry/syscalls/syscall_64.tbl and + // include/uapi/asm-generic/unistd.h. + #[cfg(target_arch = "x86_64")] + const NATIVE_ARCHITECTURE: u32 = 0xC000_003E; + #[cfg(target_arch = "x86_64")] + const SOCKET_SYSCALL: u32 = 41; + #[cfg(target_arch = "aarch64")] + const NATIVE_ARCHITECTURE: u32 = 0xC000_00B7; + #[cfg(target_arch = "aarch64")] + const SOCKET_SYSCALL: u32 = 198; + + // The kernel runs these in order and stops at the first `RETURN` it + // reaches. Read the architecture, and kill anything not built for the one + // this filter was compiled against, whose syscall numbers would not line up + // with the comparison below. Read the syscall number, and let everything + // that is not `socket` through. Read the first argument, which is the + // socket domain, and refuse `AF_PACKET`. Each jump counts forward from the + // instruction after it, taking `jt` when the comparison holds and `jf` when + // it does not. + // + // Laid out one instruction per line, which is the only way the jumps can be + // counted by eye. + #[rustfmt::skip] + let mut program = [ + libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: ARCHITECTURE }, + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 7, k: NATIVE_ARCHITECTURE }, + libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: SYSCALL_NUMBER }, + libc::sock_filter { code: JUMP_IF_AT_LEAST, jt: 5, jf: 0, k: FOREIGN_NUMBERING }, + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 2, k: SOCKET_SYSCALL }, + libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: FIRST_ARGUMENT }, + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 1, jf: 0, k: libc::AF_PACKET as u32 }, + libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_ALLOW }, + libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: REFUSE_WITH_EPERM }, + libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_KILL_PROCESS }, + ]; + + let header = libc::sock_fprog { + len: program.len() as libc::c_ushort, + filter: program.as_mut_ptr(), + }; + + // SAFETY: `libc` has no wrapper for this syscall. The kernel copies the + // program during the call and writes nothing back through the pointer, and + // both the program and the header outlive the call. + let taken = unsafe { + libc::syscall( + libc::SYS_seccomp, + libc::SECCOMP_SET_MODE_FILTER as libc::c_ulong, + NO_FLAGS, + &header as *const libc::sock_fprog, + ) + }; + + if taken != 0 { + return Err(std::io::Error::last_os_error()); + } + + Ok(()) +} + /// A `/etc/hosts` helper emits one `mxc:` diagnostic line, so this is far above /// any legitimate output. #[cfg(target_os = "linux")] @@ -1450,9 +1562,9 @@ mod tests { ); } - // The attach paths drop this capability only when chains are installed: the - // drop is privileged, so applying it to every run costs an unprivileged - // caller the whole execution. + // The attach paths confine the workload only when chains are installed: + // dropping a capability is privileged, so applying it to every run costs an + // unprivileged caller the whole execution. #[cfg(target_os = "linux")] #[test] fn confining_a_command_takes_a_privilege_an_unprivileged_caller_lacks() { diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs index 9abf2c93b..814504c27 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs @@ -2145,8 +2145,9 @@ mod tests { } #[test] - fn a_policy_that_installs_no_chain_leaves_the_capability_alone() { - // Dropping it needs CAP_SETPCAP, which an unprivileged caller lacks. + fn a_policy_that_installs_no_chain_leaves_the_workload_unconfined() { + // Confining takes CAP_SETPCAP to drop a capability, which an + // unprivileged caller lacks. assert_eq!( container_firewall(false, false), ContainerFirewall::Absent, diff --git a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs index 453a12534..903a4111d 100644 --- a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs +++ b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs @@ -9,6 +9,7 @@ use serde_json::json; use wxc_e2e_tests::{has_lxc_host, has_platform_exec, run_platform_config_value}; const CAP_NET_ADMIN: u64 = 1 << 12; +const CAP_NET_RAW: u64 = 1 << 13; /// Whether the LXC capability prerequisites are present. fn ready() -> bool { @@ -25,13 +26,24 @@ fn capability_mask(status: &str, field: &str) -> u64 { .unwrap_or_else(|| panic!("the container reported no readable {field} line\n{status}")) } +/// Read one decimal field out of the container's `/proc/self/status`. +fn status_number(status: &str, field: &str) -> u64 { + status + .lines() + .find(|line| line.starts_with(field)) + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|value| value.parse().ok()) + .unwrap_or_else(|| panic!("the container reported no readable {field} line\n{status}")) +} + #[test] -fn workload_cannot_reconfigure_the_network() { +fn workload_cannot_reconfigure_or_bypass_the_network() { if !ready() { return; } - // The drop only happens when chains exist, so this policy asks for a firewall. + // The confinement only happens when chains exist. This policy asks for a + // firewall to bring it on. let config = json!({ "version": "0.9.0-alpha", "containerId": "lxc-network-capability", @@ -72,5 +84,20 @@ fn workload_cannot_reconfigure_the_network() { 0, "{field} still carries CAP_NET_ADMIN; the workload can rewrite the firewall confining it\n{status}" ); + + assert_ne!( + capability_mask(&status, field) & CAP_NET_RAW, + 0, + "{field} lost CAP_NET_RAW; an explicit `protocol: \"icmp\"` allow needs a raw socket\n{status}" + ); } + + // 2 is SECCOMP_MODE_FILTER. The kernel writes this line too. A filter + // cannot be lifted once it is attached, and this one fails + // socket(AF_PACKET, ...) with EPERM. + assert_eq!( + status_number(&status, "Seccomp:"), + 2, + "the workload runs with no seccomp filter; AF_PACKET reaches the interface below the chains\n{status}" + ); } diff --git a/tests/configs/lxc_network_ga_egress_icmp_allowed.json b/tests/configs/lxc_network_ga_egress_icmp_allowed.json index a7712fc4e..f575b14ee 100644 --- a/tests/configs/lxc_network_ga_egress_icmp_allowed.json +++ b/tests/configs/lxc_network_ga_egress_icmp_allowed.json @@ -4,7 +4,7 @@ "containerId": "CLI-LXC-GA-Egress-Icmp-Allowed", "containment": "lxc", "process": { - "commandLine": "sh -c \"timeout 8 ping -c 1 -W 5 203.0.113.2 >/dev/null 2>&1 && echo MXC_NET_ALLOWED || echo MXC_NET_BLOCKED\"" + "commandLine": "sh -c \"command -v ping || echo MXC_PING_MISSING; timeout 8 ping -c 1 -W 5 203.0.113.2 2>&1 && echo MXC_NET_ALLOWED || echo MXC_NET_BLOCKED\"" }, "lifecycle": { "destroyOnExit": true diff --git a/tests/scripts/run_lxc_network_ga_egress_test.sh b/tests/scripts/run_lxc_network_ga_egress_test.sh index fb4af1822..5dfbd8158 100644 --- a/tests/scripts/run_lxc_network_ga_egress_test.sh +++ b/tests/scripts/run_lxc_network_ga_egress_test.sh @@ -283,13 +283,6 @@ for dns_addr in "$PEER_DNS_IP" "$PEER_DNS_SIBLING_IP"; do fi done -# The ICMP cases below read an unanswered echo as a firewall verdict, so a peer -# that ignores echo has to fail here as harness breakage instead. -if command -v ping >/dev/null 2>&1; then - ping -c 1 -W 5 "$PEER_IP" >/dev/null 2>&1 \ - || fail "the egress peer does not answer ICMP echo at $PEER_IP." -fi - # Drift guard: the peer fixtures must target this peer, or the run would probe a # stale address and prove nothing. PEER_TARGETING_CONFIGS=( @@ -376,6 +369,9 @@ assert_blocked "a destination denied by its own rule was reachable because an ea run_case "shadow-control case: the same first rule with no second deny" "$EXCEPT_SHADOW_CONTROL_CONFIG" assert_allowed "an address excluded from a deny was unreachable under egress.default allow. The exclusion is not narrowing its own rule, so the shadow case above proves only that everything was blocked." +# This case runs before the icmp-denied case below and fail() exits at once. A +# container that cannot send ICMP at all stops the run here instead of letting +# the denied case report a pass it did not earn. run_case "icmp case: egress.default deny, peer allowed on protocol icmp" "$ICMP_ALLOWED_CONFIG" assert_allowed "an ICMP echo to a peer allowed on protocol icmp was unreachable. Either the icmp selector never reached the chain, or the container cannot open a raw socket at all, which would make the icmp-denied case below pass without filtering anything." From f2aa888e5cb550b115491277a775dcb10324cc41 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Thu, 8 Oct 2026 10:58:49 -0700 Subject: [PATCH 02/17] simplifying main --- src/tools/lxc/src/linux_executor_arguments.rs | 219 +++++++++++ src/tools/lxc/src/main.rs | 345 ++++-------------- 2 files changed, 298 insertions(+), 266 deletions(-) create mode 100644 src/tools/lxc/src/linux_executor_arguments.rs diff --git a/src/tools/lxc/src/linux_executor_arguments.rs b/src/tools/lxc/src/linux_executor_arguments.rs new file mode 100644 index 000000000..dd4764c8b --- /dev/null +++ b/src/tools/lxc/src/linux_executor_arguments.rs @@ -0,0 +1,219 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use clap::Parser; +use mxc_sdk::mxc_common::config_parser::load_one_shot_request; +use mxc_sdk::mxc_common::logger::{Logger, Mode}; +use mxc_sdk::mxc_common::models::ExecutionRequest; +use std::process; + +#[derive(Parser)] +#[command(name = "lxc-exec", about = "Linux Container Executor")] +pub struct LinuxExecutorArguments { + /// Path to config JSON file (positional) + #[arg(value_name = "CONFIG_PATH")] + config_path: Option, + + /// Path to config JSON file + #[arg(long = "config")] + config: Option, + + /// Base64-encoded JSON config + #[arg(long = "config-base64")] + config_base64: Option, + + /// Enable debug/console output + #[arg(long)] + debug: bool, + + /// Delete container mode + #[arg(long)] + delete: bool, + + /// Container name (required with --delete) + #[arg(long = "containername")] + containername: Option, + + /// Enable experimental features + #[arg(long)] + experimental: bool, + + /// Allow testing-only features that must never run in production, currently + /// `network.proxy.builtinTestServer` (a bundled, deliberately-permissive + /// test HTTP proxy). Distinct from --experimental. + #[arg(long = "allow-testing-features")] + allow_testing_features: bool, + + /// Report host backend availability as JSON and exit + #[arg(long = "available-backends")] + available_backends: bool, + + /// Parse and validate config then exit without executing + #[arg(long = "dry-run")] + dry_run: bool, + + /// Path to diagnostic log file (appends, creates if missing) + #[arg(long = "log-file")] + log_file: Option, + + /// Install the warmed Hyperlight snapshot and exit, for the default + /// runtime (`agent`) or the comma-separated runtimes given with `=`: + /// `--setup-hyperlight=python,node`. Names are `agent`, `python`, + /// `python-shell`, `node`, `bash` and `dotnet-jit`. For each, pulls the + /// published rootfs from GHCR unless the image home already holds it, + /// boots it once, and writes the snapshot into the default user data + /// dir (~/.local/share/mxc-hyperlight on Linux, + /// %LOCALAPPDATA%\mxc-hyperlight on Windows). $MXC_HYPERLIGHT_HOME + /// overrides the destination if set. Intended for tool install hooks + /// so first-run has zero warmup cost. + #[arg( + long = "setup-hyperlight", + value_name = "RUNTIMES", + num_args = 0..=1, + require_equals = true, + value_delimiter = ',' + )] + setup_hyperlight: Option>, + + /// Rebuild the snapshot even if one already exists. + #[arg(long, requires = "setup_hyperlight")] + force: bool, +} + +impl LinuxExecutorArguments { + pub fn get_log_path(&self) -> Option { + self.log_file.clone() + } + + pub fn should_run_in_debug(&self) -> bool { + self.debug + } + + pub fn should_display_avalible_backends(&self) -> bool { + self.available_backends + } + + pub fn parse_and_validate(&self) -> bool { + self.dry_run + } + + pub fn get_request(&self) -> ExecutionRequest { + let (config_data, is_base64) = if let Some(ref b64) = self.config_base64 { + (b64.clone(), true) + } else if let Some(ref path) = self.config { + (path.clone(), false) + } else if let Some(ref path) = self.config_path { + (path.clone(), false) + } else { + (String::new(), false) + }; + + let mut logger = Logger::new(if self.debug { + Mode::Console + } else { + Mode::Buffer + }); + + let mut request = match load_one_shot_request(&config_data, &mut logger, is_base64) { + Ok(r) => r, + Err(_) => { + eprint!("Request error\n{}", logger.get_buffer()); + process::exit(1); + } + }; + + request.experimental_enabled = self.experimental; + request.testing_features_enabled = self.allow_testing_features; + request.dry_run = self.dry_run; + + request + } + + pub fn does_user_want_hyperlight_setup(&self) -> bool { + self.setup_hyperlight.is_some() + } + + pub fn setup_hyperlight(&self) -> Result { + if let Some(config_path) = &self.config_path { + return Err(format!( + "Error: --setup-hyperlight takes no config path; name runtimes with \ + --setup-hyperlight={config_path}" + )); + } + + #[cfg(not(all(feature = "hyperlight", target_arch = "x86_64")))] + { + return Err( + "Error: --setup-hyperlight requires x86_64 (Hyperlight needs KVM or WHP)" + .to_string(), + ); + } + + #[cfg(all(feature = "hyperlight", target_arch = "x86_64"))] + { + // WHP is delay-loaded; check before setup boots a VM. + #[cfg(target_os = "windows")] + if !mxc_sdk::hyperlight_common::is_whp_available() { + return Err( + "Error: --setup-hyperlight requires Windows Hypervisor Platform (WHP). \ + Enable the HypervisorPlatform optional feature and reboot." + .to_string(), + ); + } + + //KVM is checked before anything is pulled. + #[cfg(target_os = "linux")] + { + if !mxc_sdk::hyperlight_common::is_kvm_available() { + return Err( + "Error: --setup-hyperlight requires KVM: /dev/kvm must be readable and \ + writable by this user." + .to_string(), + ); + } + + // Setup is an interactive install: the pull and the warm-up + // report progress as they go. + let runtimes = mxc_sdk::hyperlight_common::parse_runtimes( + &self.setup_hyperlight.clone().unwrap(), + )?; + + let mut logger = mxc_sdk::mxc_common::logger::Logger::new( + mxc_sdk::mxc_common::logger::Mode::Console, + ); + match mxc_sdk::hyperlight_common::setup(self.force, &runtimes, &mut logger) { + Ok(home) => Ok(format!("hyperlight setup: image home ready at {home:?}")), + Err(msg) => Err(format!("hyperlight setup failed: {msg}")), + } + } + } + } + + pub fn does_user_want_to_delete_a_container(&self) -> bool { + self.delete + } + + pub fn delete_container(&self) -> Result { + let name = match self.containername { + Some(ref n) => n, + None => return Err("Error: --containername is required with --delete".to_string()), + }; + + Self::delete_lxc_container(name) + } + + fn delete_lxc_container(name: &str) -> Result { + use mxc_sdk::lxc_common::lxc_bindings::LxcContainer; + + let container = LxcContainer::new(name, None); + + if !container.is_defined() { + return Err(format!("Container '{}' does not exist.", name)); + } + + match container.destroy() { + Ok(()) => Ok(format!("Deleted LXC container: {}", name)), + Err(e) => Err(format!("Failed to delete LXC container '{}': {}", name, e)), + } + } +} diff --git a/src/tools/lxc/src/main.rs b/src/tools/lxc/src/main.rs index 795538f1c..3024fe2cf 100644 --- a/src/tools/lxc/src/main.rs +++ b/src/tools/lxc/src/main.rs @@ -1,294 +1,89 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +mod linux_executor_arguments; + +use clap::Parser; use std::fmt::Write; -use std::process; use std::time::Instant; +use std::{eprint, process}; -use clap::Parser; -use mxc_sdk::mxc_common::config_parser::load_one_shot_request; use mxc_sdk::mxc_common::logger::{Logger, Mode}; use mxc_sdk::mxc_common::models::{ExecutionRequest, ScriptResponse}; -use mxc_sdk::mxc_common::script_runner::handle_dry_run_exit; +use mxc_sdk::mxc_common::script_runner; use mxc_sdk::mxc_common::telemetry; use mxc_sdk::lxc_common::signal_cleanup; -#[derive(Parser)] -#[command(name = "lxc-exec", about = "Linux Container Executor")] -struct Cli { - /// Path to config JSON file (positional) - #[arg(value_name = "CONFIG_PATH")] - config_path: Option, - - /// Path to config JSON file - #[arg(long = "config")] - config: Option, - - /// Base64-encoded JSON config - #[arg(long = "config-base64")] - config_base64: Option, - - /// Enable debug/console output - #[arg(long)] - debug: bool, - - /// Delete container mode - #[arg(long)] - delete: bool, - - /// Container name (required with --delete) - #[arg(long = "containername")] - containername: Option, - - /// Enable experimental features - #[arg(long)] - experimental: bool, - - /// Report host backend availability as JSON and exit - #[arg(long = "available-backends")] - available_backends: bool, - - /// Parse and validate config then exit without executing - #[arg(long = "dry-run")] - dry_run: bool, - - /// Path to diagnostic log file (appends, creates if missing) - #[arg(long = "log-file")] - log_file: Option, - - /// Install the warmed Hyperlight snapshot and exit, for the default - /// runtime (`agent`) or the comma-separated runtimes given with `=`: - /// `--setup-hyperlight=python,node`. Names are `agent`, `python`, - /// `python-shell`, `node`, `bash` and `dotnet-jit`. For each, pulls the - /// published rootfs from GHCR unless the image home already holds it, - /// boots it once, and writes the snapshot into the default user data - /// dir (~/.local/share/mxc-hyperlight on Linux, - /// %LOCALAPPDATA%\mxc-hyperlight on Windows). $MXC_HYPERLIGHT_HOME - /// overrides the destination if set. Intended for tool install hooks - /// so first-run has zero warmup cost. - #[arg( - long = "setup-hyperlight", - value_name = "RUNTIMES", - num_args = 0..=1, - require_equals = true, - value_delimiter = ',' - )] - setup_hyperlight: Option>, - - /// Rebuild the snapshot even if one already exists. Use after - /// replacing `initrd.cpio` so the warm state matches the new - /// bits. Requires --setup-hyperlight. - #[arg(long, requires = "setup_hyperlight")] - force: bool, -} - -fn log_request(request: &ExecutionRequest, logger: &mut Logger) { - let _ = writeln!(logger, "Script code length: {}", request.script_code.len()); - let _ = writeln!(logger, "Working directory: {}", request.working_directory); - let _ = writeln!(logger, "Script timeout: {}", request.script_timeout); - let _ = writeln!(logger, "Container name: {}", request.container_id); -} - -fn display_script_results(response: &ScriptResponse, logger: &mut Logger) { - let code = response.exit_code; - let _ = writeln!(logger, "Exit code: {} (0x{:08X})", code, code as u32); - if !response.error_message.is_empty() { - let _ = writeln!(logger, "Error: {}", response.error_message); - } -} - -/// Surface warnings the run recorded (e.g. Bubblewrap reporting an IPv6 allow -/// the sandbox namespace cannot reach). -/// -/// The logger only retains these rather than writing them itself, so that -/// `mxc_engine` embedders don't get unannounced writes to a terminal they own. -/// `lxc-exec` *does* own its terminal, so it opts in here — matching wxc-exec. -/// stderr, not stdout: stdout carries the workload's own output. -fn emit_warnings(logger: &Logger) { - for warning in logger.warnings() { - eprintln!("{warning}"); - } -} - -fn delete_lxc_container(name: &str, logger: &mut Logger) -> bool { - use mxc_sdk::lxc_common::lxc_bindings::LxcContainer; - - let container = LxcContainer::new(name, None); +fn main() { + let arguments = linux_executor_arguments::LinuxExecutorArguments::parse(); - if !container.is_defined() { - logger.log_line(&format!("Container '{}' does not exist.", name)); - return false; - } + // Get the logger + let mut logger = Logger::new(if arguments.should_run_in_debug() { + Mode::Console + } else { + Mode::Buffer + }); - match container.destroy() { - Ok(()) => { - logger.log_line(&format!("Deleted LXC container: {}", name)); - true - } - Err(e) => { - logger.log_line(&format!("Failed to delete LXC container '{}': {}", name, e)); - false + // Enable the file sink if needed + if let Some(ref log_path) = arguments.get_log_path() { + if let Err(e) = logger.enable_file_sink(std::path::Path::new(log_path)) { + eprintln!("Warning: could not open log file '{}': {}", log_path, e); + process::exit(1); } } -} - -fn main() { - // Install before spawning any other threads so the signal mask propagates. - // Failure here is fatal: install() either succeeds with the watchdog - // running, or restores the original signal mask and returns Err. We - // refuse to continue without it because containers leaked on SIGTERM/INT - // are exactly the failure mode this code exists to prevent. - if let Err(e) = signal_cleanup::install() { - eprintln!("Error: failed to install signal cleanup handler: {}", e); - process::exit(1); - } - - let cli = Cli::parse(); - // Detection-only fast path used by SDK `getPlatformSupport()`; runs before - // config handling so no JSON file is needed just to ask what the host can - // do, and mutates no host state. - if cli.available_backends { + // Go through the different operating modes + if arguments.should_display_avalible_backends() { match mxc_sdk::mxc_engine::to_json_pretty(&mxc_sdk::mxc_engine::available_backends()) { - Ok(json) => println!("{json}"), + Ok(json) => { + println!("{json}"); + process::exit(0); + } Err(e) => { eprintln!("Error: probe serialization failed: {e}"); process::exit(1); } } - return; - } - - // --setup-hyperlight: eagerly warm up the snapshot and exit. Runs - // before config parsing so the user doesn't need a JSON file on - // disk just to install. - if let Some(runtime_names) = &cli.setup_hyperlight { - // Setup exits before any config is read, so a positional here is - // most likely a runtime name given with a space instead of `=`. - if let Some(stray) = &cli.config_path { - eprintln!( - "Error: --setup-hyperlight takes no config path; name runtimes with \ - --setup-hyperlight={stray}" - ); - process::exit(1); - } - #[cfg(all(feature = "hyperlight", target_arch = "x86_64"))] - { - // WHP is delay-loaded; check before setup boots a VM. - #[cfg(target_os = "windows")] - if !mxc_sdk::hyperlight_common::is_whp_available() { - eprintln!( - "Error: --setup-hyperlight requires Windows Hypervisor Platform (WHP). \ - Enable the HypervisorPlatform optional feature and reboot." - ); - process::exit(1); + } else if arguments.does_user_want_hyperlight_setup() { + match arguments.setup_hyperlight() { + Ok(message) => { + eprintln!("{message}"); + process::exit(0); } - // KVM is checked before anything is pulled. - #[cfg(target_os = "linux")] - if !mxc_sdk::hyperlight_common::is_kvm_available() { - eprintln!( - "Error: --setup-hyperlight requires KVM: /dev/kvm must be readable and \ - writable by this user." - ); + Err(message) => { + eprintln!("{message}"); process::exit(1); } - - // Setup is an interactive install: the pull and the warm-up - // report progress as they go. - let mut logger = Logger::new(Mode::Console); - let runtimes = match mxc_sdk::hyperlight_common::parse_runtimes(runtime_names) { - Ok(runtimes) => runtimes, - Err(msg) => { - eprintln!("Error: {msg}"); - process::exit(1); - } - }; - match mxc_sdk::hyperlight_common::setup(cli.force, &runtimes, &mut logger) { - Ok(home) => { - eprintln!("hyperlight setup: image home ready at {:?}", home); - process::exit(0); - } - Err(msg) => { - eprintln!("hyperlight setup failed: {msg}"); - process::exit(1); - } - } - } - #[cfg(not(all(feature = "hyperlight", target_arch = "x86_64")))] - { - let _ = runtime_names; - eprintln!("Error: --setup-hyperlight requires x86_64 (Hyperlight needs KVM or WHP)"); - process::exit(1); } - } - - // Determine config input - let (config_data, is_base64) = if let Some(ref b64) = cli.config_base64 { - (b64.clone(), true) - } else if let Some(ref path) = cli.config { - (path.clone(), false) - } else if let Some(ref path) = cli.config_path { - (path.clone(), false) - } else if !cli.delete { - eprintln!("Error: No config provided. Use a positional path, --config, or --config-base64"); - process::exit(1); - } else { - (String::new(), false) - }; - - let mut logger = Logger::new(if cli.debug { - Mode::Console - } else { - Mode::Buffer - }); - - if let Some(ref log_path) = cli.log_file { - if let Err(e) = logger.enable_file_sink(std::path::Path::new(log_path)) { - eprintln!("Warning: could not open log file '{}': {}", log_path, e); - } - } - - // Delete mode - if cli.delete { - let name = match cli.containername { - Some(ref n) => n.as_str(), - None => { - eprintln!("Error: --containername is required with --delete"); + } else if arguments.does_user_want_to_delete_a_container() { + match arguments.delete_container() { + Ok(message) => { + eprintln!("{message}"); + process::exit(0); + } + Err(message) => { + eprintln!("{message}"); process::exit(1); } - }; - let success = delete_lxc_container(name, &mut logger); - print!("{}", logger.get_buffer()); - process::exit(if success { 0 } else { 1 }); - } - - // Load request - let mut request = match load_one_shot_request(&config_data, &mut logger, is_base64) { - Ok(r) => r, - Err(_) => { - eprint!("Request error\n{}", logger.get_buffer()); - process::exit(1); } - }; + } - request.experimental_enabled = cli.experimental; - request.dry_run = cli.dry_run; + // User wants to run a request. + // Make the request. + let request = arguments.get_request(); - // ── Telemetry init ────────────────────────────────────────────── let telemetry_active = request .telemetry .as_ref() .map(|c| telemetry::init(c, &mut logger)) .unwrap_or(false); + let requested_sandbox_kind = request .telemetry .as_ref() .and_then(|config| config.requested_sandbox_kind); - // Install a crash-telemetry panic hook once telemetry is active, chaining - // the previously-installed hook so the default stderr backtrace still - // prints. The hook body is panic-free and emits no message text. if telemetry_active { telemetry::set_process_context_with_kind(&request.containment, requested_sandbox_kind); telemetry::install_panic_hook(); @@ -296,15 +91,15 @@ fn main() { log_request(&request, &mut logger); - // Dispatch by containment backend. Backend selection and runner - // construction — Bubblewrap (the Linux default for abstract intents), LXC - // (explicit `containment: "lxc"`, plus the catch-all for anything else such - // as `processcontainer`), and the experimental Hyperlight / MicroVM - // backends — live in `mxc_sdk::mxc_engine::run`, the single home for one-shot backend - // dispatch. It runs the selected backend to completion and returns the - // response; experimental backends that require `--experimental` (or that - // aren't compiled in) surface an error here. + // Setup watchdog for unexpected shutdowns + if let Err(e) = signal_cleanup::install() { + eprint!("Error: failed to install signal cleanup handler: {e}"); + process::exit(1); + } + let run_start = Instant::now(); + + // Run the request. let response = match mxc_sdk::mxc_engine::run(&request, &mut logger) { Ok(response) => response, Err(e) => { @@ -320,19 +115,20 @@ fn main() { process::exit(1); } }; + let run_elapsed = run_start.elapsed(); let _ = writeln!(logger, "Runner completed in {}ms", run_elapsed.as_millis()); - // Emitted before the dry-run branch below, which exits the process. emit_warnings(&logger); - if cli.dry_run { - handle_dry_run_exit(&response, &mut logger); + // Handle dry_run request. + if arguments.parse_and_validate() { + script_runner::handle_dry_run_exit(&response, &mut logger); } + // Show output to the user. display_script_results(&response, &mut logger); - // ── Telemetry emit ────────────────────────────────────────────── telemetry::emit_completion_with_kind( telemetry_active, &request.containment, @@ -344,11 +140,28 @@ fn main() { print!("{}", response.standard_out); eprint!("{}", response.standard_err); - // Never exit non-zero on an infrastructure failure without a diagnostic: - // `display_script_results` only writes the error into the (buffered, - // non-debug-suppressed) logger, so surface it on stderr here for parity - // with wxc-exec (issue #564). - mxc_sdk::mxc_common::script_runner::emit_backend_error_envelope(&response); + script_runner::emit_backend_error_envelope(&response); process::exit(response.exit_code); } + +fn log_request(request: &ExecutionRequest, logger: &mut Logger) { + let _ = writeln!(logger, "Script code length: {}", request.script_code.len()); + let _ = writeln!(logger, "Working directory: {}", request.working_directory); + let _ = writeln!(logger, "Script timeout: {}", request.script_timeout); + let _ = writeln!(logger, "Container name: {}", request.container_id); +} + +fn emit_warnings(logger: &Logger) { + for warning in logger.warnings() { + eprintln!("{warning}"); + } +} + +fn display_script_results(response: &ScriptResponse, logger: &mut Logger) { + let code = response.exit_code; + let _ = writeln!(logger, "Exit code: {} (0x{:08X})", code, code as u32); + if !response.error_message.is_empty() { + let _ = writeln!(logger, "Error: {}", response.error_message); + } +} From a7b725eb8976563dd94b6c35d9225146880dbce9 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Fri, 9 Oct 2026 10:46:46 -0700 Subject: [PATCH 03/17] Finishing up seccomp --- .../src/backends/lxc/common/lxc_bindings.rs | 122 ++++++++---------- src/tools/lxc/src/linux_executor_arguments.rs | 7 - 2 files changed, 55 insertions(+), 74 deletions(-) diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index 1197ac6cc..a32f13246 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -135,14 +135,11 @@ pub enum ContainerFirewall { Absent, } -/// Take away the two ways a workload can get around the chains MXC installed. -/// `CAP_NET_ADMIN` rewrites them and is dropped. Packets built on an -/// `AF_PACKET` socket go to the interface below the hook the chains hang on, -/// and a seccomp filter refuses that socket. -/// -/// `CAP_NET_RAW` stays. An `AF_INET` raw socket still goes through `OUTPUT`, -/// where the chains already filter it, and an explicit `protocol: "icmp"` -/// allow needs one. +/// Two networking capabilites LXC cares about is CAP_NET_ADMIN and CAP_NET_RAW. +/// CAP_NET_ADMIN allows a process to change the networking rules +/// CAP_NET_RAW allows a process to use RAW and PACKET sockets and those allow a process to bypass +/// a network chain. +/// Remove both capabilites here. (Technically CAP_NET_RAW isn not removed. More details inside) #[cfg(target_os = "linux")] fn confine_network_capabilities(command: &mut std::process::Command) { use std::os::unix::process::CommandExt; @@ -155,102 +152,93 @@ fn confine_network_capabilities(command: &mut std::process::Command) { // syscalls, the filter lives on the stack, and this closure allocates // nothing and captures nothing. unsafe { + // Take away CAP_NET_ADMIN. command.pre_exec(|| { if libc::prctl(libc::PR_CAPBSET_DROP, CAP_NET_ADMIN, 0, 0, 0) != 0 { return Err(std::io::Error::last_os_error()); } + + // Refuse AF_PACKET communication. refuse_packet_sockets() }); } } -/// Make `socket(AF_PACKET, ...)` fail with `EPERM` for this process and -/// everything it goes on to exec. A seccomp filter cannot be lifted once it is -/// installed, which is what holds the refusal against a workload trying to get -/// out of it. -/// -/// Needs `CAP_SYS_ADMIN`, which the runner already holds. Asking the kernel to -/// take the filter without it would mean setting `no_new_privs`, and that would -/// also strip the file capabilities a distribution's `ping` is shipped with. + +/// Removing CA_NET_RAW prevents a process from using RAW and PACKET sockets. However, ICMP is also denied. +/// LXC needs to allow ICMP and prevent programs from using sockets to +/// bypass the network guards. The solution is to give the kernel a custom filter to refuse +/// any socket that carries AF_PACKET. LXC utalizes SecComp. +/// see https://docs.kernel.org/userspace-api/seccomp_filter.html and +/// https://man7.org/linux/man-pages/man2/seccomp.2.html #[cfg(target_os = "linux")] fn refuse_packet_sockets() -> std::io::Result<()> { #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] - compile_error!( - "refusing AF_PACKET needs this target's audit architecture and socket syscall number" - ); - - // Classic BPF opcodes, from linux/bpf_common.h: BPF_LD|BPF_W|BPF_ABS, - // BPF_JMP|BPF_JEQ|BPF_K, BPF_JMP|BPF_JGE|BPF_K, and BPF_RET|BPF_K. - const LOAD_WORD: u16 = 0x20; - const JUMP_IF_EQUAL: u16 = 0x15; - const JUMP_IF_AT_LEAST: u16 = 0x35; - const RETURN: u16 = 0x06; - - // Byte offsets into the `seccomp_data` the kernel presents, from - // linux/seccomp.h. `args` holds six 64-bit values; on a little-endian - // machine the low half of the first one sits at the lower address, and a - // socket domain is an `int` that fits there. + compile_error!("LXC only understands SecComp system calls for x86_x64 and arm64."); + + // https://docs.kernel.org/networking/filter.html for more information on filtering sockets + // Filtering sockets requires assembling a small filter program with assembly like syntax. + // The below code is to consolidate some commands into easy to understand words. + + // https://github.com/torvalds/linux/blob/master/include/uapi/linux/bpf_common.h + // Common BPF words + const LOAD_WORD: u16 = (libc::BPF_LD | libc::BPF_W | libc::BPF_ABS) as u16; + const JUMP_IF_EQUAL: u16 = (libc::BPF_JMP | libc::BPF_JEQ | libc::BPF_K) as u16; + const JUMP_IF_AT_LEAST: u16 = (libc::BPF_JMP | libc::BPF_JGE | libc::BPF_K) as u16; + const RETURN: u16 = (libc::BPF_RET | libc::BPF_K) as u16; + + + // https://github.com/torvalds/linux/blob/master/include/uapi/linux/seccomp.h + // Keys of the information needed during the filter. const SYSCALL_NUMBER: u32 = 0; const ARCHITECTURE: u32 = 4; const FIRST_ARGUMENT: u32 = 16; - // x86-64 sets this bit in the syscall number to mean the x32 ABI, which - // numbers its calls differently. Neither supported architecture reaches - // this value any other way. - const FOREIGN_NUMBERING: u32 = 0x4000_0000; - const REFUSE_WITH_EPERM: u32 = libc::SECCOMP_RET_ERRNO | (libc::EPERM as u32); - // `seccomp` takes no flags here. Naming the argument keeps a bare zero out - // of a variadic call, where nothing would say what it meant. const NO_FLAGS: libc::c_ulong = 0; - // AUDIT_ARCH_* from linux/audit.h: the machine number in linux/elf-em.h - // with the 64-bit and little-endian flags set. The syscall numbers are from - // arch/x86/entry/syscalls/syscall_64.tbl and - // include/uapi/asm-generic/unistd.h. - #[cfg(target_arch = "x86_64")] - const NATIVE_ARCHITECTURE: u32 = 0xC000_003E; + // numbers come from + // https://github.com/torvalds/linux/blob/master/include/uapi/linux/audit.h + // To test the ABI the filter needs to know the endiness and arch. + const _64BIT_ABI_LITTLE_ENDINESS: u32 = 0x8000_0000 | 0x4000_0000; + + // Also from https://github.com/torvalds/linux/blob/master/include/uapi/linux/audit.h #[cfg(target_arch = "x86_64")] - const SOCKET_SYSCALL: u32 = 41; + const NATIVE_ARCHITECTURE: u32 = _64BIT_ABI_LITTLE_ENDINESS | libc::EM_X86_64 as u32; #[cfg(target_arch = "aarch64")] - const NATIVE_ARCHITECTURE: u32 = 0xC000_00B7; - #[cfg(target_arch = "aarch64")] - const SOCKET_SYSCALL: u32 = 198; - - // The kernel runs these in order and stops at the first `RETURN` it - // reaches. Read the architecture, and kill anything not built for the one - // this filter was compiled against, whose syscall numbers would not line up - // with the comparison below. Read the syscall number, and let everything - // that is not `socket` through. Read the first argument, which is the - // socket domain, and refuse `AF_PACKET`. Each jump counts forward from the - // instruction after it, taking `jt` when the comparison holds and `jf` when - // it does not. - // - // Laid out one instruction per line, which is the only way the jumps can be - // counted by eye. + const NATIVE_ARCHITECTURE: u32 = _64BIT_ABI_LITTLE_ENDINESS | libc::EM_AARCH64 as u32; + + // Filter only works for x64 and arm64 ABI's. + // Used to filter out a 32 bit process under the "Filters" section. + // check https://man7.org/linux/man-pages/man2/seccomp.2.html + const USES_X32_ABI_FLAG: u32 = 0x4000_0000; + + const SOCKET_SYSCALL: u32 = libc::SYS_socket as u32; + + + // Make the filter #[rustfmt::skip] let mut program = [ libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: ARCHITECTURE }, - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 7, k: NATIVE_ARCHITECTURE }, + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 7, k: NATIVE_ARCHITECTURE }, // test ABI libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: SYSCALL_NUMBER }, - libc::sock_filter { code: JUMP_IF_AT_LEAST, jt: 5, jf: 0, k: FOREIGN_NUMBERING }, - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 2, k: SOCKET_SYSCALL }, + libc::sock_filter { code: JUMP_IF_AT_LEAST, jt: 5, jf: 0, k: USES_X32_ABI_FLAG }, // Test 32 bit + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 2, k: SOCKET_SYSCALL }, // Only process a socket syscall libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: FIRST_ARGUMENT }, - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 1, jf: 0, k: libc::AF_PACKET as u32 }, + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 1, jf: 0, k: libc::AF_PACKET as u32 }, // return error if using AF_PACKET libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_ALLOW }, libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: REFUSE_WITH_EPERM }, libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_KILL_PROCESS }, ]; + // Make the struct let header = libc::sock_fprog { len: program.len() as libc::c_ushort, filter: program.as_mut_ptr(), }; - // SAFETY: `libc` has no wrapper for this syscall. The kernel copies the - // program during the call and writes nothing back through the pointer, and - // both the program and the header outlive the call. + // Make the sys call let taken = unsafe { libc::syscall( libc::SYS_seccomp, diff --git a/src/tools/lxc/src/linux_executor_arguments.rs b/src/tools/lxc/src/linux_executor_arguments.rs index dd4764c8b..78ee79ced 100644 --- a/src/tools/lxc/src/linux_executor_arguments.rs +++ b/src/tools/lxc/src/linux_executor_arguments.rs @@ -38,12 +38,6 @@ pub struct LinuxExecutorArguments { #[arg(long)] experimental: bool, - /// Allow testing-only features that must never run in production, currently - /// `network.proxy.builtinTestServer` (a bundled, deliberately-permissive - /// test HTTP proxy). Distinct from --experimental. - #[arg(long = "allow-testing-features")] - allow_testing_features: bool, - /// Report host backend availability as JSON and exit #[arg(long = "available-backends")] available_backends: bool, @@ -123,7 +117,6 @@ impl LinuxExecutorArguments { }; request.experimental_enabled = self.experimental; - request.testing_features_enabled = self.allow_testing_features; request.dry_run = self.dry_run; request From c6a95211aa24a3823c8349c5127b9dbc17952f46 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Fri, 9 Oct 2026 10:51:08 -0700 Subject: [PATCH 04/17] Fixing up some AI comments --- docs/backends/lxc/lxc-backend.md | 9 --------- tests/scripts/run_lxc_network_ga_egress_test.sh | 3 --- 2 files changed, 12 deletions(-) diff --git a/docs/backends/lxc/lxc-backend.md b/docs/backends/lxc/lxc-backend.md index f8239415f..775f77e1c 100644 --- a/docs/backends/lxc/lxc-backend.md +++ b/docs/backends/lxc/lxc-backend.md @@ -335,15 +335,6 @@ The zone query should answer the zone you assigned. `CAP_NET_ADMIN` dropped from its bounding set whenever chains are installed, so it cannot. Default-deny closes external reachability for a container that does not deliberately tear it down, including services the workload itself starts. -- **Only the command MXC attaches is held under the chains.** `CAP_NET_RAW` - stays, because an explicit `protocol: "icmp"` allow needs a raw socket. It - also permits `AF_PACKET` sockets, which write link-layer frames straight to - the interface without traversing the filter chain, and the attached command - carries a seccomp filter that fails `socket(AF_PACKET, ...)` with `EPERM` - whenever chains are installed. Container init and anything it starts are not - attached and are not filtered. A syscall entering under numbering the filter - was not built for, such as a 32-bit compat entry, kills the process rather - than passing unchecked. - **Policy is not in force while the container starts.** The chains are installed after the container has started and its address has settled, so container init and anything it starts run unfiltered in both directions for that interval. The diff --git a/tests/scripts/run_lxc_network_ga_egress_test.sh b/tests/scripts/run_lxc_network_ga_egress_test.sh index 5dfbd8158..d49387e92 100644 --- a/tests/scripts/run_lxc_network_ga_egress_test.sh +++ b/tests/scripts/run_lxc_network_ga_egress_test.sh @@ -369,9 +369,6 @@ assert_blocked "a destination denied by its own rule was reachable because an ea run_case "shadow-control case: the same first rule with no second deny" "$EXCEPT_SHADOW_CONTROL_CONFIG" assert_allowed "an address excluded from a deny was unreachable under egress.default allow. The exclusion is not narrowing its own rule, so the shadow case above proves only that everything was blocked." -# This case runs before the icmp-denied case below and fail() exits at once. A -# container that cannot send ICMP at all stops the run here instead of letting -# the denied case report a pass it did not earn. run_case "icmp case: egress.default deny, peer allowed on protocol icmp" "$ICMP_ALLOWED_CONFIG" assert_allowed "an ICMP echo to a peer allowed on protocol icmp was unreachable. Either the icmp selector never reached the chain, or the container cannot open a raw socket at all, which would make the icmp-denied case below pass without filtering anything." From b2949c620cb7edbcca92efdfc372b13bfbd45e72 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Fri, 9 Oct 2026 15:36:41 -0700 Subject: [PATCH 05/17] Restore hyperlight setup on non-Linux targets The setup body moved inside a target_os = "linux" block, which broke the lxc-exec build on Windows and macOS x86_64 and left --setup-hyperlight with nothing to do on either. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1654c073-27eb-43cf-82f3-417614ce845b --- .../src/backends/lxc/common/lxc_bindings.rs | 3 -- src/tools/lxc/src/linux_executor_arguments.rs | 38 +++++++++---------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index a32f13246..d60f90742 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -164,7 +164,6 @@ fn confine_network_capabilities(command: &mut std::process::Command) { } } - /// Removing CA_NET_RAW prevents a process from using RAW and PACKET sockets. However, ICMP is also denied. /// LXC needs to allow ICMP and prevent programs from using sockets to /// bypass the network guards. The solution is to give the kernel a custom filter to refuse @@ -187,7 +186,6 @@ fn refuse_packet_sockets() -> std::io::Result<()> { const JUMP_IF_AT_LEAST: u16 = (libc::BPF_JMP | libc::BPF_JGE | libc::BPF_K) as u16; const RETURN: u16 = (libc::BPF_RET | libc::BPF_K) as u16; - // https://github.com/torvalds/linux/blob/master/include/uapi/linux/seccomp.h // Keys of the information needed during the filter. const SYSCALL_NUMBER: u32 = 0; @@ -216,7 +214,6 @@ fn refuse_packet_sockets() -> std::io::Result<()> { const SOCKET_SYSCALL: u32 = libc::SYS_socket as u32; - // Make the filter #[rustfmt::skip] let mut program = [ diff --git a/src/tools/lxc/src/linux_executor_arguments.rs b/src/tools/lxc/src/linux_executor_arguments.rs index 78ee79ced..76a8a9a95 100644 --- a/src/tools/lxc/src/linux_executor_arguments.rs +++ b/src/tools/lxc/src/linux_executor_arguments.rs @@ -156,28 +156,26 @@ impl LinuxExecutorArguments { //KVM is checked before anything is pulled. #[cfg(target_os = "linux")] - { - if !mxc_sdk::hyperlight_common::is_kvm_available() { - return Err( - "Error: --setup-hyperlight requires KVM: /dev/kvm must be readable and \ + if !mxc_sdk::hyperlight_common::is_kvm_available() { + return Err( + "Error: --setup-hyperlight requires KVM: /dev/kvm must be readable and \ writable by this user." - .to_string(), - ); - } - - // Setup is an interactive install: the pull and the warm-up - // report progress as they go. - let runtimes = mxc_sdk::hyperlight_common::parse_runtimes( - &self.setup_hyperlight.clone().unwrap(), - )?; - - let mut logger = mxc_sdk::mxc_common::logger::Logger::new( - mxc_sdk::mxc_common::logger::Mode::Console, + .to_string(), ); - match mxc_sdk::hyperlight_common::setup(self.force, &runtimes, &mut logger) { - Ok(home) => Ok(format!("hyperlight setup: image home ready at {home:?}")), - Err(msg) => Err(format!("hyperlight setup failed: {msg}")), - } + } + + // Setup is an interactive install: the pull and the warm-up + // report progress as they go. + let runtimes = mxc_sdk::hyperlight_common::parse_runtimes( + &self.setup_hyperlight.clone().unwrap(), + )?; + + let mut logger = mxc_sdk::mxc_common::logger::Logger::new( + mxc_sdk::mxc_common::logger::Mode::Console, + ); + match mxc_sdk::hyperlight_common::setup(self.force, &runtimes, &mut logger) { + Ok(home) => Ok(format!("hyperlight setup: image home ready at {home:?}")), + Err(msg) => Err(format!("hyperlight setup failed: {msg}")), } } } From f9e9014a71611d330fcfbc75f9932267d5d7691f Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Fri, 9 Oct 2026 15:53:14 -0700 Subject: [PATCH 06/17] Prove the AF_PACKET refusal with a test The container check only asserted that a filter was attached, which a permissive filter passes identically. A paired unit test now opens a packet socket with and without the filter: allowed unfiltered, EPERM filtered. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1654c073-27eb-43cf-82f3-417614ce845b --- docs/backends/lxc/lxc-backend.md | 5 +- .../src/backends/lxc/common/lxc_bindings.rs | 111 ++++++++++++++++-- ...xc_e2e_tests_e2e_lxc_network_capability.rs | 7 +- 3 files changed, 110 insertions(+), 13 deletions(-) diff --git a/docs/backends/lxc/lxc-backend.md b/docs/backends/lxc/lxc-backend.md index 775f77e1c..e2fe97bc4 100644 --- a/docs/backends/lxc/lxc-backend.md +++ b/docs/backends/lxc/lxc-backend.md @@ -333,7 +333,10 @@ The zone query should answer the zone you assigned. container init keeps `CAP_NET_ADMIN` there, so a process running as root inside the container can flush or delete them. The command MXC runs is attached with `CAP_NET_ADMIN` dropped from its bounding set whenever chains are installed, so - it cannot. Default-deny closes external reachability for a container that does + it cannot. That same attach installs a seccomp filter refusing + `socket(AF_PACKET, ...)` with `EPERM`. Without it a workload holding + `CAP_NET_RAW` could put frames on the wire beneath the chains, which filter at + the IP layer. Default-deny closes external reachability for a container that does not deliberately tear it down, including services the workload itself starts. - **Policy is not in force while the container starts.** The chains are installed after the container has started and its address has settled, so container init diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index d60f90742..0ac48f339 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -135,11 +135,11 @@ pub enum ContainerFirewall { Absent, } -/// Two networking capabilites LXC cares about is CAP_NET_ADMIN and CAP_NET_RAW. +/// Two networking capabilities LXC cares about are CAP_NET_ADMIN and CAP_NET_RAW. /// CAP_NET_ADMIN allows a process to change the networking rules /// CAP_NET_RAW allows a process to use RAW and PACKET sockets and those allow a process to bypass /// a network chain. -/// Remove both capabilites here. (Technically CAP_NET_RAW isn not removed. More details inside) +/// Remove both capabilities here. (Technically CAP_NET_RAW is not removed. More details inside) #[cfg(target_os = "linux")] fn confine_network_capabilities(command: &mut std::process::Command) { use std::os::unix::process::CommandExt; @@ -164,16 +164,16 @@ fn confine_network_capabilities(command: &mut std::process::Command) { } } -/// Removing CA_NET_RAW prevents a process from using RAW and PACKET sockets. However, ICMP is also denied. +/// Removing CAP_NET_RAW prevents a process from using RAW and PACKET sockets. However, ICMP is also denied. /// LXC needs to allow ICMP and prevent programs from using sockets to /// bypass the network guards. The solution is to give the kernel a custom filter to refuse -/// any socket that carries AF_PACKET. LXC utalizes SecComp. +/// any socket that carries AF_PACKET. LXC utilizes seccomp. /// see https://docs.kernel.org/userspace-api/seccomp_filter.html and /// https://man7.org/linux/man-pages/man2/seccomp.2.html #[cfg(target_os = "linux")] fn refuse_packet_sockets() -> std::io::Result<()> { #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] - compile_error!("LXC only understands SecComp system calls for x86_x64 and arm64."); + compile_error!("LXC only understands seccomp system calls for x86_64 and arm64."); // https://docs.kernel.org/networking/filter.html for more information on filtering sockets // Filtering sockets requires assembling a small filter program with assembly like syntax. @@ -198,14 +198,14 @@ fn refuse_packet_sockets() -> std::io::Result<()> { // numbers come from // https://github.com/torvalds/linux/blob/master/include/uapi/linux/audit.h - // To test the ABI the filter needs to know the endiness and arch. - const _64BIT_ABI_LITTLE_ENDINESS: u32 = 0x8000_0000 | 0x4000_0000; + // To test the ABI the filter needs to know the endianness and arch. + const _64BIT_ABI_LITTLE_ENDIANNESS: u32 = 0x8000_0000 | 0x4000_0000; // Also from https://github.com/torvalds/linux/blob/master/include/uapi/linux/audit.h #[cfg(target_arch = "x86_64")] - const NATIVE_ARCHITECTURE: u32 = _64BIT_ABI_LITTLE_ENDINESS | libc::EM_X86_64 as u32; + const NATIVE_ARCHITECTURE: u32 = _64BIT_ABI_LITTLE_ENDIANNESS | libc::EM_X86_64 as u32; #[cfg(target_arch = "aarch64")] - const NATIVE_ARCHITECTURE: u32 = _64BIT_ABI_LITTLE_ENDINESS | libc::EM_AARCH64 as u32; + const NATIVE_ARCHITECTURE: u32 = _64BIT_ABI_LITTLE_ENDIANNESS | libc::EM_AARCH64 as u32; // Filter only works for x64 and arm64 ABI's. // Used to filter out a 32 bit process under the "Filters" section. @@ -1577,6 +1577,99 @@ mod tests { ); } + /// The probe child opened `AF_PACKET`. + #[cfg(target_os = "linux")] + const AF_PACKET_ALLOWED: i32 = 10; + + /// The probe child was refused with EPERM, which is what the filter returns. + #[cfg(target_os = "linux")] + const AF_PACKET_REFUSED_EPERM: i32 = 11; + + /// The probe child was refused, but for some other reason. + #[cfg(target_os = "linux")] + const AF_PACKET_REFUSED_OTHER: i32 = 12; + + /// Fork a child that optionally installs the packet-socket filter, opens + /// `socket(AF_PACKET, SOCK_RAW, 0)`, and reports which answer the kernel + /// gave. The child leaves through `_exit` and never reaches `exec`, so the + /// attempt runs in the same post-fork context that confines a real + /// workload. Reporting through the exit code keeps a refusal distinct from + /// a child that never ran: a missing or unspawnable program cannot forge + /// one of these three numbers. + #[cfg(target_os = "linux")] + fn probe_packet_socket(install_filter: bool) -> i32 { + use std::os::unix::process::CommandExt; + + let mut command = std::process::Command::new("/bin/true"); + + // SAFETY: `pre_exec` runs between fork and exec, where only + // async-signal-safe work is permitted. This closure makes bare + // syscalls, allocates nothing, and leaves through `_exit`, which skips + // the atexit handlers the parent still owns. + unsafe { + command.pre_exec(move || { + // Seccomp wants this or CAP_SYS_ADMIN. Production is root and + // gets it from the capability; setting it here lets the probe + // report the filter's own verdict at any privilege level. + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + + if install_filter { + refuse_packet_sockets()?; + } + + let descriptor = libc::socket(libc::AF_PACKET, libc::SOCK_RAW, 0); + if descriptor >= 0 { + libc::close(descriptor); + libc::_exit(AF_PACKET_ALLOWED); + } + + if std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) { + libc::_exit(AF_PACKET_REFUSED_EPERM) + } else { + libc::_exit(AF_PACKET_REFUSED_OTHER) + } + }); + } + + command + .status() + .expect("the probe child must fork") + .code() + .expect("the probe child reports through its exit code, never a signal") + } + + // `Seccomp: 2` in `/proc/self/status` says a filter is attached, not that it + // refuses anything: a filter permitting every syscall reports the same + // number. This opens the one socket the filter exists to stop and reads + // the kernel's answer. + #[cfg(target_os = "linux")] + #[test] + fn the_filter_refuses_a_packet_socket_with_eperm() { + assert_eq!( + probe_packet_socket(true), + AF_PACKET_REFUSED_EPERM, + "a workload under the filter must not reach AF_PACKET, which writes \ + frames below the chains confining it" + ); + } + + // The control for the test above. On its own an EPERM settles nothing, + // because a caller lacking CAP_NET_RAW is refused identically and an empty + // filter would pass just as well. + #[cfg(target_os = "linux")] + #[test] + fn a_packet_socket_is_otherwise_open_to_a_caller_holding_cap_net_raw() { + // SAFETY: `geteuid` is a thread-safe, side-effect-free libc call. + let running_as_root = unsafe { libc::geteuid() } == 0; + + assert_eq!( + probe_packet_socket(false) == AF_PACKET_ALLOWED, + running_as_root, + "an unfiltered caller holding CAP_NET_RAW has to reach AF_PACKET, or \ + the refusal above is the privilege talking rather than the filter" + ); + } + #[cfg(target_os = "linux")] #[test] fn a_failed_release_carries_the_tools_own_diagnosis() { diff --git a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs index 903a4111d..bdc01fb1c 100644 --- a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs +++ b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs @@ -92,9 +92,10 @@ fn workload_cannot_reconfigure_or_bypass_the_network() { ); } - // 2 is SECCOMP_MODE_FILTER. The kernel writes this line too. A filter - // cannot be lifted once it is attached, and this one fails - // socket(AF_PACKET, ...) with EPERM. + // 2 is SECCOMP_MODE_FILTER. The kernel writes this line too, and a filter + // cannot be lifted once it is attached. This asserts only that one is + // attached; that it refuses AF_PACKET with EPERM is proven by the + // lxc_bindings unit test the_filter_refuses_a_packet_socket_with_eperm. assert_eq!( status_number(&status, "Seccomp:"), 2, From 0aa18c8200d0e1585e1baa60435b5ed7f9b7363d Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Fri, 9 Oct 2026 15:53:15 -0700 Subject: [PATCH 07/17] Restore the error reporting the main rewrite dropped A missing config exited silently on an empty path, runtime parse failures lost their Error: prefix, the signal handler installed after work had begun, and an unreadable backend list exited 1 instead of warning and continuing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1654c073-27eb-43cf-82f3-417614ce845b --- src/tools/lxc/src/linux_executor_arguments.rs | 11 +++++++---- src/tools/lxc/src/main.rs | 17 ++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/tools/lxc/src/linux_executor_arguments.rs b/src/tools/lxc/src/linux_executor_arguments.rs index 76a8a9a95..e44835c7c 100644 --- a/src/tools/lxc/src/linux_executor_arguments.rs +++ b/src/tools/lxc/src/linux_executor_arguments.rs @@ -99,7 +99,10 @@ impl LinuxExecutorArguments { } else if let Some(ref path) = self.config_path { (path.clone(), false) } else { - (String::new(), false) + eprintln!( + "Error: No config provided. Use a positional path, --config, or --config-base64" + ); + process::exit(1); }; let mut logger = Logger::new(if self.debug { @@ -166,9 +169,9 @@ impl LinuxExecutorArguments { // Setup is an interactive install: the pull and the warm-up // report progress as they go. - let runtimes = mxc_sdk::hyperlight_common::parse_runtimes( - &self.setup_hyperlight.clone().unwrap(), - )?; + let runtimes = + mxc_sdk::hyperlight_common::parse_runtimes(&self.setup_hyperlight.clone().unwrap()) + .map_err(|message| format!("Error: {message}"))?; let mut logger = mxc_sdk::mxc_common::logger::Logger::new( mxc_sdk::mxc_common::logger::Mode::Console, diff --git a/src/tools/lxc/src/main.rs b/src/tools/lxc/src/main.rs index 3024fe2cf..c7d8594ce 100644 --- a/src/tools/lxc/src/main.rs +++ b/src/tools/lxc/src/main.rs @@ -16,6 +16,16 @@ use mxc_sdk::mxc_common::telemetry; use mxc_sdk::lxc_common::signal_cleanup; fn main() { + // Install before spawning any other threads so the signal mask propagates. + // Failure here is fatal: install() either succeeds with the watchdog + // running, or restores the original signal mask and returns Err. We refuse + // to continue without it because containers leaked on SIGTERM/INT are + // exactly the failure mode this code exists to prevent. + if let Err(e) = signal_cleanup::install() { + eprintln!("Error: failed to install signal cleanup handler: {e}"); + process::exit(1); + } + let arguments = linux_executor_arguments::LinuxExecutorArguments::parse(); // Get the logger @@ -29,7 +39,6 @@ fn main() { if let Some(ref log_path) = arguments.get_log_path() { if let Err(e) = logger.enable_file_sink(std::path::Path::new(log_path)) { eprintln!("Warning: could not open log file '{}': {}", log_path, e); - process::exit(1); } } @@ -91,12 +100,6 @@ fn main() { log_request(&request, &mut logger); - // Setup watchdog for unexpected shutdowns - if let Err(e) = signal_cleanup::install() { - eprint!("Error: failed to install signal cleanup handler: {e}"); - process::exit(1); - } - let run_start = Instant::now(); // Run the request. From 1c9893dd71ce1e0ac79ea41ef90135c19b81dedd Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Fri, 9 Oct 2026 15:53:35 -0700 Subject: [PATCH 08/17] Restore the ICMP precondition to the egress test The later cases read an unanswered echo as a firewall verdict. A peer that never answers ICMP produces that same silence, which passes the run without anything having been enforced. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1654c073-27eb-43cf-82f3-417614ce845b --- tests/scripts/run_lxc_network_ga_egress_test.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/scripts/run_lxc_network_ga_egress_test.sh b/tests/scripts/run_lxc_network_ga_egress_test.sh index d49387e92..fb4af1822 100644 --- a/tests/scripts/run_lxc_network_ga_egress_test.sh +++ b/tests/scripts/run_lxc_network_ga_egress_test.sh @@ -283,6 +283,13 @@ for dns_addr in "$PEER_DNS_IP" "$PEER_DNS_SIBLING_IP"; do fi done +# The ICMP cases below read an unanswered echo as a firewall verdict, so a peer +# that ignores echo has to fail here as harness breakage instead. +if command -v ping >/dev/null 2>&1; then + ping -c 1 -W 5 "$PEER_IP" >/dev/null 2>&1 \ + || fail "the egress peer does not answer ICMP echo at $PEER_IP." +fi + # Drift guard: the peer fixtures must target this peer, or the run would probe a # stale address and prove nothing. PEER_TARGETING_CONFIGS=( From 7c9b02fd65cf0adc87e34262c5228734335467f1 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Fri, 9 Oct 2026 16:27:55 -0700 Subject: [PATCH 09/17] Install the signal watchdog just before the container run The watchdog destroys a running container on a fatal signal, and nothing earlier in main creates one. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1654c073-27eb-43cf-82f3-417614ce845b --- src/tools/lxc/src/main.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/tools/lxc/src/main.rs b/src/tools/lxc/src/main.rs index c7d8594ce..e2e373b2f 100644 --- a/src/tools/lxc/src/main.rs +++ b/src/tools/lxc/src/main.rs @@ -16,16 +16,6 @@ use mxc_sdk::mxc_common::telemetry; use mxc_sdk::lxc_common::signal_cleanup; fn main() { - // Install before spawning any other threads so the signal mask propagates. - // Failure here is fatal: install() either succeeds with the watchdog - // running, or restores the original signal mask and returns Err. We refuse - // to continue without it because containers leaked on SIGTERM/INT are - // exactly the failure mode this code exists to prevent. - if let Err(e) = signal_cleanup::install() { - eprintln!("Error: failed to install signal cleanup handler: {e}"); - process::exit(1); - } - let arguments = linux_executor_arguments::LinuxExecutorArguments::parse(); // Get the logger @@ -100,6 +90,12 @@ fn main() { log_request(&request, &mut logger); + // Setup watchdog for unexpected shutdowns + if let Err(e) = signal_cleanup::install() { + eprintln!("Error: failed to install signal cleanup handler: {e}"); + process::exit(1); + } + let run_start = Instant::now(); // Run the request. From ebfb45de5849683913c11b2316feda45cce99a4c Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Tue, 13 Oct 2026 14:02:15 -0700 Subject: [PATCH 10/17] Refuse legacy SOCK_PACKET, AF_XDP, and io_uring under the firewall A firewalled workload could still reach a packet socket three ways the family check never sees: SOCK_PACKET, which the kernel rewrites to AF_PACKET after the family argument is read; AF_XDP; and a ring, which creates sockets without the socket system call. CAP_NET_RAW stays in place. Dropping it would take ICMP with it, which every backend promises. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fb9b6565-8ada-47a6-8b33-db655ee9a7a0 --- docs/backends/lxc/lxc-backend.md | 26 +- .../src/backends/lxc/common/lxc_bindings.rs | 312 +++++++++++++++--- ...xc_e2e_tests_e2e_lxc_network_capability.rs | 128 +++++++ 3 files changed, 420 insertions(+), 46 deletions(-) diff --git a/docs/backends/lxc/lxc-backend.md b/docs/backends/lxc/lxc-backend.md index e2fe97bc4..a383b3384 100644 --- a/docs/backends/lxc/lxc-backend.md +++ b/docs/backends/lxc/lxc-backend.md @@ -333,11 +333,33 @@ The zone query should answer the zone you assigned. container init keeps `CAP_NET_ADMIN` there, so a process running as root inside the container can flush or delete them. The command MXC runs is attached with `CAP_NET_ADMIN` dropped from its bounding set whenever chains are installed, so - it cannot. That same attach installs a seccomp filter refusing - `socket(AF_PACKET, ...)` with `EPERM`. Without it a workload holding + it cannot. That same attach installs a seccomp filter refusing three ways of + reaching a packet socket: the `AF_PACKET` family whatever the socket type, + `AF_XDP`, and the legacy `SOCK_PACKET` type, which the kernel rewrites into + `AF_PACKET` after the family argument has already been read. Each is refused with + `EPERM`, and a socket-type flag riding along does not change the answer. The + filter refuses io_uring as well, with `ENOSYS`: a ring creates sockets without the + `socket` system call the filter reads, and `ENOSYS` is the answer callers probe + for before falling back to ordinary I/O. A process issuing system calls under an + ABI other than the one the filter was built against is killed outright rather + than refused, because its arguments are not where the filter reads them. + Installing the filter at all requires MXC to hold `CAP_SYS_ADMIN`. The kernel + accepts `no_new_privs` in its place, and MXC does not set it; a deployment + lacking the capability does not lose the filter quietly, as the attach fails and + the command never runs. Without all of this a workload holding `CAP_NET_RAW` could put frames on the wire beneath the chains, which filter at the IP layer. Default-deny closes external reachability for a container that does not deliberately tear it down, including services the workload itself starts. +- **The packet-socket refusal is a creation-time check, not a transmit-time one.** + The kernel tests `CAP_NET_RAW` when a socket is opened and never again. A + descriptor opened before the filter is attached keeps sending for as long as + something holds it, and container init runs unfiltered with `CAP_NET_RAW` in hand + (see the next limitation). Taking that descriptor needs no cooperation from init: + a workload permitted to trace it can pull the descriptor out with `pidfd_getfd`. + Either route is outside what this filter covers. The filter also + names the families it refuses one by one; a family added by a later kernel is + permitted until it is named here. Both gaps close only below the socket layer, on + the host side of the container's interface, which this backend does not do today. - **Policy is not in force while the container starts.** The chains are installed after the container has started and its address has settled, so container init and anything it starts run unfiltered in both directions for that interval. The diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index 0ac48f339..ee11367fa 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -168,6 +168,11 @@ fn confine_network_capabilities(command: &mut std::process::Command) { /// LXC needs to allow ICMP and prevent programs from using sockets to /// bypass the network guards. The solution is to give the kernel a custom filter to refuse /// any socket that carries AF_PACKET. LXC utilizes seccomp. +/// A packet socket has more than one spelling, and the filter has to answer all of them. +/// `socket(PF_INET, SOCK_PACKET, ...)` becomes an AF_PACKET socket after seccomp has +/// already read the family, so the filter reads the socket type as well. AF_XDP puts +/// frames on the wire the same way AF_PACKET does. io_uring builds a socket without +/// issuing the syscall this filter inspects, so the ring itself is refused. /// see https://docs.kernel.org/userspace-api/seccomp_filter.html and /// https://man7.org/linux/man-pages/man2/seccomp.2.html #[cfg(target_os = "linux")] @@ -184,6 +189,7 @@ fn refuse_packet_sockets() -> std::io::Result<()> { const LOAD_WORD: u16 = (libc::BPF_LD | libc::BPF_W | libc::BPF_ABS) as u16; const JUMP_IF_EQUAL: u16 = (libc::BPF_JMP | libc::BPF_JEQ | libc::BPF_K) as u16; const JUMP_IF_AT_LEAST: u16 = (libc::BPF_JMP | libc::BPF_JGE | libc::BPF_K) as u16; + const BITWISE_AND: u16 = (libc::BPF_ALU | libc::BPF_AND | libc::BPF_K) as u16; const RETURN: u16 = (libc::BPF_RET | libc::BPF_K) as u16; // https://github.com/torvalds/linux/blob/master/include/uapi/linux/seccomp.h @@ -192,8 +198,16 @@ fn refuse_packet_sockets() -> std::io::Result<()> { const ARCHITECTURE: u32 = 4; const FIRST_ARGUMENT: u32 = 16; + // The arguments are 64 bits wide and the filter reads 32 bits at a time, so + // the second one starts two words after the first. + const SECOND_ARGUMENT: u32 = 24; + const REFUSE_WITH_EPERM: u32 = libc::SECCOMP_RET_ERRNO | (libc::EPERM as u32); + // io_uring answers "this kernel has no such call" rather than "you may not", + // because that is the answer a library probing for io_uring falls back on. + const REFUSE_WITH_ENOSYS: u32 = libc::SECCOMP_RET_ERRNO | (libc::ENOSYS as u32); + const NO_FLAGS: libc::c_ulong = 0; // numbers come from @@ -214,19 +228,44 @@ fn refuse_packet_sockets() -> std::io::Result<()> { const SOCKET_SYSCALL: u32 = libc::SYS_socket as u32; + // A ring can create a socket on its own, and the request never reaches the + // socket syscall this filter reads. Refusing all three entry points leaves + // no ring to submit that request to. + const IO_URING_SETUP_SYSCALL: u32 = libc::SYS_io_uring_setup as u32; + const IO_URING_ENTER_SYSCALL: u32 = libc::SYS_io_uring_enter as u32; + const IO_URING_REGISTER_SYSCALL: u32 = libc::SYS_io_uring_register as u32; + + // `libc` marks SOCK_PACKET deprecated, and this crate builds warnings as + // errors, so naming it here is the only way to compare against it. The + // value is from linux/net.h and matches what `libc` defines. + const SOCK_PACKET: u32 = 10; + + // A caller may fold SOCK_CLOEXEC or SOCK_NONBLOCK into the socket type, and + // the kernel masks them off before reading it, so the filter masks them off + // too. + const SOCKET_TYPE_MASK: u32 = 0xF; + // Make the filter #[rustfmt::skip] let mut program = [ - libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: ARCHITECTURE }, - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 7, k: NATIVE_ARCHITECTURE }, // test ABI - libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: SYSCALL_NUMBER }, - libc::sock_filter { code: JUMP_IF_AT_LEAST, jt: 5, jf: 0, k: USES_X32_ABI_FLAG }, // Test 32 bit - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 2, k: SOCKET_SYSCALL }, // Only process a socket syscall - libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: FIRST_ARGUMENT }, - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 1, jf: 0, k: libc::AF_PACKET as u32 }, // return error if using AF_PACKET - libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_ALLOW }, - libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: REFUSE_WITH_EPERM }, - libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_KILL_PROCESS }, + libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: ARCHITECTURE }, + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 15, k: NATIVE_ARCHITECTURE }, // test ABI + libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: SYSCALL_NUMBER }, + libc::sock_filter { code: JUMP_IF_AT_LEAST, jt: 13, jf: 0, k: USES_X32_ABI_FLAG }, // Test 32 bit + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 11, jf: 0, k: IO_URING_SETUP_SYSCALL }, // leave no ring to ask with + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 10, jf: 0, k: IO_URING_ENTER_SYSCALL }, + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 9, jf: 0, k: IO_URING_REGISTER_SYSCALL }, + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 6, k: SOCKET_SYSCALL }, // Only process a socket syscall + libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: FIRST_ARGUMENT }, + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 5, jf: 0, k: libc::AF_PACKET as u32 }, // return error if using AF_PACKET + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 4, jf: 0, k: libc::AF_XDP as u32 }, // AF_XDP reaches the wire the same way + libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: SECOND_ARGUMENT }, + libc::sock_filter { code: BITWISE_AND, jt: 0, jf: 0, k: SOCKET_TYPE_MASK }, // drop SOCK_CLOEXEC and SOCK_NONBLOCK + libc::sock_filter { code: JUMP_IF_EQUAL, jt: 1, jf: 0, k: SOCK_PACKET }, // the kernel turns this one into AF_PACKET + libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_ALLOW }, + libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: REFUSE_WITH_EPERM }, + libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: REFUSE_WITH_ENOSYS }, + libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_KILL_PROCESS }, ]; // Make the struct @@ -1577,29 +1616,61 @@ mod tests { ); } - /// The probe child opened `AF_PACKET`. + /// The probe child got the descriptor it asked for. #[cfg(target_os = "linux")] - const AF_PACKET_ALLOWED: i32 = 10; + const ATTEMPT_ALLOWED: i32 = 10; - /// The probe child was refused with EPERM, which is what the filter returns. + /// The probe child was refused with EPERM, which is what the filter returns + /// for a socket that reaches the wire below the chains. #[cfg(target_os = "linux")] - const AF_PACKET_REFUSED_EPERM: i32 = 11; + const ATTEMPT_REFUSED_EPERM: i32 = 11; + + /// The probe child was refused with ENOSYS, which is what the filter + /// returns for io_uring. + #[cfg(target_os = "linux")] + const ATTEMPT_REFUSED_ENOSYS: i32 = 12; /// The probe child was refused, but for some other reason. #[cfg(target_os = "linux")] - const AF_PACKET_REFUSED_OTHER: i32 = 12; - - /// Fork a child that optionally installs the packet-socket filter, opens - /// `socket(AF_PACKET, SOCK_RAW, 0)`, and reports which answer the kernel - /// gave. The child leaves through `_exit` and never reaches `exec`, so the - /// attempt runs in the same post-fork context that confines a real - /// workload. Reporting through the exit code keeps a refusal distinct from - /// a child that never ran: a missing or unspawnable program cannot forge - /// one of these three numbers. + const ATTEMPT_REFUSED_OTHER: i32 = 13; + + /// What the probe child asks the kernel for. #[cfg(target_os = "linux")] - fn probe_packet_socket(install_filter: bool) -> i32 { + #[derive(Clone, Copy, Debug)] + enum Attempt { + /// The socket the filter exists to stop. + PacketSocket, + /// The same socket under a family the filter permits, which the kernel + /// swaps over once seccomp has read it. + LegacyPacketSocket, + /// That spelling with a flag folded into the socket type. + LegacyPacketSocketWithFlags, + /// Another family that reaches the wire below the chains. + ExpressDataPathSocket, + /// A ring, which creates any of the above without a socket syscall. + IoUringRing, + /// What an `icmp` egress allow needs. + RawIcmpSocket, + /// What every ordinary workload needs. + TcpSocket, + } + + /// Fork a child that optionally installs the packet-socket filter, makes + /// one request of the kernel, and reports which answer it got. The child + /// leaves through `_exit` and never reaches `exec`, so the attempt runs in + /// the same post-fork context that confines a real workload. Reporting + /// through the exit code keeps a refusal distinct from a child that never + /// ran: a missing or unspawnable program cannot forge one of these numbers. + #[cfg(target_os = "linux")] + fn probe_socket(attempt: Attempt, install_filter: bool) -> i32 { use std::os::unix::process::CommandExt; + // `libc` marks SOCK_PACKET deprecated, and this crate builds warnings as + // errors, so naming it here is the only way to probe with it. Both + // values match what `libc` defines. + const AF_XDP: libc::c_int = libc::AF_XDP; + const SOCK_PACKET: libc::c_int = 10; + let mut command = std::process::Command::new("/bin/true"); // SAFETY: `pre_exec` runs between fork and exec, where only @@ -1617,16 +1688,38 @@ mod tests { refuse_packet_sockets()?; } - let descriptor = libc::socket(libc::AF_PACKET, libc::SOCK_RAW, 0); + let descriptor = match attempt { + Attempt::PacketSocket => libc::socket(libc::AF_PACKET, libc::SOCK_RAW, 0), + Attempt::LegacyPacketSocket => libc::socket(libc::PF_INET, SOCK_PACKET, 0), + Attempt::LegacyPacketSocketWithFlags => { + libc::socket(libc::PF_INET, SOCK_PACKET | libc::SOCK_CLOEXEC, 0) + } + Attempt::ExpressDataPathSocket => libc::socket(AF_XDP, libc::SOCK_RAW, 0), + Attempt::IoUringRing => { + // The kernel writes a fixed-size struct back through + // this buffer, which is larger than the struct is. + let mut parameters = [0u8; 256]; + libc::syscall( + libc::SYS_io_uring_setup, + 1 as libc::c_ulong, + parameters.as_mut_ptr(), + ) as libc::c_int + } + Attempt::RawIcmpSocket => { + libc::socket(libc::AF_INET, libc::SOCK_RAW, libc::IPPROTO_ICMP) + } + Attempt::TcpSocket => libc::socket(libc::AF_INET, libc::SOCK_STREAM, 0), + }; + if descriptor >= 0 { libc::close(descriptor); - libc::_exit(AF_PACKET_ALLOWED); + libc::_exit(ATTEMPT_ALLOWED); } - if std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) { - libc::_exit(AF_PACKET_REFUSED_EPERM) - } else { - libc::_exit(AF_PACKET_REFUSED_OTHER) + match std::io::Error::last_os_error().raw_os_error() { + Some(libc::EPERM) => libc::_exit(ATTEMPT_REFUSED_EPERM), + Some(libc::ENOSYS) => libc::_exit(ATTEMPT_REFUSED_ENOSYS), + _ => libc::_exit(ATTEMPT_REFUSED_OTHER), } }); } @@ -1639,34 +1732,165 @@ mod tests { } // `Seccomp: 2` in `/proc/self/status` says a filter is attached, not that it + // Reports whether the filter is what refuses `attempt` on this runner, + // skipping or failing when the question cannot be settled here. + // + // The kernel refuses a packet socket with EPERM to a caller lacking + // CAP_NET_RAW, which is the errno the filter returns. Asserting the + // refusal on its own therefore passes under an unprivileged runner with the + // filter having done nothing, and would keep passing if the filter were + // deleted. Reading the unfiltered answer first is what separates the two, + // and it has to happen in the same test: a control standing alone passes + // under that runner too. + #[cfg(target_os = "linux")] + fn the_filter_is_what_refuses(attempt: Attempt, refusal: i32) -> bool { + let unfiltered = probe_socket(attempt, false); + + if unfiltered != refusal { + return true; + } + + let reason = format!( + "{attempt:?} is already refused ({unfiltered}) with no filter installed, \ + so a refusal under the filter would prove nothing — run as a caller \ + holding CAP_NET_RAW on a kernel offering the route" + ); + + // A lane provisioned for this suite reports a skip as success, so the + // gate would go green having tested nothing. + if std::env::var("MXC_LXC_TESTS_REQUIRE_EXECUTION").is_ok_and(|value| value != "0") { + panic!("strict mode: {reason}"); + } + + println!("SKIPPED: {reason}"); + false + } + // refuses anything: a filter permitting every syscall reports the same - // number. This opens the one socket the filter exists to stop and reads - // the kernel's answer. + // number. These open the sockets the filter exists to stop and read the + // kernel's answer. #[cfg(target_os = "linux")] #[test] fn the_filter_refuses_a_packet_socket_with_eperm() { + if !the_filter_is_what_refuses(Attempt::PacketSocket, ATTEMPT_REFUSED_EPERM) { + return; + } + assert_eq!( - probe_packet_socket(true), - AF_PACKET_REFUSED_EPERM, + probe_socket(Attempt::PacketSocket, true), + ATTEMPT_REFUSED_EPERM, "a workload under the filter must not reach AF_PACKET, which writes \ frames below the chains confining it" ); } - // The control for the test above. On its own an EPERM settles nothing, - // because a caller lacking CAP_NET_RAW is refused identically and an empty - // filter would pass just as well. + // `socket(PF_INET, SOCK_PACKET, ...)` names a family the filter permits and + // hands back a packet socket anyway, because the kernel swaps the family + // over after seccomp has read it. Reading the socket type catches it. #[cfg(target_os = "linux")] #[test] - fn a_packet_socket_is_otherwise_open_to_a_caller_holding_cap_net_raw() { - // SAFETY: `geteuid` is a thread-safe, side-effect-free libc call. - let running_as_root = unsafe { libc::geteuid() } == 0; + fn the_filter_refuses_a_legacy_packet_socket_with_eperm() { + if !the_filter_is_what_refuses(Attempt::LegacyPacketSocket, ATTEMPT_REFUSED_EPERM) { + return; + } assert_eq!( - probe_packet_socket(false) == AF_PACKET_ALLOWED, - running_as_root, - "an unfiltered caller holding CAP_NET_RAW has to reach AF_PACKET, or \ - the refusal above is the privilege talking rather than the filter" + probe_socket(Attempt::LegacyPacketSocket, true), + ATTEMPT_REFUSED_EPERM, + "SOCK_PACKET reaches AF_PACKET under a family the filter allows" + ); + } + + // The kernel masks the flags off the socket type before reading it, so a + // filter comparing the type whole is passed by adding one. + #[cfg(target_os = "linux")] + #[test] + fn the_filter_refuses_a_legacy_packet_socket_carrying_a_flag() { + if !the_filter_is_what_refuses(Attempt::LegacyPacketSocketWithFlags, ATTEMPT_REFUSED_EPERM) + { + return; + } + + assert_eq!( + probe_socket(Attempt::LegacyPacketSocketWithFlags, true), + ATTEMPT_REFUSED_EPERM, + "SOCK_CLOEXEC must not carry a packet socket past the type check" + ); + } + + // A kernel built without XDP sockets answers EAFNOSUPPORT rather than + // handing back a descriptor, which is a different answer from the filter's + // and settles this as well as an open route would. + #[cfg(target_os = "linux")] + #[test] + fn the_filter_refuses_an_express_data_path_socket_with_eperm() { + if !the_filter_is_what_refuses(Attempt::ExpressDataPathSocket, ATTEMPT_REFUSED_EPERM) { + return; + } + + assert_eq!( + probe_socket(Attempt::ExpressDataPathSocket, true), + ATTEMPT_REFUSED_EPERM, + "AF_XDP transmits on the interface below the chains, as AF_PACKET does" + ); + } + + // A ring creates a socket on its own, and that request is not a socket + // syscall, so no family check can see it. Refusing the ring is what closes + // the route. A kernel offering no io_uring answers ENOSYS by itself, which + // the check above rules out. + #[cfg(target_os = "linux")] + #[test] + fn the_filter_refuses_an_io_uring_ring() { + if !the_filter_is_what_refuses(Attempt::IoUringRing, ATTEMPT_REFUSED_ENOSYS) { + return; + } + + assert_eq!( + probe_socket(Attempt::IoUringRing, true), + ATTEMPT_REFUSED_ENOSYS, + "a ring opens AF_PACKET without issuing the syscall this filter reads" + ); + } + + // CAP_NET_RAW stays so that an `icmp` egress allow has the raw socket it + // needs, and the filter is what makes keeping it safe. A runner that + // cannot open the socket unfiltered cannot show the filter left it open, + // so it reports a skip rather than a pass. + #[cfg(target_os = "linux")] + #[test] + fn the_filter_leaves_a_raw_icmp_socket_alone() { + let unfiltered = probe_socket(Attempt::RawIcmpSocket, false); + + if unfiltered != ATTEMPT_ALLOWED { + let reason = format!( + "raw ICMP answers {unfiltered} with no filter installed, so this \ + runner cannot show the filter leaves it open — run as a caller \ + holding CAP_NET_RAW" + ); + + if std::env::var("MXC_LXC_TESTS_REQUIRE_EXECUTION").is_ok_and(|value| value != "0") { + panic!("strict mode: {reason}"); + } + + println!("SKIPPED: {reason}"); + return; + } + + assert_eq!( + probe_socket(Attempt::RawIcmpSocket, true), + ATTEMPT_ALLOWED, + "the filter closed raw ICMP, which an `icmp` egress allow depends on" + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_filter_leaves_an_ordinary_socket_alone() { + assert_eq!( + probe_socket(Attempt::TcpSocket, true), + ATTEMPT_ALLOWED, + "a confined workload still has to reach the network it is allowed" ); } diff --git a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs index bdc01fb1c..919d13969 100644 --- a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs +++ b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs @@ -26,6 +26,56 @@ fn capability_mask(status: &str, field: &str) -> u64 { .unwrap_or_else(|| panic!("the container reported no readable {field} line\n{status}")) } +/// Ask the kernel for every socket the filter is meant to refuse, for two it is +/// meant to leave alone, and print one tagged line per answer. ctypes calls +/// `socket(2)` and `io_uring_setup(2)` by number, so nothing between the test +/// and the kernel can substitute one route for another. The two permitted +/// sockets are the control: they come back from the same mechanism in the same +/// run, so a refusal above them is the filter choosing, not the probe failing. +const PACKET_SOCKET_PROBE: &str = r#"python3 - <<'MXC_PROBE' +import ctypes, errno + +libc = ctypes.CDLL(None, use_errno=True) +libc.syscall.restype = ctypes.c_long + +def report(tag, fd): + if fd >= 0: + libc.close(fd) + print(tag + " ALLOWED") + else: + code = ctypes.get_errno() + print(tag + " REFUSED " + errno.errorcode.get(code, str(code))) + +def sock(tag, domain, kind, protocol): + ctypes.set_errno(0) + report(tag, libc.socket(domain, kind, protocol)) + +CLOEXEC = 0o2000000 + +sock("PACKET_RAW", 17, 3, 0) +sock("PACKET_DGRAM", 17, 2, 0) +sock("PACKET_RAW_CLOEXEC", 17, 3 | CLOEXEC, 0) +sock("LEGACY_SOCK_PACKET", 2, 10, 0) +sock("LEGACY_SOCK_PACKET_CLOEXEC", 2, 10 | CLOEXEC, 0) +sock("XDP", 44, 3, 0) +sock("ICMP_RAW", 2, 3, 1) +sock("TCP", 2, 1, 0) + +params = (ctypes.c_ubyte * 120)() +ctypes.set_errno(0) +report("IO_URING", int(libc.syscall(425, 1, ctypes.byref(params)))) +MXC_PROBE +"#; + +/// Pick one tagged line out of the probe's output. +fn probe_line<'a>(output: &'a str, tag: &str) -> &'a str { + output + .lines() + .map(str::trim) + .find(|line| line.split_whitespace().next() == Some(tag)) + .unwrap_or_else(|| panic!("the probe printed no {tag} line\n{output}")) +} + /// Read one decimal field out of the container's `/proc/self/status`. fn status_number(status: &str, field: &str) -> u64 { status @@ -102,3 +152,81 @@ fn workload_cannot_reconfigure_or_bypass_the_network() { "the workload runs with no seccomp filter; AF_PACKET reaches the interface below the chains\n{status}" ); } + +#[test] +fn workload_cannot_open_a_packet_capable_socket_by_any_route() { + if !ready() { + return; + } + + // Same firewall as above, because the confinement only happens when chains + // exist. Ubuntu rather than Alpine: the probe needs an interpreter that + // can call socket(2) by number, and Alpine's image is busybox alone. + let config = json!({ + "version": "0.9.0-alpha", + "containerId": "lxc-packet-socket-refusal", + "containment": "lxc", + "process": { "commandLine": PACKET_SOCKET_PROBE, "timeout": 120000 }, + "lifecycle": { "destroyOnExit": true }, + "lxc": { "distribution": "ubuntu", "release": "noble" }, + "network": { + "egress": { + "default": "deny", + "allow": [ + { + "to": [{ "cidr": "140.82.112.0/20" }], + "ports": [{ "protocol": "tcp", "port": 443 }] + } + ] + } + } + }); + + let result = run_platform_config_value("lxc packet socket refusal", &config, &[], None); + let output = result.combined_output(); + + assert_eq!( + result.code, + Some(0), + "lxc-exec did not run the probe\n--- stderr ---\n{}", + result.stderr + ); + + // AF_PACKET by its own family, the legacy SOCK_PACKET spelling that the + // kernel rewrites into that family after seccomp has read the argument, and + // AF_XDP, each with and without a socket-type flag riding along. + for tag in [ + "PACKET_RAW", + "PACKET_DGRAM", + "PACKET_RAW_CLOEXEC", + "LEGACY_SOCK_PACKET", + "LEGACY_SOCK_PACKET_CLOEXEC", + "XDP", + ] { + assert_eq!( + probe_line(&output, tag), + format!("{tag} REFUSED EPERM"), + "the workload opened a packet-capable socket below the egress chains\n{output}" + ); + } + + // io_uring creates sockets without the socket(2) syscall the filter reads, + // so the ring itself is refused. ENOSYS rather than EPERM because callers + // probe for io_uring and fall back when it reports unsupported. + assert_eq!( + probe_line(&output, "IO_URING"), + "IO_URING REFUSED ENOSYS", + "io_uring is reachable, and it creates sockets the filter never sees\n{output}" + ); + + // The control. These two come back from the same ctypes call in the same + // container, so the refusals above are the filter and not a broken probe. + // Raw ICMP is also the socket an explicit `protocol: "icmp"` allow needs. + for tag in ["ICMP_RAW", "TCP"] { + assert_eq!( + probe_line(&output, tag), + format!("{tag} ALLOWED"), + "the filter took a socket it is supposed to leave alone\n{output}" + ); + } +} From 3150b36e03541dfe113772ef3e3c00e779bcb2e3 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Tue, 13 Oct 2026 14:36:19 -0700 Subject: [PATCH 11/17] Shorten the refuse_packet_sockets doc comment The route-by-route detail is already in the backend documentation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fb9b6565-8ada-47a6-8b33-db655ee9a7a0 --- src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index ee11367fa..334cada08 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -168,11 +168,6 @@ fn confine_network_capabilities(command: &mut std::process::Command) { /// LXC needs to allow ICMP and prevent programs from using sockets to /// bypass the network guards. The solution is to give the kernel a custom filter to refuse /// any socket that carries AF_PACKET. LXC utilizes seccomp. -/// A packet socket has more than one spelling, and the filter has to answer all of them. -/// `socket(PF_INET, SOCK_PACKET, ...)` becomes an AF_PACKET socket after seccomp has -/// already read the family, so the filter reads the socket type as well. AF_XDP puts -/// frames on the wire the same way AF_PACKET does. io_uring builds a socket without -/// issuing the syscall this filter inspects, so the ring itself is refused. /// see https://docs.kernel.org/userspace-api/seccomp_filter.html and /// https://man7.org/linux/man-pages/man2/seccomp.2.html #[cfg(target_os = "linux")] From ccc6b1b173f457713ea4b38e83d67756053c9bc4 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Tue, 13 Oct 2026 14:41:01 -0700 Subject: [PATCH 12/17] Group SECOND_ARGUMENT with the other seccomp_data offsets Both arguments are byte offsets into args[], not one derived from the other. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fb9b6565-8ada-47a6-8b33-db655ee9a7a0 --- src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index 334cada08..554d3a47a 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -175,6 +175,10 @@ fn refuse_packet_sockets() -> std::io::Result<()> { #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] compile_error!("LXC only understands seccomp system calls for x86_64 and arm64."); + // Use raw SecComp calls instead of the libseccomp crate. + // Using the libseccomp crate pulls in a dependency to the c libseccomp library. + // Use raw called here. + // https://docs.kernel.org/networking/filter.html for more information on filtering sockets // Filtering sockets requires assembling a small filter program with assembly like syntax. // The below code is to consolidate some commands into easy to understand words. @@ -192,9 +196,6 @@ fn refuse_packet_sockets() -> std::io::Result<()> { const SYSCALL_NUMBER: u32 = 0; const ARCHITECTURE: u32 = 4; const FIRST_ARGUMENT: u32 = 16; - - // The arguments are 64 bits wide and the filter reads 32 bits at a time, so - // the second one starts two words after the first. const SECOND_ARGUMENT: u32 = 24; const REFUSE_WITH_EPERM: u32 = libc::SECCOMP_RET_ERRNO | (libc::EPERM as u32); From 076376b40a08ca6ef0f14e0a9eb98e6f60d56845 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Tue, 13 Oct 2026 14:45:47 -0700 Subject: [PATCH 13/17] Say what the ENOSYS refusal means instead of guessing at callers The old wording claimed liburing falls back on ENOSYS, which was never measured. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fb9b6565-8ada-47a6-8b33-db655ee9a7a0 --- src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index 554d3a47a..26c91dfd4 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -200,8 +200,8 @@ fn refuse_packet_sockets() -> std::io::Result<()> { const REFUSE_WITH_EPERM: u32 = libc::SECCOMP_RET_ERRNO | (libc::EPERM as u32); - // io_uring answers "this kernel has no such call" rather than "you may not", - // because that is the answer a library probing for io_uring falls back on. + // ENOSYS is the errno from a kernel built without io_uring. The refusal + // borrows it to read as absence rather than denial. const REFUSE_WITH_ENOSYS: u32 = libc::SECCOMP_RET_ERRNO | (libc::ENOSYS as u32); const NO_FLAGS: libc::c_ulong = 0; From d008e69fb52db921a6c26c80bdd6f2be49ab54c0 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Thu, 15 Oct 2026 10:45:14 -0700 Subject: [PATCH 14/17] Removing SecComp. Dropping CAP_NET_RAW. Adding ping group range --- .../src/backends/lxc/common/lxc_bindings.rs | 442 +----------------- .../src/backends/lxc/common/lxc_runner.rs | 56 +-- .../backends/lxc/common/network_iptables.rs | 32 ++ ...xc_e2e_tests_e2e_lxc_network_capability.rs | 183 ++------ 4 files changed, 107 insertions(+), 606 deletions(-) diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index 26c91dfd4..ab6d9d9ea 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -135,156 +135,32 @@ pub enum ContainerFirewall { Absent, } -/// Two networking capabilities LXC cares about are CAP_NET_ADMIN and CAP_NET_RAW. -/// CAP_NET_ADMIN allows a process to change the networking rules -/// CAP_NET_RAW allows a process to use RAW and PACKET sockets and those allow a process to bypass -/// a network chain. -/// Remove both capabilities here. (Technically CAP_NET_RAW is not removed. More details inside) +/// Drop Network capabilities for sandboxing purposes. #[cfg(target_os = "linux")] fn confine_network_capabilities(command: &mut std::process::Command) { use std::os::unix::process::CommandExt; - // `libc` does not export this; the value is from linux/capability.h. + // `libc` does not export these; the values are from linux/capability.h. const CAP_NET_ADMIN: libc::c_ulong = 12; + const CAP_NET_RAW: libc::c_ulong = 13; // SAFETY: `pre_exec` runs between fork and exec, where only - // async-signal-safe work is permitted. `prctl` and `seccomp` are bare - // syscalls, the filter lives on the stack, and this closure allocates - // nothing and captures nothing. + // async-signal-safe work is permitted. `prctl` is a bare syscall, and this + // closure allocates nothing and captures nothing. unsafe { - // Take away CAP_NET_ADMIN. command.pre_exec(|| { + // Drop CAP_NET_ADMIN and CAP_NET_RAW from the bounding set. if libc::prctl(libc::PR_CAPBSET_DROP, CAP_NET_ADMIN, 0, 0, 0) != 0 { return Err(std::io::Error::last_os_error()); } - // Refuse AF_PACKET communication. - refuse_packet_sockets() - }); - } -} - -/// Removing CAP_NET_RAW prevents a process from using RAW and PACKET sockets. However, ICMP is also denied. -/// LXC needs to allow ICMP and prevent programs from using sockets to -/// bypass the network guards. The solution is to give the kernel a custom filter to refuse -/// any socket that carries AF_PACKET. LXC utilizes seccomp. -/// see https://docs.kernel.org/userspace-api/seccomp_filter.html and -/// https://man7.org/linux/man-pages/man2/seccomp.2.html -#[cfg(target_os = "linux")] -fn refuse_packet_sockets() -> std::io::Result<()> { - #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] - compile_error!("LXC only understands seccomp system calls for x86_64 and arm64."); - - // Use raw SecComp calls instead of the libseccomp crate. - // Using the libseccomp crate pulls in a dependency to the c libseccomp library. - // Use raw called here. - - // https://docs.kernel.org/networking/filter.html for more information on filtering sockets - // Filtering sockets requires assembling a small filter program with assembly like syntax. - // The below code is to consolidate some commands into easy to understand words. - - // https://github.com/torvalds/linux/blob/master/include/uapi/linux/bpf_common.h - // Common BPF words - const LOAD_WORD: u16 = (libc::BPF_LD | libc::BPF_W | libc::BPF_ABS) as u16; - const JUMP_IF_EQUAL: u16 = (libc::BPF_JMP | libc::BPF_JEQ | libc::BPF_K) as u16; - const JUMP_IF_AT_LEAST: u16 = (libc::BPF_JMP | libc::BPF_JGE | libc::BPF_K) as u16; - const BITWISE_AND: u16 = (libc::BPF_ALU | libc::BPF_AND | libc::BPF_K) as u16; - const RETURN: u16 = (libc::BPF_RET | libc::BPF_K) as u16; - - // https://github.com/torvalds/linux/blob/master/include/uapi/linux/seccomp.h - // Keys of the information needed during the filter. - const SYSCALL_NUMBER: u32 = 0; - const ARCHITECTURE: u32 = 4; - const FIRST_ARGUMENT: u32 = 16; - const SECOND_ARGUMENT: u32 = 24; - - const REFUSE_WITH_EPERM: u32 = libc::SECCOMP_RET_ERRNO | (libc::EPERM as u32); - - // ENOSYS is the errno from a kernel built without io_uring. The refusal - // borrows it to read as absence rather than denial. - const REFUSE_WITH_ENOSYS: u32 = libc::SECCOMP_RET_ERRNO | (libc::ENOSYS as u32); - - const NO_FLAGS: libc::c_ulong = 0; - - // numbers come from - // https://github.com/torvalds/linux/blob/master/include/uapi/linux/audit.h - // To test the ABI the filter needs to know the endianness and arch. - const _64BIT_ABI_LITTLE_ENDIANNESS: u32 = 0x8000_0000 | 0x4000_0000; - - // Also from https://github.com/torvalds/linux/blob/master/include/uapi/linux/audit.h - #[cfg(target_arch = "x86_64")] - const NATIVE_ARCHITECTURE: u32 = _64BIT_ABI_LITTLE_ENDIANNESS | libc::EM_X86_64 as u32; - #[cfg(target_arch = "aarch64")] - const NATIVE_ARCHITECTURE: u32 = _64BIT_ABI_LITTLE_ENDIANNESS | libc::EM_AARCH64 as u32; - - // Filter only works for x64 and arm64 ABI's. - // Used to filter out a 32 bit process under the "Filters" section. - // check https://man7.org/linux/man-pages/man2/seccomp.2.html - const USES_X32_ABI_FLAG: u32 = 0x4000_0000; - - const SOCKET_SYSCALL: u32 = libc::SYS_socket as u32; - - // A ring can create a socket on its own, and the request never reaches the - // socket syscall this filter reads. Refusing all three entry points leaves - // no ring to submit that request to. - const IO_URING_SETUP_SYSCALL: u32 = libc::SYS_io_uring_setup as u32; - const IO_URING_ENTER_SYSCALL: u32 = libc::SYS_io_uring_enter as u32; - const IO_URING_REGISTER_SYSCALL: u32 = libc::SYS_io_uring_register as u32; - - // `libc` marks SOCK_PACKET deprecated, and this crate builds warnings as - // errors, so naming it here is the only way to compare against it. The - // value is from linux/net.h and matches what `libc` defines. - const SOCK_PACKET: u32 = 10; - - // A caller may fold SOCK_CLOEXEC or SOCK_NONBLOCK into the socket type, and - // the kernel masks them off before reading it, so the filter masks them off - // too. - const SOCKET_TYPE_MASK: u32 = 0xF; - - // Make the filter - #[rustfmt::skip] - let mut program = [ - libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: ARCHITECTURE }, - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 15, k: NATIVE_ARCHITECTURE }, // test ABI - libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: SYSCALL_NUMBER }, - libc::sock_filter { code: JUMP_IF_AT_LEAST, jt: 13, jf: 0, k: USES_X32_ABI_FLAG }, // Test 32 bit - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 11, jf: 0, k: IO_URING_SETUP_SYSCALL }, // leave no ring to ask with - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 10, jf: 0, k: IO_URING_ENTER_SYSCALL }, - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 9, jf: 0, k: IO_URING_REGISTER_SYSCALL }, - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 0, jf: 6, k: SOCKET_SYSCALL }, // Only process a socket syscall - libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: FIRST_ARGUMENT }, - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 5, jf: 0, k: libc::AF_PACKET as u32 }, // return error if using AF_PACKET - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 4, jf: 0, k: libc::AF_XDP as u32 }, // AF_XDP reaches the wire the same way - libc::sock_filter { code: LOAD_WORD, jt: 0, jf: 0, k: SECOND_ARGUMENT }, - libc::sock_filter { code: BITWISE_AND, jt: 0, jf: 0, k: SOCKET_TYPE_MASK }, // drop SOCK_CLOEXEC and SOCK_NONBLOCK - libc::sock_filter { code: JUMP_IF_EQUAL, jt: 1, jf: 0, k: SOCK_PACKET }, // the kernel turns this one into AF_PACKET - libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_ALLOW }, - libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: REFUSE_WITH_EPERM }, - libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: REFUSE_WITH_ENOSYS }, - libc::sock_filter { code: RETURN, jt: 0, jf: 0, k: libc::SECCOMP_RET_KILL_PROCESS }, - ]; - - // Make the struct - let header = libc::sock_fprog { - len: program.len() as libc::c_ushort, - filter: program.as_mut_ptr(), - }; - - // Make the sys call - let taken = unsafe { - libc::syscall( - libc::SYS_seccomp, - libc::SECCOMP_SET_MODE_FILTER as libc::c_ulong, - NO_FLAGS, - &header as *const libc::sock_fprog, - ) - }; + if libc::prctl(libc::PR_CAPBSET_DROP, CAP_NET_RAW, 0, 0, 0) != 0 { + return Err(std::io::Error::last_os_error()); + } - if taken != 0 { - return Err(std::io::Error::last_os_error()); + Ok(()) + }); } - - Ok(()) } /// A `/etc/hosts` helper emits one `mxc:` diagnostic line, so this is far above @@ -671,7 +547,7 @@ impl LxcContainer { env: &[String], force_clear_env: bool, timeout: Option, - firewall: ContainerFirewall, + _firewall: ContainerFirewall, ) -> Result<(i32, String, String), String> { use crate::mxc_pty::{run_with_pty, PtyOptions, PtyOutcome, Signal}; @@ -687,9 +563,7 @@ impl LxcContainer { force_clear_env, )); - if firewall == ContainerFirewall::Installed { - confine_network_capabilities(&mut cmd); - } + confine_network_capabilities(&mut cmd); let options = PtyOptions { unblock_signals: UNBLOCK, @@ -733,7 +607,7 @@ impl LxcContainer { env: &[String], force_clear_env: bool, timeout: Option, - firewall: ContainerFirewall, + _firewall: ContainerFirewall, ) -> Result<(i32, String, String), String> { use crate::mxc_common::interruptible_reader::wrap_pipe; use crate::mxc_common::sandbox_process::{wait_with_timeout, StreamCloser, WaitError}; @@ -752,9 +626,7 @@ impl LxcContainer { unblock_fatal_signals(&mut cmd); - if firewall == ContainerFirewall::Installed { - confine_network_capabilities(&mut cmd); - } + confine_network_capabilities(&mut cmd); let mut child = cmd .spawn() @@ -918,7 +790,7 @@ impl LxcContainer { working_directory: &str, env: &[String], force_clear_env: bool, - firewall: ContainerFirewall, + _firewall: ContainerFirewall, ) -> std::process::Command { let mut cmd = self.lxc_command("lxc-attach"); cmd.args(build_attach_args_with_env_control( @@ -927,9 +799,7 @@ impl LxcContainer { command, force_clear_env, )); - if firewall == ContainerFirewall::Installed { - confine_network_capabilities(&mut cmd); - } + confine_network_capabilities(&mut cmd); cmd } @@ -1612,284 +1482,6 @@ mod tests { ); } - /// The probe child got the descriptor it asked for. - #[cfg(target_os = "linux")] - const ATTEMPT_ALLOWED: i32 = 10; - - /// The probe child was refused with EPERM, which is what the filter returns - /// for a socket that reaches the wire below the chains. - #[cfg(target_os = "linux")] - const ATTEMPT_REFUSED_EPERM: i32 = 11; - - /// The probe child was refused with ENOSYS, which is what the filter - /// returns for io_uring. - #[cfg(target_os = "linux")] - const ATTEMPT_REFUSED_ENOSYS: i32 = 12; - - /// The probe child was refused, but for some other reason. - #[cfg(target_os = "linux")] - const ATTEMPT_REFUSED_OTHER: i32 = 13; - - /// What the probe child asks the kernel for. - #[cfg(target_os = "linux")] - #[derive(Clone, Copy, Debug)] - enum Attempt { - /// The socket the filter exists to stop. - PacketSocket, - /// The same socket under a family the filter permits, which the kernel - /// swaps over once seccomp has read it. - LegacyPacketSocket, - /// That spelling with a flag folded into the socket type. - LegacyPacketSocketWithFlags, - /// Another family that reaches the wire below the chains. - ExpressDataPathSocket, - /// A ring, which creates any of the above without a socket syscall. - IoUringRing, - /// What an `icmp` egress allow needs. - RawIcmpSocket, - /// What every ordinary workload needs. - TcpSocket, - } - - /// Fork a child that optionally installs the packet-socket filter, makes - /// one request of the kernel, and reports which answer it got. The child - /// leaves through `_exit` and never reaches `exec`, so the attempt runs in - /// the same post-fork context that confines a real workload. Reporting - /// through the exit code keeps a refusal distinct from a child that never - /// ran: a missing or unspawnable program cannot forge one of these numbers. - #[cfg(target_os = "linux")] - fn probe_socket(attempt: Attempt, install_filter: bool) -> i32 { - use std::os::unix::process::CommandExt; - - // `libc` marks SOCK_PACKET deprecated, and this crate builds warnings as - // errors, so naming it here is the only way to probe with it. Both - // values match what `libc` defines. - const AF_XDP: libc::c_int = libc::AF_XDP; - const SOCK_PACKET: libc::c_int = 10; - - let mut command = std::process::Command::new("/bin/true"); - - // SAFETY: `pre_exec` runs between fork and exec, where only - // async-signal-safe work is permitted. This closure makes bare - // syscalls, allocates nothing, and leaves through `_exit`, which skips - // the atexit handlers the parent still owns. - unsafe { - command.pre_exec(move || { - // Seccomp wants this or CAP_SYS_ADMIN. Production is root and - // gets it from the capability; setting it here lets the probe - // report the filter's own verdict at any privilege level. - libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); - - if install_filter { - refuse_packet_sockets()?; - } - - let descriptor = match attempt { - Attempt::PacketSocket => libc::socket(libc::AF_PACKET, libc::SOCK_RAW, 0), - Attempt::LegacyPacketSocket => libc::socket(libc::PF_INET, SOCK_PACKET, 0), - Attempt::LegacyPacketSocketWithFlags => { - libc::socket(libc::PF_INET, SOCK_PACKET | libc::SOCK_CLOEXEC, 0) - } - Attempt::ExpressDataPathSocket => libc::socket(AF_XDP, libc::SOCK_RAW, 0), - Attempt::IoUringRing => { - // The kernel writes a fixed-size struct back through - // this buffer, which is larger than the struct is. - let mut parameters = [0u8; 256]; - libc::syscall( - libc::SYS_io_uring_setup, - 1 as libc::c_ulong, - parameters.as_mut_ptr(), - ) as libc::c_int - } - Attempt::RawIcmpSocket => { - libc::socket(libc::AF_INET, libc::SOCK_RAW, libc::IPPROTO_ICMP) - } - Attempt::TcpSocket => libc::socket(libc::AF_INET, libc::SOCK_STREAM, 0), - }; - - if descriptor >= 0 { - libc::close(descriptor); - libc::_exit(ATTEMPT_ALLOWED); - } - - match std::io::Error::last_os_error().raw_os_error() { - Some(libc::EPERM) => libc::_exit(ATTEMPT_REFUSED_EPERM), - Some(libc::ENOSYS) => libc::_exit(ATTEMPT_REFUSED_ENOSYS), - _ => libc::_exit(ATTEMPT_REFUSED_OTHER), - } - }); - } - - command - .status() - .expect("the probe child must fork") - .code() - .expect("the probe child reports through its exit code, never a signal") - } - - // `Seccomp: 2` in `/proc/self/status` says a filter is attached, not that it - // Reports whether the filter is what refuses `attempt` on this runner, - // skipping or failing when the question cannot be settled here. - // - // The kernel refuses a packet socket with EPERM to a caller lacking - // CAP_NET_RAW, which is the errno the filter returns. Asserting the - // refusal on its own therefore passes under an unprivileged runner with the - // filter having done nothing, and would keep passing if the filter were - // deleted. Reading the unfiltered answer first is what separates the two, - // and it has to happen in the same test: a control standing alone passes - // under that runner too. - #[cfg(target_os = "linux")] - fn the_filter_is_what_refuses(attempt: Attempt, refusal: i32) -> bool { - let unfiltered = probe_socket(attempt, false); - - if unfiltered != refusal { - return true; - } - - let reason = format!( - "{attempt:?} is already refused ({unfiltered}) with no filter installed, \ - so a refusal under the filter would prove nothing — run as a caller \ - holding CAP_NET_RAW on a kernel offering the route" - ); - - // A lane provisioned for this suite reports a skip as success, so the - // gate would go green having tested nothing. - if std::env::var("MXC_LXC_TESTS_REQUIRE_EXECUTION").is_ok_and(|value| value != "0") { - panic!("strict mode: {reason}"); - } - - println!("SKIPPED: {reason}"); - false - } - - // refuses anything: a filter permitting every syscall reports the same - // number. These open the sockets the filter exists to stop and read the - // kernel's answer. - #[cfg(target_os = "linux")] - #[test] - fn the_filter_refuses_a_packet_socket_with_eperm() { - if !the_filter_is_what_refuses(Attempt::PacketSocket, ATTEMPT_REFUSED_EPERM) { - return; - } - - assert_eq!( - probe_socket(Attempt::PacketSocket, true), - ATTEMPT_REFUSED_EPERM, - "a workload under the filter must not reach AF_PACKET, which writes \ - frames below the chains confining it" - ); - } - - // `socket(PF_INET, SOCK_PACKET, ...)` names a family the filter permits and - // hands back a packet socket anyway, because the kernel swaps the family - // over after seccomp has read it. Reading the socket type catches it. - #[cfg(target_os = "linux")] - #[test] - fn the_filter_refuses_a_legacy_packet_socket_with_eperm() { - if !the_filter_is_what_refuses(Attempt::LegacyPacketSocket, ATTEMPT_REFUSED_EPERM) { - return; - } - - assert_eq!( - probe_socket(Attempt::LegacyPacketSocket, true), - ATTEMPT_REFUSED_EPERM, - "SOCK_PACKET reaches AF_PACKET under a family the filter allows" - ); - } - - // The kernel masks the flags off the socket type before reading it, so a - // filter comparing the type whole is passed by adding one. - #[cfg(target_os = "linux")] - #[test] - fn the_filter_refuses_a_legacy_packet_socket_carrying_a_flag() { - if !the_filter_is_what_refuses(Attempt::LegacyPacketSocketWithFlags, ATTEMPT_REFUSED_EPERM) - { - return; - } - - assert_eq!( - probe_socket(Attempt::LegacyPacketSocketWithFlags, true), - ATTEMPT_REFUSED_EPERM, - "SOCK_CLOEXEC must not carry a packet socket past the type check" - ); - } - - // A kernel built without XDP sockets answers EAFNOSUPPORT rather than - // handing back a descriptor, which is a different answer from the filter's - // and settles this as well as an open route would. - #[cfg(target_os = "linux")] - #[test] - fn the_filter_refuses_an_express_data_path_socket_with_eperm() { - if !the_filter_is_what_refuses(Attempt::ExpressDataPathSocket, ATTEMPT_REFUSED_EPERM) { - return; - } - - assert_eq!( - probe_socket(Attempt::ExpressDataPathSocket, true), - ATTEMPT_REFUSED_EPERM, - "AF_XDP transmits on the interface below the chains, as AF_PACKET does" - ); - } - - // A ring creates a socket on its own, and that request is not a socket - // syscall, so no family check can see it. Refusing the ring is what closes - // the route. A kernel offering no io_uring answers ENOSYS by itself, which - // the check above rules out. - #[cfg(target_os = "linux")] - #[test] - fn the_filter_refuses_an_io_uring_ring() { - if !the_filter_is_what_refuses(Attempt::IoUringRing, ATTEMPT_REFUSED_ENOSYS) { - return; - } - - assert_eq!( - probe_socket(Attempt::IoUringRing, true), - ATTEMPT_REFUSED_ENOSYS, - "a ring opens AF_PACKET without issuing the syscall this filter reads" - ); - } - - // CAP_NET_RAW stays so that an `icmp` egress allow has the raw socket it - // needs, and the filter is what makes keeping it safe. A runner that - // cannot open the socket unfiltered cannot show the filter left it open, - // so it reports a skip rather than a pass. - #[cfg(target_os = "linux")] - #[test] - fn the_filter_leaves_a_raw_icmp_socket_alone() { - let unfiltered = probe_socket(Attempt::RawIcmpSocket, false); - - if unfiltered != ATTEMPT_ALLOWED { - let reason = format!( - "raw ICMP answers {unfiltered} with no filter installed, so this \ - runner cannot show the filter leaves it open — run as a caller \ - holding CAP_NET_RAW" - ); - - if std::env::var("MXC_LXC_TESTS_REQUIRE_EXECUTION").is_ok_and(|value| value != "0") { - panic!("strict mode: {reason}"); - } - - println!("SKIPPED: {reason}"); - return; - } - - assert_eq!( - probe_socket(Attempt::RawIcmpSocket, true), - ATTEMPT_ALLOWED, - "the filter closed raw ICMP, which an `icmp` egress allow depends on" - ); - } - - #[cfg(target_os = "linux")] - #[test] - fn the_filter_leaves_an_ordinary_socket_alone() { - assert_eq!( - probe_socket(Attempt::TcpSocket, true), - ATTEMPT_ALLOWED, - "a confined workload still has to reach the network it is allowed" - ); - } - #[cfg(target_os = "linux")] #[test] fn a_failed_release_carries_the_tools_own_diagnosis() { diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs index 814504c27..7f845784a 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs @@ -616,12 +616,23 @@ impl LxcScriptRunner { } }; - let firewall = container_firewall( - fw_manager.rules_applied(), - ingress_manager + // figure out if a firewall needs to be installed. + let is_firewall_installed = if fw_manager.rules_applied() + || ingress_manager .as_ref() - .is_some_and(|mgr| mgr.rules_applied()), - ); + .is_some_and(|mgr| mgr.rules_applied()) + { + ContainerFirewall::Installed + } else { + ContainerFirewall::Absent + }; + + // Dropping CAP_NET_RAW also removed raw ICMP. Widen the container's + // ping_group_range so unprivileged ping still works. + if let Err(e) = fw_manager.allow_ping_sockets(logger) { + self.release_after_failure(&container, container_created, logger); + return Err(ScriptResponse::error(&e)); + } // A reused container can carry a pin left by an older run. let stale_pin_error = Self::clear_stale_hosts_pin(container_created, |command| { @@ -632,7 +643,7 @@ impl LxcScriptRunner { &[], true, Some(HOSTS_COMMAND_TIMEOUT), - firewall, + is_firewall_installed, ) .map_err(|e| e.to_string()) }); @@ -659,7 +670,7 @@ impl LxcScriptRunner { ingress_manager, container, container_created, - firewall, + firewall: is_firewall_installed, released: false, force_stopped: false, stop_failed: false, @@ -1553,15 +1564,6 @@ fn launch_failure_message(reason: &str, cleanup_failures: &[String]) -> String { ) } -/// Whether the workload must be kept away from chains in its own namespace. -fn container_firewall(egress_applied: bool, ingress_applied: bool) -> ContainerFirewall { - if egress_applied || ingress_applied { - ContainerFirewall::Installed - } else { - ContainerFirewall::Absent - } -} - #[cfg(test)] mod tests { use super::*; @@ -2133,28 +2135,6 @@ mod tests { )); } - #[test] - fn either_installed_chain_confines_the_workload() { - for (egress, ingress) in [(true, false), (false, true), (true, true)] { - assert_eq!( - container_firewall(egress, ingress), - ContainerFirewall::Installed, - "egress={egress} ingress={ingress} leaves a chain the workload could flush" - ); - } - } - - #[test] - fn a_policy_that_installs_no_chain_leaves_the_workload_unconfined() { - // Confining takes CAP_SETPCAP to drop a capability, which an - // unprivileged caller lacks. - assert_eq!( - container_firewall(false, false), - ContainerFirewall::Absent, - "a run with no firewall must not be confined" - ); - } - /// `process.env` resolution, which schema 0.9 gave a default block. mod env { use super::*; diff --git a/src/mxc-sdk/src/backends/lxc/common/network_iptables.rs b/src/mxc-sdk/src/backends/lxc/common/network_iptables.rs index 983452633..55729fffe 100644 --- a/src/mxc-sdk/src/backends/lxc/common/network_iptables.rs +++ b/src/mxc-sdk/src/backends/lxc/common/network_iptables.rs @@ -1169,6 +1169,16 @@ impl NetworkIptablesManager { Ok(()) } + /// The CAP_NET_RAW drop removes raw ICMP. Opening net.ipv4.ping_group_range + /// to every gid lets the workload still ping through a datagram ICMP socket. + pub fn allow_ping_sockets(&self, logger: &mut Logger) -> Result { + self.run_firewall_command( + "sysctl", + &["-w", "net.ipv4.ping_group_range=0 2147483647"], + logger, + ) + } + // Clean up the iptables state recorded by another thread. // The ownership record keeps signal-time cleanup from flushing a later // run's chain under the same container name. @@ -1329,6 +1339,28 @@ mod tests { } } + #[test] + fn allowing_ping_opens_the_ping_group_range_in_the_container_netns() { + let fake = super::test_firewall::install(); + let manager = + NetworkIptablesManager::new("lxc-ping", EgressHookPoint::ContainerNetns(4242)); + let mut logger = Logger::new(Mode::Buffer); + + manager + .allow_ping_sockets(&mut logger) + .expect("the fake firewall accepts every command"); + + assert_eq!( + fake.issued(), + vec![vec![ + "sysctl".to_string(), + "-w".to_string(), + "net.ipv4.ping_group_range=0 2147483647".to_string(), + ]], + "ping restore must open the ping group range to every gid" + ); + } + #[test] fn an_unhooked_caller_is_not_refused_in_firewall_mode() { let _fake = super::test_firewall::install(); diff --git a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs index 919d13969..0c065ee47 100644 --- a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs +++ b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs @@ -9,7 +9,6 @@ use serde_json::json; use wxc_e2e_tests::{has_lxc_host, has_platform_exec, run_platform_config_value}; const CAP_NET_ADMIN: u64 = 1 << 12; -const CAP_NET_RAW: u64 = 1 << 13; /// Whether the LXC capability prerequisites are present. fn ready() -> bool { @@ -26,92 +25,19 @@ fn capability_mask(status: &str, field: &str) -> u64 { .unwrap_or_else(|| panic!("the container reported no readable {field} line\n{status}")) } -/// Ask the kernel for every socket the filter is meant to refuse, for two it is -/// meant to leave alone, and print one tagged line per answer. ctypes calls -/// `socket(2)` and `io_uring_setup(2)` by number, so nothing between the test -/// and the kernel can substitute one route for another. The two permitted -/// sockets are the control: they come back from the same mechanism in the same -/// run, so a refusal above them is the filter choosing, not the probe failing. -const PACKET_SOCKET_PROBE: &str = r#"python3 - <<'MXC_PROBE' -import ctypes, errno - -libc = ctypes.CDLL(None, use_errno=True) -libc.syscall.restype = ctypes.c_long - -def report(tag, fd): - if fd >= 0: - libc.close(fd) - print(tag + " ALLOWED") - else: - code = ctypes.get_errno() - print(tag + " REFUSED " + errno.errorcode.get(code, str(code))) - -def sock(tag, domain, kind, protocol): - ctypes.set_errno(0) - report(tag, libc.socket(domain, kind, protocol)) - -CLOEXEC = 0o2000000 - -sock("PACKET_RAW", 17, 3, 0) -sock("PACKET_DGRAM", 17, 2, 0) -sock("PACKET_RAW_CLOEXEC", 17, 3 | CLOEXEC, 0) -sock("LEGACY_SOCK_PACKET", 2, 10, 0) -sock("LEGACY_SOCK_PACKET_CLOEXEC", 2, 10 | CLOEXEC, 0) -sock("XDP", 44, 3, 0) -sock("ICMP_RAW", 2, 3, 1) -sock("TCP", 2, 1, 0) - -params = (ctypes.c_ubyte * 120)() -ctypes.set_errno(0) -report("IO_URING", int(libc.syscall(425, 1, ctypes.byref(params)))) -MXC_PROBE -"#; - -/// Pick one tagged line out of the probe's output. -fn probe_line<'a>(output: &'a str, tag: &str) -> &'a str { - output - .lines() - .map(str::trim) - .find(|line| line.split_whitespace().next() == Some(tag)) - .unwrap_or_else(|| panic!("the probe printed no {tag} line\n{output}")) -} - -/// Read one decimal field out of the container's `/proc/self/status`. -fn status_number(status: &str, field: &str) -> u64 { - status - .lines() - .find(|line| line.starts_with(field)) - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|value| value.parse().ok()) - .unwrap_or_else(|| panic!("the container reported no readable {field} line\n{status}")) -} - #[test] -fn workload_cannot_reconfigure_or_bypass_the_network() { +fn workload_cannot_reconfigure_the_network() { if !ready() { return; } - // The confinement only happens when chains exist. This policy asks for a - // firewall to bring it on. let config = json!({ "version": "0.9.0-alpha", "containerId": "lxc-network-capability", "containment": "lxc", "process": { "commandLine": "sh -c \"cat /proc/self/status\"" }, "lifecycle": { "destroyOnExit": true }, - "lxc": { "distribution": "alpine", "release": "3.23" }, - "network": { - "egress": { - "default": "deny", - "allow": [ - { - "to": [{ "cidr": "140.82.112.0/20" }], - "ports": [{ "protocol": "tcp", "port": 443 }] - } - ] - } - } + "lxc": { "distribution": "alpine", "release": "3.23" } }); let result = run_platform_config_value("lxc network capability", &config, &[], None); @@ -132,57 +58,56 @@ fn workload_cannot_reconfigure_or_bypass_the_network() { assert_eq!( capability_mask(&status, field) & CAP_NET_ADMIN, 0, - "{field} still carries CAP_NET_ADMIN; the workload can rewrite the firewall confining it\n{status}" + "{field} still carries CAP_NET_ADMIN; the workload could reconfigure the network it asked to go without\n{status}" ); + } +} - assert_ne!( - capability_mask(&status, field) & CAP_NET_RAW, - 0, - "{field} lost CAP_NET_RAW; an explicit `protocol: \"icmp\"` allow needs a raw socket\n{status}" - ); +#[test] +fn workload_can_still_ping() { + if !ready() { + return; } - // 2 is SECCOMP_MODE_FILTER. The kernel writes this line too, and a filter - // cannot be lifted once it is attached. This asserts only that one is - // attached; that it refuses AF_PACKET with EPERM is proven by the - // lxc_bindings unit test the_filter_refuses_a_packet_socket_with_eperm. + let config = json!({ + "version": "0.9.0-alpha", + "containerId": "lxc-ping-allowed", + "containment": "lxc", + "process": { "commandLine": "ping -c 1 127.0.0.1" }, + "lifecycle": { "destroyOnExit": true }, + "lxc": { "distribution": "ubuntu", "release": "noble" } + }); + + let result = run_platform_config_value("lxc ping allowed", &config, &[], None); + assert_eq!( - status_number(&status, "Seccomp:"), - 2, - "the workload runs with no seccomp filter; AF_PACKET reaches the interface below the chains\n{status}" + result.code, + Some(0), + "the workload could not ping after the capability drop\n{}", + result.combined_output() ); } #[test] -fn workload_cannot_open_a_packet_capable_socket_by_any_route() { +fn workload_cannot_open_a_raw_or_packet_socket() { if !ready() { return; } - // Same firewall as above, because the confinement only happens when chains - // exist. Ubuntu rather than Alpine: the probe needs an interpreter that - // can call socket(2) by number, and Alpine's image is busybox alone. + // Bash does not have the ability to make RAW and PACKET sockets. Use perl instead. + const RAW_AND_PACKET_PROBE: &str = r#"perl -e 'use Socket; my $raw = socket(RAW, PF_INET, SOCK_RAW, 1) ? "RAW_OPENED" : "RAW_REFUSED: $!"; my $pkt = socket(PKT, 17, SOCK_RAW, 0) ? "PACKET_OPENED" : "PACKET_REFUSED: $!"; print "$raw\n$pkt\n";'"#; + + // Use Ubuntu for pearl. let config = json!({ "version": "0.9.0-alpha", - "containerId": "lxc-packet-socket-refusal", + "containerId": "lxc-raw-socket-refusal", "containment": "lxc", - "process": { "commandLine": PACKET_SOCKET_PROBE, "timeout": 120000 }, + "process": { "commandLine": RAW_AND_PACKET_PROBE }, "lifecycle": { "destroyOnExit": true }, - "lxc": { "distribution": "ubuntu", "release": "noble" }, - "network": { - "egress": { - "default": "deny", - "allow": [ - { - "to": [{ "cidr": "140.82.112.0/20" }], - "ports": [{ "protocol": "tcp", "port": 443 }] - } - ] - } - } + "lxc": { "distribution": "ubuntu", "release": "noble" } }); - let result = run_platform_config_value("lxc packet socket refusal", &config, &[], None); + let result = run_platform_config_value("lxc raw socket refusal", &config, &[], None); let output = result.combined_output(); assert_eq!( @@ -192,41 +117,13 @@ fn workload_cannot_open_a_packet_capable_socket_by_any_route() { result.stderr ); - // AF_PACKET by its own family, the legacy SOCK_PACKET spelling that the - // kernel rewrites into that family after seccomp has read the argument, and - // AF_XDP, each with and without a socket-type flag riding along. - for tag in [ - "PACKET_RAW", - "PACKET_DGRAM", - "PACKET_RAW_CLOEXEC", - "LEGACY_SOCK_PACKET", - "LEGACY_SOCK_PACKET_CLOEXEC", - "XDP", - ] { - assert_eq!( - probe_line(&output, tag), - format!("{tag} REFUSED EPERM"), - "the workload opened a packet-capable socket below the egress chains\n{output}" - ); - } - - // io_uring creates sockets without the socket(2) syscall the filter reads, - // so the ring itself is refused. ENOSYS rather than EPERM because callers - // probe for io_uring and fall back when it reports unsupported. - assert_eq!( - probe_line(&output, "IO_URING"), - "IO_URING REFUSED ENOSYS", - "io_uring is reachable, and it creates sockets the filter never sees\n{output}" + assert!( + output.contains("RAW_REFUSED"), + "the workload opened a raw socket the capability drop must refuse\n{output}" ); - // The control. These two come back from the same ctypes call in the same - // container, so the refusals above are the filter and not a broken probe. - // Raw ICMP is also the socket an explicit `protocol: "icmp"` allow needs. - for tag in ["ICMP_RAW", "TCP"] { - assert_eq!( - probe_line(&output, tag), - format!("{tag} ALLOWED"), - "the filter took a socket it is supposed to leave alone\n{output}" - ); - } + assert!( + output.contains("PACKET_REFUSED"), + "the workload opened a packet socket the capability drop must refuse\n{output}" + ); } From 5b22da5fa7231b631927fb7ed0ff0b1d586a0b27 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Thu, 15 Oct 2026 11:39:17 -0700 Subject: [PATCH 15/17] removing return statement --- docs/backends/lxc/lxc-backend.md | 45 ++++++++----------- src/tools/lxc/src/linux_executor_arguments.rs | 4 +- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/docs/backends/lxc/lxc-backend.md b/docs/backends/lxc/lxc-backend.md index a383b3384..f67b0cf4a 100644 --- a/docs/backends/lxc/lxc-backend.md +++ b/docs/backends/lxc/lxc-backend.md @@ -333,33 +333,24 @@ The zone query should answer the zone you assigned. container init keeps `CAP_NET_ADMIN` there, so a process running as root inside the container can flush or delete them. The command MXC runs is attached with `CAP_NET_ADMIN` dropped from its bounding set whenever chains are installed, so - it cannot. That same attach installs a seccomp filter refusing three ways of - reaching a packet socket: the `AF_PACKET` family whatever the socket type, - `AF_XDP`, and the legacy `SOCK_PACKET` type, which the kernel rewrites into - `AF_PACKET` after the family argument has already been read. Each is refused with - `EPERM`, and a socket-type flag riding along does not change the answer. The - filter refuses io_uring as well, with `ENOSYS`: a ring creates sockets without the - `socket` system call the filter reads, and `ENOSYS` is the answer callers probe - for before falling back to ordinary I/O. A process issuing system calls under an - ABI other than the one the filter was built against is killed outright rather - than refused, because its arguments are not where the filter reads them. - Installing the filter at all requires MXC to hold `CAP_SYS_ADMIN`. The kernel - accepts `no_new_privs` in its place, and MXC does not set it; a deployment - lacking the capability does not lose the filter quietly, as the attach fails and - the command never runs. Without all of this a workload holding - `CAP_NET_RAW` could put frames on the wire beneath the chains, which filter at - the IP layer. Default-deny closes external reachability for a container that does - not deliberately tear it down, including services the workload itself starts. -- **The packet-socket refusal is a creation-time check, not a transmit-time one.** - The kernel tests `CAP_NET_RAW` when a socket is opened and never again. A - descriptor opened before the filter is attached keeps sending for as long as - something holds it, and container init runs unfiltered with `CAP_NET_RAW` in hand - (see the next limitation). Taking that descriptor needs no cooperation from init: - a workload permitted to trace it can pull the descriptor out with `pidfd_getfd`. - Either route is outside what this filter covers. The filter also - names the families it refuses one by one; a family added by a later kernel is - permitted until it is named here. Both gaps close only below the socket layer, on - the host side of the container's interface, which this backend does not do today. + it cannot. That same attach drops `CAP_NET_RAW`, the capability the kernel tests + when any packet-capable socket is created: `AF_PACKET` whatever the socket type, + `AF_XDP`, raw IP, and the legacy `SOCK_PACKET` the kernel rewrites into + `AF_PACKET`. The test sits in the kernel's socket-creation path, which both + `socket` and io_uring reach; the socket comes back `EPERM` by either route. + Without the drop a workload holding `CAP_NET_RAW` could put frames on the wire + beneath the chains, which filter at the IP layer. The drop also ends raw ICMP, + and MXC reopens unprivileged ping by widening `net.ipv4.ping_group_range` inside + the container; a `SOCK_DGRAM` `IPPROTO_ICMP` echo still works. Default-deny + closes external reachability for a container that does not deliberately tear it + down, including services the workload itself starts. +- **The drop covers the command MXC runs, not container init.** Init keeps + `CAP_NET_RAW` just as it keeps `CAP_NET_ADMIN`. A privileged process inside the + container can still open a packet socket. The kernel checks the capability at + creation, never at transmit, and an already-open descriptor handed to the + workload keeps sending. Nothing in a stock container image opens one; closing the + gap entirely needs a gate below the socket layer, which this backend does not add + today. - **Policy is not in force while the container starts.** The chains are installed after the container has started and its address has settled, so container init and anything it starts run unfiltered in both directions for that interval. The diff --git a/src/tools/lxc/src/linux_executor_arguments.rs b/src/tools/lxc/src/linux_executor_arguments.rs index e44835c7c..d4b506868 100644 --- a/src/tools/lxc/src/linux_executor_arguments.rs +++ b/src/tools/lxc/src/linux_executor_arguments.rs @@ -139,10 +139,10 @@ impl LinuxExecutorArguments { #[cfg(not(all(feature = "hyperlight", target_arch = "x86_64")))] { - return Err( + Err( "Error: --setup-hyperlight requires x86_64 (Hyperlight needs KVM or WHP)" .to_string(), - ); + ) } #[cfg(all(feature = "hyperlight", target_arch = "x86_64"))] From ca8b1004b33082063c66ada8ef9bda39113ff668 Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Fri, 16 Oct 2026 15:37:11 -0700 Subject: [PATCH 16/17] Addressing comments --- docs/backends/lxc/lxc-backend.md | 11 ++- .../src/backends/lxc/common/lxc_bindings.rs | 18 ++-- .../src/backends/lxc/common/lxc_runner.rs | 90 ++++--------------- ...xc_e2e_tests_e2e_lxc_network_capability.rs | 70 +++++++++++---- src/tools/lxc/src/linux_executor_arguments.rs | 42 +++++---- src/tools/lxc/src/main.rs | 17 ++-- src/tools/lxc/tests/cli.rs | 29 ++++++ tests/configs/lxc_rootless_no_network.json | 16 ++++ tests/scripts/run_lxc_all_tests.sh | 1 + tests/scripts/run_lxc_log_file_test.sh | 30 +++++++ .../run_lxc_rootless_no_network_test.sh | 36 ++++++++ 11 files changed, 233 insertions(+), 127 deletions(-) create mode 100644 src/tools/lxc/tests/cli.rs create mode 100644 tests/configs/lxc_rootless_no_network.json create mode 100644 tests/scripts/run_lxc_log_file_test.sh create mode 100644 tests/scripts/run_lxc_rootless_no_network_test.sh diff --git a/docs/backends/lxc/lxc-backend.md b/docs/backends/lxc/lxc-backend.md index f67b0cf4a..a85c09448 100644 --- a/docs/backends/lxc/lxc-backend.md +++ b/docs/backends/lxc/lxc-backend.md @@ -341,14 +341,21 @@ The zone query should answer the zone you assigned. Without the drop a workload holding `CAP_NET_RAW` could put frames on the wire beneath the chains, which filter at the IP layer. The drop also ends raw ICMP, and MXC reopens unprivileged ping by widening `net.ipv4.ping_group_range` inside - the container; a `SOCK_DGRAM` `IPPROTO_ICMP` echo still works. Default-deny + the container. The workload can then send ICMP Echo requests through a + `SOCK_DGRAM` `IPPROTO_ICMP` socket, and the kernel refuses every other ICMP type + on that socket. A `protocol: "icmp"` policy rule is broader: it matches every + ICMP type at the firewall. The rule decides which ICMP may pass; the workload + can only originate Echo. Default-deny closes external reachability for a container that does not deliberately tear it down, including services the workload itself starts. - **The drop covers the command MXC runs, not container init.** Init keeps `CAP_NET_RAW` just as it keeps `CAP_NET_ADMIN`. A privileged process inside the container can still open a packet socket. The kernel checks the capability at creation, never at transmit, and an already-open descriptor handed to the - workload keeps sending. Nothing in a stock container image opens one; closing the + workload keeps sending. An io_uring ring handed down the same way is the same + case: an operation submitted under a personality init registered runs with + init's credentials. A ring the workload creates itself carries the workload's + credentials. Nothing in a stock container image opens either; closing the gap entirely needs a gate below the socket layer, which this backend does not add today. - **Policy is not in force while the container starts.** The chains are installed diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs index ab6d9d9ea..cb04ee555 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_bindings.rs @@ -547,7 +547,7 @@ impl LxcContainer { env: &[String], force_clear_env: bool, timeout: Option, - _firewall: ContainerFirewall, + firewall: ContainerFirewall, ) -> Result<(i32, String, String), String> { use crate::mxc_pty::{run_with_pty, PtyOptions, PtyOutcome, Signal}; @@ -563,7 +563,9 @@ impl LxcContainer { force_clear_env, )); - confine_network_capabilities(&mut cmd); + if firewall == ContainerFirewall::Installed { + confine_network_capabilities(&mut cmd); + } let options = PtyOptions { unblock_signals: UNBLOCK, @@ -607,7 +609,7 @@ impl LxcContainer { env: &[String], force_clear_env: bool, timeout: Option, - _firewall: ContainerFirewall, + firewall: ContainerFirewall, ) -> Result<(i32, String, String), String> { use crate::mxc_common::interruptible_reader::wrap_pipe; use crate::mxc_common::sandbox_process::{wait_with_timeout, StreamCloser, WaitError}; @@ -626,7 +628,9 @@ impl LxcContainer { unblock_fatal_signals(&mut cmd); - confine_network_capabilities(&mut cmd); + if firewall == ContainerFirewall::Installed { + confine_network_capabilities(&mut cmd); + } let mut child = cmd .spawn() @@ -790,7 +794,7 @@ impl LxcContainer { working_directory: &str, env: &[String], force_clear_env: bool, - _firewall: ContainerFirewall, + firewall: ContainerFirewall, ) -> std::process::Command { let mut cmd = self.lxc_command("lxc-attach"); cmd.args(build_attach_args_with_env_control( @@ -799,7 +803,9 @@ impl LxcContainer { command, force_clear_env, )); - confine_network_capabilities(&mut cmd); + if firewall == ContainerFirewall::Installed { + confine_network_capabilities(&mut cmd); + } cmd } diff --git a/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs b/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs index 7f845784a..ec3cefd4a 100644 --- a/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs +++ b/src/mxc-sdk/src/backends/lxc/common/lxc_runner.rs @@ -399,31 +399,6 @@ impl LxcScriptRunner { ))) } - fn enforce_netns_discovery( - &self, - netns_pid: Option, - installs_firewall: bool, - container_created: bool, - logger: &mut Logger, - mut release_container: R, - ) -> Result - where - R: FnMut(ContainerRelease) -> Result<(), String>, - { - if let Some(hook_point) = egress_hook_point(netns_pid, installs_firewall) { - return Ok(hook_point); - } - - let release = self.release_kind(container_created); - Self::report_release_failure(release, release_container(release), logger); - - Err(ScriptResponse::error( - "Failed to discover the container init PID; cannot enter the container \ - network namespace to enforce the requested firewall. Aborting rather \ - than running with enforcement silently disabled.", - )) - } - /// Everything that must succeed before a workload can launch; a failure /// releases whatever it had already acquired. fn prepare( @@ -582,16 +557,14 @@ impl LxcScriptRunner { // The init PID names the container's network namespace. let netns_pid = container.init_pid(); - let hook_point = self.enforce_netns_discovery( - netns_pid, - plan.installs_firewall(), - container_created, - logger, - |release| match release { - ContainerRelease::Destroy => container.destroy(), - ContainerRelease::Stop => container.stop(), - }, - )?; + let Some(hook_point) = egress_hook_point(netns_pid, plan.installs_firewall()) else { + self.release_after_failure(&container, container_created, logger); + return Err(ScriptResponse::error( + "Failed to discover the container init PID; cannot enter the container \ + network namespace to install its firewall. Aborting rather than \ + running without it.", + )); + }; if let Some(pid) = netns_pid { let _ = writeln!(logger, "Container init PID: {}", pid); @@ -629,9 +602,11 @@ impl LxcScriptRunner { // Dropping CAP_NET_RAW also removed raw ICMP. Widen the container's // ping_group_range so unprivileged ping still works. - if let Err(e) = fw_manager.allow_ping_sockets(logger) { - self.release_after_failure(&container, container_created, logger); - return Err(ScriptResponse::error(&e)); + if is_firewall_installed == ContainerFirewall::Installed { + if let Err(e) = fw_manager.allow_ping_sockets(logger) { + self.release_after_failure(&container, container_created, logger); + return Err(ScriptResponse::error(&e)); + } } // A reused container can carry a pin left by an older run. @@ -1540,6 +1515,8 @@ fn uuid_simple() -> String { format!("{:08x}", (t & 0xFFFF_FFFF) as u32) } +/// Where egress rules attach, or `None` when a firewall is needed and the +/// container's network namespace cannot be found. fn egress_hook_point(netns_pid: Option, installs_firewall: bool) -> Option { match netns_pid { Some(pid) => Some(EgressHookPoint::ContainerNetns(pid)), @@ -2092,43 +2069,12 @@ mod tests { } #[test] - fn a_known_netns_hooks_the_chain_inside_the_container() { - assert!(matches!( - egress_hook_point(Some(4242), true), - Some(EgressHookPoint::ContainerNetns(4242)) - )); - } - - #[test] - fn an_unknown_netns_refuses_to_run_when_a_firewall_was_requested() { - let mut logger = Logger::new(Mode::Buffer); - - let response = runner_for_network_readiness_tests(true) - .enforce_netns_discovery(None, true, true, &mut logger, |_release| Ok(())) - .expect_err("a firewall was requested and the namespace is unknown"); - - assert_ne!(response.exit_code, 0); - assert!(response.standard_out.is_empty()); - } - - #[test] - fn a_known_netns_lets_the_run_proceed() { - let mut logger = Logger::new(Mode::Buffer); - - assert!(matches!( - runner_for_network_readiness_tests(true).enforce_netns_discovery( - Some(4242), - true, - true, - &mut logger, - |_release| Ok(()) - ), - Ok(EgressHookPoint::ContainerNetns(4242)) - )); + fn a_missing_init_pid_stops_a_run_that_needs_a_firewall() { + assert!(egress_hook_point(None, true).is_none()); } #[test] - fn an_unknown_netns_is_allowed_when_no_firewall_was_requested() { + fn a_missing_init_pid_does_not_stop_a_run_with_no_firewall() { assert!(matches!( egress_hook_point(None, false), Some(EgressHookPoint::Unhooked) diff --git a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs index 0c065ee47..bed5872b3 100644 --- a/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs +++ b/src/mxc-sdk/tests/wxc_e2e_tests_e2e_lxc_network_capability.rs @@ -9,6 +9,8 @@ use serde_json::json; use wxc_e2e_tests::{has_lxc_host, has_platform_exec, run_platform_config_value}; const CAP_NET_ADMIN: u64 = 1 << 12; +const CAP_NET_RAW: u64 = 1 << 13; +const NETWORK_CAPABILITIES: u64 = CAP_NET_ADMIN | CAP_NET_RAW; /// Whether the LXC capability prerequisites are present. fn ready() -> bool { @@ -26,7 +28,7 @@ fn capability_mask(status: &str, field: &str) -> u64 { } #[test] -fn workload_cannot_reconfigure_the_network() { +fn workload_holds_neither_network_capability() { if !ready() { return; } @@ -37,30 +39,64 @@ fn workload_cannot_reconfigure_the_network() { "containment": "lxc", "process": { "commandLine": "sh -c \"cat /proc/self/status\"" }, "lifecycle": { "destroyOnExit": true }, - "lxc": { "distribution": "alpine", "release": "3.23" } + "lxc": { "distribution": "alpine", "release": "3.23" }, + + // An allow rule puts the workload behind the firewall. + "network": { + "egress": { + "default": "deny", + "allow": [{ "to": [{ "cidr": "198.51.100.2/32" }] }] + } + } }); let result = run_platform_config_value("lxc network capability", &config, &[], None); let status = result.combined_output(); + let held = capability_mask(&status, "CapEff:") | capability_mask(&status, "CapPrm:"); + assert_eq!( - result.code, - Some(0), - "lxc-exec did not run the container\n--- stderr ---\n{}", - result.stderr + held & NETWORK_CAPABILITIES, + 0, + "the workload still holds CAP_NET_ADMIN or CAP_NET_RAW\n{status}" ); +} - // The kernel writes these masks; the workload cannot forge them. Effective - // alone would not settle it: a process raises a permitted capability into - // its effective set whenever it likes, and only a bounding-set drop - // survives execve. - for field in ["CapEff:", "CapPrm:", "CapBnd:"] { - assert_eq!( - capability_mask(&status, field) & CAP_NET_ADMIN, - 0, - "{field} still carries CAP_NET_ADMIN; the workload could reconfigure the network it asked to go without\n{status}" - ); +#[test] +fn workload_cannot_regain_a_network_capability() { + if !ready() { + return; } + + let config = json!({ + "version": "0.9.0-alpha", + "containerId": "lxc-capability-regain", + "containment": "lxc", + "process": { "commandLine": "sh -c \"cat /proc/self/status\"" }, + "lifecycle": { "destroyOnExit": true }, + "lxc": { "distribution": "alpine", "release": "3.23" }, + "network": { + "egress": { + "default": "deny", + "allow": [{ "to": [{ "cidr": "198.51.100.2/32" }] }] + } + } + }); + + let result = run_platform_config_value("lxc capability regain", &config, &[], None); + let status = result.combined_output(); + + // An exec can only grant a capability that is in one of these sets. + // https://man7.org/linux/man-pages/man7/capabilities.7.html + let regainable = capability_mask(&status, "CapBnd:") + | capability_mask(&status, "CapInh:") + | capability_mask(&status, "CapAmb:"); + + assert_eq!( + regainable & NETWORK_CAPABILITIES, + 0, + "an exec could hand the workload CAP_NET_ADMIN or CAP_NET_RAW back\n{status}" + ); } #[test] @@ -97,7 +133,7 @@ fn workload_cannot_open_a_raw_or_packet_socket() { // Bash does not have the ability to make RAW and PACKET sockets. Use perl instead. const RAW_AND_PACKET_PROBE: &str = r#"perl -e 'use Socket; my $raw = socket(RAW, PF_INET, SOCK_RAW, 1) ? "RAW_OPENED" : "RAW_REFUSED: $!"; my $pkt = socket(PKT, 17, SOCK_RAW, 0) ? "PACKET_OPENED" : "PACKET_REFUSED: $!"; print "$raw\n$pkt\n";'"#; - // Use Ubuntu for pearl. + // Use Ubuntu for perl. let config = json!({ "version": "0.9.0-alpha", "containerId": "lxc-raw-socket-refusal", diff --git a/src/tools/lxc/src/linux_executor_arguments.rs b/src/tools/lxc/src/linux_executor_arguments.rs index d4b506868..bbe5fcdc1 100644 --- a/src/tools/lxc/src/linux_executor_arguments.rs +++ b/src/tools/lxc/src/linux_executor_arguments.rs @@ -3,7 +3,7 @@ use clap::Parser; use mxc_sdk::mxc_common::config_parser::load_one_shot_request; -use mxc_sdk::mxc_common::logger::{Logger, Mode}; +use mxc_sdk::mxc_common::logger::Logger; use mxc_sdk::mxc_common::models::ExecutionRequest; use std::process; @@ -83,7 +83,7 @@ impl LinuxExecutorArguments { self.debug } - pub fn should_display_avalible_backends(&self) -> bool { + pub fn should_display_available_backends(&self) -> bool { self.available_backends } @@ -91,7 +91,7 @@ impl LinuxExecutorArguments { self.dry_run } - pub fn get_request(&self) -> ExecutionRequest { + pub fn get_request(&self, logger: &mut Logger) -> ExecutionRequest { let (config_data, is_base64) = if let Some(ref b64) = self.config_base64 { (b64.clone(), true) } else if let Some(ref path) = self.config { @@ -105,16 +105,12 @@ impl LinuxExecutorArguments { process::exit(1); }; - let mut logger = Logger::new(if self.debug { - Mode::Console - } else { - Mode::Buffer - }); - - let mut request = match load_one_shot_request(&config_data, &mut logger, is_base64) { + let mut request = match load_one_shot_request(&config_data, logger, is_base64) { Ok(r) => r, Err(_) => { - eprint!("Request error\n{}", logger.get_buffer()); + eprintln!("Request error"); + crate::emit_warnings(logger); + eprint!("{}", logger.get_buffer()); process::exit(1); } }; @@ -187,27 +183,37 @@ impl LinuxExecutorArguments { self.delete } - pub fn delete_container(&self) -> Result { + pub fn delete_container(&self, logger: &mut Logger) -> bool { let name = match self.containername { Some(ref n) => n, - None => return Err("Error: --containername is required with --delete".to_string()), + None => { + eprintln!("Error: --containername is required with --delete"); + process::exit(1); + } }; - Self::delete_lxc_container(name) + Self::delete_lxc_container(name, logger) } - fn delete_lxc_container(name: &str) -> Result { + fn delete_lxc_container(name: &str, logger: &mut Logger) -> bool { use mxc_sdk::lxc_common::lxc_bindings::LxcContainer; let container = LxcContainer::new(name, None); if !container.is_defined() { - return Err(format!("Container '{}' does not exist.", name)); + logger.log_line(&format!("Container '{}' does not exist.", name)); + return false; } match container.destroy() { - Ok(()) => Ok(format!("Deleted LXC container: {}", name)), - Err(e) => Err(format!("Failed to delete LXC container '{}': {}", name, e)), + Ok(()) => { + logger.log_line(&format!("Deleted LXC container: {}", name)); + true + } + Err(e) => { + logger.log_line(&format!("Failed to delete LXC container '{}': {}", name, e)); + false + } } } } diff --git a/src/tools/lxc/src/main.rs b/src/tools/lxc/src/main.rs index e2e373b2f..1b482ad86 100644 --- a/src/tools/lxc/src/main.rs +++ b/src/tools/lxc/src/main.rs @@ -33,7 +33,7 @@ fn main() { } // Go through the different operating modes - if arguments.should_display_avalible_backends() { + if arguments.should_display_available_backends() { match mxc_sdk::mxc_engine::to_json_pretty(&mxc_sdk::mxc_engine::available_backends()) { Ok(json) => { println!("{json}"); @@ -56,21 +56,14 @@ fn main() { } } } else if arguments.does_user_want_to_delete_a_container() { - match arguments.delete_container() { - Ok(message) => { - eprintln!("{message}"); - process::exit(0); - } - Err(message) => { - eprintln!("{message}"); - process::exit(1); - } - } + let success = arguments.delete_container(&mut logger); + print!("{}", logger.get_buffer()); + process::exit(if success { 0 } else { 1 }); } // User wants to run a request. // Make the request. - let request = arguments.get_request(); + let request = arguments.get_request(&mut logger); let telemetry_active = request .telemetry diff --git a/src/tools/lxc/tests/cli.rs b/src/tools/lxc/tests/cli.rs new file mode 100644 index 000000000..05c45571f --- /dev/null +++ b/src/tools/lxc/tests/cli.rs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::fs; +use std::path::Path; +use std::process::Command; + +#[test] +fn writes_config_parse_error_to_log_file() { + let directory = Path::new(env!("CARGO_TARGET_TMPDIR")); + let config_path = directory.join("unparseable_config.json"); + let log_path = directory.join("unparseable_config.log"); + fs::write(&config_path, "{ not json").expect("write config"); + fs::write(&log_path, "").expect("clear log left by an earlier run"); + + let output = Command::new(env!("CARGO_BIN_EXE_lxc-exec")) + .arg("--log-file") + .arg(&log_path) + .arg(&config_path) + .output() + .expect("run lxc-exec"); + + let log = fs::read_to_string(&log_path).expect("read log file"); + assert!( + log.contains("Configuration parse error"), + "lxc-exec did not write the parse error to --log-file\nlog:\n{log}\nstderr:\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/tests/configs/lxc_rootless_no_network.json b/tests/configs/lxc_rootless_no_network.json new file mode 100644 index 000000000..b416c3f64 --- /dev/null +++ b/tests/configs/lxc_rootless_no_network.json @@ -0,0 +1,16 @@ +{ + "$schema": "../../schemas/stable/mxc-config.schema.1.0.0.json", + "version": "1.0.0", + "containerId": "CLI-LXC-Rootless-No-Network", + "containment": "lxc", + "process": { + "commandLine": "echo MXC_ROOTLESS_OK" + }, + "lifecycle": { + "destroyOnExit": true + }, + "lxc": { + "distribution": "alpine", + "release": "3.23" + } +} diff --git a/tests/scripts/run_lxc_all_tests.sh b/tests/scripts/run_lxc_all_tests.sh index 4b844f22c..93eaf5a14 100644 --- a/tests/scripts/run_lxc_all_tests.sh +++ b/tests/scripts/run_lxc_all_tests.sh @@ -104,6 +104,7 @@ run_test "LXC Inbound Traffic" "$SCRIPT_DIR/run_lxc_inbound_traffic_test.sh" run_test "LXC Timeout" "$SCRIPT_DIR/run_lxc_timeout_test.sh" run_test "LXC Env+Cwd" "$SCRIPT_DIR/run_lxc_env_cwd_test.sh" run_test "LXC Environment" "$SCRIPT_DIR/run_lxc_env_test.sh" +run_test "LXC Log File" "$SCRIPT_DIR/run_lxc_log_file_test.sh" echo "================================" echo "Results: $PASSED passed, $FAILED failed, $SKIPPED skipped, $DISABLED disabled" diff --git a/tests/scripts/run_lxc_log_file_test.sh b/tests/scripts/run_lxc_log_file_test.sh new file mode 100644 index 000000000..684a60f5a --- /dev/null +++ b/tests/scripts/run_lxc_log_file_test.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# LXC --log-file test +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" +LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" + +if [ ! -f "$LXC_EXEC" ]; then + LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" +fi + +if [ ! -f "$LXC_EXEC" ]; then + echo "Error: lxc-exec not found. Run build.sh first." + exit 1 +fi + +LOG_FILE=$(mktemp) +trap 'rm -f "$LOG_FILE"' EXIT + +echo "Running LXC log file test..." +"$LXC_EXEC" --log-file "$LOG_FILE" "$REPO_DIR/tests/configs/basic_lxc.json" + +if ! grep -q "Executing script inside container" "$LOG_FILE"; then + echo "FAIL: the container run was not written to the log file." + cat "$LOG_FILE" + exit 1 +fi + +echo "LXC log file test complete." diff --git a/tests/scripts/run_lxc_rootless_no_network_test.sh b/tests/scripts/run_lxc_rootless_no_network_test.sh new file mode 100644 index 000000000..4dbc4fd82 --- /dev/null +++ b/tests/scripts/run_lxc_rootless_no_network_test.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# LXC rootless test: a config with no network section runs without root +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" +LXC_EXEC="$REPO_DIR/src/target/release/lxc-exec" + +if [ ! -f "$LXC_EXEC" ]; then + LXC_EXEC="$REPO_DIR/src/target/debug/lxc-exec" +fi + +if [ ! -f "$LXC_EXEC" ]; then + echo "Error: lxc-exec not found. Run build.sh first." + exit 1 +fi + +if [ -z "${SUDO_USER:-}" ]; then + echo "Error: run this under sudo, so the account that invoked sudo can run lxc-exec without root." + exit 1 +fi + +echo "Running LXC rootless no-network test as $SUDO_USER..." +OUTPUT=$(sudo -u "$SUDO_USER" -H "$LXC_EXEC" "$REPO_DIR/tests/configs/lxc_rootless_no_network.json" 2>&1) || { + echo "FAIL: lxc-exec without root could not run a config that has no network section." + echo "$OUTPUT" + exit 1 +} + +if ! grep -q "MXC_ROOTLESS_OK" <<< "$OUTPUT"; then + echo "FAIL: the container ran without root but the command printed nothing." + echo "$OUTPUT" + exit 1 +fi + +echo "LXC rootless no-network test complete." \ No newline at end of file From 13f08b52a9eb88a10bbca0773bfb05e9f2837d5b Mon Sep 17 00:00:00 2001 From: Darren Hoehna Date: Fri, 16 Oct 2026 15:56:31 -0700 Subject: [PATCH 17/17] Adding a test for un-privlaged LXC container scenario --- .github/workflows/lxc-e2e.yml | 6 ++++ docs/backends/lxc/lxc-backend.md | 33 ++++++++++++++++++- tests/scripts/run_lxc_all_tests.sh | 4 +++ .../run_lxc_rootless_no_network_test.sh | 26 ++++++++++++--- 4 files changed, 64 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lxc-e2e.yml b/.github/workflows/lxc-e2e.yml index 9fe298c21..2641cf284 100644 --- a/.github/workflows/lxc-e2e.yml +++ b/.github/workflows/lxc-e2e.yml @@ -105,6 +105,12 @@ jobs: MXC_LXC_TESTS_REQUIRE_EXECUTION: "1" run: sudo --preserve-env=MXC_LXC_TESTS_REQUIRE_EXECUTION bash tests/scripts/run_lxc_all_tests.sh + # Run a seperate test for non-privlaged LXC container test. + # LXC containers can run rootless, but with setup. + # To prevent tests runing state this test only runs during CI. + - name: Run LXC rootless no-network test + run: sudo bash tests/scripts/run_lxc_rootless_no_network_test.sh + # The shell suite above does not cover the capability confinement, and the # Linux build lane runs this crate without LXC installed, so these tests # skip everywhere else. diff --git a/docs/backends/lxc/lxc-backend.md b/docs/backends/lxc/lxc-backend.md index a85c09448..d2c8f5775 100644 --- a/docs/backends/lxc/lxc-backend.md +++ b/docs/backends/lxc/lxc-backend.md @@ -24,7 +24,8 @@ Creates an LXC container to provide: - Linux kernel >= 2.6.32, or >= 3.12 to run unprivileged - LXC >= 5.0 installed (`liblxc-dev` for building, `lxc-utils` for runtime) -- Root privileges, or unprivileged LXC for a policy that asks for no network. +- Root privileges, or [unprivileged LXC](#running-without-root) for a policy + that asks for no network. Filtering egress or ingress installs iptables rules in the container's network namespace, which requires root. @@ -48,6 +49,36 @@ process. sudo pacman -S lxc ``` +### Running without root + +A config with no `network` section can run without sudo once the host is set up +for unprivileged LXC. Everything else needs root. + +1. Install `newuidmap` and `newgidmap` (`sudo apt install uidmap` on + Debian/Ubuntu). +2. Make sure your account has a subordinate ID range. On Ubuntu 24.04, + `useradd` creates one. To check: + ```bash + grep "^$USER:" /etc/subuid /etc/subgid + ``` + If nothing prints, an admin adds a range: + ```bash + sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 "$USER" + ``` +3. Map the container's IDs onto that range in `~/.config/lxc/default.conf`, + using the start and count from step 2: + ``` + lxc.include = /etc/lxc/default.conf + lxc.idmap = u 0 100000 65536 + lxc.idmap = g 0 100000 65536 + ``` +4. Let the container's root user reach its files under your home directory: + `chmod o+x ~`. + +`tests/scripts/run_lxc_rootless_no_network_test.sh` checks this with a temporary +user. CI runs it, and `run_lxc_all_tests.sh` leaves it out because it adds that +user to the host. Run it by hand with `sudo` when you need it locally. + ### Container networking A policy that permits any network needs `lxcbr0` to hand the container an IPv4 diff --git a/tests/scripts/run_lxc_all_tests.sh b/tests/scripts/run_lxc_all_tests.sh index 93eaf5a14..a63c28ee3 100644 --- a/tests/scripts/run_lxc_all_tests.sh +++ b/tests/scripts/run_lxc_all_tests.sh @@ -77,6 +77,10 @@ disabled_test() { echo "" } +echo "NOTE: LXC Rootless No-Network is not run here, because it adds a temporary user account to this machine." +echo " Run it manually with: sudo bash $SCRIPT_DIR/run_lxc_rootless_no_network_test.sh" +echo "" + run_test "Basic LXC" "$SCRIPT_DIR/run_lxc_basic_test.sh" run_test "LXC Filesystem" "$SCRIPT_DIR/run_lxc_filesystem_test.sh" run_test "LXC Object Validation" "$SCRIPT_DIR/run_lxc_object_test.sh" diff --git a/tests/scripts/run_lxc_rootless_no_network_test.sh b/tests/scripts/run_lxc_rootless_no_network_test.sh index 4dbc4fd82..6bd84a03c 100644 --- a/tests/scripts/run_lxc_rootless_no_network_test.sh +++ b/tests/scripts/run_lxc_rootless_no_network_test.sh @@ -15,13 +15,31 @@ if [ ! -f "$LXC_EXEC" ]; then exit 1 fi -if [ -z "${SUDO_USER:-}" ]; then - echo "Error: run this under sudo, so the account that invoked sudo can run lxc-exec without root." +if [ "$(id -u)" -ne 0 ]; then + echo "Error: run this under sudo, so it can create the unprivileged user that runs lxc-exec." exit 1 fi -echo "Running LXC rootless no-network test as $SUDO_USER..." -OUTPUT=$(sudo -u "$SUDO_USER" -H "$LXC_EXEC" "$REPO_DIR/tests/configs/lxc_rootless_no_network.json" 2>&1) || { +RUN_USER=mxc-rootless-e2e +useradd -m "$RUN_USER" +trap 'userdel -r "$RUN_USER" 2>/dev/null' EXIT + +RUN_HOME="/home/$RUN_USER" +SUBUID_START=$(grep "^$RUN_USER:" /etc/subuid | cut -d: -f2) +SUBGID_START=$(grep "^$RUN_USER:" /etc/subgid | cut -d: -f2) + +# The container's mapped root must be able to walk into the user's home to reach its rootfs. +chmod 755 "$RUN_HOME" +mkdir -p "$RUN_HOME/.config/lxc" +cat > "$RUN_HOME/.config/lxc/default.conf" <&1) || { echo "FAIL: lxc-exec without root could not run a config that has no network section." echo "$OUTPUT" exit 1