From f856f52a1e054c1838e132230827455c95bb1da8 Mon Sep 17 00:00:00 2001 From: Leon Krix Date: Thu, 12 Nov 2026 10:20:14 +0100 Subject: [PATCH 1/2] feat: make links, addresses, routes and kernel settings by netlink instead of ip --- Cargo.lock | 33 + Cargo.toml | 5 + crates/netscenario-backend-kernel/Cargo.toml | 6 +- crates/netscenario-backend-kernel/src/lib.rs | 1 + .../netscenario-backend-kernel/src/netlink.rs | 1072 +++++++++++++++++ .../src/provisioner.rs | 349 ++---- .../src/supervisor.rs | 102 +- crates/netscenario-cli/src/main.rs | 5 +- docs/decisions/D-098.md | 2 +- 9 files changed, 1233 insertions(+), 342 deletions(-) create mode 100644 crates/netscenario-backend-kernel/src/netlink.rs diff --git a/Cargo.lock b/Cargo.lock index 25dab82..7611133 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -543,6 +543,36 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" +[[package]] +name = "netlink-packet-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6e2daf9a8c2e21302714706860a0e6be02e03d17294ecc66b354ea7d4059dd" + +[[package]] +name = "netlink-packet-route" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3c407d8e95c0128ee5729728482a369f3ba9a0d9ba8335c20949a8d8989cf6b" +dependencies = [ + "bitflags", + "libc", + "log", + "netlink-packet-core", + "zerocopy", +] + +[[package]] +name = "netlink-sys" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c99d38e00b420df49e940fe0826e667c3dad82ac68eb4c2bbf754c1c14a83a10" +dependencies = [ + "bytes", + "libc", + "log", +] + [[package]] name = "netscenario-agent" version = "0.3.0" @@ -557,6 +587,9 @@ dependencies = [ name = "netscenario-backend-kernel" version = "0.3.0" dependencies = [ + "netlink-packet-core", + "netlink-packet-route", + "netlink-sys", "netscenario-agent", "netscenario-core", "netscenario-model", diff --git a/Cargo.toml b/Cargo.toml index 2cf217e..63fb37b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,11 @@ serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["float_roundtrip"] } # The agent reads the clock and sets socket buffers through this safe interface, so no crate of ours needs `unsafe` rustix = { version = "1", default-features = false, features = ["std", "time", "net"] } +# Netlink for the Kernel backend (D-101): the messages are built by `netlink-packet-route` (no `unsafe`), the socket is `netlink-sys`. +# What the first lacks (`netem`, `htb`, `tbf`, `codel`, `pie`, the fifos) the backend encodes itself +netlink-packet-core = "0.9" +netlink-packet-route = "0.34" +netlink-sys = "0.9" # The command line stops a run in an orderly way on Ctrl-C (D-066). The handler of the signal is in this crate, so no crate of # ours needs `unsafe` signal-hook = "0.4" diff --git a/crates/netscenario-backend-kernel/Cargo.toml b/crates/netscenario-backend-kernel/Cargo.toml index 7d88243..a2f7ca6 100644 --- a/crates/netscenario-backend-kernel/Cargo.toml +++ b/crates/netscenario-backend-kernel/Cargo.toml @@ -16,7 +16,11 @@ netscenario-model.workspace = true serde.workspace = true serde_json.workspace = true # The signal for the process group of a program: a safe call, no `unsafe` (D-025) -rustix = { workspace = true, features = ["process"] } +rustix = { workspace = true, features = ["process", "thread"] } +# Netlink instead of the tools `ip` and `tc` (D-101) +netlink-packet-core.workspace = true +netlink-packet-route.workspace = true +netlink-sys.workspace = true # Handlers for the signals a program may be sent, so that they are not ignored in the programs (below) signal-hook.workspace = true diff --git a/crates/netscenario-backend-kernel/src/lib.rs b/crates/netscenario-backend-kernel/src/lib.rs index feacfe3..75cce50 100644 --- a/crates/netscenario-backend-kernel/src/lib.rs +++ b/crates/netscenario-backend-kernel/src/lib.rs @@ -14,6 +14,7 @@ mod command; pub mod doctor; mod fabric; mod netem; +mod netlink; mod programs; mod protocol; mod provisioner; diff --git a/crates/netscenario-backend-kernel/src/netlink.rs b/crates/netscenario-backend-kernel/src/netlink.rs new file mode 100644 index 0000000..8ee839e --- /dev/null +++ b/crates/netscenario-backend-kernel/src/netlink.rs @@ -0,0 +1,1072 @@ +//! Netlink instead of the tools `ip` and `tc` (BKD-05, D-101): links, addresses, neighbours and routes in the network namespace +//! of a node, and the kernel settings of that namespace. +//! +//! The messages are built by `netlink-packet-route`, which has no `unsafe`; the socket is `netlink-sys`. A node is a network +//! namespace held by a process, so the socket of a node is made by a thread that has entered that namespace with `setns` +//! (the call changes the namespace of the calling thread only). A socket stays in the namespace it was made in, so the thread +//! can end at once and the socket is used from any thread afterwards. + +// The structures of `netlink-packet-route` are `non_exhaustive`: outside that crate they can only be made by `default()` and then +// filled in, so the lint that asks for a struct expression cannot be followed +#![allow(clippy::field_reassign_with_default)] + +use std::fs; +use std::net::IpAddr; + +use netlink_packet_core::{ + NLM_F_ACK, NLM_F_CREATE, NLM_F_DUMP, NLM_F_EXCL, NLM_F_REPLACE, NLM_F_REQUEST, NetlinkHeader, + NetlinkMessage, NetlinkPayload, +}; +use netlink_packet_route::address::{AddressAttribute, AddressFlags, AddressMessage}; +use netlink_packet_route::link::{ + InfoData, InfoKind, InfoVeth, LinkAttribute, LinkFlags, LinkInfo, LinkMessage, +}; +use netlink_packet_route::neighbour::{ + NeighbourAddress, NeighbourAttribute, NeighbourMessage, NeighbourState, +}; +use netlink_packet_route::route::{ + RouteAddress, RouteAttribute, RouteHeader, RouteMessage, RouteNextHop, RouteProtocol, + RouteScope, RouteType, +}; +use netlink_packet_route::{AddressFamily, RouteNetlinkMessage}; +use netlink_sys::protocols::NETLINK_ROUTE; +use netlink_sys::{Socket, SocketAddr}; +use rustix::thread::{LinkNameSpaceType, move_into_link_name_space}; + +/// The table `main`, where `ip route` puts a route that does not say otherwise. +const TABLE_MAIN: u8 = 254; + +/// The size of the buffer for one reply. A message about one link is about 2 KB. +const REPLY_BUFFER: usize = 32 * 1024; + +/// A way out of a route: the address of the next hop, the interface it is reached by and the weight among several ways. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Hop { + /// The address of the other end of the link. + pub(crate) gateway: IpAddr, + /// The name of the interface the route leaves by. + pub(crate) interface: String, + /// The share of this way in a route with several (`ip route ... nexthop ... weight`). + pub(crate) weight: u32, +} + +/// A route of the main table (D-078): to one address or every address, with a metric, a source address and one or several ways. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RouteSpec { + /// The address the route leads to, or `None` for the default route. + pub(crate) destination: Option, + /// The metric (`ip route ... metric`). + pub(crate) metric: u32, + /// The source address of the packets that the node sends over the route. + pub(crate) source: Option, + /// The ways out. One is a plain route, several a multipath route. + pub(crate) hops: Vec, + /// The family of the route when it has no destination. + pub(crate) v6: bool, +} + +/// How a message treats an object that is there already. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Verb { + /// Make a new one; it is an error if it exists (`ip ... add`). + Add, + /// Make a new one or replace the one that is there (`ip ... replace`). + Replace, +} + +impl Verb { + fn flags(self) -> u16 { + match self { + Self::Add => NLM_F_CREATE | NLM_F_EXCL, + Self::Replace => NLM_F_CREATE | NLM_F_REPLACE, + } + } +} + +/// A netlink socket in one network namespace. +pub(crate) struct Netlink { + socket: Socket, + sequence: u32, +} + +impl std::fmt::Debug for Netlink { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Netlink") + .field("sequence", &self.sequence) + .finish_non_exhaustive() + } +} + +impl Netlink { + /// A socket in the network namespace of the calling thread. + pub(crate) fn here() -> Result { + let mut socket = Socket::new(NETLINK_ROUTE) + .map_err(|error| format!("cannot open a netlink socket: {error}"))?; + socket + .bind_auto() + .map_err(|error| format!("cannot bind the netlink socket: {error}"))?; + socket + .connect(&SocketAddr::new(0, 0)) + .map_err(|error| format!("cannot connect the netlink socket to the kernel: {error}"))?; + Ok(Self { + socket, + sequence: 1, + }) + } + + /// A socket in the network namespace that process `pid` holds. A thread enters the namespace, makes the socket and ends; + /// the process itself stays where it is. + pub(crate) fn in_namespace(pid: u32) -> Result { + let path = format!("/proc/{pid}/ns/net"); + let namespace = fs::File::open(&path) + .map_err(|error| format!("cannot open the network namespace {path}: {error}"))?; + std::thread::scope(|scope| { + scope + .spawn(|| { + move_into_link_name_space( + std::os::fd::AsFd::as_fd(&namespace), + Some(LinkNameSpaceType::Network), + ) + .map_err(|error| { + format!("cannot enter the network namespace {path}: {error}") + })?; + Self::here() + }) + .join() + .unwrap_or_else(|_| Err("the thread that enters a namespace panicked".to_owned())) + }) + } + + /// Sends one request and reads the answer: the acknowledgement of the kernel, or its error. + fn request(&mut self, payload: RouteNetlinkMessage, flags: u16) -> Result<(), String> { + let sequence = self.send(payload, NLM_F_ACK | flags)?; + match self.receive(sequence)?.payload { + NetlinkPayload::Error(error) => match error.code { + None => Ok(()), + Some(code) => Err(os_error(-code.get())), + }, + other => Err(format!("the kernel answered with {other:?}")), + } + } + + fn send(&mut self, payload: RouteNetlinkMessage, flags: u16) -> Result { + let sequence = self.sequence; + self.sequence = self.sequence.wrapping_add(1); + let mut header = NetlinkHeader::default(); + header.flags = NLM_F_REQUEST | flags; + header.sequence_number = sequence; + let mut message = NetlinkMessage::new(header, NetlinkPayload::from(payload)); + message.finalize(); + let mut buffer = vec![0_u8; message.buffer_len()]; + message.serialize(&mut buffer); + self.socket + .send(&buffer, 0) + .map_err(|error| format!("cannot send to the kernel: {error}"))?; + Ok(sequence) + } + + fn receive(&mut self, sequence: u32) -> Result, String> { + let mut buffer = vec![0_u8; REPLY_BUFFER]; + let mut view: &mut [u8] = &mut buffer; + let size = self + .socket + .recv(&mut view, 0) + .map_err(|error| format!("cannot read from the kernel: {error}"))?; + let message = NetlinkMessage::::deserialize(&buffer[..size]) + .map_err(|error| format!("cannot read the answer of the kernel: {error}"))?; + if message.header.sequence_number != sequence { + return Err(format!( + "the kernel answered request {} to request {sequence}", + message.header.sequence_number + )); + } + Ok(message) + } + + /// The index of the interface with this name, or why there is none. + pub(crate) fn link_index(&mut self, name: &str) -> Result { + let mut link = LinkMessage::default(); + link.attributes.push(LinkAttribute::IfName(name.to_owned())); + let sequence = self.send(RouteNetlinkMessage::GetLink(link), 0)?; + match self.receive(sequence)?.payload { + NetlinkPayload::InnerMessage(RouteNetlinkMessage::NewLink(link)) => { + Ok(link.header.index) + } + NetlinkPayload::Error(error) => Err(match error.code { + Some(code) => format!("the interface {name}: {}", os_error(-code.get())), + None => format!("the kernel did not describe the interface {name}"), + }), + other => Err(format!("the kernel answered with {other:?}")), + } + } + + /// The names of the interfaces of the namespace, in the order of their indexes. + pub(crate) fn link_names(&mut self) -> Result, String> { + let sequence = self.send( + RouteNetlinkMessage::GetLink(LinkMessage::default()), + NLM_F_DUMP, + )?; + let mut names = Vec::new(); + loop { + let mut buffer = vec![0_u8; REPLY_BUFFER]; + let mut view: &mut [u8] = &mut buffer; + let size = self + .socket + .recv(&mut view, 0) + .map_err(|error| format!("cannot read from the kernel: {error}"))?; + let mut offset = 0; + while offset < size { + let message = + NetlinkMessage::::deserialize(&buffer[offset..size]) + .map_err(|error| { + format!("cannot read the answer of the kernel: {error}") + })?; + if message.header.sequence_number != sequence { + return Err("the kernel answered another request".to_owned()); + } + let length = message.header.length as usize; + match message.payload { + NetlinkPayload::Done(_) => return Ok(names), + NetlinkPayload::Error(error) => { + return Err(match error.code { + Some(code) => os_error(-code.get()), + None => "the kernel ended the list of interfaces".to_owned(), + }); + } + NetlinkPayload::InnerMessage(RouteNetlinkMessage::NewLink(link)) => { + names.extend(link.attributes.into_iter().find_map(|attribute| { + match attribute { + LinkAttribute::IfName(name) => Some(name), + _ => None, + } + })); + } + _ => {} + } + if length == 0 { + break; + } + // Messages are aligned to four bytes + offset += length.div_ceil(4) * 4; + } + } + } + + /// Makes a virtual Ethernet pair: `name` with the hardware address `mac` and its peer. + pub(crate) fn add_veth( + &mut self, + name: &str, + mac: Option<&str>, + peer: &str, + peer_mac: Option<&str>, + ) -> Result<(), String> { + let mut peer_message = LinkMessage::default(); + peer_message + .attributes + .push(LinkAttribute::IfName(peer.to_owned())); + if let Some(peer_mac) = peer_mac { + peer_message + .attributes + .push(LinkAttribute::Address(parse_mac(peer_mac)?)); + } + let mut link = LinkMessage::default(); + link.attributes.push(LinkAttribute::IfName(name.to_owned())); + if let Some(mac) = mac { + link.attributes + .push(LinkAttribute::Address(parse_mac(mac)?)); + } + link.attributes.push(LinkAttribute::LinkInfo(vec![ + LinkInfo::Kind(InfoKind::Veth), + LinkInfo::Data(InfoData::Veth(InfoVeth::Peer(peer_message))), + ])); + self.request(RouteNetlinkMessage::NewLink(link), Verb::Add.flags()) + .map_err(|error| format!("cannot make the pair {name} and {peer}: {error}")) + } + + /// Moves the interface `name` into the network namespace that process `pid` holds, and names it `new_name` there (the same + /// name when it is `None`). + pub(crate) fn move_link( + &mut self, + name: &str, + pid: u32, + new_name: Option<&str>, + ) -> Result<(), String> { + let index = self.link_index(name)?; + let mut link = LinkMessage::default(); + link.header.index = index; + link.attributes.push(LinkAttribute::NetNsPid(pid)); + if let Some(new_name) = new_name { + link.attributes + .push(LinkAttribute::IfName(new_name.to_owned())); + } + self.request(RouteNetlinkMessage::SetLink(link), 0) + .map_err(|error| { + format!("cannot move {name} into the namespace of process {pid}: {error}") + }) + } + + /// Brings the interface up or takes it down (`ip link set ... up|down`). + pub(crate) fn set_state(&mut self, name: &str, up: bool) -> Result<(), String> { + let index = self.link_index(name)?; + let mut link = LinkMessage::default(); + link.header.index = index; + link.header.change_mask = LinkFlags::Up; + if up { + link.header.flags = LinkFlags::Up; + } + self.request(RouteNetlinkMessage::SetLink(link), 0) + .map_err(|error| { + format!( + "cannot set {name} {}: {error}", + if up { "up" } else { "down" } + ) + }) + } + + /// Sets the MTU of an interface. + pub(crate) fn set_mtu(&mut self, name: &str, mtu: u32) -> Result<(), String> { + let index = self.link_index(name)?; + let mut link = LinkMessage::default(); + link.header.index = index; + link.attributes.push(LinkAttribute::Mtu(mtu)); + self.request(RouteNetlinkMessage::SetLink(link), 0) + .map_err(|error| format!("cannot set the MTU of {name} to {mtu}: {error}")) + } + + /// Sets the largest segment the stack builds for the interface and how many segments one has (`gso_max_size`, + /// `gso_max_segs`). + pub(crate) fn set_segments( + &mut self, + name: &str, + size: u32, + segments: u32, + ) -> Result<(), String> { + let index = self.link_index(name)?; + let mut link = LinkMessage::default(); + link.header.index = index; + link.attributes = vec![ + LinkAttribute::GsoMaxSize(size), + LinkAttribute::GsoMaxSegs(segments), + ]; + self.request(RouteNetlinkMessage::SetLink(link), 0) + .map_err(|error| format!("cannot limit the segments of {name}: {error}")) + } + + /// Gives an interface an address (`ip addr add|replace ADDRESS/PREFIX dev NAME`). An IPv6 address is there at once, without + /// the duplicate address detection (`nodad`). + pub(crate) fn set_address( + &mut self, + name: &str, + address: IpAddr, + prefix: u8, + verb: Verb, + ) -> Result<(), String> { + let index = self.link_index(name)?; + let mut message = AddressMessage::default(); + message.header.family = family(address); + message.header.prefix_len = prefix; + message.header.index = index; + message.attributes.push(AddressAttribute::Local(address)); + // For a point to point address `Address` is the peer; for the others the kernel wants the same as `Local` + message.attributes.push(AddressAttribute::Address(address)); + if address.is_ipv6() { + message + .attributes + .push(AddressAttribute::Flags(AddressFlags::Nodad)); + } + self.request(RouteNetlinkMessage::NewAddress(message), verb.flags()) + .map_err(|error| format!("cannot give {name} the address {address}/{prefix}: {error}")) + } + + /// Fixes the hardware address of a neighbour on an interface (`ip neigh replace ADDRESS lladdr MAC dev NAME nud permanent`). + pub(crate) fn set_neighbour( + &mut self, + name: &str, + address: IpAddr, + mac: &str, + ) -> Result<(), String> { + let index = self.link_index(name)?; + let mut message = NeighbourMessage::default(); + message.header.family = family(address); + message.header.ifindex = index; + message.header.state = NeighbourState::Permanent; + message.attributes = vec![ + NeighbourAttribute::Destination(match address { + IpAddr::V4(address) => NeighbourAddress::Inet(address), + IpAddr::V6(address) => NeighbourAddress::Inet6(address), + }), + NeighbourAttribute::LinkLayerAddress(parse_mac(mac)?), + ]; + self.request( + RouteNetlinkMessage::NewNeighbour(message), + Verb::Replace.flags(), + ) + .map_err(|error| format!("cannot fix the neighbour {address} on {name}: {error}")) + } + + /// Puts a route in the main table. + pub(crate) fn set_route(&mut self, route: &RouteSpec, verb: Verb) -> Result<(), String> { + if route.hops.is_empty() { + return Err("a route needs a way out".to_owned()); + } + let v6 = route + .destination + .map_or(route.v6, |address| address.is_ipv6()); + let mut message = RouteMessage::default(); + let mut header = RouteHeader::default(); + header.address_family = if v6 { + AddressFamily::Inet6 + } else { + AddressFamily::Inet + }; + header.destination_prefix_length = match route.destination { + None => 0, + Some(IpAddr::V4(_)) => 32, + Some(IpAddr::V6(_)) => 128, + }; + header.table = TABLE_MAIN; + header.protocol = RouteProtocol::Boot; + header.scope = RouteScope::Universe; + header.kind = RouteType::Unicast; + message.header = header; + if let Some(destination) = route.destination { + message + .attributes + .push(RouteAttribute::Destination(route_address(destination))); + } + message + .attributes + .push(RouteAttribute::Priority(route.metric)); + if let Some(source) = route.source { + message + .attributes + .push(RouteAttribute::PrefSource(route_address(source))); + } + if let [hop] = route.hops.as_slice() { + let index = self.link_index(&hop.interface)?; + message + .attributes + .push(RouteAttribute::Gateway(route_address(hop.gateway))); + message.attributes.push(RouteAttribute::Oif(index)); + } else { + let mut hops = Vec::with_capacity(route.hops.len()); + for hop in &route.hops { + let index = self.link_index(&hop.interface)?; + let mut next = RouteNextHop::default(); + // `ip route ... weight W` is sent as W - 1 + next.hops = u8::try_from(hop.weight.saturating_sub(1)).unwrap_or(u8::MAX); + next.interface_index = index; + next.attributes = vec![RouteAttribute::Gateway(route_address(hop.gateway))]; + hops.push(next); + } + message.attributes.push(RouteAttribute::MultiPath(hops)); + } + self.request(RouteNetlinkMessage::NewRoute(message), verb.flags()) + .map_err(|error| { + let to = route.destination.map_or_else( + || "the default route".to_owned(), + |address| address.to_string(), + ); + format!("cannot put the route to {to}: {error}") + }) + } +} + +fn family(address: IpAddr) -> AddressFamily { + match address { + IpAddr::V4(_) => AddressFamily::Inet, + IpAddr::V6(_) => AddressFamily::Inet6, + } +} + +fn route_address(address: IpAddr) -> RouteAddress { + match address { + IpAddr::V4(address) => RouteAddress::Inet(address), + IpAddr::V6(address) => RouteAddress::Inet6(address), + } +} + +/// An address in the text of the tools (`10.100.0.1`, `fd00:100:0::1`). +pub(crate) fn parse_ip(text: &str) -> Result { + text.parse() + .map_err(|_| format!("'{text}' is not an address")) +} + +/// An address with its prefix length (`10.100.0.1/30`). +pub(crate) fn parse_cidr(text: &str) -> Result<(IpAddr, u8), String> { + let (address, prefix) = text + .split_once('/') + .ok_or_else(|| format!("'{text}' is not an address with a prefix length"))?; + let prefix = prefix + .parse() + .map_err(|_| format!("'{text}' is not an address with a prefix length"))?; + Ok((parse_ip(address)?, prefix)) +} + +/// The six bytes of `02:4e:53:00:00:01`. +pub(crate) fn parse_mac(text: &str) -> Result, String> { + let bytes: Result, _> = text + .split(':') + .map(|part| u8::from_str_radix(part, 16)) + .collect(); + match bytes { + Ok(bytes) if bytes.len() == 6 => Ok(bytes), + _ => Err(format!("'{text}' is not a hardware address")), + } +} + +/// What the kernel said, in the words of the system (`File exists`, `Operation not permitted`). +fn os_error(code: i32) -> String { + std::io::Error::from_raw_os_error(code).to_string() +} + +/// Writes kernel settings (`net.ipv4.ip_forward=1`) in the network namespace that process `pid` holds, or in the namespace of the +/// caller when `pid` is `None`. A thread enters the namespace and writes the files of `/proc/sys`, which show the tables of the +/// namespace of the thread that opens them. Only names of the networking tables are accepted (D-078). +pub(crate) fn write_settings(pid: Option, settings: &[&str]) -> Result<(), String> { + let mut writes = Vec::with_capacity(settings.len()); + for setting in settings { + let (name, value) = split_setting(setting)?; + writes.push((format!("/proc/sys/{}", name.replace('.', "/")), value)); + } + let write_all = move || -> Result<(), String> { + for (path, value) in &writes { + fs::write(path, value) + .map_err(|error| format!("cannot write {value} to {path}: {error}"))?; + } + Ok(()) + }; + let Some(pid) = pid else { + return write_all(); + }; + let path = format!("/proc/{pid}/ns/net"); + let namespace = fs::File::open(&path) + .map_err(|error| format!("cannot open the network namespace {path}: {error}"))?; + std::thread::scope(|scope| { + scope + .spawn(|| { + move_into_link_name_space( + std::os::fd::AsFd::as_fd(&namespace), + Some(LinkNameSpaceType::Network), + ) + .map_err(|error| format!("cannot enter the network namespace {path}: {error}"))?; + write_all() + }) + .join() + .unwrap_or_else(|_| Err("the thread that enters a namespace panicked".to_owned())) + }) +} + +/// The name and the value of `net.ipv4.ip_forward=1`, or why it is refused: it has to be `name=value`, and the name has to be in +/// the tables of the network with parts that are plain words (no empty part, no `..`, nothing that is not a letter, a digit, `_` +/// or `-`), so that the path in `/proc/sys` cannot leave them. +pub(crate) fn split_setting(setting: &str) -> Result<(&str, &str), String> { + let Some((name, value)) = setting.split_once('=') else { + return Err(format!("'{setting}' is not name=value")); + }; + if !name.starts_with("net.") + || name.split('.').any(|part| { + part.is_empty() + || part == ".." + || !part + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + }) + { + return Err(format!("'{name}' is not the name of a network setting")); + } + Ok((name, value)) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + use std::fmt::Write as _; + use std::process::{Child, Command, Stdio}; + use std::time::{Duration, Instant}; + + use super::*; + + /// Set in the copy of the test binary that runs inside the namespaces. + const INSIDE: &str = "NETSCENARIO_NETLINK_TEST_INSIDE"; + + /// Runs the test `name` again in a user namespace of its own with a network namespace (`unshare --user --map-root-user + /// --net`), where the test may make namespaces and interfaces without touching the machine (SAF-01). `false` in the copy + /// that runs inside, where the body goes on; `true` in the first copy, which has asserted that the second passed. Where the + /// machine does not allow user namespaces the test has nothing to show and returns at once: the CI is the judge there. + fn rerun_inside(name: &str) -> bool { + if std::env::var_os(INSIDE).is_some() { + return false; + } + let probe = Command::new("unshare") + .args(["--user", "--map-root-user", "--net", "--", "true"]) + .stderr(Stdio::null()) + .status(); + if !probe.is_ok_and(|status| status.success()) { + return true; + } + let output = Command::new("unshare") + .args(["--user", "--map-root-user", "--net", "--"]) + .arg(std::env::current_exe().unwrap()) + .args(["--exact", name, "--nocapture", "--test-threads=1"]) + .env(INSIDE, "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{name} failed inside the namespaces:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + true + } + + /// A network namespace held by a process that sleeps; it ends with the value. + struct Node(Child); + + impl Node { + fn new() -> Self { + let before = fs::read_link("/proc/self/ns/net").unwrap(); + let child = Command::new("unshare") + .args(["--net", "--", "sleep", "300"]) + .spawn() + .unwrap(); + let path = format!("/proc/{}/ns/net", child.id()); + let start = Instant::now(); + // `unshare` is the process until the namespace is made + while fs::read_link(&path).ok().as_ref() == Some(&before) { + assert!(start.elapsed() < Duration::from_secs(10), "no namespace"); + std::thread::sleep(Duration::from_millis(5)); + } + Self(child) + } + + fn pid(&self) -> u32 { + self.0.id() + } + + /// What `program` prints in the namespace of the node. + fn show(&self, program: &str, arguments: &[&str]) -> String { + let namespace = format!("--net=/proc/{}/ns/net", self.pid()); + let output = Command::new("nsenter") + .arg(namespace) + .arg("--") + .arg(program) + .args(arguments) + .output() + .unwrap(); + assert!( + output.status.success(), + "{program} {arguments:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + } + + /// Runs the tool `ip` in the namespace: the oracle that the layer is compared with. + fn ip(&self, arguments: &[&str]) { + self.show("ip", arguments); + } + } + + impl Drop for Node { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + /// Everything the layer sets, as `ip` shows it. + fn state(node: &Node) -> String { + let mut text = String::new(); + for arguments in [ + &["-d", "-o", "link", "show"][..], + &["-o", "address", "show"], + &["-6", "-o", "address", "show"], + &["-d", "route", "show"], + &["-d", "-6", "route", "show"], + &["neigh", "show"], + &["-6", "neigh", "show"], + ] { + writeln!(text, "$ ip {}", arguments.join(" ")).unwrap(); + let mut lines: Vec = node + .show("ip", arguments) + .lines() + .map(|line| { + // The times of the addresses and the neighbours differ between two namespaces made at different moments + line.split(" valid_lft") + .next() + .unwrap_or(line) + .split(" used ") + .next() + .unwrap_or(line) + .trim_end() + .to_owned() + }) + .collect(); + lines.sort(); + text.push_str(&lines.join("\n")); + text.push('\n'); + } + text + } + + /// A setting is written only into the tables of the network (D-078): a name outside them, with a part that could leave the + /// directory or with a character that is no part of a name, is refused before a file is opened; a plain name is taken as it + /// is, which is the canary for the refusals (a check that refuses everything would pass the rest). + #[test] + fn a_setting_outside_the_network_tables_is_refused_before_a_file_is_opened() { + for (setting, why) in [ + ( + "kernel.hostname=x", + "'kernel.hostname' is not the name of a network setting", + ), + ( + "net..ipv4.ip_forward=1", + "'net..ipv4.ip_forward' is not the name of a network setting", + ), + ( + "net.ipv4.ip_forward.=1", + "'net.ipv4.ip_forward.' is not the name of a network setting", + ), + ( + "net.ipv4./.x=1", + "'net.ipv4./.x' is not the name of a network setting", + ), + ( + "net.ipv4.ip forward=1", + "'net.ipv4.ip forward' is not the name of a network setting", + ), + ("net.=1", "'net.' is not the name of a network setting"), + ( + "net.ipv4.ip_forward", + "'net.ipv4.ip_forward' is not name=value", + ), + ] { + assert_eq!(split_setting(setting), Err(why.to_owned()), "{setting}"); + } + for (setting, parts) in [ + ("net.ipv4.ip_forward=1", ("net.ipv4.ip_forward", "1")), + ( + "net.ipv6.conf.all.disable_ipv6=0", + ("net.ipv6.conf.all.disable_ipv6", "0"), + ), + ( + "net.ipv4.conf.veth-1.rp_filter=", + ("net.ipv4.conf.veth-1.rp_filter", ""), + ), + ] { + assert_eq!(split_setting(setting), Ok(parts), "{setting}"); + } + // A refusal stops the rest, and a good name that is not there fails at the write, with no file made + assert!( + write_settings(None, &["kernel.x=1", "net.netscenario_no_such_setting=1"]).is_err() + ); + assert!(write_settings(None, &["net.netscenario_no_such_setting.x=1"]).is_err()); + } + + /// The first thing the layer needs: a socket made by a thread that entered the namespace of another process works in that + /// namespace from every thread, and the process that asked stays in its own. + #[test] + fn a_socket_made_by_a_thread_in_a_node_sees_that_node_and_nothing_else() { + if rerun_inside( + "netlink::tests::a_socket_made_by_a_thread_in_a_node_sees_that_node_and_nothing_else", + ) { + return; + } + let node = Node::new(); + let mut ours = Netlink::here().unwrap(); + ours.add_veth( + "here0", + Some("02:4e:53:00:00:01"), + "here1", + Some("02:4e:53:00:00:02"), + ) + .unwrap(); + let mut theirs = Netlink::in_namespace(node.pid()).unwrap(); + // The interfaces of this namespace are not in the node, and the node has none of them + assert!(theirs.link_index("here0").is_err()); + assert!(theirs.link_index("lo").is_ok()); + // Used from another thread than the one that made it + let mut theirs = std::thread::spawn(move || { + theirs.set_state("lo", true).unwrap(); + theirs + }) + .join() + .unwrap(); + assert!( + node.show("ip", &["-o", "link", "show", "lo"]) + .contains("LOOPBACK,UP") + ); + // The calling thread did not move + assert!(ours.link_index("here0").is_ok()); + assert!(ours.link_index("lo").is_ok()); + // A link goes into the node and is renamed there + ours.move_link("here1", node.pid(), Some("l0")).unwrap(); + assert!(ours.link_index("here1").is_err()); + assert!(theirs.link_index("l0").is_ok()); + // An error of the kernel comes back with its words + let error = ours + .add_veth("here0", Some("02:4e:53:00:00:03"), "here9", None) + .unwrap_err(); + assert!(error.contains("File exists"), "{error}"); + let error = theirs.link_index("nothing").unwrap_err(); + assert!( + error.contains("nothing") && error.contains("No such device"), + "{error}" + ); + } + + /// What a link of the provisioner gets, made once by the tool `ip` in one node and once by the layer in another: the dumps + /// of `ip` of the two are equal. This is the criterion of the layer (BKD-05): the same state as the old path, for both + /// families, a plain route and a multipath route, and a down and up again. + #[test] + #[allow(clippy::too_many_lines)] // two lists of the same steps, one for each way, side by side + fn the_layer_leaves_the_state_that_ip_leaves() { + if rerun_inside("netlink::tests::the_layer_leaves_the_state_that_ip_leaves") { + return; + } + let by_tool = Node::new(); + let by_layer = Node::new(); + let (mac_a, mac_b) = ("02:4e:53:00:00:01", "02:4e:53:00:00:02"); + let (mac_c, mac_d) = ("02:4e:53:00:01:01", "02:4e:53:00:01:02"); + + // The old path + for (inside, peer, own, other, a, b, v4, v6) in [ + ( + "l0", + "p0", + mac_a, + mac_b, + "10.100.0.1/30", + "10.100.0.2", + "fd00:100:0::1/126", + "fd00:100:0::2", + ), + ( + "l1", + "p1", + mac_c, + mac_d, + "10.100.1.1/30", + "10.100.1.2", + "fd00:100:1::1/126", + "fd00:100:1::2", + ), + ] { + by_tool.ip(&[ + "link", "add", inside, "address", own, "type", "veth", "peer", "name", peer, + "address", other, + ]); + by_tool.ip(&["addr", "add", a, "dev", inside]); + by_tool.ip(&["addr", "add", v4, "dev", inside, "nodad"]); + by_tool.ip(&[ + "neigh", + "replace", + b, + "lladdr", + other, + "dev", + inside, + "nud", + "permanent", + ]); + by_tool.ip(&[ + "neigh", + "replace", + v6, + "lladdr", + other, + "dev", + inside, + "nud", + "permanent", + ]); + by_tool.ip(&["link", "set", inside, "up"]); + by_tool.ip(&[ + "link", + "set", + inside, + "gso_max_size", + "1514", + "gso_max_segs", + "1", + ]); + by_tool.ip(&["link", "set", inside, "mtu", "1400"]); + } + by_tool.ip(&[ + "route", + "add", + "10.9.9.9/32", + "metric", + "7", + "src", + "10.100.0.1", + "via", + "10.100.0.2", + "dev", + "l0", + ]); + by_tool.ip(&[ + "-6", + "route", + "add", + "fd00:9::9/128", + "metric", + "7", + "via", + "fd00:100:0::2", + "dev", + "l0", + ]); + by_tool.ip(&[ + "route", + "add", + "default", + "metric", + "9", + "nexthop", + "via", + "10.100.0.2", + "dev", + "l0", + "weight", + "1", + "nexthop", + "via", + "10.100.1.2", + "dev", + "l1", + "weight", + "3", + ]); + by_tool.ip(&["link", "set", "l0", "down"]); + by_tool.ip(&["link", "set", "l0", "up"]); + by_tool.ip(&["addr", "replace", "10.100.0.1/30", "dev", "l0"]); + + // The layer + let mut layer = Netlink::in_namespace(by_layer.pid()).unwrap(); + let ip = |text: &str| text.parse::().unwrap(); + for (inside, peer, own, other, v4, v6, number) in [ + ( + "l0", + "p0", + mac_a, + mac_b, + ("10.100.0.1", "10.100.0.2"), + ("fd00:100:0::1", "fd00:100:0::2"), + 0, + ), + ( + "l1", + "p1", + mac_c, + mac_d, + ("10.100.1.1", "10.100.1.2"), + ("fd00:100:1::1", "fd00:100:1::2"), + 1, + ), + ] { + let _ = number; + layer + .add_veth(inside, Some(own), peer, Some(other)) + .unwrap(); + layer.set_address(inside, ip(v4.0), 30, Verb::Add).unwrap(); + layer.set_address(inside, ip(v6.0), 126, Verb::Add).unwrap(); + layer.set_neighbour(inside, ip(v4.1), other).unwrap(); + layer.set_neighbour(inside, ip(v6.1), other).unwrap(); + layer.set_state(inside, true).unwrap(); + layer.set_segments(inside, 1514, 1).unwrap(); + layer.set_mtu(inside, 1400).unwrap(); + } + let hop = |gateway: &str, interface: &str, weight| Hop { + gateway: ip(gateway), + interface: interface.to_owned(), + weight, + }; + layer + .set_route( + &RouteSpec { + destination: Some(ip("10.9.9.9")), + metric: 7, + source: Some(ip("10.100.0.1")), + hops: vec![hop("10.100.0.2", "l0", 1)], + v6: false, + }, + Verb::Add, + ) + .unwrap(); + layer + .set_route( + &RouteSpec { + destination: Some(ip("fd00:9::9")), + metric: 7, + source: None, + hops: vec![hop("fd00:100:0::2", "l0", 1)], + v6: true, + }, + Verb::Add, + ) + .unwrap(); + layer + .set_route( + &RouteSpec { + destination: None, + metric: 9, + source: None, + hops: vec![hop("10.100.0.2", "l0", 1), hop("10.100.1.2", "l1", 3)], + v6: false, + }, + Verb::Add, + ) + .unwrap(); + layer.set_state("l0", false).unwrap(); + layer.set_state("l0", true).unwrap(); + layer + .set_address("l0", ip("10.100.0.1"), 30, Verb::Replace) + .unwrap(); + + let expected = state(&by_tool); + let found = state(&by_layer); + assert!( + expected.contains("nexthop via 10.100.1.2 dev l1 weight 3"), + "{expected}" + ); + assert_eq!(found, expected); + // The canary: one more attribute in the layer's node and the comparison sees it + layer.set_mtu("l1", 1300).unwrap(); + assert_ne!(state(&by_layer), expected); + + // The settings of the kernel in the namespace of a node + // A new namespace takes the value of the machine, so both start from a value that is written + write_settings(Some(by_tool.pid()), &["net.ipv4.ip_forward=0"]).unwrap(); + write_settings( + Some(by_layer.pid()), + &["net.ipv4.ip_forward=1", "net.ipv4.conf.all.rp_filter=0"], + ) + .unwrap(); + assert_eq!( + by_layer + .show("cat", &["/proc/sys/net/ipv4/ip_forward"]) + .trim(), + "1" + ); + assert_eq!( + by_tool + .show("cat", &["/proc/sys/net/ipv4/ip_forward"]) + .trim(), + "0" + ); + // The names are checked before a file is opened + for setting in [ + "kernel.hostname=x", + "net..ipv4.ip_forward=1", + "net.ipv4.ip_forward", + ] { + assert!( + write_settings(Some(by_layer.pid()), &[setting]).is_err(), + "{setting}" + ); + } + } +} diff --git a/crates/netscenario-backend-kernel/src/provisioner.rs b/crates/netscenario-backend-kernel/src/provisioner.rs index d3cd6dc..6e4d6bc 100644 --- a/crates/netscenario-backend-kernel/src/provisioner.rs +++ b/crates/netscenario-backend-kernel/src/provisioner.rs @@ -2,6 +2,7 @@ //! pairs, impairments as `netem`. It runs in the supervisor, which is the first process of a PID namespace and //! lives in a network namespace of its own (D-022), so nothing here can reach the network of the host (SAF-01). +use std::cell::RefCell; use std::collections::HashMap; use std::fs; use std::io::{BufRead, BufReader, Write}; @@ -17,6 +18,7 @@ use netscenario_model::Impairments; use crate::batch::{ANSWER_WITHIN, Batch, Failure}; use crate::command::{Toolbox, run, run_with_env}; use crate::netem; +use crate::netlink::{self, Hop, Netlink, RouteSpec, Verb}; use crate::programs::{Output, read_stderr, read_stdout, socket_is_bound}; use crate::protocol::{OutputCounts, Way}; use crate::tables::TableDirectory; @@ -203,6 +205,9 @@ pub(crate) struct Provisioner { /// The tables of the jitter that the run made from samples, once there is one (D-084). Every `tc` of the run gets this /// directory as `TC_LIB_DIR` from then on. tables: Option, + /// The netlink sockets of the run: one in the namespace of the supervisor (`None`) and one in the namespace of each node and of + /// the fabric, by the process that holds it. A socket stays in the namespace it was made in (BKD-05). + sockets: RefCell, Netlink>>, } fn id(index: usize) -> Result { @@ -240,6 +245,7 @@ impl Provisioner { batches: HashMap::new(), notes: Vec::new(), tables: None, + sockets: RefCell::new(HashMap::new()), }) } @@ -253,12 +259,7 @@ impl Provisioner { "the supervisor is process {pid}, not the first process of a PID namespace; it refuses to run (SAF-01)" )); } - let listing = run(&self.tools.ip, &["-o", "link", "show"])?; - let interfaces: Vec<&str> = listing - .lines() - .filter_map(|line| line.split(": ").nth(1)) - .map(|name| name.split('@').next().unwrap_or(name)) - .collect(); + let interfaces = self.netlink(None, Netlink::link_names)?; if interfaces != ["lo"] { return Err(format!( "the supervisor sees the network interfaces {}: it is not in a network namespace of its own and refuses to touch the network (SAF-01)", @@ -285,14 +286,27 @@ impl Provisioner { Ok(()) } - /// Writes kernel settings (`net.ipv4.ip_forward=1`) in the network namespace of a node, by the program itself (the - /// hidden call `sysctl`). - fn sysctl(&self, pid: u32, settings: &[&str]) -> Result<(), String> { - let namespace = format!("--net=/proc/{pid}/ns/net"); - let exe = self.exe.to_string_lossy().into_owned(); - let mut arguments = vec![namespace.as_str(), "--", exe.as_str(), "sysctl"]; - arguments.extend_from_slice(settings); - run(&self.tools.nsenter, &arguments).map(|_| ()) + /// Writes kernel settings (`net.ipv4.ip_forward=1`) in the network namespace of a node. + fn sysctl(pid: u32, settings: &[&str]) -> Result<(), String> { + netlink::write_settings(Some(pid), settings) + } + + /// Runs `action` with the netlink socket of the namespace that process `pid` holds (`None`: the namespace of the supervisor). + /// The socket is made when it is first needed and kept until the teardown. + fn netlink( + &self, + pid: Option, + action: impl FnOnce(&mut Netlink) -> Result, + ) -> Result { + let mut sockets = self.sockets.borrow_mut(); + let socket = match sockets.entry(pid) { + std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => entry.insert(match pid { + Some(pid) => Netlink::in_namespace(pid)?, + None => Netlink::here()?, + }), + }; + action(socket) } /// Waits until the packets that the interfaces of a network with IPv6 send of their own are over (D-081), so that the @@ -353,18 +367,12 @@ impl Provisioner { return Err(format!("the node '{name}' exists already")); } let pid = self.spawn_holder(name)?; - self.in_node(pid, &self.tools.ip, &["link", "set", "lo", "up"])?; + self.netlink(Some(pid), |socket| socket.set_state("lo", true))?; // A route through an interface whose carrier is gone is not used (the other path takes over, D-078), and a packet is // not dropped because it arrives on another interface than the one the reverse route leaves by - let namespace = format!("--net=/proc/{pid}/ns/net"); - let exe = self.exe.to_string_lossy().into_owned(); - run( - &self.tools.nsenter, + Self::sysctl( + pid, &[ - namespace.as_str(), - "--", - exe.as_str(), - "sysctl", "net.ipv4.conf.all.ignore_routes_with_linkdown=1", "net.ipv4.conf.all.rp_filter=0", "net.ipv4.conf.default.rp_filter=0", @@ -390,7 +398,7 @@ impl Provisioner { } settings.push("net.ipv6.conf.all.ignore_routes_with_linkdown=1".to_owned()); let settings: Vec<&str> = settings.iter().map(String::as_str).collect(); - self.sysctl(pid, &settings)?; + Self::sysctl(pid, &settings)?; } self.failpoints.hit("node:end"); id(self.nodes.len() - 1) @@ -424,27 +432,15 @@ impl Provisioner { for (addresses, prefix) in families.iter().zip(&prefixes) { let address = format!("{}/{prefix}", addresses[end]); // IPv6: no duplicate address detection (`nodad`), the address is there at once - let mut add = vec!["addr", "add", address.as_str(), "dev", &inside]; - if addresses[end].contains(':') { - add.push("nodad"); - } - self.in_node(pid, &self.tools.ip, &add)?; + let (local, length) = netlink::parse_cidr(&address)?; + self.netlink(Some(pid), |socket| { + socket.set_address(inside, local, length, Verb::Add) + })?; // The neighbour is fixed instead of learned: no ARP or neighbour solicitation may shift the loss pattern - self.in_node( - pid, - &self.tools.ip, - &[ - "neigh", - "replace", - &addresses[other], - "lladdr", - &macs[other], - "dev", - inside, - "nud", - "permanent", - ], - )?; + let neighbour = netlink::parse_ip(&addresses[other])?; + self.netlink(Some(pid), |socket| { + socket.set_neighbour(inside, neighbour, &macs[other]) + })?; neighbours[end] .0 .push([addresses[other].clone(), macs[other].clone()]); @@ -452,7 +448,7 @@ impl Provisioner { neighbours[end].1 = Some(address.clone()); } } - self.in_node(pid, &self.tools.ip, &["link", "set", inside, "up"])?; + self.netlink(Some(pid), |socket| socket.set_state(inside, true))?; self.limit_segments(pid, inside, DEFAULT_MTU)?; } Ok(neighbours) @@ -464,21 +460,9 @@ impl Provisioner { /// machine may not have; the size and the number of segments that the device accepts are properties of the link that `ip` /// sets. fn limit_segments(&self, pid: u32, interface: &str, mtu: u32) -> Result<(), String> { - let size = (mtu + ETHERNET_HEADER).to_string(); - self.in_node( - pid, - &self.tools.ip, - &[ - "link", - "set", - interface, - "gso_max_size", - &size, - "gso_max_segs", - "1", - ], - )?; - Ok(()) + self.netlink(Some(pid), |socket| { + socket.set_segments(interface, mtu + ETHERNET_HEADER, 1) + }) } /// A veth pair between two nodes, moved into them (the direct build of a link). @@ -489,37 +473,14 @@ impl Provisioner { macs: &[String; 2], pids: [u32; 2], ) -> Result<(), String> { - run( - &self.tools.ip, - &[ - "link", - "add", - &outside[0], - "address", - &macs[0], - "type", - "veth", - "peer", - "name", - &outside[1], - "address", - &macs[1], - ], - )?; + self.netlink(None, |socket| { + socket.add_veth(&outside[0], Some(&macs[0]), &outside[1], Some(&macs[1])) + })?; self.failpoints.hit("link:created"); for (end, pid) in pids.iter().enumerate() { - run( - &self.tools.ip, - &[ - "link", - "set", - &outside[end], - "netns", - &pid.to_string(), - "name", - inside, - ], - )?; + self.netlink(None, |socket| { + socket.move_link(&outside[end], *pid, Some(inside)) + })?; } Ok(()) } @@ -538,59 +499,23 @@ impl Provisioner { ) -> Result<[String; 2], String> { let fabric = self.fabric_pid()?; let ports = [format!("nsf{index}a"), format!("nsf{index}b")]; - run( - &self.tools.ip, - &[ - "link", - "add", - &outside[0], - "address", - &macs[0], - "type", - "veth", - "peer", - "name", - &ports[0], - ], - )?; - run( - &self.tools.ip, - &[ - "link", - "add", - &ports[1], - "type", - "veth", - "peer", - "name", - &outside[1], - "address", - &macs[1], - ], - )?; + self.netlink(None, |socket| { + socket.add_veth(&outside[0], Some(&macs[0]), &ports[0], None) + })?; + self.netlink(None, |socket| { + socket.add_veth(&ports[1], None, &outside[1], Some(&macs[1])) + })?; self.failpoints.hit("link:created"); for (end, pid) in pids.iter().enumerate() { - run( - &self.tools.ip, - &[ - "link", - "set", - &outside[end], - "netns", - &pid.to_string(), - "name", - inside, - ], - )?; + self.netlink(None, |socket| { + socket.move_link(&outside[end], *pid, Some(inside)) + })?; } for port in &ports { - run( - &self.tools.ip, - &["link", "set", port, "netns", &fabric.to_string()], - )?; + self.netlink(None, |socket| socket.move_link(port, fabric, None))?; } for (from, to) in [(0, 1), (1, 0)] { - self.in_node(fabric, &self.tools.ip, &["link", "set", &ports[from], "up"])?; + self.netlink(Some(fabric), |socket| socket.set_state(&ports[from], true))?; self.in_node( fabric, &self.tools.tc, @@ -692,26 +617,11 @@ impl Provisioner { pub(crate) fn set_node_address(&mut self, node: u32, address: &str) -> Result<(), String> { let pid = self.node_pid(node)?; self.failpoints.hit("address:start"); - if address.contains(':') { - self.in_node( - pid, - &self.tools.ip, - &[ - "addr", - "add", - &format!("{address}/128"), - "dev", - "lo", - "nodad", - ], - )?; - } else { - self.in_node( - pid, - &self.tools.ip, - &["addr", "add", &format!("{address}/32"), "dev", "lo"], - )?; - } + let local = netlink::parse_ip(address)?; + let length = if local.is_ipv6() { 128 } else { 32 }; + self.netlink(Some(pid), |socket| { + socket.set_address("lo", local, length, Verb::Add) + })?; self.node_addresses .insert((node, address.contains(':')), address.to_owned()); self.failpoints.hit("address:end"); @@ -732,11 +642,7 @@ impl Provisioner { .collect(); self.failpoints.hit("mtu:start"); for (pid, interface) in &ends { - self.in_node( - *pid, - &self.tools.ip, - &["link", "set", interface, "mtu", &mtu.to_string()], - )?; + self.netlink(Some(*pid), |socket| socket.set_mtu(interface, mtu))?; self.limit_segments(*pid, interface, mtu)?; } self.failpoints.hit("mtu:end"); @@ -751,9 +657,9 @@ impl Provisioner { let v4 = format!("net.ipv4.ip_forward={}", u8::from(on)); let v6 = format!("net.ipv6.conf.all.forwarding={}", u8::from(on)); if self.v6 { - self.sysctl(pid, &[v4.as_str(), v6.as_str()])?; + Self::sysctl(pid, &[v4.as_str(), v6.as_str()])?; } else { - self.sysctl(pid, &[v4.as_str()])?; + Self::sysctl(pid, &[v4.as_str()])?; } self.failpoints.hit("forwarding:end"); Ok(()) @@ -777,17 +683,17 @@ impl Provisioner { metric, next_hops: next_hops.to_vec(), }; - self.install(&route, "add", &route.next_hops)?; + self.install(&route, Verb::Add, &route.next_hops)?; self.routes.push(route); self.failpoints.hit("route:end"); Ok(()) } - /// Runs `ip route ` for a route with the given ways out (all of its own, or those that are up). + /// Puts a route in the main table of its node, with the given ways out (all of its own, or those that are up). fn install( &self, route: &Route, - verb: &str, + verb: Verb, next_hops: &[crate::protocol::WireHop], ) -> Result<(), String> { let node = route.node; @@ -795,25 +701,20 @@ impl Provisioner { if next_hops.is_empty() { return Err("a route needs a way out".to_owned()); } - let mut arguments: Vec = Vec::new(); - if route.v6 { - arguments.push("-6".into()); - } - arguments.extend(["route".to_owned(), verb.to_owned()]); - let host_prefix = if route.v6 { 128 } else { 32 }; - arguments.push( - route - .destination - .as_ref() - .map_or_else(|| "default".to_owned(), |a| format!("{a}/{host_prefix}")), - ); - arguments.extend(["metric".to_owned(), route.metric.to_string()]); + let destination = route + .destination + .as_deref() + .map(netlink::parse_ip) + .transpose()?; // A packet that the node sends over the route has the address of the node as its source, not the address of the link it // leaves by: the other nodes have a route to the node address only, so a reply to a link address would have none // (a TCP connection through a router never got its answer) - if let Some(address) = self.node_addresses.get(&(node, route.v6)) { - arguments.extend(["src".to_owned(), address.clone()]); - } + let source = self + .node_addresses + .get(&(node, route.v6)) + .map(|address| netlink::parse_ip(address)) + .transpose()?; + let mut hops = Vec::with_capacity(next_hops.len()); for hop in next_hops { let link = self .links @@ -827,45 +728,36 @@ impl Provisioner { return Err(format!("the link {} is not at the node {node}", hop.link)); } let gateway = if route.v6 { - link_address_v6(hop.link, 1 - hop.end).to_string() + std::net::IpAddr::V6(link_address_v6(hop.link, 1 - hop.end)) } else { - link_address(hop.link, 1 - hop.end).to_string() + std::net::IpAddr::V4(link_address(hop.link, 1 - hop.end)) }; - if next_hops.len() > 1 { - arguments.push("nexthop".into()); - } - arguments.extend([ - "via".to_owned(), + hops.push(Hop { gateway, - "dev".to_owned(), - end.interface.clone(), - ]); - if next_hops.len() > 1 { - arguments.extend(["weight".to_owned(), hop.weight.to_string()]); - } + interface: end.interface.clone(), + weight: hop.weight, + }); } - let words: Vec<&str> = arguments.iter().map(String::as_str).collect(); if next_hops.len() > 1 { // The kernel chooses a way of a multipath route by the addresses only unless told to look at the ports as well, // so every flow between two nodes would take the same way; with the ports the flows share the ways (D-078) - let namespace = format!("--net=/proc/{pid}/ns/net"); - let exe = self.exe.to_string_lossy().into_owned(); - run( - &self.tools.nsenter, - &[ - namespace.as_str(), - "--", - exe.as_str(), - "sysctl", - if route.v6 { - "net.ipv6.fib_multipath_hash_policy=1" - } else { - "net.ipv4.fib_multipath_hash_policy=1" - }, - ], + Self::sysctl( + pid, + &[if route.v6 { + "net.ipv6.fib_multipath_hash_policy=1" + } else { + "net.ipv4.fib_multipath_hash_policy=1" + }], )?; } - self.in_node(pid, &self.tools.ip, &words).map(|_| ()) + let spec = RouteSpec { + destination, + metric: route.metric, + source, + hops, + v6: route.v6, + }; + self.netlink(Some(pid), |socket| socket.set_route(&spec, verb)) } /// Puts the routes back that lead out of a link that came up. The kernel removed them when the interface went down (the @@ -883,7 +775,7 @@ impl Provisioner { .copied() .collect(); if !ways.is_empty() { - self.install(route, "replace", &ways)?; + self.install(route, Verb::Replace, &ways)?; } } Ok(()) @@ -1076,7 +968,6 @@ impl Provisioner { .links .get(link as usize) .ok_or_else(|| format!("the link {link} does not exist in this run"))?; - let state = if up { "up" } else { "down" }; let ends: Vec = link .ends .iter() @@ -1091,7 +982,7 @@ impl Provisioner { .collect(); // One end after the other: for a moment a sender on the first side is refused and one on the second is not for (pid, interface, _, _) in &ends { - self.in_node(*pid, &self.tools.ip, &["link", "set", interface, state])?; + self.netlink(Some(*pid), |socket| socket.set_state(interface, up))?; } if up { // The kernel removed the fixed neighbours with the down (a fixed one as well as a learned one); without them the @@ -1100,30 +991,18 @@ impl Provisioner { // with it, so the address comes first for (pid, interface, _, v6_address) in &ends { if let Some(address) = v6_address { - self.in_node( - *pid, - &self.tools.ip, - &["addr", "replace", address, "dev", interface, "nodad"], - )?; + let (local, length) = netlink::parse_cidr(address)?; + self.netlink(Some(*pid), |socket| { + socket.set_address(interface, local, length, Verb::Replace) + })?; } } for (pid, interface, neighbours, _) in &ends { for [address, mac] in neighbours { - self.in_node( - *pid, - &self.tools.ip, - &[ - "neigh", - "replace", - address, - "lladdr", - mac, - "dev", - interface, - "nud", - "permanent", - ], - )?; + let address = netlink::parse_ip(address)?; + self.netlink(Some(*pid), |socket| { + socket.set_neighbour(interface, address, mac) + })?; } } self.down.remove(&link_index); diff --git a/crates/netscenario-backend-kernel/src/supervisor.rs b/crates/netscenario-backend-kernel/src/supervisor.rs index 3516f48..5c8a362 100644 --- a/crates/netscenario-backend-kernel/src/supervisor.rs +++ b/crates/netscenario-backend-kernel/src/supervisor.rs @@ -28,10 +28,7 @@ pub fn main(arguments: impl Iterator) -> ExitCode { Err(_) => fail(&format!("'{max_nodes}' is not a number of nodes")), }, ["hold", _run_id] => hold(), - ["sysctl", settings @ ..] if !settings.is_empty() => sysctl(settings), - _ => { - fail("usage: supervise | hold | sysctl ...") - } + _ => fail("usage: supervise | hold "), } } @@ -71,44 +68,6 @@ fn hold() -> ExitCode { } } -/// Writes kernel settings of the network namespace this process is in: `net.ipv4.ip_forward=1` writes `1` to -/// `/proc/sys/net/ipv4/ip_forward`. The backend starts it with `nsenter` in a node, because it has no shell and no -/// `sysctl` to rely on (D-078). Only names of the networking tables are accepted. -fn sysctl(settings: &[&str]) -> ExitCode { - for setting in settings { - let (name, value) = match split_setting(setting) { - Ok(parts) => parts, - Err(message) => return fail(&message), - }; - let path = format!("/proc/sys/{}", name.replace('.', "/")); - if let Err(error) = std::fs::write(&path, value) { - return fail(&format!("cannot write {value} to {path}: {error}")); - } - } - ExitCode::SUCCESS -} - -/// The name and the value of `net.ipv4.ip_forward=1`, or why it is refused: it has to be `name=value`, and the name has to be in -/// the tables of the network with parts that are plain words (no empty part, no `..`, nothing that is not a letter, a digit, `_` -/// or `-`), so that the path in `/proc/sys` cannot leave them. -fn split_setting(setting: &str) -> Result<(&str, &str), String> { - let Some((name, value)) = setting.split_once('=') else { - return Err(format!("'{setting}' is not name=value")); - }; - if !name.starts_with("net.") - || name.split('.').any(|part| { - part.is_empty() - || part == ".." - || !part - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') - }) - { - return Err(format!("'{name}' is not the name of a network setting")); - } - Ok((name, value)) -} - /// Puts the signals that a program may be sent (`TERM` by default, and the others of `shutdown`) back to what a program /// expects. The supervisor is the first process of a pid namespace and may have been started with `SIGTERM` and `SIGINT` /// ignored (the `unshare` of util-linux 2.37, on Ubuntu 22.04, does). An ignored signal stays ignored across `exec`, so @@ -339,68 +298,9 @@ mod tests { &["supervise", "run", "-1"], &["supervise", "run", "4", "extra"], &["hold"], - &["sysctl"], &["restart", "run"], ] { assert_eq!(main_with(arguments), ExitCode::FAILURE, "{arguments:?}"); } } - - /// `sysctl` writes only into the tables of the network (D-078): a name outside them, with a part that could leave the - /// directory or with a character that is no part of a name, is refused before a file is opened; a plain name is taken as it - /// is, which is the canary for the refusals (a check that refuses everything would pass the rest). - #[test] - fn a_setting_outside_the_network_tables_is_refused_before_a_file_is_opened() { - for (setting, why) in [ - ( - "kernel.hostname=x", - "'kernel.hostname' is not the name of a network setting", - ), - ( - "net..ipv4.ip_forward=1", - "'net..ipv4.ip_forward' is not the name of a network setting", - ), - ( - "net.ipv4.ip_forward.=1", - "'net.ipv4.ip_forward.' is not the name of a network setting", - ), - ( - "net.ipv4./.x=1", - "'net.ipv4./.x' is not the name of a network setting", - ), - ( - "net.ipv4.ip forward=1", - "'net.ipv4.ip forward' is not the name of a network setting", - ), - ("net.=1", "'net.' is not the name of a network setting"), - ( - "net.ipv4.ip_forward", - "'net.ipv4.ip_forward' is not name=value", - ), - ] { - assert_eq!(split_setting(setting), Err(why.to_owned()), "{setting}"); - } - for (setting, parts) in [ - ("net.ipv4.ip_forward=1", ("net.ipv4.ip_forward", "1")), - ( - "net.ipv6.conf.all.disable_ipv6=0", - ("net.ipv6.conf.all.disable_ipv6", "0"), - ), - ( - "net.ipv4.conf.veth-1.rp_filter=", - ("net.ipv4.conf.veth-1.rp_filter", ""), - ), - ] { - assert_eq!(split_setting(setting), Ok(parts), "{setting}"); - } - // A refusal stops the rest, and a good name that is not there fails at the write, with no file made - assert_eq!( - sysctl(&["kernel.x=1", "net.netscenario_no_such_setting=1"]), - ExitCode::FAILURE - ); - assert_eq!( - sysctl(&["net.netscenario_no_such_setting.x=1"]), - ExitCode::FAILURE - ); - } } diff --git a/crates/netscenario-cli/src/main.rs b/crates/netscenario-cli/src/main.rs index fb3da61..230dc4e 100644 --- a/crates/netscenario-cli/src/main.rs +++ b/crates/netscenario-cli/src/main.rs @@ -10,10 +10,7 @@ fn main() -> ExitCode { let args: Vec = std::env::args_os() .map(|arg| arg.to_string_lossy().into_owned()) .collect(); - if matches!( - args.get(1).map(String::as_str), - Some("supervise" | "hold" | "sysctl") - ) { + if matches!(args.get(1).map(String::as_str), Some("supervise" | "hold")) { return netscenario_backend_kernel::supervisor::main(args.into_iter().skip(1)); } let code = netscenario_cli::run( diff --git a/docs/decisions/D-098.md b/docs/decisions/D-098.md index 32c3b64..258c3ea 100644 --- a/docs/decisions/D-098.md +++ b/docs/decisions/D-098.md @@ -5,7 +5,7 @@ Accepted (2026-11-12), for BKD-06. Measured by S21 and S23 on Debian 13 (kernel - **The shape.** One engine process for a run, in the fabric namespace of D-079: `node A -- veth -- engine -- veth -- node B`, a packet socket (`AF_PACKET`) on each veth end. Nodes are ordinary namespaces, so routing, TCP, IPv6, fragments, programs and capture are the kernel's and are not rebuilt. The supervisor, the crash guarantee (D-022), `clean` and the limits are those of the Kernel backend; only the middle of a link differs (the engine where `netem` and the shaper were). The engine needs no kernel module and no root, so its tests also run where `netem` is missing. - **The data path** is one thread for all links of a process (or one thread per link on request): non-blocking sockets, a heap of frames ordered by the moment they are due with the arrival number as tie-break (so a jitter can reorder, and equal moments keep their order), and a timer. S21: 1,000 to 100,000 packets per second of 200 bytes, delay 20 ms, 0 packets lost up to 50,000 per second with every timer and up to 400 of 400,000 lost at 100,000 per second with the sleeping timers (none with the spinning ones); a uniform jitter of 5 ms reordered the packets (99,573 of 200,000 left out of arrival order) and gave the mean, the minimum and the standard deviation of the model (20.0, 15.0 and 2.89 ms); one TCP flow through the engine reached 2.0 to 2.4 Gbit/s against 4.5 Gbit/s on a plain veth with one segment per frame, which is far more than a test needs. - **The timer, measured as the lateness of a release** (the time between the moment a packet was due and the moment it was handed to the interface), delay 20 ms, 10,000 packets per second: `epoll_pwait2` with a nanosecond timeout (through `rustix`) has a median of 9 to 10 microseconds, a 99th percentile of 0.26 to 0.5 ms and rare outliers of 4 to 7 ms (a core that sleeps wakes late; the load of this desktop shows in it), 2 % of a CPU at 1,000 packets per second, 6 % at 10,000, 30 % at 50,000 and 41 % at 100,000; a timer with a millisecond timeout is 1 ms late at low rates and is not used; `timerfd` is like `epoll_pwait2` and a little later; a spinning loop has a 99th percentile of 5 to 8 microseconds and a maximum of 0.1 ms at the price of a whole CPU; **spin-sleep** (sleep until 100 microseconds before the due moment, then spin) has a 99th percentile of 5 to 13 microseconds, as tight as the kernel (its standard deviation of the delay is 1 to 2 microseconds, the engine's 3), at 6 % of a CPU at 1,000 packets per second and 63 % at 10,000 and more. Its weakness is a low rate: at 1,000 packets per second it sleeps for most of 20 ms and still woke late now and then (99.9th percentile 2.6 ms, maximum 4 ms), because the window of 100 microseconds is shorter than the late wake-ups of this machine. The window therefore adapts: the engine measures how late its sleeps end and spins for that long (the step of the data path builds and tests it). The kernel's own delay is tighter than any timer of the engine; the engine adds 6 to 12 microseconds of base delay for its hop (S23). -- **The timer is a setting, and its default is `auto`.** The engine always measures the lateness of every release and puts the distribution and an overload flag in the result (ENV-08). `engine.timer` is `auto`, `epoll`, `spin-sleep` or `spin`, set with `--engine-timer`, `engine_timer=` and the plugin option, and recorded. `auto` takes spin-sleep when the run has at least 4 CPUs available (by the CPU mask of the process) and `epoll` otherwise, and the result says which and why. The reason: a test author on a laptop or a 4-core runner gets the exact timing the release is named for; a 2-core runner, where a spinning engine would take a core from the sender, the receiver and the programs, gets the cheap timer and an honest report of the lateness. The threshold is one constant, to be confirmed with the numbers of the runners. The setting changes the precision of the timing and never what the engine decides (D-099). +- **The timer is a setting, and its default is `auto`.** The engine always measures the lateness of every release and puts the distribution and an overload flag in the result (ENV-08). `engine.timer` is `auto`, `epoll`, `spin-sleep` or `spin`, set with `--engine-timer`, `engine_timer=` and the plugin option, and recorded. `auto` takes spin-sleep when the run has at least 4 CPUs available (by the CPU mask of the process) and `epoll` otherwise, and the result says which and why. The reason: a test author on a laptop or a 4-core runner gets the exact timing the release is named for; a 2-core runner, where a spinning engine would take a core from the sender, the receiver and the programs, gets the cheap timer and an honest report of the lateness. The threshold is one constant. The runners of the CI have exactly 4 CPUs and the spin-sleep timer ran there with the sender, the receiver and the TCP programs without a lost packet and with a 99th percentile of 7 to 23 microseconds, so 4 holds; a machine with 2 CPUs was not measured and gets the cheap timer by the rule. The setting changes the precision of the timing and never what the engine decides (D-099). - **`unsafe`.** Everything is safe Rust (`nix`, `rustix`: receive, send, `epoll`, `timerfd`, the clock, the CPU time) but one module of the engine crate, `sys`, about 35 lines: the `bind` of a packet socket to an interface and the two options `PACKET_IGNORE_OUTGOING` (the engine must not read what it sends) and `PACKET_VNET_HDR`. `nix` 0.31.3 cannot build the address that `bind` and `send` need (the field of `LinkAddr` is private and there is no constructor) and neither `nix` nor `rustix` knows the options; there is no safe way today. The module has a `SAFETY` comment for each call and a test, the crate has `#![deny(unsafe_code)]` with the one allowed module, and every other crate keeps `forbid`. A pull request to `nix` was considered and not made (the engine would wait on someone else's release). - **Checksums, solved without a tool.** A frame that leaves a node over a veth with checksum offload on carries an unfinished checksum; a packet socket that re-sends it hands the receiver a frame it drops (S21: a TCP connection through the engine with the offloads on never opens). `PACKET_VNET_HDR` keeps the unfinished checksum unfinished: each frame is read and written with a 10-byte header that says where to complete it, and the interface that receives it completes it, as from a real wire. TCP through the engine with it (and one segment per frame, D-086) ran at 2.1 Gbit/s; with the transmit offload switched off by the `ethtool` ioctl instead it ran at 2.0 to 2.4 Gbit/s. So no `ethtool` and no extra netlink message is needed, and the engine does not depend on a setting of the nodes. - **The binary.** The engine is a program, `netscenario-engine`, installed next to the agent (the wheel carries it too, as the agent), started by the supervisor in the fabric namespace; the control channel is lines of JSON, like the agent's. From bea2229878281a90f16e4896f1267c221875c540 Mon Sep 17 00:00:00 2001 From: Leon Krix Date: Thu, 12 Nov 2026 10:31:18 +0100 Subject: [PATCH 2/2] fix: run the netlink tests in a plain network namespace when they run as root --- crates/netscenario-backend-kernel/src/netlink.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/netscenario-backend-kernel/src/netlink.rs b/crates/netscenario-backend-kernel/src/netlink.rs index 8ee839e..0690c23 100644 --- a/crates/netscenario-backend-kernel/src/netlink.rs +++ b/crates/netscenario-backend-kernel/src/netlink.rs @@ -590,7 +590,7 @@ mod tests { /// Set in the copy of the test binary that runs inside the namespaces. const INSIDE: &str = "NETSCENARIO_NETLINK_TEST_INSIDE"; - /// Runs the test `name` again in a user namespace of its own with a network namespace (`unshare --user --map-root-user + /// Runs the test `name` again in a network namespace of its own (`unshare --user --map-root-user --net`, as root `unshare /// --net`), where the test may make namespaces and interfaces without touching the machine (SAF-01). `false` in the copy /// that runs inside, where the body goes on; `true` in the first copy, which has asserted that the second passed. Where the /// machine does not allow user namespaces the test has nothing to show and returns at once: the CI is the judge there. @@ -598,15 +598,24 @@ mod tests { if std::env::var_os(INSIDE).is_some() { return false; } + // As root a network namespace is enough. A user namespace would make the test binary unreachable when it lies in a + // directory of another user (`/home/runner/...` on the CI): root of the new user namespace has no rights over files + // whose owner is not mapped. + let arguments: &[&str] = if rustix::process::geteuid().is_root() { + &["--net", "--"] + } else { + &["--user", "--map-root-user", "--net", "--"] + }; let probe = Command::new("unshare") - .args(["--user", "--map-root-user", "--net", "--", "true"]) + .args(arguments) + .arg("true") .stderr(Stdio::null()) .status(); if !probe.is_ok_and(|status| status.success()) { return true; } let output = Command::new("unshare") - .args(["--user", "--map-root-user", "--net", "--"]) + .args(arguments) .arg(std::env::current_exe().unwrap()) .args(["--exact", name, "--nocapture", "--test-threads=1"]) .env(INSIDE, "1")