From bbd191c7dccebdcca3b9592804db570af0ad8ebf Mon Sep 17 00:00:00 2001 From: Leon Krix Date: Wed, 11 Nov 2026 12:08:45 +0100 Subject: [PATCH] test: remove the leftover directories, make two timing tests sound and close gaps of the command line --- crates/netscenario-agent/tests/loopback.rs | 12 +- .../src/supervisor.rs | 123 ++++++++- .../netscenario-backend-kernel/tests/live.rs | 6 + .../tests/profiles.rs | 84 ++++--- .../tests/support/mod.rs | 31 ++- .../tests/traces.rs | 24 +- crates/netscenario-cli/src/lib.rs | 67 +++++ crates/netscenario-cli/src/render.rs | 192 ++++++++++++++ crates/netscenario-cli/src/run_cmd.rs | 60 +++++ crates/netscenario-cli/tests/binary.rs | 5 +- .../tests/fixtures/all-sections.result.json | 238 ++++++++++++++++++ crates/netscenario-core/src/files.rs | 4 +- crates/netscenario-core/src/lib.rs | 2 + crates/netscenario-core/src/replay.rs | 4 +- crates/netscenario-core/src/report/tests.rs | 5 +- crates/netscenario-core/src/run.rs | 5 +- crates/netscenario-core/src/runset.rs | 2 +- crates/netscenario-core/src/testdir.rs | 90 +++++++ docs/TESTING.md | 2 + 19 files changed, 889 insertions(+), 67 deletions(-) create mode 100644 crates/netscenario-cli/tests/fixtures/all-sections.result.json create mode 100644 crates/netscenario-core/src/testdir.rs diff --git a/crates/netscenario-agent/tests/loopback.rs b/crates/netscenario-agent/tests/loopback.rs index 5a684b8..f91d934 100644 --- a/crates/netscenario-agent/tests/loopback.rs +++ b/crates/netscenario-agent/tests/loopback.rs @@ -456,10 +456,12 @@ fn a_hold_on_the_input_stops_the_stream_between_two_packets() { let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); socket.set_nonblocking(true).unwrap(); let target = socket.local_addr().unwrap().to_string(); - // 800 kbit with 100 byte packets: 1,000 packets a second, for 400 ms + // 800 kbit with 100 byte packets: 1,000 packets a second, for 1.2 s. The run is long so that a machine under load, which + // answers the first hold late (the sleeps below are minimums, not exact), still has the second hold before the end: a run + // of 400 ms failed once in a full run that went out of the bounds of the packet below. let mut process = Command::new(agent) .args(["send", "--to", &target, "--rate", "800000", "--size", "100"]) - .args(["--duration-ns", "400000000"]) + .args(["--duration-ns", "1200000000"]) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .spawn() @@ -473,7 +475,9 @@ fn a_hold_on_the_input_stops_the_stream_between_two_packets() { panic!("a barrier was expected, the sender said {line}"); }; assert_eq!(id, 5); - assert!((90..=130).contains(&packet), "held before packet {packet}"); + // The hold came while the stream ran: after its start (a packet is due every millisecond, the sender needs a few to start) + // and before its end. Where exactly depends on the load, so the packets that arrive are compared with `packet`, not with a time. + assert!((1..1200).contains(&packet), "held before packet {packet}"); // Held: everything before the packet is there and nothing else arrives while the answer is held back std::thread::sleep(Duration::from_millis(100)); let seen = drain(&socket); @@ -505,7 +509,7 @@ fn a_hold_on_the_input_stops_the_stream_between_two_packets() { assert!(process.wait().unwrap().success()); assert_eq!( sent.packets + sent.errors, - 400, + 1200, "no slot is lost or repeated" ); assert_eq!(sent.barrier_timeouts, 0); diff --git a/crates/netscenario-backend-kernel/src/supervisor.rs b/crates/netscenario-backend-kernel/src/supervisor.rs index 62b2ea4..3516f48 100644 --- a/crates/netscenario-backend-kernel/src/supervisor.rs +++ b/crates/netscenario-backend-kernel/src/supervisor.rs @@ -76,20 +76,10 @@ fn hold() -> ExitCode { /// `sysctl` to rely on (D-078). Only names of the networking tables are accepted. fn sysctl(settings: &[&str]) -> ExitCode { for setting in settings { - let Some((name, value)) = setting.split_once('=') else { - return fail(&format!("'{setting}' is not name=value")); + let (name, value) = match split_setting(setting) { + Ok(parts) => parts, + Err(message) => return fail(&message), }; - if !name.starts_with("net.") - || name.split('.').any(|part| { - part.is_empty() - || part == ".." - || !part - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') - }) - { - return fail(&format!("'{name}' is not the name of a network setting")); - } let path = format!("/proc/sys/{}", name.replace('.', "/")); if let Err(error) = std::fs::write(&path, value) { return fail(&format!("cannot write {value} to {path}: {error}")); @@ -98,6 +88,27 @@ fn sysctl(settings: &[&str]) -> ExitCode { ExitCode::SUCCESS } +/// The name and the value of `net.ipv4.ip_forward=1`, or why it is refused: it has to be `name=value`, and the name has to be in +/// the tables of the network with parts that are plain words (no empty part, no `..`, nothing that is not a letter, a digit, `_` +/// or `-`), so that the path in `/proc/sys` cannot leave them. +fn split_setting(setting: &str) -> Result<(&str, &str), String> { + let Some((name, value)) = setting.split_once('=') else { + return Err(format!("'{setting}' is not name=value")); + }; + if !name.starts_with("net.") + || name.split('.').any(|part| { + part.is_empty() + || part == ".." + || !part + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + }) + { + return Err(format!("'{name}' is not the name of a network setting")); + } + Ok((name, value)) +} + /// Puts the signals that a program may be sent (`TERM` by default, and the others of `shutdown`) back to what a program /// expects. The supervisor is the first process of a pid namespace and may have been started with `SIGTERM` and `SIGINT` /// ignored (the `unshare` of util-linux 2.37, on Ubuntu 22.04, does). An ignored signal stays ignored across `exec`, so @@ -307,3 +318,89 @@ fn supervise(run_id: &str, max_nodes: usize) -> ExitCode { let _ = provisioner.teardown(); ExitCode::SUCCESS } + +#[cfg(test)] +mod tests { + use super::*; + + fn main_with(arguments: &[&str]) -> ExitCode { + main(arguments.iter().map(|argument| (*argument).to_owned())) + } + + /// The command line of the helper: anything but the three forms, and a number of nodes that is no number, is refused + /// before anything is made (nothing is started, nothing is written). + #[test] + fn a_command_line_that_is_none_of_the_three_forms_is_refused() { + for arguments in [ + &[][..], + &["supervise"], + &["supervise", "run"], + &["supervise", "run", "many"], + &["supervise", "run", "-1"], + &["supervise", "run", "4", "extra"], + &["hold"], + &["sysctl"], + &["restart", "run"], + ] { + assert_eq!(main_with(arguments), ExitCode::FAILURE, "{arguments:?}"); + } + } + + /// `sysctl` writes only into the tables of the network (D-078): a name outside them, with a part that could leave the + /// directory or with a character that is no part of a name, is refused before a file is opened; a plain name is taken as it + /// is, which is the canary for the refusals (a check that refuses everything would pass the rest). + #[test] + fn a_setting_outside_the_network_tables_is_refused_before_a_file_is_opened() { + for (setting, why) in [ + ( + "kernel.hostname=x", + "'kernel.hostname' is not the name of a network setting", + ), + ( + "net..ipv4.ip_forward=1", + "'net..ipv4.ip_forward' is not the name of a network setting", + ), + ( + "net.ipv4.ip_forward.=1", + "'net.ipv4.ip_forward.' is not the name of a network setting", + ), + ( + "net.ipv4./.x=1", + "'net.ipv4./.x' is not the name of a network setting", + ), + ( + "net.ipv4.ip forward=1", + "'net.ipv4.ip forward' is not the name of a network setting", + ), + ("net.=1", "'net.' is not the name of a network setting"), + ( + "net.ipv4.ip_forward", + "'net.ipv4.ip_forward' is not name=value", + ), + ] { + assert_eq!(split_setting(setting), Err(why.to_owned()), "{setting}"); + } + for (setting, parts) in [ + ("net.ipv4.ip_forward=1", ("net.ipv4.ip_forward", "1")), + ( + "net.ipv6.conf.all.disable_ipv6=0", + ("net.ipv6.conf.all.disable_ipv6", "0"), + ), + ( + "net.ipv4.conf.veth-1.rp_filter=", + ("net.ipv4.conf.veth-1.rp_filter", ""), + ), + ] { + assert_eq!(split_setting(setting), Ok(parts), "{setting}"); + } + // A refusal stops the rest, and a good name that is not there fails at the write, with no file made + assert_eq!( + sysctl(&["kernel.x=1", "net.netscenario_no_such_setting=1"]), + ExitCode::FAILURE + ); + assert_eq!( + sysctl(&["net.netscenario_no_such_setting.x=1"]), + ExitCode::FAILURE + ); + } +} diff --git a/crates/netscenario-backend-kernel/tests/live.rs b/crates/netscenario-backend-kernel/tests/live.rs index 5d4ab9b..bafaf72 100644 --- a/crates/netscenario-backend-kernel/tests/live.rs +++ b/crates/netscenario-backend-kernel/tests/live.rs @@ -75,6 +75,10 @@ fn a_change_from_test_code_lands_between_two_packets_and_a_replay_loses_the_same return; } let (result, dir) = live_run("live-a", Duration::from_secs(1), |handle| { + // The links the code may change, and how old the measured phase is when the wait is over (the wait is on the clock of + // the phase, so it is at least what was waited for) + assert_eq!(handle.links(), [access()]); + assert!(handle.elapsed().unwrap() >= Duration::from_secs(1)); let records = handle.change(LiveChange::set(access(), loss_30())).unwrap(); assert_eq!(records.len(), 1); assert!(records[0].live && records[0].packet.is_some()); @@ -331,6 +335,8 @@ fn a_run_that_fails_does_not_leave_the_code_waiting() { let error = run(&mut failing, &scenario(), &options).unwrap_err(); assert!(error.to_string().contains("agent"), "{error}"); let began = std::time::Instant::now(); + // The workloads never began, so the phase has no age + assert_eq!(handle.elapsed(), None); assert!(handle.change(LiveChange::down(access())).is_err()); assert!(handle.wait_until(Duration::from_secs(3_600)).is_err()); assert!( diff --git a/crates/netscenario-backend-kernel/tests/profiles.rs b/crates/netscenario-backend-kernel/tests/profiles.rs index 908e148..bf47d3b 100644 --- a/crates/netscenario-backend-kernel/tests/profiles.rs +++ b/crates/netscenario-backend-kernel/tests/profiles.rs @@ -14,6 +14,7 @@ mod support; +use std::fmt::Write as _; use std::time::Duration; use netscenario_agent::records::{self, Record}; @@ -97,44 +98,61 @@ const DELAYS: &[(&str, f64, f64)] = &[ /// What a few milliseconds of two interfaces, a receiver that is not scheduled at once and a busy machine add to a delay. const ROOM: f64 = 0.015; -fn measured_delays(id: &str) { +/// How many profiles one run holds: a run may use 8 network namespaces (SAF-02), the network and 7 terminals. +const PER_RUN: usize = 7; + +/// The profiles of the library in as few runs as the limit of namespaces allows (`PER_RUN` each): the network is a star with +/// one link for each profile and a stream of 20 packets a second in each direction of each link. A run holds several profiles +/// because the links do not touch each other and what a profile adds to the kernel (its delay, jitter, loss and rate on its +/// own link) does not depend on the others; a run for each profile costs 14 times the set-up and the wait for nothing more. +/// A failure names the profile and the direction. +fn delays_of_the_profiles_in_run(run_index: usize) { if !namespaces_available() || agent_exe().is_none() { return; } - let &(_, up, down) = DELAYS.iter().find(|(name, ..)| *name == id).unwrap(); - let (_, records, dir) = go(&scenario(id, "", TWO_STREAMS), &format!("profile-{id}")); - // `up` is the first workload, from the terminal, which is the first node of the link: its delay is `forward` - delay_holds(&records[0], up, ROOM).unwrap_or_else(|why| panic!("{id}, up: {why}")); - delay_holds(&records[1], down, ROOM).unwrap_or_else(|why| panic!("{id}, down: {why}")); + let profiles = DELAYS.chunks(PER_RUN).nth(run_index).unwrap(); + let mut nodes = String::from("network"); + let mut links = String::new(); + let mut streams = String::new(); + for &(id, ..) in profiles { + write!(nodes, ", t-{id}").unwrap(); + writeln!( + links, + " - {{ id: l-{id}, between: [t-{id}, network], profile: {id} }}" + ) + .unwrap(); + // `up` is from the terminal, which is the first node of the link: its delay is `forward` + write!( + streams, + " - {{ id: up-{id}, kind: udp-cbr, from: t-{id}, to: network, rate: 160kbit, packet-size: 1000b }}\n - {{ id: down-{id}, kind: udp-cbr, from: network, to: t-{id}, rate: 160kbit, packet-size: 1000b }}\n" + ) + .unwrap(); + } + let yaml = format!( + "version: 0\nname: profile-test\nduration: 3s\ntopology:\n nodes: [{nodes}]\n links:\n{links}workloads:\n{streams}" + ); + let mut scenario = Scenario::from_yaml(&yaml, "profiles.yaml").unwrap(); + scenario.measure.packets = true; + let (_, records, dir) = go(&scenario, &format!("profile-all-{run_index}")); + assert_eq!(records.len(), 2 * profiles.len()); + for (index, &(id, up, down)) in profiles.iter().enumerate() { + delay_holds(&records[2 * index], up, ROOM).unwrap_or_else(|why| panic!("{id}, up: {why}")); + delay_holds(&records[2 * index + 1], down, ROOM) + .unwrap_or_else(|why| panic!("{id}, down: {why}")); + } let _ = std::fs::remove_dir_all(dir); } -macro_rules! delay_tests { - ($($name:ident: $id:literal),* $(,)?) => { - $( - #[test] - fn $name() { - measured_delays($id); - } - )* - }; +/// The first seven profiles of the table. +#[test] +fn the_delays_of_the_first_profiles_are_the_published_ones() { + delays_of_the_profiles_in_run(0); } -delay_tests! { - the_delays_of_3g_good_are_the_published_medians: "3g-good", - the_delays_of_3g_typical_are_half_of_the_round_trip: "3g-typical", - the_delays_of_lte_good_differ_by_direction: "lte-good", - the_delays_of_lte_typical_are_half_of_the_round_trip: "lte-typical", - the_delays_of_5g_typical_are_half_of_the_round_trip: "5g-typical", - the_delays_of_5g_poor_are_the_published_average: "5g-poor", - the_delays_of_dsl_good_are_half_of_the_round_trip: "dsl-good", - the_delays_of_dsl_poor_are_half_of_the_round_trip: "dsl-poor", - the_delays_of_geo_good_are_half_of_the_round_trip: "geo-good", - the_delays_of_geo_typical_are_half_of_the_median: "geo-typical", - the_delays_of_geo_poor_are_half_of_the_median: "geo-poor", - the_delays_of_leo_typical_are_half_of_the_middle_median: "leo-typical", - the_delays_of_datacenter_rack_are_half_of_the_round_trip: "datacenter-rack", - the_delays_of_datacenter_fabric_are_half_of_the_round_trip: "datacenter-fabric", +/// The next seven. +#[test] +fn the_delays_of_the_next_profiles_are_the_published_ones() { + delays_of_the_profiles_in_run(1); } /// Canary for the list above: a profile in the library without a test here fails, and so does a test for a profile that is gone. @@ -148,6 +166,12 @@ fn every_profile_of_the_library_is_tested_and_the_table_is_the_library() { tested, library_ids, "a profile without a test, or a test without a profile" ); + // ... and the runs above hold all of them (two tests, `PER_RUN` profiles each) + assert_eq!( + DELAYS.chunks(PER_RUN).count(), + 2, + "a profile is in no run: add a test for the next run" + ); // The numbers of the table are those of the library: a changed number fails here until the table is changed on purpose for &(id, up, down) in DELAYS { let profile = library().get(id).unwrap(); diff --git a/crates/netscenario-backend-kernel/tests/support/mod.rs b/crates/netscenario-backend-kernel/tests/support/mod.rs index f3cf7ab..d75b28d 100644 --- a/crates/netscenario-backend-kernel/tests/support/mod.rs +++ b/crates/netscenario-backend-kernel/tests/support/mod.rs @@ -180,11 +180,40 @@ pub(crate) fn assert_nothing_left(run_id: &str, context: &str) { } } -/// A directory for the files that programs of a test write. +/// A directory for the files that programs of a test write. It is removed when the thread of the test ends if the test passed; +/// the directory of a test that failed is kept for a look (`NETSCENARIO_KEEP_TEST_DIRS=1` keeps all of them). pub(crate) fn work_dir(name: &str) -> PathBuf { + use std::cell::{Cell, RefCell}; + use std::sync::Once; + + struct Made(RefCell>); + impl Drop for Made { + fn drop(&mut self) { + if std::env::var_os("NETSCENARIO_KEEP_TEST_DIRS").is_some() || FAILED.with(Cell::get) { + return; + } + for path in self.0.get_mut().drain(..) { + let _ = std::fs::remove_dir_all(path); + } + } + } + thread_local! { + static MADE: Made = const { Made(RefCell::new(Vec::new())) }; + static FAILED: Cell = const { Cell::new(false) }; + } + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + let before = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + FAILED.with(|failed| failed.set(true)); + before(info); + })); + }); + let dir = std::env::temp_dir().join(format!("netscenario-{name}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); + MADE.with(|made| made.0.borrow_mut().push(dir.clone())); dir } diff --git a/crates/netscenario-backend-kernel/tests/traces.rs b/crates/netscenario-backend-kernel/tests/traces.rs index 5ffb3b5..daec62a 100644 --- a/crates/netscenario-backend-kernel/tests/traces.rs +++ b/crates/netscenario-backend-kernel/tests/traces.rs @@ -386,12 +386,30 @@ fn a_row_without_a_delivery_is_a_stall_that_the_scenario_must_accept() { .iter() .filter(|r| r.receive_ns >= from && r.receive_ns < to) .count() as i64; - let late = 2.0 * 2000.0 * slack(&result); - let (upper, lower) = (3602.0 + 8.0 + late, 3602.0 - 8.0 - 400.0 - late); + // A step that starts a stall and is applied late lets the link run on at the full rate (a surplus); a step that ends one + // and is late keeps the link stalled (a deficit). Each direction has its own bound from the steps of its own kind, at the + // rate of 2,000 packets a second. (The steps that end a stall were once counted for the surplus too: one 128 ms late, on a + // loaded machine, made the bound useless and the test failed with 3,404 packets arrived, which is the right number.) + let late_at = |offset_ms: u64| -> f64 { + result + .events + .iter() + .filter(|e| e.scheduled_ns >= 1_000_000_000 && e.scheduled_ns < 3_000_000_000) + .filter(|e| e.scheduled_ns % 1_000_000_000 == offset_ms * 1_000_000) + .map(|e| e.applied_ns.saturating_sub(e.scheduled_ns) as f64 / 1e9 + 0.005) + .sum() + }; + // The stall of a pass begins at 400 ms and ends at 500 ms + let (surplus, deficit) = (2000.0 * late_at(400), 2000.0 * late_at(500)); + let (upper, lower) = (3602.0 + 8.0 + surplus, 3602.0 - 8.0 - 400.0 - deficit); // Canary: the surplus allowed leaves out the 4,000 of a link that does not stall, so the bound can fail assert!( upper < 4000.0, - "the lateness of the steps ({late}) makes the bound useless" + "the lateness of the steps that start a stall ({surplus}) makes the bound useless" + ); + assert!( + late_at(400) > 0.0 && late_at(500) > 0.0, + "the steps of the stalls were not found in the events" ); assert!( (lower..=upper).contains(&(arrived as f64)), diff --git a/crates/netscenario-cli/src/lib.rs b/crates/netscenario-cli/src/lib.rs index b11fad5..7340829 100644 --- a/crates/netscenario-cli/src/lib.rs +++ b/crates/netscenario-cli/src/lib.rs @@ -460,6 +460,73 @@ mod tests { assert_eq!(json["topology"]["links"].as_array().unwrap().len(), 5); } + /// `expand --seed` writes the events that a model or a repeating event makes with that seed, which is what a run with it + /// does: the file with `every:` has more steps, and the same seed gives the same text. A seed that is no number, and a + /// help or a missing file, are told apart from a good call. + #[test] + fn expand_with_a_seed_writes_the_events_that_the_repeats_make() { + let path = scenarios().join("flapping-link.yaml"); + let path = path.to_str().unwrap(); + let (code, plain, err) = call(&["netscenario", "expand", path]); + assert_eq!((code, err.as_str()), (exit::OK, "")); + let (code, seeded, err) = call(&["netscenario", "expand", path, "--seed", "4"]); + assert_eq!((code, err.as_str()), (exit::OK, "")); + let steps = |text: &str| text.matches("- at:").count(); + assert!( + steps(&seeded) > steps(&plain), + "{} steps against {}", + steps(&seeded), + steps(&plain) + ); + assert!(seeded.contains("seed: 4"), "{seeded}"); + assert!(!seeded.contains("every:"), "{seeded}"); + let (_, again, _) = call(&["netscenario", "expand", path, "--seed", "4"]); + assert_eq!(seeded, again); + // Canary: the seed has to be a number, and a call with no file is a mistake of the command line + let (code, out, err) = call(&["netscenario", "expand", path, "--seed", "four"]); + assert_eq!((code, out.as_str()), (exit::USAGE, "")); + assert!( + err.contains("--seed needs a whole number, found 'four'"), + "{err}" + ); + let (code, _, err) = call(&["netscenario", "expand"]); + assert_eq!(code, exit::USAGE); + assert!(err.contains("the scenario file is missing"), "{err}"); + let (code, out, _) = call(&["netscenario", "expand", "--help"]); + assert_eq!(code, exit::OK); + assert!(out.starts_with("Usage: netscenario expand"), "{out}"); + let (code, _, err) = call(&["netscenario", "expand", "/nonexistent/x.yaml"]); + assert_eq!(code, exit::USAGE); + assert!(err.contains("cannot read '/nonexistent/x.yaml'"), "{err}"); + } + + /// `show` names the model of a link: the files of a trace with their format and rows, and the states of a chain. + #[test] + fn show_names_the_model_of_a_link() { + let show = |name: &str| { + let path = scenarios().join(name); + let (code, out, err) = call(&["netscenario", "show", path.to_str().unwrap()]); + assert_eq!((code, err.as_str()), (exit::OK, ""), "{name}"); + out + }; + let trace = show("cell-drive.yaml"); + assert!( + trace.contains("trace both cell-drive.csv (csv, "), + "{trace}" + ); + assert!(trace.contains(" rows, "), "{trace}"); + let walk = show("mahimahi-walk.yaml"); + assert!(walk.contains("(mahimahi, "), "{walk}"); + let chain = show("signal-states.yaml"); + assert!(chain.contains(" states"), "{chain}"); + let (code, out, _) = call(&["netscenario", "show", "--help"]); + assert_eq!(code, exit::OK); + assert!(out.starts_with("Usage: netscenario show"), "{out}"); + let (code, _, err) = call(&["netscenario", "show"]); + assert_eq!(code, exit::USAGE); + assert!(err.contains("the scenario file is missing"), "{err}"); + } + /// `show` lists the nodes, the links, the routes and the path of each workload; as Graphviz and as JSON too. #[test] fn show_lists_the_network_the_routes_and_the_paths() { diff --git a/crates/netscenario-cli/src/render.rs b/crates/netscenario-cli/src/render.rs index d3194cb..2edc5b6 100644 --- a/crates/netscenario-cli/src/render.rs +++ b/crates/netscenario-cli/src/render.rs @@ -1617,4 +1617,196 @@ result: out/result.json assert_eq!(rate(1_234_567_890.0), "1.23 gbit/s"); assert_eq!(ms(20_016_000), "20.0 ms"); } + /// A real run with a TCP transfer, a program and the capture of a link (`tests/fixtures`): every section of the summary + /// that a user reads for them, and then the warnings, each one from the field that raises it. The fixture is a result as + /// `netscenario run` wrote it; a test changes the field and reads the summary again. + const ALL: &str = include_str!("../tests/fixtures/all-sections.result.json"); + + fn all_with(change: impl FnOnce(&mut serde_json::Value)) -> String { + let mut value: serde_json::Value = serde_json::from_str(ALL).unwrap(); + change(&mut value); + summary( + &RunResult::from_json(&value.to_string()).unwrap(), + Path::new("out"), + ) + } + + fn lacks_warning(text: &str) { + assert!(!text.contains("warning:"), "{text}"); + } + + #[test] + fn the_summary_of_a_run_with_tcp_a_program_and_a_capture_has_a_section_for_each() { + let text = all_with(|_| {}); + for expected in [ + "tcp download server -> client tcp-bulk, 2 connections, cubic", + " sent 400000 bytes received 400000 bytes", + " 2 of 2 transfers complete, the last after ", + " goodput 2.59 gbit/s on average over the transfer", + " round trip time of the kernel min 0.0 ms median 0.0 ms p95 0.0 ms max 0.0 ms", + " retransmitted segments 0", + "program tool in server sh -c echo 'metric latency 1.5'; echo hello; sleep 30", + "(stopped, signal 15)", + " output 25 bytes in program-tool.stdout.log, 0 bytes of standard error in program-tool.stderr.log", + "capture link access in client 560 frames, 437098 bytes, in capture-access-client.pcap", + " from 0.000 s to 0.002 s of the measured phase", + " whole: the interface l0 counted 560 frames, as many as the file has", + ] { + assert!(text.contains(expected), "{expected}\n{text}"); + } + lacks_warning(&text); + } + + #[test] + fn a_failed_or_interrupted_transfer_is_a_warning_with_the_reason() { + let text = all_with(|v| { + v["tcp"][0]["failed_connections"] = 1.into(); + v["tcp"][0]["first_error"] = "connection reset".into(); + v["tcp"][0]["stopped"] = true.into(); + }); + assert!( + text.contains(" warning: 1 of 2 connections failed: connection reset"), + "{text}" + ); + assert!( + text.contains(" the run was interrupted before the client was done"), + "{text}" + ); + let text = all_with(|v| v["tcp"][0]["failed_connections"] = 2.into()); + assert!( + text.contains(" warning: 2 of 2 connections failed: no reason was given"), + "{text}" + ); + // One connection reads in the singular + let text = all_with(|v| v["tcp"][0]["connections"] = 1.into()); + assert!(text.contains("tcp-bulk, 1 connection, cubic"), "{text}"); + } + + #[test] + fn the_requests_of_a_workload_are_counted_and_timed() { + let summary_of_delays = |ns: u64| { + serde_json::json!({"min_ns": ns, "mean_ns": ns, "p50_ns": ns, "p90_ns": ns, "p95_ns": ns, + "p99_ns": ns, "p999_ns": ns, "max_ns": ns}) + }; + let text = all_with(|v| { + v["tcp"][0]["kind"] = "tcp-request".into(); + v["tcp"][0]["requests"] = serde_json::json!({ + "sent": 10, "answered": 9, "failed": 1, "max_wait_ns": 5_000_000, + "latency": summary_of_delays(2_000_000), "round_trip": summary_of_delays(1_000_000) + }); + }); + for expected in [ + " requests sent 10 answered 9 failed 1", + " latency from the moment due min 2.0 ms median 2.0 ms p95 2.0 ms p99 2.0 ms max 2.0 ms", + " round trip from the moment sent min 1.0 ms median 1.0 ms p95 1.0 ms p99 1.0 ms max 1.0 ms", + ] { + assert!(text.contains(expected), "{expected}\n{text}"); + } + // The bytes of a transfer are not shown for requests + assert!(!text.contains("transfers complete"), "{text}"); + } + + #[test] + fn what_went_wrong_with_a_program_is_a_warning_on_a_line_of_its_own() { + let text = all_with(|v| { + let program = &mut v["programs"][0]; + program["ended"] = "on-its-own".into(); + program["exit_code"] = 3.into(); + program["exit_signal"] = serde_json::Value::Null; + program["output"]["stdout_dropped_bytes"] = 7.into(); + program["output"]["stderr_dropped_bytes"] = 9.into(); + program["output"]["long_lines"] = 2.into(); + program["lines_dropped"] = true.into(); + program["bad_metric_lines"] = 4.into(); + program["bad_metric_examples"] = serde_json::json!(["metric latency fast"]); + program["metrics"] = serde_json::json!([{ + "name": "latency", "unit": "ms", "count": 3, "first": 1.0, "last": 1.5, "min": 1.0, "max": 2.0, + "mean": 1.5, "sum": 4.5 + }]); + }); + for expected in [ + "(on-its-own, exit code 3)", + " metric latency 3 readings, last 1.5 ms, min 1, max 2", + " warning: it ended by itself with an error, before the run asked it to stop", + " warning: the files hold the first megabyte; 7 bytes of standard output and 9 of standard error are not in them", + " warning: 2 lines of standard output were longer than 64 KiB and were cut", + " warning: lines of standard output were dropped before the run read them; metrics in them are lost", + " warning: 4 lines looked like metrics and were not valid, the first: metric latency fast", + ] { + assert!(text.contains(expected), "{expected}\n{text}"); + } + // A program that ends by itself with 0 has finished its work + let text = all_with(|v| { + v["programs"][0]["ended"] = "on-its-own".into(); + v["programs"][0]["exit_code"] = 0.into(); + }); + lacks_warning(&text); + // No exit status at all + let text = all_with(|v| v["programs"][0]["exit_signal"] = serde_json::Value::Null); + assert!(text.contains("(stopped, no exit status)"), "{text}"); + } + + #[test] + fn a_capture_that_lost_frames_or_could_not_check_itself_is_a_warning() { + let text = all_with(|v| { + v["captures"][0]["missing"] = 12.into(); + v["captures"][0]["packets"] = 548.into(); + }); + assert!( + text.contains(" warning: 12 frames are missing: the interface l0 counted 560 and the file has 548 (the receive buffer of the capture was 425984 bytes; `net.core.rmem_max` limits it)"), + "{text}" + ); + let text = all_with(|v| { + v["captures"][0]["missing"] = serde_json::Value::Null; + v["captures"][0]["interface_frames"] = serde_json::Value::Null; + v["captures"][0]["first_ns"] = serde_json::Value::Null; + }); + assert!( + text.contains(" warning: the counters of the interface could not be read, so the capture could not check that it is whole"), + "{text}" + ); + // Without a first frame there is no span to show for the client's file + assert!( + !text.contains("capture link access in client 560 frames, 437098 bytes, in capture-access-client.pcap\n from"), + "{text}" + ); + } + + #[test] + fn the_timeline_lists_its_steps_with_how_late_each_was_and_cuts_a_long_one() { + let step = |index: u64, packet: Option, model: bool, live: bool| { + serde_json::json!({ + "event": index, "kind": "set", "link": "access", "direction": if index == 1 { serde_json::Value::Null } else { "forward".into() }, + "scheduled_ns": index * 1_000_000_000, "started_ns": index * 1_000_000_000, + "applied_ns": index * 1_000_000_000 + 2_500_000, "packet": packet, + "held_ns": packet.map(|_| 1_200_000_u64), "from_model": model, "live": live + }) + }; + let text = all_with(|v| { + v["events"] = serde_json::json!([ + step(0, None, false, false), + step(1, Some(42), true, false), + step(2, None, false, true) + ]); + }); + for expected in [ + "timeline\n", + " at 0.000 s link access forward set done 2.5 ms late", + " at 1.000 s link access (model) both ways set before packet 42, which waited 1.200 ms", + " at 2.000 s link access (from code) forward set done 2.5 ms late", + ] { + assert!(text.contains(expected), "{expected}\n{text}"); + } + // More than the summary lists: the first ones and the count + let text = all_with(|v| { + v["events"] = + serde_json::Value::Array((0..25).map(|i| step(i, None, true, false)).collect()); + }); + assert!( + text.contains(" and 5 more steps (all of them are in result.json)"), + "{text}" + ); + assert!(!text.contains("at 20.000 s"), "{text}"); + assert!(text.contains("at 19.000 s"), "{text}"); + } } diff --git a/crates/netscenario-cli/src/run_cmd.rs b/crates/netscenario-cli/src/run_cmd.rs index b44d7b5..8ed9523 100644 --- a/crates/netscenario-cli/src/run_cmd.rs +++ b/crates/netscenario-cli/src/run_cmd.rs @@ -720,6 +720,66 @@ mod tests { Parsed::parse(&all, &RUN_SPEC).unwrap() } + /// What went wrong is told with the exit code that fits: a scenario the backend refuses or that is not valid is a wrong input + /// (2), a backend that fails is the machine (3, with the way to `doctor`), any other failure of the run is 4. A canary: each + /// kind has its own code, so a mix-up of two arms fails. + #[test] + fn each_kind_of_failure_has_its_exit_code_and_its_message() { + let loaded = Loaded { + name: "bad.yaml".to_owned(), + text: "not a scenario".to_owned(), + scenario: netscenario_model::Scenario::from_yaml("not a scenario", "bad.yaml"), + }; + assert!(loaded.scenario.is_err()); + let say = |error: Error| { + let mut err = Vec::new(); + let code = report(&error, &loaded, &mut err); + (code, String::from_utf8(err).unwrap()) + }; + let (code, text) = say(Error::Backend { + backend: "kernel".to_owned(), + action: "create the node 'a'".to_owned(), + message: "no room".to_owned(), + }); + assert_eq!(code, exit::ENVIRONMENT); + assert!( + text.contains("the kernel backend failed to create the node 'a': no room") + && text.contains("run `netscenario doctor`"), + "{text}" + ); + let (code, text) = say(Error::InvalidScenario(Vec::new())); + assert_eq!(code, exit::USAGE); + assert!( + text.starts_with("netscenario: the scenario is not valid"), + "{text}" + ); + let (code, text) = say(Error::Run { + action: "start the agent".to_owned(), + message: "it did not answer".to_owned(), + }); + assert_eq!(code, exit::RUN_FAILED); + assert_eq!( + text, + "netscenario: the run failed to start the agent: it did not answer\n" + ); + // A refusal of the backend names the feature and the fix, with the place in the file where there is one + let refusal = netscenario_core::Refusal { + feature: netscenario_core::Feature::DifferentValuesPerDirection, + path: "topology.links[0].both.loss".to_owned(), + reason: "this backend cannot do it".to_owned(), + fix: "use the kernel backend".to_owned(), + }; + let (code, text) = say(Error::Capability(netscenario_core::CapabilityError::new( + "other", + vec![refusal], + ))); + assert_eq!(code, exit::USAGE); + assert!( + text.contains("this backend cannot do it") && text.contains("use the kernel backend"), + "{text}" + ); + } + /// The defaults are those of the library, and every option moves its own parameter (D-070). #[test] fn the_options_of_the_measure_section_are_read() { diff --git a/crates/netscenario-cli/tests/binary.rs b/crates/netscenario-cli/tests/binary.rs index 703e990..2d8730f 100644 --- a/crates/netscenario-cli/tests/binary.rs +++ b/crates/netscenario-cli/tests/binary.rs @@ -1396,9 +1396,12 @@ fn a_set_judged_by_the_file_exits_with_the_verdict() { } let dir = work_dir("expect-set-run"); let out = dir.join("result"); + // Five repetitions are the least a set may have; one second each is enough for a verdict (the exit code follows `passed`). + // Two sets of five runs of two seconds were the slowest test of the command line (33 s in a full run) + let short = SCENARIO.replace("duration: 2s", "duration: 1s"); for (bound, code) in [("5%", 0), ("0%", 1)] { let text = format!( - "{SCENARIO}expect:\n repetitions: 5\n checks:\n - {{ metric: media.loss_ratio, max: \"< {bound}\" }}\n" + "{short}expect:\n repetitions: 5\n checks:\n - {{ metric: media.loss_ratio, max: \"< {bound}\" }}\n" ); let file = write_scenario(&dir, &text); let output = netscenario_in(&dir, &["run", &file, "--out", out.to_str().unwrap()]).unwrap(); diff --git a/crates/netscenario-cli/tests/fixtures/all-sections.result.json b/crates/netscenario-cli/tests/fixtures/all-sections.result.json new file mode 100644 index 0000000..ed5daa9 --- /dev/null +++ b/crates/netscenario-cli/tests/fixtures/all-sections.result.json @@ -0,0 +1,238 @@ +{ + "format": "netscenario-result", + "version": 0, + "scenario": { + "name": "all-sections", + "hash": "sha256:a672b5e6ea3eebb2ccbb6bfdb32476c2fb1b3f34daa1bf629fdc495a60b64fdc" + }, + "provenance": { + "netscenario": "0.2.0", + "started_at": "2026-11-11T10:08:14Z", + "backend": "kernel", + "backend_details": { + "rights": "user namespace", + "tc": "iproute2 6.15.0", + "tcp_congestion_controls": "reno cubic" + }, + "kernel": "6.12.116+deb13-amd64", + "cpu_model": "AMD Ryzen 7 5800X 8-Core Processor", + "cpus": 16, + "timers": { + "clock_source": "tsc", + "clock_read_ns": 30, + "clock_step_ns": 30, + "timer_slack_ns": 50000, + "cpu_governor": "performance", + "load_1m_hundredths": 122, + "runnable_tasks": 2, + "sleeps": [ + { + "asked_ns": 200000, + "samples": 50, + "p50_ns": 54070, + "p99_ns": 56580, + "max_ns": 56580 + }, + { + "asked_ns": 1000000, + "samples": 20, + "p50_ns": 54680, + "p99_ns": 61690, + "max_ns": 61690 + } + ] + } + }, + "seed": { + "value": 3, + "drawn": false, + "loss_reproducible": true, + "queue_checks": [], + "not_reproducible_because": [] + }, + "accepted_restrictions": [], + "addresses": [ + { + "node": "client", + "link": "access", + "address": "10.100.0.1" + }, + { + "node": "server", + "link": "access", + "address": "10.100.0.2" + } + ], + "nodes": [ + { + "node": "client", + "role": "host", + "address": "10.200.0.1" + }, + { + "node": "server", + "role": "host", + "address": "10.200.0.2" + } + ], + "routes": [ + { + "node": "client", + "to": "server", + "link": "access", + "next_hop": "server", + "metric": 100, + "weight": 1, + "given": false + }, + { + "node": "server", + "to": "client", + "link": "access", + "next_hop": "client", + "metric": 100, + "weight": 1, + "given": false + } + ], + "timing": { + "duration_ns": 2000000000, + "settle_ns": 301098608, + "interrupted_ns": null + }, + "workloads": [], + "directions": [], + "queues": [], + "events": [], + "notes": [], + "programs": [ + { + "id": "tool", + "node": "server", + "run": [ + "sh", + "-c", + "echo 'metric latency 1.5'; echo hello; sleep 30" + ], + "started_ns": -604207739, + "ready_ns": -502054420, + "ended_ns": 2320533857, + "ended": "stopped", + "exit_code": null, + "exit_signal": 15, + "stdout_file": "program-tool.stdout.log", + "stderr_file": "program-tool.stderr.log", + "output": { + "stdout_bytes": 25, + "stdout_dropped_bytes": 0, + "stderr_bytes": 0, + "stderr_dropped_bytes": 0, + "long_lines": 0 + }, + "lines_dropped": false, + "metrics": [], + "metrics_file": null, + "metric_readings": 0, + "bad_metric_lines": 0, + "bad_metric_examples": [] + } + ], + "captures": [ + { + "link": "access", + "node": "client", + "interface": "l0", + "file": "capture-access-client.pcap", + "packets": 560, + "bytes": 437098, + "first_ns": 171921, + "last_ns": 2475010, + "realtime_minus_monotonic_ns": 1794356634991674252, + "receive_buffer_bytes": 425984, + "interface_frames": 560, + "missing": 0 + }, + { + "link": "access", + "node": "server", + "interface": "l0", + "file": "capture-access-server.pcap", + "packets": 560, + "bytes": 437098, + "first_ns": 166121, + "last_ns": 2470770, + "realtime_minus_monotonic_ns": 1794356634991674472, + "receive_buffer_bytes": 425984, + "interface_frames": 560, + "missing": 0 + } + ], + "tcp": [ + { + "id": "download", + "kind": "tcp-bulk", + "from": "server", + "to": "client", + "from_address": "10.200.0.2", + "to_address": "10.200.0.1:6000", + "congestion": "cubic", + "connections": 2, + "failed_connections": 0, + "start_ns": 0, + "window_ns": 2000000000, + "bytes_sent": 400000, + "bytes_received": 400000, + "goodput": { + "bin_ns": 1000000000, + "bits_per_second": [ + 3200000 + ], + "mean_bits_per_second": 2586109360 + }, + "transfers": [ + { + "connection": 0, + "started_ns": 138891, + "established_ns": 230441, + "done_ns": 1330241, + "bytes_sent": 200000, + "bytes_received": 0, + "ok": true + }, + { + "connection": 1, + "started_ns": 147621, + "established_ns": 231151, + "done_ns": 1376271, + "bytes_sent": 200000, + "bytes_received": 0, + "ok": true + } + ], + "retransmitted_segments": 0, + "rtt": { + "min_ns": 30000, + "mean_ns": 32000, + "p50_ns": 30000, + "p90_ns": 34000, + "p95_ns": 34000, + "p99_ns": 34000, + "p999_ns": 34000, + "max_ns": 34000 + }, + "requests": null, + "first_error": null, + "info_error": null, + "stopped": false, + "client_file": "tcp-0.client.streams", + "server_file": "tcp-0.server.streams", + "recoveries": [] + } + ], + "reproducibility": { + "class": "statistical", + "because": [ + "TCP is driven by feedback (the windows, the timers and the acknowledgements of the kernel of this machine), so the same file gives a different run each time; repeat the run and read the interval" + ] + } +} diff --git a/crates/netscenario-core/src/files.rs b/crates/netscenario-core/src/files.rs index 27e8e07..f6625ab 100644 --- a/crates/netscenario-core/src/files.rs +++ b/crates/netscenario-core/src/files.rs @@ -165,9 +165,7 @@ mod tests { use super::*; fn dir(name: &str) -> PathBuf { - let path = - std::env::temp_dir().join(format!("netscenario-files-{name}-{}", std::process::id())); - let _ = fs::remove_dir_all(&path); + let path = crate::testdir::fresh("files", name); fs::create_dir_all(&path).unwrap(); path } diff --git a/crates/netscenario-core/src/lib.rs b/crates/netscenario-core/src/lib.rs index 28f8219..8b422cd 100644 --- a/crates/netscenario-core/src/lib.rs +++ b/crates/netscenario-core/src/lib.rs @@ -82,6 +82,8 @@ pub mod series; mod special; pub mod stats; mod tcp; +#[cfg(test)] +mod testdir; pub mod timeline; pub mod timers; mod traffic; diff --git a/crates/netscenario-core/src/replay.rs b/crates/netscenario-core/src/replay.rs index a926701..981f2d1 100644 --- a/crates/netscenario-core/src/replay.rs +++ b/crates/netscenario-core/src/replay.rs @@ -541,9 +541,7 @@ mod tests { } fn directory(name: &str) -> PathBuf { - let dir = - std::env::temp_dir().join(format!("netscenario-replay-{name}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); + let dir = crate::testdir::fresh("replay", name); std::fs::create_dir_all(&dir).unwrap(); dir } diff --git a/crates/netscenario-core/src/report/tests.rs b/crates/netscenario-core/src/report/tests.rs index 1d63850..cb10243 100644 --- a/crates/netscenario-core/src/report/tests.rs +++ b/crates/netscenario-core/src/report/tests.rs @@ -148,10 +148,7 @@ pub(super) fn sample_run(dir: &Path) -> RunResult { } fn directory(name: &str) -> PathBuf { - let dir = - std::env::temp_dir().join(format!("netscenario-report-{name}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - dir + crate::testdir::fresh("report", name) } fn load(name: &str, detail: bool) -> Report { diff --git a/crates/netscenario-core/src/run.rs b/crates/netscenario-core/src/run.rs index 1c57f1b..a162266 100644 --- a/crates/netscenario-core/src/run.rs +++ b/crates/netscenario-core/src/run.rs @@ -3244,10 +3244,7 @@ mod tests { } fn dir(name: &str) -> PathBuf { - let dir = - std::env::temp_dir().join(format!("netscenario-run-{name}-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - dir + crate::testdir::fresh("run", name) } fn options(name: &str) -> RunOptions { diff --git a/crates/netscenario-core/src/runset.rs b/crates/netscenario-core/src/runset.rs index 0afa330..95b38fc 100644 --- a/crates/netscenario-core/src/runset.rs +++ b/crates/netscenario-core/src/runset.rs @@ -600,7 +600,7 @@ mod tests { /// Where the set of the test `name` writes. fn written_to(name: &str) -> PathBuf { - std::env::temp_dir().join(format!("netscenario-set-{name}-{}", std::process::id())) + crate::testdir::path("set", name) } /// The same, emptied first. diff --git a/crates/netscenario-core/src/testdir.rs b/crates/netscenario-core/src/testdir.rs new file mode 100644 index 0000000..0c937eb --- /dev/null +++ b/crates/netscenario-core/src/testdir.rs @@ -0,0 +1,90 @@ +//! Directories of the unit tests: named by the test and the process, and removed when the thread of the test ends. +//! +//! The tests used to leave their directories in the temporary directory (thousands after a few weeks of work). A directory +//! is removed when the test thread ends if the test passed; the directories of a test that failed are kept for a look (and +//! `NETSCENARIO_KEEP_TEST_DIRS=1` keeps all of them). Nothing is created here: the test makes the directory, or the code it +//! calls does. (A test that catches a panic on purpose, as a canary of a check, counts as failed and keeps its directory.) + +use std::cell::{Cell, RefCell}; +use std::path::PathBuf; +use std::sync::Once; + +/// The directories this thread asked for. +struct Made(RefCell>); + +impl Drop for Made { + fn drop(&mut self) { + if std::env::var_os("NETSCENARIO_KEEP_TEST_DIRS").is_some() || FAILED.with(Cell::get) { + return; + } + for path in self.0.get_mut().drain(..) { + let _ = std::fs::remove_dir_all(path); + } + } +} + +thread_local! { + static MADE: Made = const { Made(RefCell::new(Vec::new())) }; + /// Whether this thread panicked: the test failed. + static FAILED: Cell = const { Cell::new(false) }; +} + +/// Notes in the thread that panics that it did, before the message is printed. +fn watch_for_failure() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + let before = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + FAILED.with(|failed| failed.set(true)); + before(info); + })); + }); +} + +/// The path of a directory for the test `name` (not made): `netscenario---`. Asking again gives the same path. +pub(crate) fn path(kind: &str, name: &str) -> PathBuf { + let path = + std::env::temp_dir().join(format!("netscenario-{kind}-{name}-{}", std::process::id())); + watch_for_failure(); + MADE.with(|made| { + let mut made = made.0.borrow_mut(); + if !made.contains(&path) { + made.push(path.clone()); + } + }); + path +} + +/// The same, and emptied if it is there. +pub(crate) fn fresh(kind: &str, name: &str) -> PathBuf { + let path = path(kind, name); + let _ = std::fs::remove_dir_all(&path); + path +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A directory is gone when the thread of a passed test ends and stays when the test failed (a canary: with the wrong + /// check, one of the two asserts fails). + #[test] + fn a_directory_goes_with_a_passed_test_and_stays_with_a_failed_one() { + let made = |name: &'static str, fail: bool| { + std::thread::spawn(move || { + let dir = fresh("testdir", name); + std::fs::create_dir_all(&dir).unwrap(); + assert!(!fail, "the test fails"); + dir + }) + .join() + }; + let passed = made("passed", false).unwrap(); + assert!(!passed.exists(), "the directory of a passed test is left"); + let _ = made("failed", true).unwrap_err(); + let kept = + std::env::temp_dir().join(format!("netscenario-testdir-failed-{}", std::process::id())); + assert!(kept.exists(), "the directory of a failed test is gone"); + std::fs::remove_dir_all(kept).unwrap(); + } +} diff --git a/docs/TESTING.md b/docs/TESTING.md index 9e88fac..9a5f5d1 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -68,9 +68,11 @@ A cargo-mutants result is relative to the tests it runs: by default the tests of ## The rules for a new test +- **Clean up.** A unit test gets its directory from `testdir` (core) or `work_dir` (Kernel backend tests): it is removed when the test passed and kept when it failed, so the temporary directory does not fill up and a failure can still be looked at (`NETSCENARIO_KEEP_TEST_DIRS=1` keeps all of them). - **Name the bound.** Bounds are derived, not picked by eye: the model, its deviation and the size of the run are written next to the number. A delay has an exact lower bound and room only upward. - **Test the edge.** For a limit, the value on the edge is accepted and the next one is refused with the reason (`crates/netscenario-model/tests/limits.rs`). - **Look for the words of a refusal**, so that another fault in the same file cannot stand for it. - **Say why a test is slow** if it takes longer than five seconds, in its doc comment, and mark it `once` (Python) or let it run alone (`.config/nextest.toml`) if it measures time. +- **One run for what does not interact.** Tests that only differ in the entry of a table (the delays of each profile) share a run when the limits of a run allow it (8 namespaces); a failure still names the entry. - **A test that does not depend on the version runs once.** A new CI job goes into `needs` and, if code changes need it, into `must` of `ci-ok`. - **Link the test where the ledger asks for it.** A row of [FEATURES.md](../FEATURES.md) that says `done` names the test that proves it, and a faithful mark in [COVERAGE.md](COVERAGE.md) links to it; tests check both.