From f437f8cdc40c19848bb7922813c8e92b6feb44aa Mon Sep 17 00:00:00 2001 From: Leon Krix Date: Wed, 11 Nov 2026 10:11:54 +0100 Subject: [PATCH] test: close the gaps that mutation tests found in the judging code --- CONTRIBUTING.md | 1 + README.md | 1 + crates/netscenario-core/src/capability.rs | 32 + crates/netscenario-core/src/expect.rs | 123 ++++ crates/netscenario-core/src/recovery.rs | 92 +++ crates/netscenario-core/src/stats.rs | 26 + crates/netscenario-model/src/reserved.rs | 7 +- crates/netscenario-model/src/units.rs | 9 + crates/netscenario-model/tests/jitter.rs | 15 + crates/netscenario-model/tests/limits.rs | 690 ++++++++++++++++++ ...napshot@planned-key-link-address.yaml.snap | 12 + ..._its_snapshot@planned-key-matrix.yaml.snap | 12 + ...ts_snapshot@planned-key-mobility.yaml.snap | 12 + docs/TESTING.md | 76 ++ docs/book/src/reference.md | 1 + justfile | 18 + .../invalid/planned-key-link-address.yaml | 16 + scenarios/invalid/planned-key-matrix.yaml | 16 + scenarios/invalid/planned-key-mobility.yaml | 16 + 19 files changed, 1174 insertions(+), 1 deletion(-) create mode 100644 crates/netscenario-model/tests/limits.rs create mode 100644 crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-link-address.yaml.snap create mode 100644 crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-matrix.yaml.snap create mode 100644 crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-mobility.yaml.snap create mode 100644 docs/TESTING.md create mode 100644 scenarios/invalid/planned-key-link-address.yaml create mode 100644 scenarios/invalid/planned-key-matrix.yaml create mode 100644 scenarios/invalid/planned-key-mobility.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 119f562..2d77fba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,6 +47,7 @@ The development toolchain is the current stable Rust (`rust-toolchain.toml` inst | `just fidelity` | Run the fidelity tests alone and rewrite the measured block of [docs/FIDELITY.md](docs/FIDELITY.md) with what your machine measures | | `just test-quick`, `just fidelity-test [part of]` | The tests but the fidelity ones (a minute), and the fidelity tests without touching the page (four and a half minutes; `just fidelity-test 1 2` is the first half). The CI runs them as separate jobs | | `just msrv` | Build with the lowest supported Rust | +| `just coverage`, `just mutants ` | How much of the code the tests run, and whether they would notice a change of it ([docs/TESTING.md](docs/TESTING.md)) | | `just schema`, `just snapshots` | After a change to the scenario model: write the JSON Schema; accept changed error messages (read the diff before you commit it) | | `just test-kernel` (and `just test`) | The tests of `netscenario-backend-kernel` create real network namespaces as an unprivileged user. On a machine that does not allow that (Ubuntu 24.04 until `sudo sysctl kernel.apparmor_restrict_unprivileged_userns=0`) they say `SKIPPED` and pass; on the CI (`CI` is set) they fail instead. The CI runs them a second time as root; `unshare --user --map-root-user just test-kernel` takes the same code path on your machine. Needs `ip`, `tc`, `unshare`, `nsenter` and `ping`. | | `just spikes` | Run the experiments of [spikes/](spikes/) (they need `iproute2`, `python3` and user namespaces) | diff --git a/README.md b/README.md index a045439..82ab1db 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ A Rust core (scenario model, scheduler, backends, traffic generators, probes, st - [docs/PRIOR-ART.md](docs/PRIOR-ART.md): what exists, and where netscenario differs. - [docs/PROFILES.md](docs/PROFILES.md): the draft of the link profiles (3G to LEO satellite) with the source of every number. - [docs/ACCEPTANCE.md](docs/ACCEPTANCE.md): how each criterion of a release is proven, and what the proof does not cover. +- [docs/TESTING.md](docs/TESTING.md): how the tests are organised, where each kind runs, how coverage and mutation tests are measured, and the rules for a new test. - [docs/VERSIONING.md](docs/VERSIONING.md): what a version number promises, and what a release may break. - [docs/RELEASING.md](docs/RELEASING.md): how a release is made, and the one-time setup of the registries. - [docs/DECISIONS.md](docs/DECISIONS.md): the decisions made so far, with reasons (one file each in [docs/decisions/](docs/decisions/)). diff --git a/crates/netscenario-core/src/capability.rs b/crates/netscenario-core/src/capability.rs index 23038b8..6e2cc38 100644 --- a/crates/netscenario-core/src/capability.rs +++ b/crates/netscenario-core/src/capability.rs @@ -462,4 +462,36 @@ mod tests { assert_eq!(Capabilities::new("none").iter().count(), 0); assert_eq!(Capabilities::new("x").backend(), "x"); } + + /// Several refusals are shown one after the other with an empty line between them, and none before the first or after the last. + /// The mutation test found that a display of one refusal was all that was checked. + #[test] + fn several_refusals_are_separated_by_an_empty_line() { + let refusal = |feature, path: &str| Refusal { + feature, + path: path.to_owned(), + reason: "no".to_owned(), + fix: "none".to_owned(), + }; + let error = CapabilityError::new( + "fake", + vec![ + refusal(Feature::FixedDelay, "a"), + refusal(Feature::SeededLoss, "b"), + ], + ); + let text = error.to_string(); + let blocks: Vec<&str> = text.trim_end().split("\n\n").collect(); + assert_eq!(blocks.len(), 2, "{text:?}"); + assert!( + blocks[0].contains("path: a") && !blocks[0].contains("path: b"), + "{text:?}" + ); + assert!( + blocks[1].starts_with("error: ") && blocks[1].contains("path: b"), + "{text:?}" + ); + assert!(text.starts_with("error: ") && text.ends_with('\n')); + assert!(!text.ends_with("\n\n")); + } } diff --git a/crates/netscenario-core/src/expect.rs b/crates/netscenario-core/src/expect.rs index a701291..f80b661 100644 --- a/crates/netscenario-core/src/expect.rs +++ b/crates/netscenario-core/src/expect.rs @@ -560,4 +560,127 @@ mod tests { assert_eq!(show_value(MetricUnit::Number, 0.5), "0.5"); assert_eq!(show_value(MetricUnit::Ratio, f64::NAN), "n/a"); } + + /// The median and the smallest of a set are judged with their own values, not with the mean or the largest. The mutation test + /// found both statistics unchecked: a set of eight losses with an outlier tells them apart. + #[test] + fn the_median_and_the_minimum_of_a_set_are_judged_with_their_own_values() { + let (metrics, confidence) = set_of(&[0.01, 0.01, 0.01, 0.01, 0.02, 0.02, 0.02, 0.09]); + let set = metrics + .iter() + .find(|set| set.name == "w.loss_ratio") + .unwrap(); + assert!(set.min < set.median && set.median < set.mean && set.mean < set.max); + let scenario = scenario( + "expect:\n repetitions: 8\n checks:\n - { name: m, metric: w.loss_ratio, median: \"< 2%\", min: \">= 1%\", mean: \"> 2%\", max: \"<= 9%\" }\n", + ); + let verdict = judge_set(&scenario, &metrics, &[], &confidence, 8).unwrap(); + assert!(verdict.passed, "{verdict:?}"); + let value_of = |statistic| { + verdict + .checks + .iter() + .find(|check| check.statistic == statistic) + .unwrap() + .actual + .unwrap() + }; + assert!((value_of(Statistic::Median) - set.median).abs() < 1e-15); + assert!((value_of(Statistic::Min) - set.min).abs() < 1e-15); + assert!((value_of(Statistic::Mean) - set.mean).abs() < 1e-15); + assert!((value_of(Statistic::Max) - set.max).abs() < 1e-15); + } + + /// Canary: the interval of a set that was made another way than the 95 % interval around the mean is not judged, whichever of the + /// two differs, and the check fails with the reason. + #[test] + fn an_interval_of_another_level_or_around_the_median_is_not_judged_as_the_95_interval() { + let (metrics, confidence) = set_of(&[0.01; 8]); + let scenario = scenario( + "expect:\n repetitions: 8\n checks:\n - { metric: w.loss_ratio, ci95-high: \"< 5%\", ci95-low: \">= 0%\" }\n", + ); + assert!( + judge_set(&scenario, &metrics, &[], &confidence, 8) + .unwrap() + .passed + ); + let other_level = Confidence { + level: 0.9, + ..confidence.clone() + }; + let other_statistic = Confidence { + statistic: IntervalAround::Median, + ..confidence + }; + for (name, other) in [("level", other_level), ("statistic", other_statistic)] { + let verdict = judge_set(&scenario, &metrics, &[], &other, 8).unwrap(); + assert!(!verdict.passed, "{name}"); + assert_eq!(verdict.failures().count(), 2, "{name}: both ends"); + for check in verdict.failures() { + assert!( + check + .note + .as_deref() + .unwrap() + .contains("another interval than the 95 %"), + "{name}: {check:?}" + ); + } + } + } + + /// What a missing metric usually means is said for the metrics where that is known, and not for the others. + #[test] + fn a_missing_metric_says_what_it_usually_means() { + assert_eq!( + why_missing("w.delay.p99"), + "no packet or response arrived, so there is no delay to judge" + ); + assert_eq!( + why_missing("f.rtt.p50"), + "the kernel gave no samples of the round trip time" + ); + assert_eq!(why_missing("f.completion"), "no connection did its work"); + assert_eq!( + why_missing("w.loss_ratio"), + "the workload did not report it" + ); + // `completion` is a name only at the end: a delay of a workload with that name is a delay + assert_eq!( + why_missing("completion.delay.p99"), + "no packet or response arrived, so there is no delay to judge" + ); + } + + /// The edges of the units in which a value is shown: the value that is exactly one unit is shown in that unit. + #[test] + fn a_value_at_the_edge_of_a_unit_is_shown_in_that_unit() { + let shown = [ + (MetricUnit::Seconds, 1.0, "1s"), + (MetricUnit::Seconds, 0.001, "1ms"), + (MetricUnit::Seconds, 0.000_001, "1us"), + (MetricUnit::Seconds, 0.999, "999ms"), + (MetricUnit::BitsPerSecond, 1e9, "1gbit"), + (MetricUnit::BitsPerSecond, 1e6, "1mbit"), + (MetricUnit::BitsPerSecond, 1e3, "1kbit"), + (MetricUnit::BitsPerSecond, 999.0, "999bit"), + (MetricUnit::BitsPerSecond, 999_000.0, "999kbit"), + ]; + for (unit, value, text) in shown { + assert_eq!(show_value(unit, value), text, "{value}"); + } + } + + /// Four significant digits, whatever the size of the number: a large one has no decimals, a small one has as many as it needs. + #[test] + fn a_value_is_shown_with_four_significant_digits() { + assert_eq!(show_value(MetricUnit::Number, 1234.5678), "1235"); + assert_eq!(show_value(MetricUnit::Number, 123.456_78), "123.5"); + assert_eq!(show_value(MetricUnit::Number, 12.3456), "12.35"); + assert_eq!(show_value(MetricUnit::Number, 1.234_56), "1.235"); + assert_eq!(show_value(MetricUnit::Number, 0.123_456), "0.1235"); + assert_eq!(show_value(MetricUnit::Number, 0.012_345_6), "0.01235"); + assert_eq!(show_value(MetricUnit::Number, 100.0), "100"); + assert_eq!(show_value(MetricUnit::Number, -12.3456), "-12.35"); + } } diff --git a/crates/netscenario-core/src/recovery.rs b/crates/netscenario-core/src/recovery.rs index d6753c0..cc7f880 100644 --- a/crates/netscenario-core/src/recovery.rs +++ b/crates/netscenario-core/src/recovery.rs @@ -861,4 +861,96 @@ mod tests { (Some(10e6), Some(0)) ); } + + /// The median of an even number of values is the mean of the two in the middle, of an odd number the one in the middle, and of + /// none there is none. The values are not in order. + #[test] + fn the_median_is_the_middle_value_or_the_mean_of_the_two_in_the_middle() { + assert_eq!(median(vec![4.0, 1.0, 3.0, 2.0]), Some(2.5)); + assert_eq!(median(vec![1.0, 100.0]), Some(50.5)); + assert_eq!(median(vec![9.0, 1.0, 5.0]), Some(5.0)); + assert_eq!(median(vec![7.0]), Some(7.0)); + assert_eq!(median(Vec::new()), None); + } + + /// Two outages on the same link: the end of each belongs to its own start. With the same direction and another event the start + /// of the first would be taken for both. + #[test] + fn the_end_of_an_outage_belongs_to_the_start_of_the_same_event() { + let events = [ + event("outage-start", 0, 2, Some("forward")), + event("outage-end", 0, 3, Some("forward")), + event("outage-start", 1, 5, Some("forward")), + event("outage-end", 1, 6, Some("forward")), + event("outage-start", 2, 8, Some("reverse")), + event("outage-end", 2, 9, Some("reverse")), + ]; + let at = |index: usize| disturbance_of(&events[index], &events) / SECOND; + assert_eq!((at(1), at(3), at(5)), (2, 5, 8)); + // The same event in the other direction is another outage + let both = [ + event("outage-start", 0, 2, Some("forward")), + event("outage-start", 0, 4, Some("reverse")), + event("outage-end", 0, 7, Some("reverse")), + ]; + assert_eq!(disturbance_of(&both[2], &both) / SECOND, 4); + // An end without a start is a disturbance of its own moment + let alone = [event("outage-end", 3, 6, Some("forward"))]; + assert_eq!(disturbance_of(&alone[0], &alone) / SECOND, 6); + } + + /// The level to reach is the share of the baseline, and not the share divided by it: a stream that delivered half of what it sent + /// before the disturbance has recovered when it delivers 45 % again, and does not have to reach 180 %. + #[test] + fn the_level_is_the_fraction_of_a_baseline_that_is_not_one() { + let mut delivered = vec![50; 10]; + delivered.extend([0, 0, 0]); + delivered.extend([50; 7]); + let events = [ + event("outage-start", 0, 10, Some("forward")), + event("outage-end", 0, 13, Some("forward")), + ]; + let found = recoveries( + &events, + &series(&delivered), + SECOND, + &RecoveryOptions::default(), + ); + let r = &found[0]; + assert_eq!(r.baseline, Some(0.5)); + assert!((r.level.unwrap() - 0.45).abs() < 1e-12, "{:?}", r.level); + assert_eq!(time_of(&found), [Some(0)]); + // 40 % is not enough + let mut weak = vec![50; 10]; + weak.extend([0, 0, 0]); + weak.extend([40; 7]); + let found = recoveries(&events, &series(&weak), SECOND, &RecoveryOptions::default()); + assert_eq!(time_of(&found), [None]); + } + + /// A window in which the stream sent nothing in one of the bins is not a recovery, even if the bins around it did well: the + /// link was down in that bin and the sender was refused. + #[test] + fn a_window_with_a_bin_in_which_nothing_was_sent_is_not_a_recovery() { + let mut bins = series(&[100; 20]); + // The link is down from 10 s to 13 s: nothing arrives in the bins of 10 s and 11 s, and in the bin of 12 s the sender is refused + for (index, slot) in bins.iter_mut().enumerate().take(12).skip(10) { + *slot = bin(index as u64, 0, 0); + } + bins[12].by_send.sent = 0; + bins[12].by_send.lost = 0; + bins[12].by_send.received = 0; + bins[12].by_send.delivered_ratio = None; + let events = [event("down", 0, 10, None), event("up", 1, 13, None)]; + let options = RecoveryOptions { + window_bins: 3, + ..RecoveryOptions::default() + }; + let found = recoveries(&events, &bins, SECOND, &options); + assert_eq!(found.len(), 2, "{found:?}"); + // From the `down` at 10 s: the window 12..15 would hold 200 packets sent and 200 delivered, but it has a bin of nothing sent, + // so the first window that counts is the one from 13 s + // From the `up` at 13 s: the baseline is taken before the link went down, and the windows from 13 s on are all good + assert_eq!(time_of(&found), [Some(3), Some(0)]); + } } diff --git a/crates/netscenario-core/src/stats.rs b/crates/netscenario-core/src/stats.rs index e34b395..108c594 100644 --- a/crates/netscenario-core/src/stats.rs +++ b/crates/netscenario-core/src/stats.rs @@ -537,6 +537,32 @@ mod tests { } } + /// The edges of the ranges and of the sequence numbers that a sender used: a run of refused numbers hides its first and its last + /// number, and the number `slots` itself was never sent. (The mutation test found the edges unchecked.) + #[test] + fn the_edges_of_a_refused_run_and_of_the_slots_are_exact() { + let seen: Vec = [2, 3, 4, 5, 9, 10, 11].into_iter().map(at).collect(); + // Ten slots, 3 and 4 refused: 3 and 4 are inside the run, 5 just after it, 9 the last slot, 10 and 11 were never used + assert_eq!(name_damaged_refused(10, &[[3, 4]], &[], &seen), [2, 5, 9]); + // Two runs, a single number as a run, and a run at the very end + assert_eq!( + name_damaged_refused(10, &[[2, 2], [4, 5], [9, 9]], &[], &seen), + [3] + ); + // Without a refusal every number below the slots is named, the edge included + assert_eq!(name_damaged_refused(10, &[], &[], &seen), [2, 3, 4, 5, 9]); + assert_eq!(name_damaged_refused(9, &[], &[], &seen), [2, 3, 4, 5]); + } + + /// A rank beyond the last delay is the last delay (the defence of the function against a permille above 1000). + #[test] + fn the_nearest_rank_never_goes_past_the_last_delay() { + let sorted = [10, 20, 30, 40]; + assert_eq!(nearest_rank(&sorted, 1000), 40); + assert_eq!(nearest_rank(&sorted, 5000), 40); + assert_eq!(nearest_rank(&[7], 999), 7); + } + #[test] fn the_nearest_rank_is_a_measured_delay() { // Four delays: the median is the second one (rank 2), not the mean of the middle two diff --git a/crates/netscenario-model/src/reserved.rs b/crates/netscenario-model/src/reserved.rs index bcea1ec..a7b04bb 100644 --- a/crates/netscenario-model/src/reserved.rs +++ b/crates/netscenario-model/src/reserved.rs @@ -49,7 +49,12 @@ const RESERVED_VALUES: &[(&str, &str, &str)] = &[ ]; fn planned(what: &str, release: &str) -> String { - format!("{what} is planned for release {release} (see FEATURES.md)") + // "release 0.5", or "a later release" when it is not decided which + if release.starts_with("a later") { + format!("{what} is planned for {release} (see FEATURES.md)") + } else { + format!("{what} is planned for release {release} (see FEATURES.md)") + } } /// Finds everything in the file that belongs to a later release. diff --git a/crates/netscenario-model/src/units.rs b/crates/netscenario-model/src/units.rs index 713ad78..8e47e52 100644 --- a/crates/netscenario-model/src/units.rs +++ b/crates/netscenario-model/src/units.rs @@ -751,6 +751,15 @@ mod tests { text.parse() } + /// A name is its text, for the functions that take anything that is a `str`. + #[test] + fn a_name_is_seen_as_its_text() { + let name = Name::new("client-1").unwrap(); + let text: &str = name.as_ref(); + assert_eq!(text, "client-1"); + assert_eq!(name.as_str(), text); + } + #[test] fn durations_are_exact_and_print_in_the_largest_exact_unit() { assert_eq!(duration("40ms").unwrap().as_nanos(), 40_000_000); diff --git a/crates/netscenario-model/tests/jitter.rs b/crates/netscenario-model/tests/jitter.rs index 7d1eb0d..3a36890 100644 --- a/crates/netscenario-model/tests/jitter.rs +++ b/crates/netscenario-model/tests/jitter.rs @@ -272,6 +272,21 @@ fn the_reach_of_a_table_made_from_samples_is_checked_when_it_is_read() { ); } +/// The bound is exact down to the nanosecond: `pareto` reaches 5461 / 8192 of the amount below the mean, so with an amount of +/// 8192 ns a delay of 5461 ns is accepted and 5460 ns is refused (the mutation test found the edge unchecked). +#[test] +fn a_delay_exactly_as_long_as_the_reach_of_the_table_is_accepted() { + let with = |delay: &str| { + scenario( + &format!("{{ delay: {delay}, jitter: {{ amount: 8192ns, distribution: pareto }} }}"), + "", + ) + }; + assert!(with("5461ns").is_ok()); + let refused = with("5460ns").unwrap_err(); + assert!(refused.contains("must be at least 0.67 times"), "{refused}"); +} + const STEP_BASE: &str = "{ delay: 80ms, jitter: { amount: 20ms, distribution: table, table: { file: d.txt } } }"; diff --git a/crates/netscenario-model/tests/limits.rs b/crates/netscenario-model/tests/limits.rs new file mode 100644 index 0000000..0f1c367 --- /dev/null +++ b/crates/netscenario-model/tests/limits.rs @@ -0,0 +1,690 @@ +//! The edges of the rules of `validate.rs`: for each limit the value on the edge is accepted and the next one is refused with the +//! reason. A rule that is tested only with a value far outside its limit passes when its comparison is off by one, which the mutation +//! test of the reader showed for 56 of them. A refusal is looked for by its words, so that another fault of the generated scenario +//! cannot stand for it. + +// A test file: a failing unwrap is a failing test +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use std::fmt::Write as _; + +use netscenario_model::{ + MAX_BINS, MAX_BURST_PACKETS, MAX_LINKS, MAX_NODES, MAX_PACKETS_PER_SECOND, MAX_QUEUE_BYTES, + MAX_QUEUE_PACKETS, MAX_ROUTES, Scenario, +}; + +/// What the reader says about a scenario: empty if it accepts it. +fn said(yaml: &str) -> String { + Scenario::from_yaml(yaml, "t.yaml") + .err() + .map(|error| error.to_string()) + .unwrap_or_default() +} + +/// A scenario of two nodes and a link; `link` are more lines of the link (6 spaces deep), `workloads` the lines of the workloads +/// (2 spaces deep, each one a line that starts with `- `), `rest` more top-level keys, and `duration` how long it runs. +fn scenario(duration: &str, link: &str, workloads: &str, rest: &str) -> String { + format!( + "version: 0\nname: t\nduration: {duration}\ntopology:\n nodes: [a, b]\n links:\n - id: l\n between: [a, b]\n{link}workloads:\n{workloads}{rest}" + ) +} + +const STREAM: &str = + " - { id: w, kind: udp-cbr, from: a, to: b, rate: 1mbit, packet-size: 1000b }\n"; + +/// A refusal that has to be there at one value and has to be absent at the value just before. +fn edge(words: &str, accepted: &str, refused: &str) { + let before = said(accepted); + assert!( + !before.contains(words), + "the value on the edge was refused with '{words}':\n{before}\n{accepted}" + ); + let after = said(refused); + assert!( + after.contains(words), + "the value after the edge was not refused with '{words}':\n{after}\n{refused}" + ); +} + +#[test] +fn the_number_of_nodes_has_an_edge() { + let nodes = |count: usize| { + let names: Vec = (0..count).map(|i| format!("n{i}")).collect(); + format!( + "version: 0\nname: t\nduration: 10s\ntopology:\n nodes: [{}]\n links:\n - {{ id: l, between: [n0, n1] }}\nworkloads:\n - {{ id: w, kind: udp-cbr, from: n0, to: n1, rate: 1mbit, packet-size: 1000b }}\n", + names.join(", ") + ) + }; + edge( + &format!("at most {MAX_NODES} nodes"), + &nodes(MAX_NODES), + &nodes(MAX_NODES + 1), + ); +} + +#[test] +fn the_number_of_links_has_an_edge() { + let links = |count: usize| { + let list: Vec = (0..count) + .map(|i| format!(" - {{ id: l{i}, between: [a, b] }}\n")) + .collect(); + format!( + "version: 0\nname: t\nduration: 10s\ntopology:\n nodes: [a, b]\n links:\n{}workloads:\n{STREAM}", + list.concat() + ) + }; + edge( + &format!("at most {MAX_LINKS} links"), + &links(MAX_LINKS), + &links(MAX_LINKS + 1), + ); +} + +/// It takes about 12 seconds (two scenarios of more than 8,000 routes, which the reader checks one against the other): the edge of +/// the limit cannot be shown with fewer routes, and a limit that is off by one is what the mutation test found here. +#[test] +fn the_number_of_routes_has_an_edge() { + let routes = |count: usize| { + let list: Vec = (0..count) + .map(|_| " - { node: a, to: b, via: l }\n".to_owned()) + .collect(); + scenario("10s", "", STREAM, "").replace( + "workloads:", + &format!(" routes:\n{}workloads:", list.concat()), + ) + }; + edge( + &format!("at most {MAX_ROUTES} routes"), + &routes(MAX_ROUTES), + &routes(MAX_ROUTES + 1), + ); +} + +#[test] +fn a_request_every_millisecond_is_the_fastest_there_is() { + let requests = |interval: &str| { + scenario( + "10s", + "", + &format!( + " - {{ id: api, kind: request-response, from: a, to: b, request: 100b, response: 1kb, interval: {interval} }}\n" + ), + "", + ) + }; + edge( + "between requests is below", + &requests("1ms"), + &requests("999us"), + ); +} + +#[test] +fn a_hash_is_64_hexadecimal_digits_and_nothing_else() { + let hash = |text: &str| { + scenario( + "10s", + "", + &format!( + " - {{ id: call, kind: trace-replay, from: a, to: b, trace: {{ file: call.csv, sha256: \"{text}\" }} }}\n" + ), + "", + ) + }; + let good = "ab".repeat(32); + let words = "`sha256` is 64 hexadecimal digits"; + edge(words, &hash(&good), &hash(&good[..63])); + // 64 characters that are not all digits of hexadecimal, and few characters that are all + assert!(said(&hash(&"g".repeat(64))).contains(words)); + assert!(said(&hash("abc")).contains(words)); + assert!(said(&hash(&format!("{good}0"))).contains(words)); + // The same for the hash of the table of a jitter + let table = |text: &str| { + scenario( + "10s", + &format!( + " both:\n delay: 40ms\n jitter: {{ amount: 9ms, distribution: table, table: {{ file: samples.txt, sha256: \"{text}\" }} }}\n" + ), + STREAM, + "", + ) + }; + edge(words, &table(&good), &table(&good[..63])); + assert!(said(&table(&"g".repeat(64))).contains(words)); + assert!(said(&table("abc")).contains(words)); +} + +#[test] +fn the_packets_per_second_of_a_pattern_have_an_edge() { + // 100,000 packets of 1000 bytes a second are 800 mbit/s + let at = |rate: &str, kind: &str| { + scenario( + "10s", + "", + &format!( + " - {{ id: w, kind: {kind}, from: a, to: b, rate: {rate}, packet-size: 1000b }}\n" + ), + "", + ) + }; + assert_eq!(MAX_PACKETS_PER_SECOND * 1000 * 8, 800_000_000); + for kind in ["udp-cbr", "udp-poisson"] { + edge( + "packets per second", + &at("800000000bit", kind), + &at("800000001bit", kind), + ); + } + // A burst: ten microseconds between two packets is 100,000 a second + let burst = |packets: u32, gap: &str| { + scenario( + "10s", + "", + &format!( + " - {{ id: w, kind: udp-bursty, from: a, to: b, packet-size: 1000b, burst-packets: {packets}, gap: {gap}, pause: 100ms }}\n" + ), + "", + ) + }; + edge( + "between the packets of a burst", + &burst(10, "10us"), + &burst(10, "9us"), + ); + // One packet has no gap to another + assert!(!said(&burst(1, "1us")).contains("between the packets of a burst")); + edge( + "outside 1 to", + &burst(MAX_BURST_PACKETS, "10us"), + &burst(MAX_BURST_PACKETS + 1, "10us"), + ); +} + +#[test] +fn a_run_may_have_a_million_bins() { + let bins = + |duration: &str| scenario(duration, "", STREAM, "measure:\n series: { bin: 1ms }\n"); + edge( + &format!("more than {MAX_BINS} bins"), + &bins("1000s"), + &bins("1000001ms"), + ); +} + +#[test] +fn the_limits_of_a_queue_have_edges() { + let queue = |keys: &str| { + scenario( + "10s", + &format!(" both: {{ rate: 10mbit, queue: {{ {keys} }} }}\n"), + STREAM, + "", + ) + }; + edge( + "outside 1 byte to 1gib", + &queue(&format!("discipline: fifo, limit: {MAX_QUEUE_BYTES}b")), + &queue(&format!( + "discipline: fifo, limit: {}b", + MAX_QUEUE_BYTES + 1 + )), + ); + assert!(!said(&queue("discipline: fifo, limit: 1b")).contains("outside 1 byte")); + edge( + "more than 1000000 packets", + &queue(&format!("discipline: codel, limit: {MAX_QUEUE_PACKETS}p")), + &queue(&format!( + "discipline: codel, limit: {}p", + MAX_QUEUE_PACKETS + 1 + )), + ); + edge( + "is an amount of data", + &queue("discipline: fifo, limit: 100kb"), + &queue("discipline: codel, limit: 100kb"), + ); + // The times of a discipline: more than zero, at most ten seconds + for key in ["target", "interval"] { + edge( + "is outside 1 microsecond to 10 seconds", + &queue(&format!("discipline: codel, {key}: 10s")), + &queue(&format!("discipline: codel, {key}: 10001ms")), + ); + assert!( + said(&queue(&format!("discipline: codel, {key}: 1us"))).is_empty() + || !said(&queue(&format!("discipline: codel, {key}: 1us"))) + .contains("outside 1 microsecond") + ); + } + edge( + "is outside 1 microsecond to 10 seconds", + &queue("discipline: pie, t-update: 10s"), + &queue("discipline: pie, t-update: 10001ms"), + ); + // `flows` and `quantum` of fq-codel + edge( + "outside 1 to 65536", + &queue("discipline: fq-codel, flows: 65536"), + &queue("discipline: fq-codel, flows: 65537"), + ); + assert!(said(&queue("discipline: fq-codel, flows: 0")).contains("outside 1 to 65536")); + edge( + "outside 1 byte to 1mib", + &queue("discipline: fq-codel, quantum: 1mib"), + &queue("discipline: fq-codel, quantum: 1048577b"), + ); + // A key of another discipline names the discipline it belongs to + let foreign = said(&queue("discipline: fifo, t-update: 10ms")); + assert!(foreign.contains("belongs to `pie`"), "{foreign}"); + let foreign = said(&queue("discipline: codel, flows: 4")); + assert!( + foreign.contains("`flows` and `quantum` belong to `fq-codel`"), + "{foreign}" + ); + let foreign = said(&queue("discipline: fifo, interval: 10ms")); + assert!( + foreign.contains("`interval` belongs to `codel` and `fq-codel`"), + "{foreign}" + ); +} + +#[test] +fn the_chances_out_of_one_state_add_up_to_at_most_a_hundred_percent() { + let loss = |p13: &str, p14: &str, p31: &str, p32: &str| { + scenario( + "10s", + &format!( + " both: {{ delay: 10ms, loss: {{ model: markov4, p13: {p13}, p31: {p31}, p32: {p32}, p23: 20%, p14: {p14} }} }}\n" + ), + STREAM, + "", + ) + }; + edge( + "add up to more than 100%", + &loss("60%", "40%", "10%", "10%"), + &loss("60%", "40.000001%", "10%", "10%"), + ); + edge( + "add up to more than 100%", + &loss("2%", "1%", "50%", "50%"), + &loss("2%", "1%", "50%", "50.000001%"), + ); +} + +#[test] +fn a_jitter_may_reach_the_delay_and_not_go_below_it() { + let jitter = |amount: &str, distribution: &str, delay: &str| { + scenario( + "10s", + &format!( + " both:\n delay: {delay}\n jitter: {{ amount: {amount}, distribution: {distribution} }}\n" + ), + STREAM, + "", + ) + }; + // Uniform: up to the delay + edge( + "some delays would be below zero", + &jitter("10ms", "uniform", "10ms"), + &jitter("10001us", "uniform", "10ms"), + ); + // Normal: the delay is at least four times the jitter + edge( + "some delays would be below zero", + &jitter("10ms", "normal", "40ms"), + &jitter("10001us", "normal", "40ms"), + ); +} + +// ---- outages ---- + +#[test] +fn two_outages_overlap_only_if_their_directions_and_their_times_do() { + let outages = |first: &str, second: &str| { + scenario( + "10s", + "", + STREAM, + &format!( + "timeline:\n - {{ at: 1s, link: l, outage: 2s{first} }}\n - {{ at: {second} }}\n" + ), + ) + }; + let overlap = "overlaps the outage of timeline[0]"; + // Directions: the same direction, or `both` with any, overlap; forward and reverse do not + let direction = |first: &str, second: &str| { + said(&outages( + &format!(", direction: {first}"), + &format!("2s, link: l, outage: 2s, direction: {second}"), + )) + .contains(overlap) + }; + assert!(direction("forward", "forward")); + assert!(direction("reverse", "reverse")); + assert!(direction("both", "forward")); + assert!(direction("both", "reverse")); + assert!(direction("forward", "both")); + assert!(direction("reverse", "both")); + assert!(direction("both", "both")); + assert!(!direction("forward", "reverse")); + assert!(!direction("reverse", "forward")); + // No direction written is both + assert!(said(&outages("", "2s, link: l, outage: 2s, direction: forward")).contains(overlap)); + // Times: an outage that begins when the other ends does not overlap it, one that begins a nanosecond before does + let times = |second: &str| { + said(&outages("", &format!("{second}, link: l, outage: 1s"))).contains(overlap) + }; + assert!(!times("3s"), "an outage may begin when the other ends"); + assert!(times("2999999999ns")); + assert!(times("1s")); + // The outage that begins before the other and ends when it begins + let before = said(&scenario( + "10s", + "", + STREAM, + "timeline:\n - { at: 1s, link: l, outage: 2s }\n - { at: 0s, link: l, outage: 1s }\n", + )); + assert!(!before.contains(overlap), "{before}"); + let before = said(&scenario( + "10s", + "", + STREAM, + "timeline:\n - { at: 1s, link: l, outage: 2s }\n - { at: 0s, link: l, outage: 1000000001ns }\n", + )); + assert!( + before.contains("overlaps the outage of timeline["), + "{before}" + ); +} + +#[test] +fn an_outage_may_end_with_the_run_and_not_after_it() { + let outage = |length: &str| { + scenario( + "10s", + "", + STREAM, + &format!("timeline:\n - {{ at: 8s, link: l, outage: {length} }}\n"), + ) + }; + edge( + "after the end of the measured phase", + &outage("2s"), + &outage("2000000001ns"), + ); + assert!(said(&outage("0s")).contains("must last longer than zero")); +} + +// ---- a queue waits for a rate that a step of the timeline may bring ---- + +/// A queue of a direction needs a rate to wait for, and a step of the timeline may set that rate: for `forward` a step on `forward` +/// or on `both`, for `reverse` the same, and for the queue written on `both` a step on `both` or the two directions in one step. +#[test] +fn a_queue_without_a_rate_waits_for_the_step_that_brings_one() { + let needs = "holds the packets that wait for the rate"; + let case = |side: &str, steps: &[&str]| { + let timeline: String = steps.iter().fold(String::new(), |mut text, step| { + let _ = writeln!(text, " - {{ at: 1s, link: l, set: {{ {step} }} }}"); + text + }); + let rest = if steps.is_empty() { + String::new() + } else { + format!("timeline:\n{timeline}") + }; + said(&scenario( + "10s", + &format!(" {side}: {{ queue: {{ limit: 10p }} }}\n"), + STREAM, + &rest, + )) + .contains(needs) + }; + let rate = "rate: 1mbit"; + for side in ["forward", "reverse"] { + let other = if side == "forward" { + "reverse" + } else { + "forward" + }; + assert!(case(side, &[]), "{side}: no rate"); + assert!( + !case(side, &[&format!("{side}: {{ {rate} }}")]), + "{side}: its own step" + ); + assert!( + !case(side, &[&format!("both: {{ {rate} }}")]), + "{side}: both" + ); + assert!( + case(side, &[&format!("{other}: {{ {rate} }}")]), + "{side}: the other direction" + ); + } + // A queue on both directions needs a rate in both + assert!(case("both", &[])); + assert!(!case("both", &[&format!("both: {{ {rate} }}")])); + assert!(case("both", &[&format!("forward: {{ {rate} }}")])); + assert!(case("both", &[&format!("reverse: {{ {rate} }}")])); + assert!(!case( + "both", + &[&format!("forward: {{ {rate} }}, reverse: {{ {rate} }}")] + )); + // Two steps that each set one direction are not one step that sets both: the rule looks at one step at a time + assert!(case( + "both", + &[ + &format!("forward: {{ {rate} }}"), + &format!("reverse: {{ {rate} }}") + ] + )); +} + +// ---- paths, packets and the MTU ---- + +/// Two islands of two nodes: a workload from one to the other has no route, whatever it sends. +#[test] +fn a_workload_to_a_node_that_cannot_be_reached_is_refused_with_the_missing_route() { + let to_c = |workload: &str| { + format!( + "version: 0\nname: t\nduration: 10s\ntopology:\n nodes: [a, b, c, d]\n links:\n - {{ id: l, between: [a, b] }}\n - {{ id: m, between: [c, d] }}\nworkloads:\n - {workload}\n" + ) + }; + for workload in [ + "{ id: w, kind: tcp-bulk, from: a, to: c, bytes: 1mb }", + "{ id: w, kind: request-response, from: a, to: c, request: 100b, response: 1kb }", + "{ id: w, kind: udp-cbr, from: a, to: c, rate: 1mbit, packet-size: 1000b }", + ] { + let text = said(&to_c(workload)); + assert!( + text.contains("'a' has no route to 'c'"), + "{workload}: {text}" + ); + } + // To a node it reaches, it has one + let reached = to_c("{ id: w, kind: tcp-bulk, from: a, to: b, bytes: 1mb }"); + assert!( + !said(&reached).contains("has no route"), + "{}", + said(&reached) + ); +} + +/// A packet that the first link cuts into fragments must be carried whole by the later links: a router that would cut it again is +/// refused. A sender that must not cut (`dont-fragment`) refuses a packet that does not fit the first link, so nothing reaches a +/// router; but a packet that fits the first link exactly and not the second is cut by the router, `dont-fragment` or not. +#[test] +fn a_packet_that_a_router_would_cut_is_refused_unless_the_sender_refuses_it_first() { + let cut = "would be cut by a router"; + let packet = |size: u32, dont_fragment: bool| { + format!( + "version: 0\nname: t\nduration: 10s\ntopology:\n nodes: [client, {{ id: edge, role: router }}, server]\n links:\n - {{ id: access, between: [client, edge], mtu: 1000 }}\n - {{ id: core, between: [edge, server], mtu: 600 }}\nworkloads:\n - {{ id: w, kind: udp-cbr, from: client, to: server, rate: 1mbit, packet-size: {size}b, dont-fragment: {dont_fragment} }}\n" + ) + }; + // 1400 bytes do not fit the first link (1000): the sender cuts them to 1000-byte frames, which the second link (600) cannot carry + assert!(said(&packet(1400, false)).contains(cut)); + // ... unless the sender must not cut: it refuses the packet, and no router sees it + assert!(!said(&packet(1400, true)).contains(cut)); + // 972 bytes of payload are exactly 1000 bytes of packet: they fit the first link and not the second + assert!( + said(&packet(972, true)).contains(cut), + "a packet that fits the first link exactly" + ); + assert!(said(&packet(972, false)).contains(cut)); + // One byte more does not fit the first link any more + assert!(!said(&packet(973, true)).contains(cut)); + // A packet that fits both is no problem + assert!( + said(&packet(500, true)).is_empty(), + "{}", + said(&packet(500, true)) + ); +} + +/// The smallest payload is the header of the agent, and with `udp-checksum: false` it is four bytes more, for the CRC-32. +#[test] +fn the_smallest_packet_of_a_stream_depends_on_the_checksum() { + let small = |size: u32, checksum: bool| { + scenario( + "10s", + "", + &format!( + " - {{ id: w, kind: udp-cbr, from: a, to: b, rate: 1mbit, packet-size: {size}b, udp-checksum: {checksum} }}\n" + ), + "", + ) + }; + edge("is outside 16 to", &small(16, true), &small(15, true)); + edge( + "is too small for `udp-checksum: false`", + &small(20, false), + &small(19, false), + ); + // With the checksum on 19 bytes are fine + assert_eq!(said(&small(19, true)), ""); +} + +/// The same for the packets of a trace in the scenario (the form that a file makes when it is read, and that a replay reads): the +/// smallest of them is judged like a size. A CSV file with a packet below the smallest is refused when it is read; here the packets +/// are in the scenario already, and the stream is told not to use the checksum afterwards. +#[test] +fn the_smallest_packet_of_a_trace_in_the_scenario_is_judged_like_a_size() { + let canonical = |smallest: u32| { + let text = "version: 0\nname: t\nduration: 10s\ntopology:\n nodes: [a, b]\n links:\n - id: l\n between: [a, b]\nworkloads:\n - { id: call, kind: trace-replay, from: a, to: b, trace: { file: call.csv } }\n"; + let mut parsed = Scenario::from_yaml(text, "t.yaml").unwrap(); + let file = format!("time,size\n0ms,{smallest}b\n20ms,200b\n40ms,300b\n").into_bytes(); + let found = parsed.resolve_files(|_| Ok(file.clone())); + assert!(found.is_empty(), "{found:?}"); + parsed.to_yaml().unwrap() + }; + let without_checksum = + |yaml: String| yaml.replacen(" to: b\n", " to: b\n udp-checksum: false\n", 1); + let below = "below the smallest payload"; + // With the checksum on the smallest payload is 16 bytes, and 16 is allowed + assert_eq!(said(&canonical(16)), ""); + assert_eq!(said(&canonical(17)), ""); + // Without it 20, and 20 is allowed; 19 is not, and 16 is not + assert_eq!(said(&without_checksum(canonical(20))), ""); + let message = said(&without_checksum(canonical(19))); + assert!( + message.contains( + "a packet of 19 bytes, below the smallest payload of 20 bytes for `udp-checksum: false`" + ), + "{message}" + ); + assert!(said(&without_checksum(canonical(16))).contains(below)); +} + +// ---- what a step of the timeline makes of a link ---- + +/// A step is judged on the state it makes, for each direction: a step on `reverse` that makes the reverse direction wrong is a fault of +/// the step even if the forward direction is fine, and a fault the link had before the step is not blamed on the step. +#[test] +fn a_step_is_blamed_for_the_direction_it_breaks_and_not_for_a_fault_that_was_there() { + let after = "after this step"; + let with_step = |link: &str, step: &str| { + said(&scenario( + "10s", + link, + STREAM, + &format!("timeline:\n - {{ at: 1s, link: l, set: {{ {step} }} }}\n"), + )) + }; + let link = " both: { delay: 10ms }\n"; + // The reverse direction gets a jitter that is longer than its delay; the forward direction is untouched and fine + let jitter = "jitter: { amount: 20ms, distribution: uniform }"; + let message = with_step(link, &format!("reverse: {{ {jitter} }}")); + assert!( + message.contains("longer than the delay") && message.contains(after), + "{message}" + ); + // The same on the forward direction, and on both (one problem, not two) + assert!(with_step(link, &format!("forward: {{ {jitter} }}")).contains(after)); + let both = with_step(link, &format!("both: {{ {jitter} }}")); + assert_eq!(both.matches("longer than the delay").count(), 1, "{both}"); + // A link that is wrong from the start: the step that only sets a rate is not blamed + let wrong = " both:\n delay: 10ms\n jitter: { amount: 20ms, distribution: uniform }\n"; + let message = with_step(wrong, "both: { rate: 1mbit }"); + assert!( + message.contains("longer than the delay"), + "the link itself is wrong: {message}" + ); + assert!( + !message.contains(after), + "the step is not blamed: {message}" + ); +} + +/// An event with both `until` and `count` is wrong as it is; it is not made into its copies on top of that, which would add the +/// faults of the copies (here outages that overlap) to the one fault that matters. +#[test] +fn an_event_with_until_and_count_is_not_expanded_into_copies() { + let message = said(&scenario( + "20s", + "", + STREAM, + "timeline:\n - { at: 1s, link: l, outage: 4s, every: 3s, until: 9s, count: 3 }\n", + )); + assert!( + message.contains("until") && message.contains("count"), + "{message}" + ); + assert!(!message.contains("overlaps the outage"), "{message}"); + // The same event with one of the two keys is expanded, and its copies overlap + let copies = said(&scenario( + "20s", + "", + STREAM, + "timeline:\n - { at: 1s, link: l, outage: 4s, every: 3s, count: 3 }\n", + )); + assert!(copies.contains("overlaps the outage"), "{copies}"); +} + +/// A jitter from a table without a table says that the table is missing, and nothing about delays below zero, which it cannot work out. +#[test] +fn a_jitter_from_a_table_that_is_not_there_is_not_judged_against_the_delay() { + let message = said(&scenario( + "10s", + " both:\n delay: 1ms\n jitter: { amount: 9ms, distribution: table }\n", + STREAM, + "", + )); + assert!(message.contains("table"), "{message}"); + assert!( + !message.contains("some delays would be below zero"), + "{message}" + ); +} + +/// The limits are the ones that the documents state; the tests above use the names, and these are the numbers. +#[test] +fn the_limits_are_the_documented_numbers() { + assert_eq!((MAX_NODES, MAX_LINKS, MAX_ROUTES), (256, 1024, 8192)); + assert_eq!(MAX_PACKETS_PER_SECOND, 100_000); + assert_eq!(MAX_BURST_PACKETS, 100_000); + assert_eq!(MAX_QUEUE_PACKETS, 1_000_000); + assert_eq!(MAX_QUEUE_BYTES, 1_073_741_824); + assert_eq!(MAX_BINS, 1_000_000); +} diff --git a/crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-link-address.yaml.snap b/crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-link-address.yaml.snap new file mode 100644 index 0000000..39413b3 --- /dev/null +++ b/crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-link-address.yaml.snap @@ -0,0 +1,12 @@ +--- +source: crates/netscenario-model/tests/corpus.rs +expression: error.render() +input_file: scenarios/invalid/planned-key-link-address.yaml +--- +error: `address` is planned for a later release (see FEATURES.md) + --> planned-key-link-address.yaml:9:7 + | +8 | between: [client, server] +9 | address: 10.0.0.0/30 + | ^^^^^^^ + = path: topology.links[0].address diff --git a/crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-matrix.yaml.snap b/crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-matrix.yaml.snap new file mode 100644 index 0000000..c660d03 --- /dev/null +++ b/crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-matrix.yaml.snap @@ -0,0 +1,12 @@ +--- +source: crates/netscenario-model/tests/corpus.rs +expression: error.render() +input_file: scenarios/invalid/planned-key-matrix.yaml +--- +error: `matrix` is planned for release 0.5 (see FEATURES.md) + --> planned-key-matrix.yaml:16:1 + | +15 | packet-size: 1200b +16 | matrix: {} + | ^^^^^^ + = path: matrix diff --git a/crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-mobility.yaml.snap b/crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-mobility.yaml.snap new file mode 100644 index 0000000..2497cdc --- /dev/null +++ b/crates/netscenario-model/tests/snapshots/corpus__every_faulty_scenario_is_refused_with_the_message_in_its_snapshot@planned-key-mobility.yaml.snap @@ -0,0 +1,12 @@ +--- +source: crates/netscenario-model/tests/corpus.rs +expression: error.render() +input_file: scenarios/invalid/planned-key-mobility.yaml +--- +error: `mobility` is planned for release 0.6 (see FEATURES.md) + --> planned-key-mobility.yaml:16:1 + | +15 | packet-size: 1200b +16 | mobility: {} + | ^^^^^^^^ + = path: mobility diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 0000000..9e88fac --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,76 @@ +# How the tests are organised + +What is tested, where each kind of test runs, how to measure how good the tests are, and what the rules for a new test are. The commands are in [CONTRIBUTING.md](../CONTRIBUTING.md); the decisions behind the CI are in the comments of [`ci.yml`](../.github/workflows/ci.yml). + +## The kinds of test + +| Kind | Where | What it proves | +| --- | --- | --- | +| Unit tests | in the `src` of every crate (`mod tests`) | The logic of one module: the statistics, the judging of `expect`, the recovery, the capability checks, the readers | +| Tests of a crate | `crates/*/tests/` | The model and the core without a network: the scenario file, its errors (`scenarios/invalid/`, one fault per file with its message in a snapshot), the limits (`limits.rs`), the formats | +| Tests in namespaces | `crates/netscenario-backend-kernel/tests/` | A real run: the network does what the scenario says, the agent measures it, the result says so | +| Fidelity tests | `crates/netscenario-backend-kernel/tests/fidelity.rs` | The network against its configuration to within bounds that are derived and written next to them ([FIDELITY.md](FIDELITY.md)); each runs alone on its machine | +| Python tests | `python/*/tests/`, `examples/` | The Python layer and the plugin; the acceptance of the release (TCP fairness, the formula of Mathis, two paths, the queues) | +| Tests of the documents | `python/netscenario/tests/test_ledger.py`, `test_book.py`, `crates/netscenario-backend-kernel/tests/coverage.rs` | The ledger names a test for every `done` row, the matrix of coverage agrees with the code, the scenarios and the Python of the guide run | +| Fuzzing | `fuzz/` (weekly, [`fuzz.yml`](../.github/workflows/fuzz.yml)) | The readers of input that is not trusted do not panic | + +A test that judges something (a tolerance, a statistic, a capability check, a limit) has a canary: a test that shows it can fail, with a value on the wrong side of the bound. + +## Where a test runs + +The CI runs what a change needs (`.github/scripts/ci_changes.py`). A test that does not depend on the version of Python or of Rust runs once. + +| Job | Runs | Why there | +| --- | --- | --- | +| Tests (2 images, 2 parts) | the quick tests of all crates, on the current stable Rust | Behaviour of the kernel differs by image | +| Tests as root | the Kernel backend and agent tests again with the rights of root | The two ways of running a scenario, D-022; not cut down to chosen tests | +| Lowest supported Rust (1.89) | everything compiles; the tests of the model and the core | The compiler is what the version changes | +| Fidelity (2 images, 2 parts) | the fidelity tests | They measure the network, which does not depend on the Rust version | +| Python 3.11 to 3.14 | `just py-test-versions`: all but the tests marked `once` | Threads, `asyncio` and typing differ by version | +| Acceptance (2 jobs) | the tests marked `once`: TCP acceptance, the run of the guide, the release scripts | Long, statistical, and the same on every version | + +## How good are the tests? + +Two measures, and neither is a goal in itself: coverage says what the tests never run, a mutation test says whether what they run is checked. + +### Coverage + +```bash +just coverage # Rust: lines of the crates as the CI runs the tests +uv run --with coverage coverage run --branch -m pytest python -q && uv run --with coverage coverage report -m # Python +``` + +The tool counts the `#[cfg(test)]` modules in `src` as code that is covered, which flatters it; leave those lines out when you judge a crate. Measured on 2026-11-11 without the tests marked `once` and without the fidelity tests: **90.7 % of the production lines of the Rust crates** (agent 93.5, Kernel backend 91.6, core 91.9, model 90.4, command line 82.6), and **95 % of the Python layer** with branches. The gaps are mostly error branches (a failing read, a bad option at the command line), which are tested where a user meets them and not everywhere. Not measured: the code that runs only with the rights of root, and the Python extension (`netscenario-py`), which the Python tests cover. + +### Mutation tests + +A mutation test changes the code in small ways (`<` to `<=`, a result replaced by an empty string, a `match` arm deleted) and runs the tests of the crate on each change. A change that no test notices is a survivor: either a test is missing, or the change does not change what the code does (an equivalent mutant). + +```bash +cargo install cargo-mutants --locked +just mutants netscenario-core crates/netscenario-core/src/stats.rs # the survivors are in missed.txt +``` + +It is slow, so it is run on the files that judge, one at a time, by hand and not in the CI: `stats`, `expect`, `recovery` and `capability` of the core, and `validate`, `units` and `reserved` of the model (about 700 mutants, an hour and a half with two workers). The first run found 81 survivors; each one got a test or an explanation. The second run left 8, all explained below (the timeouts of two loop mutants in `stats.rs` are not survivors: the mutant never ends). + +**Do not run it with the default settings on a machine with little memory.** Each worker builds its own copy of the tree, by default in `/tmp`, which is a `tmpfs` (memory) on many systems; three workers and their compilers used all memory of the development machine. `just mutants` puts the copies on the disk and keeps the build and the tests to a few threads; for a long run start it under `systemd-run --user` with `-p MemoryHigh=12G -p MemoryMax=16G -p MemorySwapMax=0 -p CPUQuota=800%`, which also lets it survive the end of a session. + +**Equivalent mutants**, which no test can notice and which are left alone: + +- `stats.rs`, the division by the width of a throughput bin: the bin is fixed at one second, so a division and a multiplication give the same number. +- `expect.rs`, the tolerance of `1e-12` in the comparison of the level of an interval: the comparison is made on levels that are 0.95 or differ by far more. +- `expect.rs`, the arm for `value` in the judging of a set: a file that asks a set for `value` is refused when it is read, so the arm is a defence that no scenario reaches. +- `validate.rs`, the check that an outage starts before another one ends: the steps are walked in time order, so the other outage never starts later than this one and the comparison is true whatever the operator. +- `validate.rs`, the arm for a `table` jitter without a table: the reach of a missing table is zero, which accepts every delay, as the arm does. +- `validate.rs`, the `&&` in the filter that decides whether the path of a workload is looked up: for a workload with an unknown node or the same node at both ends the lookup fails and gives no path, as the filter does. + +A cargo-mutants result is relative to the tests it runs: by default the tests of the crate of the file. A survivor may be caught by the tests of another crate (the command line, Python), which is a weaker test than one next to the code. + +## The rules for a new test + +- **Name the bound.** Bounds are derived, not picked by eye: the model, its deviation and the size of the run are written next to the number. A delay has an exact lower bound and room only upward. +- **Test the edge.** For a limit, the value on the edge is accepted and the next one is refused with the reason (`crates/netscenario-model/tests/limits.rs`). +- **Look for the words of a refusal**, so that another fault in the same file cannot stand for it. +- **Say why a test is slow** if it takes longer than five seconds, in its doc comment, and mark it `once` (Python) or let it run alone (`.config/nextest.toml`) if it measures time. +- **A test that does not depend on the version runs once.** A new CI job goes into `needs` and, if code changes need it, into `must` of `ci-ok`. +- **Link the test where the ledger asks for it.** A row of [FEATURES.md](../FEATURES.md) that says `done` names the test that proves it, and a faithful mark in [COVERAGE.md](COVERAGE.md) links to it; tests check both. diff --git a/docs/book/src/reference.md b/docs/book/src/reference.md index 95a4e29..01102f4 100644 --- a/docs/book/src/reference.md +++ b/docs/book/src/reference.md @@ -30,6 +30,7 @@ The pages of this book explain and show. The exact definition of every key, fiel | --- | --- | | [Architecture](https://github.com/leonkrix/netscenario/blob/main/docs/ARCHITECTURE.md) | The crates, the backend and the agents. | | [Coverage](https://github.com/leonkrix/netscenario/blob/main/docs/COVERAGE.md) | What each backend reproduces exactly, in part, or not at all. | +| [Testing](https://github.com/leonkrix/netscenario/blob/main/docs/TESTING.md) | The kinds of test, where they run, coverage and mutation tests. | | [Fidelity](https://github.com/leonkrix/netscenario/blob/main/docs/FIDELITY.md) | How the Kernel backend was checked against the kernel. | | [Decisions](https://github.com/leonkrix/netscenario/blob/main/docs/DECISIONS.md) | Every decision that is hard to undo, with its reasons. | | [Features](https://github.com/leonkrix/netscenario/blob/main/FEATURES.md) | The ledger of what exists and what is planned. | diff --git a/justfile b/justfile index 79a49c9..6478c8b 100644 --- a/justfile +++ b/justfile @@ -93,6 +93,24 @@ book: book-check *args: uv run python docs/book/check.py {{args}} +# One-time: rustup component add llvm-tools-preview and cargo install cargo-llvm-cov --locked. The instrumented build is in +# target/llvm-cov-target and takes 6 GB: remove it afterwards +# Line coverage of the Rust crates as the CI runs the tests (not the fidelity ones) +coverage: + cargo llvm-cov nextest --workspace --all-features --no-tests=pass -E 'not binary(fidelity)' --summary-only --ignore-filename-regex '(/tests/|/fuzz/|/spikes/)' + +# One-time: cargo install cargo-mutants --locked. Each worker builds a copy of the tree: the copies go to the disk and not to /tmp, which +# may be a tmpfs in memory, and the build and the tests are kept to a few threads. For a long run use `systemd-run --user` with MemoryMax +# Mutation tests of one file of a crate (docs/TESTING.md): `just mutants netscenario-core crates/netscenario-core/src/stats.rs` +mutants crate file: + #!/usr/bin/env bash + set -euo pipefail + base="${XDG_CACHE_HOME:-$HOME/.cache}/netscenario-mutants" + mkdir -p "$base/tmp" + TMPDIR="$base/tmp" CARGO_BUILD_JOBS=4 NEXTEST_TEST_THREADS=4 nice -n 10 \ + cargo mutants -p {{crate}} -f {{file}} -j 2 --test-tool nextest --minimum-test-timeout 60 --output "$base/out/$(basename {{file}} .rs)" + echo "survivors: $base/out/$(basename {{file}} .rs)/mutants.out/missed.txt" + # Licences, advisories, sources and bans (deny.toml) deny: cargo deny check diff --git a/scenarios/invalid/planned-key-link-address.yaml b/scenarios/invalid/planned-key-link-address.yaml new file mode 100644 index 0000000..d217ae0 --- /dev/null +++ b/scenarios/invalid/planned-key-link-address.yaml @@ -0,0 +1,16 @@ +version: 0 +name: t +duration: 10s +topology: + nodes: [client, server] + links: + - id: access + between: [client, server] + address: 10.0.0.0/30 +workloads: + - id: media + kind: udp-cbr + from: client + to: server + rate: 1mbit + packet-size: 1200b diff --git a/scenarios/invalid/planned-key-matrix.yaml b/scenarios/invalid/planned-key-matrix.yaml new file mode 100644 index 0000000..606776f --- /dev/null +++ b/scenarios/invalid/planned-key-matrix.yaml @@ -0,0 +1,16 @@ +version: 0 +name: t +duration: 10s +topology: + nodes: [client, server] + links: + - id: access + between: [client, server] +workloads: + - id: media + kind: udp-cbr + from: client + to: server + rate: 1mbit + packet-size: 1200b +matrix: {} diff --git a/scenarios/invalid/planned-key-mobility.yaml b/scenarios/invalid/planned-key-mobility.yaml new file mode 100644 index 0000000..26b4a8b --- /dev/null +++ b/scenarios/invalid/planned-key-mobility.yaml @@ -0,0 +1,16 @@ +version: 0 +name: t +duration: 10s +topology: + nodes: [client, server] + links: + - id: access + between: [client, server] +workloads: + - id: media + kind: udp-cbr + from: client + to: server + rate: 1mbit + packet-size: 1200b +mobility: {}