diff --git a/.github/workflows/spikes.yml b/.github/workflows/spikes.yml
index 3296039..2e945de 100644
--- a/.github/workflows/spikes.yml
+++ b/.github/workflows/spikes.yml
@@ -17,6 +17,10 @@ on:
description: "Run only S20 (the fabric link of release 0.3)"
type: boolean
default: false
+ only_engine:
+ description: "Run only S21 to S24 (release 0.4: the engine, netlink, the comparison, ECN)"
+ type: boolean
+ default: false
permissions:
contents: read
@@ -39,15 +43,15 @@ jobs:
# Three cases: as the runner user with the defaults of the image, as root, and as the runner
# user after relaxing the AppArmor rule that blocks unprivileged user namespaces on Ubuntu 24.04.
- name: Run the spikes as an unprivileged user
- if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: bash spikes/run-all.sh
continue-on-error: true
- name: Run the spikes as root
- if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: sudo SPIKE_OUT="$PWD/spikes/out-root" bash spikes/run-all.sh
continue-on-error: true
- name: Run the spikes as an unprivileged user, AppArmor restriction off
- if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
if sysctl -n kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
@@ -72,7 +76,7 @@ jobs:
sudo env PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SPIKE_OUT="$PWD/spikes/out-newtc" bash spikes/s2-netem-seed.sh
continue-on-error: true
- name: Which options does tc know?
- if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
tc -V || true
ip -V || true
@@ -82,7 +86,7 @@ jobs:
# The Rust and Python experiments (S7 to S10) run as the runner user. On Ubuntu 24.04 the AppArmor
# restriction was relaxed by the step above, and the setting stays for the rest of the job.
- name: S10 Toolchain of this image
- if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
mkdir -p spikes/out-rust
{
@@ -102,13 +106,13 @@ jobs:
} 2>&1 | tee spikes/out-rust/s10.txt
continue-on-error: true
- name: S7 Netlink (nlink) against tc
- if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
mkdir -p spikes/out-rust
bash spikes/rust-netlink/run.sh 2>&1 | tee spikes/out-rust/s7.txt
continue-on-error: true
- name: S9 PyO3 and maturin
- if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
mkdir -p spikes/out-rust
bash spikes/rust-pyo3/run.sh 2>&1 | tee spikes/out-rust/s9.txt
@@ -116,7 +120,7 @@ jobs:
# The engine series pins three cores; on a runner with fewer than four it is skipped. A shared virtual
# machine is noisier than a development machine, which is the point of running it here as well.
- name: S8 Packet engine
- if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
mkdir -p spikes/out-rust
if [ "$(nproc)" -lt 4 ]; then
@@ -128,21 +132,21 @@ jobs:
# S11 to S14 (release 0.2). They use `netem ... seed`, which the tc of the images does not know, so they run
# with the tc built above, as the runner user (the AppArmor restriction was relaxed by an earlier step).
- name: S11 Timeline precision
- if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 30
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SPIKE_OUT="$PWD/spikes/out-rust/s11" bash spikes/rust-timeline/run.sh 2>&1 | tee spikes/out-rust/s11.txt
continue-on-error: true
- name: S12 The netem models against their theory
- if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 20
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SPIKE_OUT="$PWD/spikes/out-rust/s12" bash spikes/s12-netem-models.sh 2>&1 | tee spikes/out-rust/s12.txt
continue-on-error: true
- name: S13 Capture
- if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 10
run: |
mkdir -p spikes/out-rust
@@ -151,6 +155,7 @@ jobs:
continue-on-error: true
# tcpdump of Ubuntu 22.04 cannot run in a user namespace; then S13 keeps the pcap and it is read here
- name: S13 Read the pcap outside the namespace
+ if: ${{ !inputs.only_engine }}
timeout-minutes: 2
run: |
f="$PWD/spikes/out-rust/s13-ours.pcap"
@@ -166,7 +171,7 @@ jobs:
rm -f "$f"
continue-on-error: true
- name: S13b Corruption accounting
- if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 5
run: |
mkdir -p spikes/out-rust
@@ -174,14 +179,14 @@ jobs:
cat spikes/out-rust/s13b.txt
continue-on-error: true
- name: S15 A duplicating netem in a tree with another netem
- if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 3
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SPIKE_OUT="$PWD/spikes/out-rust/s15" bash spikes/s15-duplicate-in-tree.sh 2>&1 | tee spikes/out-rust/s15.txt
continue-on-error: true
- name: S14 A program as a node
- if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
+ if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 5
run: |
mkdir -p spikes/out-rust
@@ -189,39 +194,64 @@ jobs:
continue-on-error: true
# S16 to S19 (release 0.3): many nodes, queue disciplines, TCP, the steps of a trace. They use the tc built above.
- name: S16 Many nodes
- if: ${{ !inputs.only_fabric }}
+ if: ${{ !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 10
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s16" bash spikes/s16-many-nodes.sh 2>&1 | tee spikes/out-rust/s16.txt
continue-on-error: true
- name: S17 The queue lab
- if: ${{ !inputs.only_fabric }}
+ if: ${{ !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 15
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s17" bash spikes/s17-queue-lab.sh 2>&1 | tee spikes/out-rust/s17.txt
continue-on-error: true
- name: S18 TCP
- if: ${{ !inputs.only_fabric }}
+ if: ${{ !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 20
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s18" bash spikes/s18-tcp.sh 2>&1 | tee spikes/out-rust/s18.txt
continue-on-error: true
- name: S19 The steps of a trace
- if: ${{ !inputs.only_fabric }}
+ if: ${{ !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 15
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s19" bash spikes/s19-trace-steps.sh 2>&1 | tee spikes/out-rust/s19.txt
continue-on-error: true
- name: S20 A link through a fabric namespace
+ if: ${{ !inputs.only_engine }}
timeout-minutes: 15
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s20" bash spikes/s20-fabric.sh 2>&1 | tee spikes/out-rust/s20.txt
continue-on-error: true
+ # S21 to S24 (release 0.4): the data path of the engine, netlink by a layer of our own, the same link on the kernel and on the
+ # engine, and ECN. They use the tc built above. The runners have 4 CPUs: the cores are set for that.
+ - name: S21 The data path of the engine (timers, a jitter, the same seed twice, TCP), and S23 the same link on the kernel
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ timeout-minutes: 30
+ run: |
+ mkdir -p spikes/out-rust
+ PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SECS=4 CORE_ENGINE=1 CORE_RECV=2 CORE_SEND=3 bash spikes/rust-engine-safe/run.sh 2>&1 | tee spikes/out-rust/s21.txt
+ PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SECTION=compare SECS=4 CORE_ENGINE=1 CORE_RECV=2 CORE_SEND=3 bash spikes/rust-engine-safe/run.sh 2>&1 | tee spikes/out-rust/s23.txt
+ continue-on-error: true
+ - name: S22 Netlink by a layer of our own against tc
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ timeout-minutes: 10
+ run: |
+ mkdir -p spikes/out-rust
+ PATH="$RUNNER_TEMP/iproute2/tc:$PATH" bash spikes/rust-netlink-own/run.sh 2>&1 | tee spikes/out-rust/s22.txt
+ continue-on-error: true
+ - name: S24 ECN
+ if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
+ timeout-minutes: 10
+ run: |
+ mkdir -p spikes/out-rust
+ PATH="$RUNNER_TEMP/iproute2/tc:$PATH" bash spikes/s24-ecn.sh 2>&1 | tee spikes/out-rust/s24.txt
+ continue-on-error: true
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
with:
diff --git a/FEATURES.md b/FEATURES.md
index 510a296..4906a3f 100644
--- a/FEATURES.md
+++ b/FEATURES.md
@@ -75,7 +75,7 @@ A line moves to `in progress` when the first code of the feature exists, and to
| LNK-10 | MTU and fragmentation | S | 0.3 | done (`mtu` on a link, the sender cuts a packet that does not fit, a router never cuts, the account is by frames, `dont-fragment` makes the sender refuse, [D-081](docs/decisions/D-081.md); `tests/network.rs`, `tests/fidelity.rs`: a lost fragment loses the packet) |
| LNK-11 | Different values per direction | M | 0.1 | done (`tests/provision.rs`: `the_impairments_are_in_the_queues_of_the_right_interfaces`; `tests/agent.rs`) |
| LNK-12 | Outages (windows without any delivery, also of the packets on their way) | M | 0.2 | done (format with D-050; the Kernel backend since D-064; `tests/timeline.rs`: nothing arrives in the window and the packets in flight are dropped; the edges are placed by the clock) |
-| LNK-13 | ECN marking | C | 0.4 | idea |
+| LNK-13 | ECN marking: a queue marks an ECN-capable packet instead of dropping it, and TCP in the nodes uses it (`tcp_ecn`) ([D-100](docs/decisions/D-100.md); spike S24: `codel`, `fq_codel`, `pie` and `netem ... ecn` mark, TCP needs no retransmission for a mark) | S | 0.4 | planned |
## Link models over time (LMD)
@@ -165,15 +165,16 @@ A line moves to `in progress` when the first code of the feature exists, and to
| BKD-02 | Kernel backend in an unprivileged user namespace | M | 0.1 | done (the CI runs the tests as an ordinary user on `ubuntu-22.04` and `ubuntu-24.04`; tests: [`doctor.rs`](crates/netscenario-backend-kernel/tests/doctor.rs), [`run.rs`](crates/netscenario-backend-kernel/tests/run.rs)) |
| BKD-03 | Every backend declares what it reproduces faithfully, and refuses the rest with an error | M | 0.1 | done (`tests/refusal.rs`, `tests/coverage.rs`, `capability.rs`; D-023; a restriction that a scenario accepts with `allow-limited` is recorded in the result, D-092) |
| BKD-04 | Cleanup guarantee: no leftovers after a crash (PID namespace, D-022), and `netscenario clean` as a safety net | M | 0.1 | done (the guarantee is tested at 18 steps in `tests/crash.rs`; `clean` in `tests/clean.rs`, D-024) |
-| BKD-05 | Netlink instead of calls to `ip` and `tc` | S | 0.4 | planned |
-| BKD-06 | Packet engine in userspace (deterministic, no root) | M | 0.4? | planned |
-| BKD-07 | Kernel and engine compared by differential tests | M | 0.4 | planned |
+| BKD-05 | Netlink instead of calls to `ip` and `tc`: a thin layer of our own over `netlink-packet-route`, the options of the qdiscs encoded by us ([D-101](docs/decisions/D-101.md); spike S22: 17 of 17 dumps of the kernel equal those of `tc`) | S | 0.4 | designed |
+| BKD-06 | Packet engine in userspace (deterministic, no root): a process in the fabric namespace with packet sockets on the veth ends, safe Rust but for one small module ([D-098](docs/decisions/D-098.md), [D-099](docs/decisions/D-099.md), [D-100](docs/decisions/D-100.md); spikes S21 and S23) | M | 0.4 | designed |
+| BKD-07 | Kernel and engine compared by differential tests: equivalence inside a margin derived from what each backend measures about itself ([D-102](docs/decisions/D-102.md)) | M | 0.4 | designed |
| BKD-08 | Portable proxy backend (UDP exact, TCP at stream level) | M | 0.5 | planned |
| BKD-09 | Docker backend (containers as nodes) | S | 0.5 | planned |
| BKD-10 | Nodes on several machines | C | 1.x | idea |
| BKD-11 | Hardware in the loop (a real device behind a physical or virtual port) | C | 1.x | idea |
| BKD-12 | Exchange of traces with ns-3 | C | 1.x | idea |
| BKD-13 | An engine based on eBPF/XDP | C | 1.x | idea |
+| BKD-14 | The run chooses the backend (`--backend kernel\|engine\|auto`, `backend=`, a plugin option); the default is the kernel and `auto` says what it chose ([D-103](docs/decisions/D-103.md)) | S | 0.4 | designed |
## Reproducibility and environment (ENV)
@@ -186,6 +187,7 @@ A line moves to `in progress` when the first code of the feature exists, and to
| ENV-05 | Report on timer and clock precision | S | 0.3 | done (`provenance.timers` of every result, `netscenario doctor --timers`, `ns.doctor(timers=True)`: [DOCTOR.md](docs/DOCTOR.md), D-097; tests in [`timers.rs`](crates/netscenario-core/src/timers.rs) and [`timers.rs`](crates/netscenario-backend-kernel/src/timers.rs)) |
| ENV-06 | Result format with a version | M | 0.1 | done (`format` and `version`, 0 until 1.0; tests: [`test_results.py`](python/netscenario/tests/test_results.py), [`run.rs`](crates/netscenario-backend-kernel/tests/run.rs)) |
| ENV-07 | Reproducibility check: the run compares the packet counter of each queue with the packets of the workload, and marks a run in which something else entered the queue as not reproducible; `doctor` checks that `tc` knows `seed` | M | 0.1 | done (`doctor` checks that `tc` knows `seed`; a run counts the packets of the queue of each direction with loss and compares them with the packets the workloads sent: `check_queues` in `run.rs`, `tests/run.rs` with a canary that sends one extra packet, D-036; two workloads in one direction whose queue draws are said not to be reproducible, D-065, because the count cannot see the order of their packets) |
+| ENV-08 | The engine reports itself: the lateness of its releases as a distribution, an overload flag, its timer and the reason for it, and a log of its decisions ([D-098](docs/decisions/D-098.md), [D-099](docs/decisions/D-099.md)) | M | 0.4 | planned |
## Safety (SAF)
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 4babc77..47ad5a7 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -100,7 +100,7 @@ The layout of the repository: a Cargo workspace plus Python packages. The two ba
A backend implements a small interface: report capabilities, create a node, create a link, apply impairments to a link direction, start a program in a node, capture, tear down. The engine uses nothing else, so a new backend means implementing that interface and filling its column in the coverage matrix.
- **Kernel** (reference). One network namespace per node, virtual Ethernet pairs for links, `tc` with `netem` on each direction; a direction with a `queue` goes through a fabric namespace with an `htb` shaper and the discipline of the queue (D-079, D-082). Time-varying behaviour is done by changing the parameters at the steps of a trace. It is real, and its limits are the kernel's: a change of a running `netem` takes about 12 microseconds in a batch (spike S5, and through netlink in S7), so the resolution is set by the scheduler and not by `tc`, and the seed of `netem` guides only the loss and corruption events and needs a quiet namespace (see [COVERAGE.md](COVERAGE.md)).
-- **Engine** (from 0.4). A Rust process that forwards packets between virtual devices inside a user namespace, with its own queues and impairment code. It needs no root, can be seeded exactly, can log every packet, and can implement behaviour that `netem` lacks (traces per packet, any loss model). Its correctness is not assumed: it is compared with the Kernel backend by differential tests. A prototype (spike S8) forwarded 100,000 packets per second on one thread with an exact delay and a deterministic seeded loss; the timer, the completion of checksums for TCP and the queue for jitter are still to be designed (see D-014 in [DECISIONS.md](DECISIONS.md)).
+- **Engine** (from 0.4). A Rust process that forwards packets between virtual devices inside a user namespace, with its own queues and impairment code. It needs no root, can be seeded exactly, can log every packet, and can implement behaviour that `netem` lacks (traces per packet, any loss model). Its correctness is not assumed: it is compared with the Kernel backend by differential tests. It is a process in the fabric namespace of a run, with packet sockets on the veth ends, in safe Rust but for one small module that binds a packet socket ([D-098](decisions/D-098.md)); its decisions depend on the seed and the order of the packets only, and it writes a log of them ([D-099](decisions/D-099.md)); it is compared with the Kernel backend by equivalence inside a margin ([D-102](decisions/D-102.md)). The spikes S8, S21 and S23 measured it: 100,000 packets per second on one thread with an exact delay, and a lateness that depends on the timer and that every run reports.
- **Proxy** (from 0.5). Your program talks to a proxy process. Exact for UDP, stream-level for TCP. It exists so that people on Windows and macOS, or in containers without rights, can run the scenarios that are meaningful there. Everything it cannot do is refused.
**Rule:** a backend never approximates silently. If it can do something only in part, the scenario must say `allow-limited` for that feature, and the result records that it did.
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index 03c6253..30f1ce2 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -60,7 +60,7 @@ Docker is not a column: it is a way to start programs and to reach the Kernel ba
| MTU, fragmentation | ✔ 0.3 ([test](../crates/netscenario-backend-kernel/tests/network.rs): the sender cuts the packet, the router carries the fragments; [fidelity](../crates/netscenario-backend-kernel/tests/fidelity.rs): a lost fragment loses the packet) | ✔ 0.4 | ✘ |
| Different values per direction | ✔ 0.1 ([test](../crates/netscenario-backend-kernel/tests/provision.rs)) | ✔ 0.4 | ✔ 0.5 |
| Outages | ✔ 0.2 ([test](../crates/netscenario-backend-kernel/tests/timeline.rs): nothing arrives in the window, the packets that were on their way are dropped, the edges are placed by the clock) | ✔ 0.4 | ✔ 0.5 |
-| ECN marking | ? | ✔ 0.4 | ✘ |
+| ECN marking | ✔ 0.4 (spike S24: `codel`, `fq_codel`, `pie` and `netem ... ecn` mark, and TCP with `tcp_ecn` needs no retransmission for a mark) | ✔ 0.4 | ✘ |
**What the Kernel backend does that a reader may not expect** (measured, [FIDELITY.md](FIDELITY.md)):
@@ -101,7 +101,7 @@ Docker is not a column: it is a way to start programs and to reach the Kernel ba
| Feature | Kernel | Engine | Proxy |
| ------------------------------ | ------ | ------ | ----- |
| Scheduled steps | ✔ 0.2 ([test](../crates/netscenario-backend-kernel/tests/timeline.rs): a rate that drops in the course of the run is in the throughput, the same seed gives the same lost packets whatever the change takes) | ✔ 0.4 | ✔ 0.5 |
-| A change from test code while the run goes | ✔ 0.3 ([test](../crates/netscenario-backend-kernel/tests/live.rs): the change is done between two packets, the recorded scenario holds it at that packet, and a replay loses the same packets; what the queues cannot follow is refused to the code) | ✘ | ✘ |
+| A change from test code while the run goes | ✔ 0.3 ([test](../crates/netscenario-backend-kernel/tests/live.rs): the change is done between two packets, the recorded scenario holds it at that packet, and a replay loses the same packets; what the queues cannot follow is refused to the code) | ✔ 0.4 | ✘ |
| Link down and up | ✔ 0.2 ([test](../crates/netscenario-backend-kernel/tests/timeline.rs): the sender is refused for exactly the packets of the window and the statistics leave them out) | ✔ 0.4 | ? |
| Trace-driven rate, delay and loss (CSV, Mahimahi) | ✔ 0.3 ([test](../crates/netscenario-backend-kernel/tests/traces.rs): a staircase of rates from a file is followed step by step, a Mahimahi file of a known rate gives that rate, the delay and the loss of a trace act in their phases; [test](../crates/netscenario-model/tests/link_traces.rs): the two formats, what the reader refuses, the events) | ✔ 0.4 | ✔ 0.5 |
| A rate that changes during the run (steps, models, traces) | ✔ 0.3 ([test](../crates/netscenario-backend-kernel/tests/traces.rs): the link is built through the fabric with a class of `htb` that is changed at each step, so the bytes that leave follow the rate) | ✔ 0.4 | ✔ 0.5 |
diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md
index f014c5d..0c57bd0 100644
--- a/docs/DECISIONS.md
+++ b/docs/DECISIONS.md
@@ -103,3 +103,9 @@ Status: **accepted**, **proposed** (to be confirmed by an experiment or a first
| [D-095](decisions/D-095.md) | A profile is a named set of sourced values for a link, resolved when the file is read and recorded with the version of the library | Accepted 2026-11-10 |
| [D-096](decisions/D-096.md) | A change from test code is a step of the timeline made at a hold of the sender, on a link that was declared live, and the async API is the same run handle | Accepted 2026-11-10 |
| [D-097](decisions/D-097.md) | The precision of the timers is measured by the process that starts a run, recorded in its provenance, and reported in full by `doctor --timers` | Accepted 2026-11-10 |
+| [D-098](decisions/D-098.md) | The engine is a process in the fabric namespace of a run, with packet sockets on the veth ends, in safe Rust but for one small module, and its timer is chosen by `auto` and can always be set | Accepted 2026-11-12 |
+| [D-099](decisions/D-099.md) | The engine's decisions depend on the seed and on the order of the packets, never on the clock; it writes a decision log; "virtual time" means the moments it computes, not a clock for the programs | Accepted 2026-11-12 |
+| [D-100](decisions/D-100.md) | What the engine does in 0.4: the 25 cells of its own, ECN, the live change; queue disciplines last, and anything it does not do it refuses by name | Accepted 2026-11-12 |
+| [D-101](decisions/D-101.md) | Netlink is a thin layer of our own over `netlink-packet-route` and `netlink-sys`, with the options of the qdiscs encoded by us and checked against `tc` | Accepted 2026-11-12 |
+| [D-102](decisions/D-102.md) | Two backends are compared by equivalence inside a margin that is derived from what each backend measures about itself, not by a test of "no difference" | Accepted 2026-11-12 |
+| [D-103](decisions/D-103.md) | The run chooses the backend, the scenario does not; the default is the kernel; `auto` picks the one that is faithful and says so | Accepted 2026-11-12 |
diff --git a/docs/decisions/D-098.md b/docs/decisions/D-098.md
new file mode 100644
index 0000000..32c3b64
--- /dev/null
+++ b/docs/decisions/D-098.md
@@ -0,0 +1,13 @@
+# D-098: The engine is a process in the fabric namespace of a run, with packet sockets on the veth ends, in safe Rust but for one small module, and its timer is chosen by `auto` and can always be set
+
+Accepted (2026-11-12), for BKD-06. Measured by S21 and S23 on Debian 13 (kernel 6.12, Ryzen 7 5800X) in an unprivileged user namespace; the runners of the CI are measured by the workflow `Spikes` (input `only_engine`) and the numbers are added to the log of the plan.
+
+- **The shape.** One engine process for a run, in the fabric namespace of D-079: `node A -- veth -- engine -- veth -- node B`, a packet socket (`AF_PACKET`) on each veth end. Nodes are ordinary namespaces, so routing, TCP, IPv6, fragments, programs and capture are the kernel's and are not rebuilt. The supervisor, the crash guarantee (D-022), `clean` and the limits are those of the Kernel backend; only the middle of a link differs (the engine where `netem` and the shaper were). The engine needs no kernel module and no root, so its tests also run where `netem` is missing.
+- **The data path** is one thread for all links of a process (or one thread per link on request): non-blocking sockets, a heap of frames ordered by the moment they are due with the arrival number as tie-break (so a jitter can reorder, and equal moments keep their order), and a timer. S21: 1,000 to 100,000 packets per second of 200 bytes, delay 20 ms, 0 packets lost up to 50,000 per second with every timer and up to 400 of 400,000 lost at 100,000 per second with the sleeping timers (none with the spinning ones); a uniform jitter of 5 ms reordered the packets (99,573 of 200,000 left out of arrival order) and gave the mean, the minimum and the standard deviation of the model (20.0, 15.0 and 2.89 ms); one TCP flow through the engine reached 2.0 to 2.4 Gbit/s against 4.5 Gbit/s on a plain veth with one segment per frame, which is far more than a test needs.
+- **The timer, measured as the lateness of a release** (the time between the moment a packet was due and the moment it was handed to the interface), delay 20 ms, 10,000 packets per second: `epoll_pwait2` with a nanosecond timeout (through `rustix`) has a median of 9 to 10 microseconds, a 99th percentile of 0.26 to 0.5 ms and rare outliers of 4 to 7 ms (a core that sleeps wakes late; the load of this desktop shows in it), 2 % of a CPU at 1,000 packets per second, 6 % at 10,000, 30 % at 50,000 and 41 % at 100,000; a timer with a millisecond timeout is 1 ms late at low rates and is not used; `timerfd` is like `epoll_pwait2` and a little later; a spinning loop has a 99th percentile of 5 to 8 microseconds and a maximum of 0.1 ms at the price of a whole CPU; **spin-sleep** (sleep until 100 microseconds before the due moment, then spin) has a 99th percentile of 5 to 13 microseconds, as tight as the kernel (its standard deviation of the delay is 1 to 2 microseconds, the engine's 3), at 6 % of a CPU at 1,000 packets per second and 63 % at 10,000 and more. Its weakness is a low rate: at 1,000 packets per second it sleeps for most of 20 ms and still woke late now and then (99.9th percentile 2.6 ms, maximum 4 ms), because the window of 100 microseconds is shorter than the late wake-ups of this machine. The window therefore adapts: the engine measures how late its sleeps end and spins for that long (the step of the data path builds and tests it). The kernel's own delay is tighter than any timer of the engine; the engine adds 6 to 12 microseconds of base delay for its hop (S23).
+- **The timer is a setting, and its default is `auto`.** The engine always measures the lateness of every release and puts the distribution and an overload flag in the result (ENV-08). `engine.timer` is `auto`, `epoll`, `spin-sleep` or `spin`, set with `--engine-timer`, `engine_timer=` and the plugin option, and recorded. `auto` takes spin-sleep when the run has at least 4 CPUs available (by the CPU mask of the process) and `epoll` otherwise, and the result says which and why. The reason: a test author on a laptop or a 4-core runner gets the exact timing the release is named for; a 2-core runner, where a spinning engine would take a core from the sender, the receiver and the programs, gets the cheap timer and an honest report of the lateness. The threshold is one constant, to be confirmed with the numbers of the runners. The setting changes the precision of the timing and never what the engine decides (D-099).
+- **`unsafe`.** Everything is safe Rust (`nix`, `rustix`: receive, send, `epoll`, `timerfd`, the clock, the CPU time) but one module of the engine crate, `sys`, about 35 lines: the `bind` of a packet socket to an interface and the two options `PACKET_IGNORE_OUTGOING` (the engine must not read what it sends) and `PACKET_VNET_HDR`. `nix` 0.31.3 cannot build the address that `bind` and `send` need (the field of `LinkAddr` is private and there is no constructor) and neither `nix` nor `rustix` knows the options; there is no safe way today. The module has a `SAFETY` comment for each call and a test, the crate has `#![deny(unsafe_code)]` with the one allowed module, and every other crate keeps `forbid`. A pull request to `nix` was considered and not made (the engine would wait on someone else's release).
+- **Checksums, solved without a tool.** A frame that leaves a node over a veth with checksum offload on carries an unfinished checksum; a packet socket that re-sends it hands the receiver a frame it drops (S21: a TCP connection through the engine with the offloads on never opens). `PACKET_VNET_HDR` keeps the unfinished checksum unfinished: each frame is read and written with a 10-byte header that says where to complete it, and the interface that receives it completes it, as from a real wire. TCP through the engine with it (and one segment per frame, D-086) ran at 2.1 Gbit/s; with the transmit offload switched off by the `ethtool` ioctl instead it ran at 2.0 to 2.4 Gbit/s. So no `ethtool` and no extra netlink message is needed, and the engine does not depend on a setting of the nodes.
+- **The binary.** The engine is a program, `netscenario-engine`, installed next to the agent (the wheel carries it too, as the agent), started by the supervisor in the fabric namespace; the control channel is lines of JSON, like the agent's.
+
+Alternatives: TAP devices (the program would talk to a TAP, the nodes would no longer be ordinary namespaces and the kernel's routing would not be used); one process per link (more processes, no gain at test rates); XDP or eBPF (BKD-13, not rootless); `epoll` or spin-sleep as the only choice (one is cheap and loose, the other exact and costly: neither is right for every user); a pull request to `nix` first (waiting for a release that is not ours).
diff --git a/docs/decisions/D-099.md b/docs/decisions/D-099.md
new file mode 100644
index 0000000..5cdedef
--- /dev/null
+++ b/docs/decisions/D-099.md
@@ -0,0 +1,10 @@
+# D-099: The engine's decisions depend on the seed and on the order of the packets, never on the clock; it writes a decision log; "virtual time" means the moments it computes, not a clock for the programs
+
+Accepted (2026-11-12), for BKD-06 and the cells "same seed, same loss pattern" and "same seed, same timing" of the Engine column.
+
+- **A stream for each link, direction and kind of decision.** Loss, jitter, reordering, duplication and corruption each draw from their own stream, seeded from the seed of the run, the link, the direction and the kind (the derivation is written out in `docs/ENGINE.md`). The generator is the one of the agent (SplitMix64 to seed, xoshiro256++ to draw, `ln` made of IEEE operations, D-071), so the same seed gives the same numbers on every machine and in every version. It moves to a place both crates can use. The k-th decision of a stream depends on k and the seed only, so adding a delay or a rate to a link does not shift its losses (a test says so, with a canary that makes one stream draw from another).
+- **What is promised.** The same seed and the same sequence of packets give the same decisions: the same packets lost, corrupted, duplicated and reordered, the same jitter samples. With a UDP workload (the packets of a sender are the same each time, D-065) that is the same lost packets on every run, on every machine, whatever the load. For TCP the packets depend on feedback and the run is `statistical` (D-086).
+- **What is not promised.** The same moments. The engine runs in real time next to programs that run in real time; a virtual clock for them does not exist. What the engine does have is the moment it computes: from the arrival stamp of a frame (`SO_TIMESTAMPNS`) it computes the moment the frame is due, and it writes both to the **decision log** (the number of the packet, the stamp of arrival, the decision, the due moment, the moment of release). The decisions do not depend on the lateness of the release; the lateness is in the log and in the distribution of the result (ENV-08). The cell "same seed, same timing" is therefore `~`: the due moments are a function of the arrival stamps; the release is as exact as the timer (D-098).
+- **The log** is a file of binary records next to the result (`engine--.log`), written through a buffer by a thread of its own so that it does not slow the path, with a reader in the core and a fuzz target. A run says in the result how many records it dropped when the buffer was full (never silently incomplete, as the capture, D-047).
+
+Alternatives: one stream for everything (a delay would shift the losses); a clock that the engine controls and that the programs read (impossible for a real kernel stack); no log (the cell would be a claim nobody can check).
diff --git a/docs/decisions/D-100.md b/docs/decisions/D-100.md
new file mode 100644
index 0000000..4218dbf
--- /dev/null
+++ b/docs/decisions/D-100.md
@@ -0,0 +1,12 @@
+# D-100: What the engine does in 0.4: the 25 cells of its own, ECN, the live change; queue disciplines last, and anything it does not do it refuses by name
+
+Accepted (2026-11-12), for BKD-06, LNK-13 and TML-05. `docs/COVERAGE.md` has 53 cells `✔ 0.4` in the Engine column; 28 follow from the build of the Kernel backend (nodes are namespaces) and need the existing tests to run on the engine, 25 are the engine itself.
+
+- **The 25:** the impairments (fixed delay, jitter in five distributions, independent and burst loss, reordering, duplication, corruption, rate in Ethernet frames, queue limit, MTU as the node's business, values per direction, outages), change over time (steps, down and up, traces, a rate that changes during the run at any width, models, periodic patterns), and the reproducibility of loss and of the due moments.
+- **Rate steps of any width.** The engine has its own shaper, so the restrictions of the kernel (`fast-rate-steps` below 100 ms, `stalled-link`, D-091, D-092) do not apply to it and no `allow-limited` is needed; a test with a staircase of 1 ms steps has a canary that an averaging shaper fails.
+- **The live change** (TML-05) is `✔ 0.4` for the engine: a change of a link from test code is a step of the timeline sent over the control channel (D-096), so the cell that said `✘` becomes a plan.
+- **ECN** (LNK-13) is `planned` for 0.4, for both backends. S24 on the kernel: `codel`, `fq_codel`, `pie` with `ecn` and `netem loss ... ecn` mark an ECN-capable packet instead of dropping it (the counter `ecn_mark` and the CE count at the receiver agree to the packet: 63, 64, 22, 101), TCP with `net.ipv4.tcp_ecn=1` in the nodes sees the marks and needs no retransmission for them (`netem` with `ecn`: 0 retransmissions, without: 102); the key and the sysctl of the nodes are designed in the step of the queues. The Kernel cell `?` becomes `✔ 0.4`.
+- **Queue disciplines last.** CoDel (RFC 8289), FQ-CoDel (RFC 8290) and PIE (RFC 8033) are the largest piece. They are built after everything else; if one is not ready when the rest is, it is refused by name, its Engine cell becomes `✘` with the reason, the ledger moves it to 0.5, and nothing is approximated. The decision is taken at the end of that step.
+- **Refusals.** The engine declares what it can do as the Kernel backend does (`Capabilities`, D-023); what a scenario asks for and the engine cannot do is an error that names the feature, the backend and one that can. A cell stays `✔ 0.4` only when it links to a test that runs on the engine.
+
+Alternatives: everything or nothing (a cut would be a surprise at the end); the engine without ECN (CoDel without the mark is half of CoDel); keeping the live change `✘` (the control channel makes it free).
diff --git a/docs/decisions/D-101.md b/docs/decisions/D-101.md
new file mode 100644
index 0000000..38b680a
--- /dev/null
+++ b/docs/decisions/D-101.md
@@ -0,0 +1,13 @@
+# D-101: Netlink is a thin layer of our own over `netlink-packet-route` and `netlink-sys`, with the options of the qdiscs encoded by us and checked against `tc`
+
+Accepted (2026-11-12), for BKD-05. S22 built the messages for `netem` (delay, jitter, limit, loss with correlation, Gilbert-Elliott, duplication, corruption, reordering with a gap, rate, `ecn`, `seed`), `pfifo`, `bfifo`, `codel`, `pie`, `fq_codel`, `tbf` and `htb` with a class and a leaf, applied them in a user namespace, and compared the dump of the kernel (`tc -d qdisc show`, `class show`) with what the `tc` command makes of the same settings: **17 of 17 are identical** (handles that the kernel hands out and drawn seeds left out). A change of a running `netem` took **2.8 microseconds** (20,000 changes; D-014 measured 12 for `nlink` and `tc -batch`).
+
+- **The libraries.** `netlink-packet-route` 0.34.0 (2026-11-03): 52,000 lines, no `unsafe`, synchronous, dependencies `bitflags`, `libc`, `log`, `netlink-packet-core`, `zerocopy`; it builds links, addresses, routes, `fq_codel`, `ingress`, `mirred` and the `u32`, `flower` and `matchall` filters, and it has **no `netem`, `htb`, `tbf`, `codel`, `pie` or fifos** (checked on 2026-11-12). The options of those are encoded by us from `linux/pkt_sched.h` (about 400 lines) and sent as raw attributes (`TcAttribute::Other`; `netem` is the one qdisc whose options begin with a plain struct, not an attribute). `netlink-sys` 0.9.0 is the socket. The 64-bit attributes for the latency and the jitter of `netem` carry nanoseconds, so no conversion to scheduler ticks is needed there; `tbf` and `htb` need the ticks of 64 nanoseconds, which the layer computes.
+- **Not `nlink`.** `nlink` 0.33.0 (2026-10-23, one maintainer, 26,000 downloads) has 227,000 lines, 106 places with `unsafe`, requires `tokio` and `tracing`, and changes its API between minor versions; the backend is synchronous and would take an async runtime for it.
+- **Namespaces.** A socket belongs to the namespace in which it was made, so the layer makes the socket of a node in a thread that has entered the namespace (`setns`, in `nix`, safe) and leaves it again; the `sysctl`s are written to `/proc/sys` the same way. This was not part of S22 and is the first thing the step builds and tests.
+- **What stays outside.** `unshare`, `nsenter` and `prlimit` of util-linux are still used by the supervisor; replacing them is not part of this release. `ethtool` is not needed (D-098).
+- **The jitter tables** (`normal`, `pareto`, `paretonormal`) are read from the files of the system as now (D-084) and sent in the attribute of the distribution, or made by us from their definition; the step chooses by comparing the tables byte for byte.
+- **What it removes.** The requirement for iproute2 6.6.0 or newer (the `seed` is an attribute the kernel reads, not a feature of `tc`), the CI step that builds iproute2 6.15 for every job (the step that stopped the CI for a day on 2026-11-11 when the mirror of the packages could not be reached), and the text of ENV-07 about `tc` knowing `seed`.
+- **The proof.** Every test of the Kernel backend passes unchanged; a golden test per message against what `tc` sends; the state read back equals the state of the `tc` path for each scenario of `scenarios/`.
+
+Alternatives: `nlink` pinned exactly (a very large asynchronous dependency with `unsafe` inside); `rtnetlink` (has no `netem`); keeping `ip` and `tc` (a tool that must be installed, a new enough version for the seed, a process for every change).
diff --git a/docs/decisions/D-102.md b/docs/decisions/D-102.md
new file mode 100644
index 0000000..fcd6576
--- /dev/null
+++ b/docs/decisions/D-102.md
@@ -0,0 +1,12 @@
+# D-102: Two backends are compared by equivalence inside a margin that is derived from what each backend measures about itself, not by a test of "no difference"
+
+Accepted (2026-11-12), for BKD-07 and the second acceptance criterion of 0.4. S23 ran the same UDP link (10,000 packets per second, 4 s, three runs) on the kernel (`netem` on the client's interface) and on the engine with two timers (the three links: a delay of 20 ms; a delay of 20 ms with a uniform jitter of 5 ms; a loss of 10 %).
+
+- **What S23 showed.** The distributions are the same in shape: delay 20 ms, jitter 5 ms: mean 19.98 to 20.01 ms (kernel) against 19.98 to 20.04 (engine), standard deviation 2.88 to 2.90 against 2.89 to 2.92 (the model gives 2.887), minimum 15.00 against 15.00 to 15.01, 99th percentile 24.90 to 24.91 against 24.91 to 24.97; loss 9.77 to 10.09 % against 10.14 to 10.27 % of 40,000 packets (one standard deviation of a binomial count is 0.15 %). But the tails and the base differ: with a fixed delay the kernel has a standard deviation of 1 to 2 microseconds and a maximum 0.03 to 0.05 ms above the delay, the engine with spin-sleep 3 microseconds and a maximum of 0.09 to 0.2 ms, the engine with `epoll` 0.11 to 0.25 ms and a maximum of 3.6 to 4.1 ms above; and the hop of the engine adds a base delay of 6 microseconds (spin-sleep) to 12 (`epoll`). Any test of "the two distributions are equal" rejects such a pair at a sample of a few thousand packets, and would pass an engine that is wrong by less than the noise of its own timer.
+- **The method.** For each quantile (0.5, 0.9, 0.99) of the delay, the difference of the engine's and the kernel's quantile has a percentile-bootstrap interval (D-074) over the runs of a set, and the comparison passes when the interval lies inside **plus or minus the margin**. Loss, duplication and corruption are counts: the rates are compared by their intervals (the exact binomial), with a margin of zero plus the interval's own width. Throughput is compared on the goodput of a bin. A rank test (Mann-Whitney) is printed as a diagnostic and is not judged.
+- **The margin is derived, not picked.** For a quantile q it is `L(q) + B + K`: L(q) is the lateness quantile that the engine reports in the very same run (D-098, so a loose timer widens its own margin and says so); B is the base delay of the hop, measured by a comparison with no impairment in the same set; K is the allowance for the kernel path, the spread of kernel against kernel measured in the set (30 pairs of runs). The margin is written next to each comparison in `docs/DIFFERENTIAL.md` with the numbers it came from.
+- **The level.** Each comparison gets a level so that one run of the CI has a false alarm below 1e-6 over all of them (the Bonferroni share); the sizes of the sets follow from the level and the margins.
+- **Canaries.** An engine built with the feature `fault-injection` (only for tests: a delay too long by 5 %, a loss rate 1.1 times too high, a jitter 20 % too wide, a rate 5 % too high) must fail exactly the comparison that it should; the kernel against the kernel passes 30 times in 30.
+- **What "statistically equal for UDP" then means** in `docs/ACCEPTANCE.md`: for the scenarios listed there, no quantile or rate differs by more than its margin, at the stated level, and the margins are the ones that the machines produced. It does not mean that the engine is as tight as the kernel; the tables say how much looser it is.
+
+Alternatives: a two-sample test of equality (rejects a good pair and passes a bad one); fixed tolerances picked by eye (the rule of this project is that bounds are derived); comparing packet by packet (the generators differ by design, D-099).
diff --git a/docs/decisions/D-103.md b/docs/decisions/D-103.md
new file mode 100644
index 0000000..914cae5
--- /dev/null
+++ b/docs/decisions/D-103.md
@@ -0,0 +1,11 @@
+# D-103: The run chooses the backend, the scenario does not; the default is the kernel; `auto` picks the one that is faithful and says so
+
+Accepted (2026-11-12), for BKD-14.
+
+- **The run chooses.** `netscenario run --backend kernel|engine|auto`, `backend=` in `run`, `start`, `run_async` and for a set, and an option and a marker of the plugin. The scenario file describes a network and never names the tool that emulates it: the same file runs on both, which is what the comparison of D-102 needs, and a replay uses the backend recorded in the result.
+- **The default is `kernel`.** It is the reference (the first principle) and it is what 0.1 to 0.3 users have. Changing the default later is a decision of its own.
+- **`auto`** takes the kernel when `doctor` says it can do what the scenario asks faithfully (the modules are there, the rights are there) and the engine otherwise, shows the choice in the output and records it in the result (`provenance.backend` and `provenance.backend_choice`: the requested name, the used name and the reason). It is never silent, and a scenario that neither can do is refused with the reasons of both.
+- **The settings of a backend** belong to the run, too: `engine.timer` (D-098), the limits of the queues of the engine, whether the decision log is written. They are recorded in the result and the scenario hash is the same for both backends.
+- **A result says what made it.** The version of the engine, its timer, its lateness distribution and its overload flag (ENV-08) are in the provenance; a report prints the backend in its first lines.
+
+Alternatives: a key `backend:` in the file (the file would no longer be one network that two tools run); `auto` as the default (the same file would give different measurements on two machines before anyone asked); no `auto` (a laptop without `netem` would need to know about the engine before it could run anything).
diff --git a/spikes/README.md b/spikes/README.md
index d8e293c..37993dc 100644
--- a/spikes/README.md
+++ b/spikes/README.md
@@ -71,3 +71,12 @@ Five more are for release 0.3 and are about networks of many nodes, queue discip
| S18 `s18-tcp.sh` | Which congestion controls can a node select? What do segmentation offloads do on a veth? Goodput of one flow, the shares of two, TCP_INFO, and reno against the model of Mathis | TCP workloads, what is measured and promised (WRK-04, WRK-05, PRB-12) | `bash spikes/s18-tcp.sh` |
| S19 `s19-trace-steps.sh` | How well does a shaper follow a staircase of rates in steps of 100 down to 1 ms: `tbf`, `tbf` with the smallest burst, `htb`, `netem`? | How a trace is stepped, the restriction that goes with it (LMD-03) | `bash spikes/s19-trace-steps.sh` |
| S20 `s20-fabric.sh` | Can a link be built as node A, a fabric namespace, node B, so that loss and the queue sit at A and delay and jitter in the fabric? What does the extra hop cost (bridge or `mirred`), is the order of the stages exact (a jitter with a saturated shaper), does the sender feel the queue, is the seeded loss the same, what do the nodes see when a port goes down? | How a link with a queue discipline or a changing rate is built (LNK-09, LMD-03) | `bash spikes/s20-fabric.sh` |
+
+Four more are for release 0.4 and are about the engine and netlink. They need a `tc` (to compare against) and `rust-engine-safe/` needs 4 CPUs at least (8 for the section of several links); set `CORE_ENGINE`, `CORE_RECV` and `CORE_SEND` for the machine. The workflow `Spikes` runs them with the input `only_engine`.
+
+| Spike | Question | Decides | Run |
+| --- | --- | --- | --- |
+| S21 `rust-engine-safe/` | Can the data path of the engine be safe Rust? How late is a release for `epoll` with a millisecond and a nanosecond timeout, `timerfd`, a spinning loop and spin-sleep, at 1,000 to 100,000 packets per second? One thread or one per link? Does a jitter reorder through a sorted queue, does the same seed give the same losses, and does TCP pass a packet socket with the checksum offload on, off, or kept by `PACKET_VNET_HDR`? | The shape of the engine, its timer and its `unsafe` (D-098), the contract of determinism (D-099) | `bash spikes/rust-engine-safe/run.sh` (about 10 minutes; `SECTION=timers\|links\|jitter\|loss\|tcp`) |
+| S22 `rust-netlink-own/` | Can a layer of our own over `netlink-packet-route` (no `unsafe`) build `netem`, the fifos, `codel`, `pie`, `fq_codel`, `tbf` and `htb` so that the kernel's dump equals what `tc` makes? How long does a change of a running `netem` take? | Netlink: the library and the layer (D-101) | `bash spikes/rust-netlink-own/run.sh` (under a minute) |
+| S23 `rust-engine-safe/` (`SECTION=compare`) | The same UDP link on the kernel (`netem`) and on the engine with two timers, three runs each: how far apart are they, and from themselves? | How backends are compared: the margin (D-102) | `SECTION=compare bash spikes/rust-engine-safe/run.sh` (about 4 minutes) |
+| S24 `s24-ecn.sh` | Do `codel`, `fq_codel`, `pie` and `netem` mark instead of dropping, and does a TCP with `tcp_ecn` in a namespace use the mark? | Whether ECN is in 0.4 (D-100, LNK-13) | `bash spikes/s24-ecn.sh` (about 1 minute) |
diff --git a/spikes/rust-engine-safe/Cargo.lock b/spikes/rust-engine-safe/Cargo.lock
new file mode 100644
index 0000000..d9c4e3c
--- /dev/null
+++ b/spikes/rust-engine-safe/Cargo.lock
@@ -0,0 +1,129 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "bitflags"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+
+[[package]]
+name = "cfg_aliases"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
+
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.191"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "nix"
+version = "0.31.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+ "memoffset",
+]
+
+[[package]]
+name = "rustix"
+version = "1.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
+[[package]]
+name = "signal-hook"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49f7447afd772ca79ad17ae0a3972a6b0df0cb4b80990fd82707593e9e012a74"
+dependencies = [
+ "libc",
+ "signal-hook-registry",
+]
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
+dependencies = [
+ "errno",
+ "libc",
+]
+
+[[package]]
+name = "spike-engine-safe"
+version = "0.0.0"
+dependencies = [
+ "libc",
+ "nix",
+ "rustix",
+ "signal-hook",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
diff --git a/spikes/rust-engine-safe/Cargo.toml b/spikes/rust-engine-safe/Cargo.toml
new file mode 100644
index 0000000..6dc3082
--- /dev/null
+++ b/spikes/rust-engine-safe/Cargo.toml
@@ -0,0 +1,20 @@
+# Spike S21: the data path of a packet engine, in Rust that is safe except for one small module. A scratch crate, not part of
+# any workspace.
+[package]
+name = "spike-engine-safe"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+nix = { version = "0.31", default-features = false, features = ["socket", "net", "uio", "process", "resource"] }
+rustix = { version = "1", default-features = false, features = ["std", "time", "event", "process", "linux_5_11"] }
+signal-hook = "0.4"
+# Only for the module `sys` (bind and two socket options of a packet socket, which nix and rustix cannot set safely)
+libc = "0.2"
+
+[profile.release]
+debug = false
+opt-level = 3
+
+[workspace]
diff --git a/spikes/rust-engine-safe/ethtool_off.py b/spikes/rust-engine-safe/ethtool_off.py
new file mode 100644
index 0000000..61fe63f
--- /dev/null
+++ b/spikes/rust-engine-safe/ethtool_off.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+"""Switch off transmit checksum offload (and with it TSO and GSO) of interfaces, with the ioctl `ethtool` uses.
+
+ ethtool_off.py IFACE...
+
+Only for the spike: `ethtool` is not installed on every machine, and the question of S21 is whether frames that leave a node with
+a finished checksum pass a packet engine (the engine does not finish a checksum that the sender left to the device)."""
+import array
+import fcntl
+import socket
+import struct
+import sys
+
+SIOCETHTOOL = 0x8946
+ETHTOOL_STXCSUM = 0x00000017
+ETHTOOL_STSO = 0x0000001F
+ETHTOOL_SGSO = 0x00000024
+ETHTOOL_SGRO = 0x0000002C
+
+
+def set_value(sock, name, command, value):
+ buffer = array.array("B", struct.pack("II", command, value))
+ address, _ = buffer.buffer_info()
+ request = struct.pack("16sP", name.encode(), address)
+ fcntl.ioctl(sock.fileno(), SIOCETHTOOL, request)
+
+
+sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+for name in sys.argv[1:]:
+ for command in (ETHTOOL_SGSO, ETHTOOL_STSO, ETHTOOL_SGRO, ETHTOOL_STXCSUM):
+ set_value(sock, name, command, 0)
+print("offloads off:", " ".join(sys.argv[1:]))
diff --git a/spikes/rust-engine-safe/run.sh b/spikes/rust-engine-safe/run.sh
new file mode 100755
index 0000000..f584d2f
--- /dev/null
+++ b/spikes/rust-engine-safe/run.sh
@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+# S21: a packet engine in (nearly) safe Rust: the rate it keeps up with, the lateness of its releases for five ways to wait, one
+# thread or one per link, a jitter that reorders, the same seed twice, and a TCP flow through it.
+#
+# client_k namespace --veth-- ENGINE (here) --veth-- server_k namespace for k = 1..N
+#
+# Everything happens in namespaces that this script creates (an unprivileged user namespace, or root without touching the host).
+# bash spikes/rust-engine-safe/run.sh everything (about 12 minutes)
+# SECTION=timers|links|jitter|loss|tcp bash ... one part
+# QUICK=1 bash ... short runs
+set -u
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export PATH="$PATH:/usr/sbin:/sbin"
+
+if [ "${1:-outer}" = outer ]; then
+ cargo build --release --quiet --manifest-path "$here/Cargo.toml" || exit 1
+ cargo build --release --quiet --manifest-path "$here/../rust-engine/Cargo.toml" || exit 1
+ if [ "$(id -u)" -eq 0 ]; then exec unshare -n bash "$0" inner; fi
+ exec unshare -Urn bash "$0" inner
+fi
+
+engine="$here/target/release/spike-engine-safe"
+tool="$here/../rust-engine/target/release/spike-engine" # the sender and the receiver of S8
+secs="${SECS:-4}"; [ -n "${QUICK:-}" ] && secs=2
+section="${SECTION:-all}"
+# One physical core each for the engine, the receiver and the sender (a Ryzen 7 5800X has 8; the runners have 4: set them)
+CE="${CORE_ENGINE:-2}"; CR="${CORE_RECV:-3}"; CS="${CORE_SEND:-4}"
+cpus="$(nproc)"
+want() { [ "$section" = all ] || [ "$section" = "$1" ]; }
+
+pids=()
+cleanup() { for p in "${pids[@]}"; do kill "$p" 2>/dev/null; done; wait 2>/dev/null; }
+trap cleanup EXIT
+ns_new() { # VARIABLE: a namespace held by a sleeping process
+ unshare -n sleep 3600 >/dev/null 2>&1 &
+ pids+=($!)
+ printf -v "$1" '%s' "$!"
+ for _ in $(seq 50); do nsenter -t "$!" -n true 2>/dev/null && return 0; sleep 0.1; done
+ return 1
+}
+nsmac() { nsenter -t "$1" -n ip -br link show dev "$2" | awk '{print $3}'; }
+
+sysctl -qw net.ipv6.conf.default.disable_ipv6=1 net.ipv6.conf.all.disable_ipv6=1
+declare -A PC PS
+for k in 1 2 3 4; do ns_new "PC$k"; ns_new "PS$k"; done
+for k in 1 2 3 4; do
+ for v in PC$k PS$k; do nsenter -t "${!v}" -n sysctl -qw net.ipv6.conf.default.disable_ipv6=1; done
+done
+
+# topology MODE N: engine = client_k -- ec_k [engine] es_k -- server_k ; direct = client_k -- server_k
+topology() {
+ local mode="$1" n="$2" k
+ for k in 1 2 3 4; do
+ ip link del "ec$k" 2>/dev/null; ip link del "es$k" 2>/dev/null
+ nsenter -t "$(eval echo \$PC$k)" -n ip link del "c$k" 2>/dev/null
+ nsenter -t "$(eval echo \$PS$k)" -n ip link del "s$k" 2>/dev/null
+ done
+ for k in $(seq "$n"); do
+ local pc pcs; pc="$(eval echo \$PC$k)"; pcs="$(eval echo \$PS$k)"
+ if [ "$mode" = engine ]; then
+ ip link add "ec$k" type veth peer name "c$k" && ip link set "c$k" netns "$pc"
+ ip link add "es$k" type veth peer name "s$k" && ip link set "s$k" netns "$pcs"
+ ip link set "ec$k" up; ip link set "es$k" up
+ else
+ ip link add "c$k" type veth peer name "s$k" && ip link set "c$k" netns "$pc" && ip link set "s$k" netns "$pcs"
+ fi
+ nsenter -t "$pc" -n ip addr add "10.9.$k.1/24" dev "c$k"
+ nsenter -t "$pcs" -n ip addr add "10.9.$k.2/24" dev "s$k"
+ nsenter -t "$pc" -n ip neigh replace "10.9.$k.2" lladdr "$(nsmac "$pcs" "s$k")" dev "c$k" nud permanent
+ nsenter -t "$pcs" -n ip neigh replace "10.9.$k.1" lladdr "$(nsmac "$pc" "c$k")" dev "s$k" nud permanent
+ nsenter -t "$pc" -n ip link set "c$k" up
+ nsenter -t "$pcs" -n ip link set "s$k" up
+ done
+}
+offload_off() { # N: the node ends of N links
+ local k
+ for k in $(seq "$1"); do
+ nsenter -t "$(eval echo \$PC$k)" -n python3 "$here/ethtool_off.py" "c$k" >/dev/null
+ nsenter -t "$(eval echo \$PS$k)" -n python3 "$here/ethtool_off.py" "s$k" >/dev/null
+ done
+}
+
+SCRATCH="$(mktemp -d)"; trap 'cleanup; rm -rf "$SCRATCH"' EXIT
+
+# udp_case LABEL N THREADS TIMER PPS_PER_LINK DELAY_US JITTER_US LOSS SEED
+udp_case() {
+ local label="$1" n="$2" threads="$3" timer="$4" pps="$5" delay="$6" jitter="$7" loss="$8" seed="$9"
+ local links="" k epid=""
+ for k in $(seq "$n"); do links="$links,ec$k:es$k"; done; links="${links#,}"
+ if [ "$timer" != direct ]; then
+ local cores="$CE"; [ "$threads" -gt 1 ] && cores="$CE-$((CE + threads - 1))"
+ taskset -c "$cores" "$engine" --links "$links" --delay-us "$delay" --jitter-us "$jitter" --loss "$loss" --seed "$seed" \
+ --timer "$timer" --threads "$threads" >"$SCRATCH/engine.json" &
+ epid=$!; sleep 0.4
+ fi
+ local rp=() sp=()
+ for k in $(seq "$n"); do
+ nsenter -t "$(eval echo \$PS$k)" -n taskset -c "$((CR + 2 * (k - 1)))" "$tool" recv "10.9.$k.2" 9000 "$((secs + 10))" >"$SCRATCH/recv$k.json" &
+ rp+=($!)
+ done
+ sleep 0.3
+ for k in $(seq "$n"); do
+ nsenter -t "$(eval echo \$PC$k)" -n taskset -c "$((CS + 2 * (k - 1)))" "$tool" send "10.9.$k.2" 9000 "$pps" "$secs" 200 >"$SCRATCH/send$k.json" &
+ sp+=($!)
+ done
+ wait "${sp[@]}"; wait "${rp[@]}"
+ [ -n "$epid" ] && { kill "$epid"; wait "$epid" 2>/dev/null; }
+ echo "=== $label (links=$n threads=$threads timer=$timer pps/link=$pps delay=${delay}us jitter=${jitter}us loss=${loss}% seed=$seed)"
+ for k in $(seq "$n"); do
+ python3 - "$SCRATCH/send$k.json" "$SCRATCH/recv$k.json" "$k" <<'PY'
+import json, sys
+s = json.load(open(sys.argv[1])); r = json.load(open(sys.argv[2]))
+keep = ["received", "lost", "loss_percent", "lost_hash", "mean_ms", "stdev_ms", "min_ms", "p50_ms", "p99_ms", "p999_ms", "max_ms"]
+print(f" link {sys.argv[3]}: sent {s.get('sent')} at {s.get('achieved_pps')} pps; " + ", ".join(f"{k}={r[k]}" for k in keep if k in r))
+PY
+ done
+ [ -n "$epid" ] && cat "$SCRATCH/engine.json"
+}
+
+echo "# S21: kernel $(uname -r), $(grep -m1 'model name' /proc/cpuinfo | cut -d: -f2 | xargs), $(nproc) threads"
+if want timers; then
+ topology direct 1
+ udp_case "baseline: no engine, direct veth" 1 1 direct 100000 0 0 0 1
+ topology engine 1
+ for timer in epoll_ms epoll_ns timerfd spin spinsleep; do
+ for pps in 1000 10000 50000 100000; do
+ udp_case "delay 20 ms" 1 1 "$timer" "$pps" 20000 0 0 1
+ done
+ done
+fi
+if want links && [ "$cpus" -lt 8 ]; then
+ echo "INFO links skipped: $cpus CPUs, the section of several links needs 8"
+elif want links; then
+ topology engine 4
+ for timer in epoll_ns spinsleep; do
+ udp_case "2 links, one thread" 2 1 "$timer" 50000 20000 0 0 1
+ udp_case "2 links, one thread each" 2 2 "$timer" 50000 20000 0 0 1
+ udp_case "4 links, one thread" 4 1 "$timer" 25000 20000 0 0 1
+ udp_case "4 links, one thread each" 4 4 "$timer" 25000 20000 0 0 1
+ done
+fi
+if want jitter; then
+ topology engine 1
+ udp_case "uniform jitter of 5 ms around 20 ms (a sorted queue; gaps of 20 us at 50000 pps)" 1 1 epoll_ns 50000 20000 5000 0 1
+fi
+if want loss; then
+ topology engine 1
+ udp_case "loss 10 percent, seed 7, run 1" 1 1 epoll_ns 10000 20000 0 10 7
+ udp_case "loss 10 percent, seed 7, run 2" 1 1 epoll_ns 10000 20000 0 10 7
+ udp_case "loss 10 percent, seed 8" 1 1 epoll_ns 10000 20000 0 10 8
+fi
+if want compare; then
+ # S23: the same link on the kernel (netem on the client's interface) and on the engine, three runs each, to see how far two
+ # backends are from each other and from themselves
+ kernel_case() { # LABEL PPS NETEM_ARGS...
+ local label="$1" pps="$2"; shift 2
+ topology direct 1
+ nsenter -t "$PC1" -n tc qdisc add dev c1 root netem "$@" || return 1
+ nsenter -t "$PS1" -n taskset -c "$CR" "$tool" recv 10.9.1.2 9000 "$((secs + 10))" >"$SCRATCH/recv1.json" &
+ local rp=$!; sleep 0.3
+ nsenter -t "$PC1" -n taskset -c "$CS" "$tool" send 10.9.1.2 9000 "$pps" "$secs" 200 >"$SCRATCH/send1.json"
+ wait "$rp"
+ echo "=== $label (links=1 threads=1 timer=kernel-netem pps/link=$pps)"
+ python3 - "$SCRATCH/send1.json" "$SCRATCH/recv1.json" <<'PY'
+import json, sys
+s = json.load(open(sys.argv[1])); r = json.load(open(sys.argv[2]))
+keep = ["received", "lost", "loss_percent", "mean_ms", "stdev_ms", "min_ms", "p50_ms", "p99_ms", "p999_ms", "max_ms"]
+print(f" link 1: sent {s.get('sent')} at {s.get('achieved_pps')} pps; " + ", ".join(f"{k}={r[k]}" for k in keep if k in r))
+PY
+ echo '{"role":"engine","timer":"kernel","threads":0,"forwarded":0,"dropped":0,"send_errors":0,"reordered":0,"max_queue":0,"wakeups":0,"cpu_seconds":0,"wall_seconds":0,"late_mean_us":0,"late_p50_us":0,"late_p99_us":0,"late_p999_us":0,"late_max_us":0,"late_over_20ms":0}'
+ }
+ for rep in 1 2 3; do
+ topology engine 1
+ udp_case "A delay 20ms rep $rep" 1 1 epoll_ns 10000 20000 0 0 $rep
+ udp_case "A delay 20ms rep $rep" 1 1 spinsleep 10000 20000 0 0 $rep
+ kernel_case "A delay 20ms rep $rep" 10000 delay 20ms
+ topology engine 1
+ udp_case "B delay 20ms jitter 5ms rep $rep" 1 1 epoll_ns 10000 20000 5000 0 $rep
+ udp_case "B delay 20ms jitter 5ms rep $rep" 1 1 spinsleep 10000 20000 5000 0 $rep
+ kernel_case "B delay 20ms jitter 5ms rep $rep" 10000 delay 20ms 5ms
+ topology engine 1
+ udp_case "C loss 10pct rep $rep" 1 1 epoll_ns 10000 0 0 10 $rep
+ udp_case "C loss 10pct rep $rep" 1 1 spinsleep 10000 0 0 10 $rep
+ kernel_case "C loss 10pct rep $rep" 10000 loss 10%
+ done
+fi
+if want tcp; then
+ # tcp_case LABEL MODE OFFLOAD DELAY_US: MODE direct|engine; OFFLOAD on (left alone), off (transmit checksum offload switched off at
+ # the node ends), segs (one segment per frame, as the backend does with D-086) or vnet (segs, and the engine keeps the
+ # unfinished checksum with PACKET_VNET_HDR)
+ tcp_case() {
+ local label="$1" mode="$2" off="$3" delay="$4" epid="" extra=""
+ topology "$mode" 1
+ [ "$off" = off ] && offload_off 1
+ if [ "$off" = segs ] || [ "$off" = vnet ]; then
+ nsenter -t "$PC1" -n ip link set c1 gso_max_size 1514 gso_max_segs 1
+ nsenter -t "$PS1" -n ip link set s1 gso_max_size 1514 gso_max_segs 1
+ fi
+ [ "$off" = vnet ] && extra="--vnet-hdr"
+ if [ "$mode" = engine ]; then
+ taskset -c "$CE" "$engine" --links ec1:es1 --delay-us "$delay" --timer "${TCP_TIMER:-spinsleep}" $extra >"$SCRATCH/engine.json" &
+ epid=$!; sleep 0.4
+ fi
+ : >"$SCRATCH/tcp-server.json"; : >"$SCRATCH/tcp-client.json"
+ nsenter -t "$PS1" -n taskset -c "$CR" timeout $((secs + 8)) python3 "$here/../tcpflow.py" server 10.9.1.2 9100 1 "$secs" >"$SCRATCH/tcp-server.json" 2>&1 &
+ local sp=$!; sleep 0.5
+ nsenter -t "$PC1" -n taskset -c "$CS" timeout $((secs + 8)) python3 "$here/../tcpflow.py" client 10.9.1.2 9100 "$((secs + 1))" cubic >"$SCRATCH/tcp-client.json" 2>&1
+ wait "$sp"
+ [ -n "$epid" ] && { kill "$epid"; wait "$epid" 2>/dev/null; }
+ echo "=== TCP: $label (mode=$mode offloads=$off delay=${delay}us each way)"
+ python3 - "$SCRATCH/tcp-server.json" "$SCRATCH/tcp-client.json" <<'PY'
+import json, sys
+def last(path):
+ lines = [l for l in open(path).read().strip().splitlines() if l.startswith("{")]
+ return json.loads(lines[-1]) if lines else None
+server, client = last(sys.argv[1]), last(sys.argv[2])
+if server:
+ rates = server.get("bytes_per_second") or []
+ flat = [x for r in rates for x in (r if isinstance(r, list) else [r])]
+ print(" server: goodput of the connection", [round(x * 8 / 1e6, 1) for x in flat[:1]], "mbit/s over its seconds:", [round(x * 8 / 1e6) for x in flat[:6]])
+else:
+ print(" server: no result (no connection arrived)")
+print(" client:", "no result (the connection did not work)" if not client else json.dumps({k: client.get(k) for k in ("retransmits", "retrans", "snd_cwnd", "rtt")}))
+PY
+ [ -n "$epid" ] && cat "$SCRATCH/engine.json"
+ }
+ tcp_case "direct veth, offloads as they are" direct on 0
+ tcp_case "direct veth, one segment per frame (D-086)" direct segs 0
+ tcp_case "engine, offloads left on" engine on 0
+ tcp_case "engine, transmit checksum offload off (ioctl)" engine off 0
+ tcp_case "engine, PACKET_VNET_HDR, one segment per frame" engine vnet 0
+ tcp_case "engine, PACKET_VNET_HDR, 10 ms each way" engine vnet 10000
+fi
diff --git a/spikes/rust-engine-safe/src/main.rs b/spikes/rust-engine-safe/src/main.rs
new file mode 100644
index 0000000..8bd8eb7
--- /dev/null
+++ b/spikes/rust-engine-safe/src/main.rs
@@ -0,0 +1,333 @@
+//! Spike S21: a packet engine data path. Everything is safe Rust except `sys`, the one module that binds a packet socket.
+//!
+//! s21 --links C1:S1,C2:S2 --delay-us N --jitter-us N --loss PERCENT --seed N --timer T --threads N --batch N
+//!
+//! Forwards Ethernet frames between the two ends of each link (AF_PACKET sockets) with a fixed delay, a uniform jitter (a sorted
+//! queue: a jitter larger than the gap reorders) and a seeded loss. T is one of epoll_ms, epoll_ns, timerfd, spin, spinsleep.
+//! Stops on SIGTERM and prints one JSON line with the lateness of every release (the time between the moment a packet was due
+//! and the moment it was handed to the interface), the CPU time and the counters.
+
+#![deny(unsafe_code)]
+
+mod sys;
+
+use std::cmp::Reverse;
+use std::collections::BinaryHeap;
+use std::os::fd::{AsFd, OwnedFd};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+use nix::errno::Errno;
+use nix::sys::socket::{recv, send, MsgFlags};
+use rustix::event::epoll;
+use rustix::event::epoll::{CreateFlags, EventData, EventFlags};
+use rustix::time::{
+ clock_gettime, timerfd_create, timerfd_settime, ClockId, Itimerspec, TimerfdClockId, TimerfdFlags, TimerfdTimerFlags,
+ Timespec,
+};
+
+const MAX_FRAME: usize = 2100; // a frame of up to 2048 bytes and the 10 byte header of --vnet-hdr
+const BUCKETS: usize = 20_000; // one microsecond each, up to 20 ms
+
+fn now_ns() -> u64 {
+ let t = clock_gettime(ClockId::Monotonic);
+ t.tv_sec as u64 * 1_000_000_000 + t.tv_nsec as u64
+}
+
+struct Rng(u64);
+impl Rng {
+ fn next(&mut self) -> u64 {
+ self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
+ let mut z = self.0;
+ z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+ z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+ z ^ (z >> 31)
+ }
+ fn unit(&mut self) -> f64 {
+ (self.next() >> 11) as f64 / (1u64 << 53) as f64
+ }
+}
+
+#[derive(Clone, Copy, PartialEq, Debug)]
+enum Timer {
+ EpollMs,
+ EpollNs,
+ Timerfd,
+ Spin,
+ SpinSleep,
+}
+
+#[derive(Clone)]
+struct Config {
+ delay_ns: u64,
+ jitter_ns: u64,
+ loss: f64,
+ seed: u64,
+ timer: Timer,
+}
+
+#[derive(Default, Clone)]
+struct Stats {
+ forwarded: u64,
+ dropped: u64,
+ send_errors: u64,
+ reordered: u64,
+ max_queue: usize,
+ wakeups: u64,
+ late_over: u64,
+ hist: Vec,
+ late_max_ns: u64,
+ late_sum_ns: u128,
+ cpu_seconds: f64,
+}
+
+struct Slot {
+ data: [u8; MAX_FRAME],
+ len: usize,
+}
+
+/// One thread: the links it owns, a heap of packets ordered by the moment they are due, and a timer.
+fn run_thread(links: Vec<(OwnedFd, OwnedFd)>, config: Config, stop: Arc) -> Stats {
+ // A timer that is allowed to fire late by 50 microseconds by default; one nanosecond asks for the exact moment
+ let _ = nix::sys::prctl::set_timerslack(1);
+ let mut fds: Vec = Vec::new();
+ for (a, b) in links {
+ fds.push(a);
+ fds.push(b);
+ }
+ let poll = epoll::create(CreateFlags::CLOEXEC).expect("epoll");
+ for (i, fd) in fds.iter().enumerate() {
+ epoll::add(&poll, fd, EventData::new_u64(i as u64), EventFlags::IN).expect("epoll add");
+ }
+ let tfd = timerfd_create(TimerfdClockId::Monotonic, TimerfdFlags::NONBLOCK | TimerfdFlags::CLOEXEC).expect("timerfd");
+ epoll::add(&poll, &tfd, EventData::new_u64(u64::MAX), EventFlags::IN).expect("epoll add timerfd");
+
+ let mut slab: Vec = Vec::new();
+ let mut free: Vec = Vec::new();
+ let mut heap: BinaryHeap> = BinaryHeap::new(); // (due, number, slot, out)
+ let mut stats = Stats { hist: vec![0; BUCKETS], ..Stats::default() };
+ let mut rng = Rng(config.seed);
+ let mut number = 0u64;
+ let mut last_released = 0u64;
+ let mut events: Vec = Vec::with_capacity(64);
+
+ while !stop.load(Ordering::Relaxed) {
+ // 1. hand over what is due
+ let now = now_ns();
+ while let Some(&Reverse((due, n, slot, out))) = heap.peek() {
+ if due > now {
+ break;
+ }
+ heap.pop();
+ let s = &slab[slot as usize];
+ match send(std::os::fd::AsRawFd::as_raw_fd(&fds[out as usize]), &s.data[..s.len], MsgFlags::empty()) {
+ Ok(_) => stats.forwarded += 1,
+ Err(_) => stats.send_errors += 1,
+ }
+ let late = now_ns().saturating_sub(due);
+ let bucket = (late / 1000) as usize;
+ if bucket < BUCKETS {
+ stats.hist[bucket] += 1;
+ } else {
+ stats.late_over += 1;
+ }
+ stats.late_max_ns = stats.late_max_ns.max(late);
+ stats.late_sum_ns += late as u128;
+ if n < last_released {
+ stats.reordered += 1;
+ }
+ last_released = n;
+ free.push(slot);
+ }
+ // 2. take in what has arrived
+ for side in 0..fds.len() {
+ for _ in 0..64 {
+ let raw = std::os::fd::AsRawFd::as_raw_fd(&fds[side]);
+ let slot = free.pop().unwrap_or_else(|| {
+ slab.push(Slot { data: [0; MAX_FRAME], len: 0 });
+ (slab.len() - 1) as u32
+ });
+ match recv(raw, &mut slab[slot as usize].data, MsgFlags::empty()) {
+ Ok(0) | Err(Errno::EAGAIN) => {
+ free.push(slot);
+ break;
+ }
+ Ok(n) => {
+ let arrival = now_ns();
+ if config.loss > 0.0 && rng.unit() < config.loss {
+ stats.dropped += 1;
+ free.push(slot);
+ continue;
+ }
+ let jitter = if config.jitter_ns > 0 {
+ let j = (rng.unit() * 2.0 - 1.0) * config.jitter_ns as f64;
+ j as i64
+ } else {
+ 0
+ };
+ let due = (arrival as i64 + config.delay_ns as i64 + jitter).max(arrival as i64) as u64;
+ slab[slot as usize].len = n;
+ number += 1;
+ // the other end of the link: 0 <-> 1, 2 <-> 3, ...
+ heap.push(Reverse((due, number, slot, (side ^ 1) as u32)));
+ }
+ Err(_) => {
+ free.push(slot);
+ break;
+ }
+ }
+ }
+ }
+ stats.max_queue = stats.max_queue.max(heap.len());
+ // 3. wait for a frame or for the next due moment
+ let wait_ns = heap.peek().map(|Reverse((due, ..))| due.saturating_sub(now_ns()));
+ match config.timer {
+ Timer::Spin => {}
+ Timer::SpinSleep => {
+ stats.wakeups += 1;
+ match wait_ns {
+ Some(w) if w > 150_000 => {
+ let t = Timespec { tv_sec: 0, tv_nsec: (w - 100_000) as _ };
+ wait_report(epoll::wait(&poll, rustix::buffer::spare_capacity(&mut events), Some(&t)));
+ }
+ Some(_) => {}
+ None => {
+ let t = Timespec { tv_sec: 0, tv_nsec: 50_000_000 };
+ wait_report(epoll::wait(&poll, rustix::buffer::spare_capacity(&mut events), Some(&t)));
+ }
+ }
+ }
+ Timer::EpollMs | Timer::EpollNs => {
+ stats.wakeups += 1;
+ let w = wait_ns.unwrap_or(50_000_000).max(1);
+ let w = if config.timer == Timer::EpollMs { w.div_ceil(1_000_000) * 1_000_000 } else { w };
+ let t = Timespec { tv_sec: (w / 1_000_000_000) as _, tv_nsec: (w % 1_000_000_000) as _ };
+ wait_report(epoll::wait(&poll, rustix::buffer::spare_capacity(&mut events), Some(&t)));
+ }
+ Timer::Timerfd => {
+ stats.wakeups += 1;
+ let w = wait_ns.unwrap_or(50_000_000).max(1);
+ let spec = Itimerspec {
+ it_interval: Timespec { tv_sec: 0, tv_nsec: 0 },
+ it_value: Timespec { tv_sec: (w / 1_000_000_000) as _, tv_nsec: (w % 1_000_000_000) as _ },
+ };
+ let _ = timerfd_settime(&tfd, TimerfdTimerFlags::empty(), &spec);
+ wait_report(epoll::wait(&poll, rustix::buffer::spare_capacity(&mut events), None));
+ let mut sink = [0u8; 8];
+ let _ = rustix::io::read(tfd.as_fd(), &mut sink);
+ }
+ }
+ events.clear();
+ }
+ if let Ok(usage) = nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_THREAD) {
+ let (user, system) = (usage.user_time(), usage.system_time());
+ stats.cpu_seconds = user.tv_sec() as f64 + user.tv_usec() as f64 / 1e6 + system.tv_sec() as f64 + system.tv_usec() as f64 / 1e6;
+ }
+ stats
+}
+
+fn wait_report(result: rustix::io::Result) {
+ use std::sync::atomic::AtomicBool;
+ static SAID: AtomicBool = AtomicBool::new(false);
+ if let Err(e) = result {
+ if !SAID.swap(true, Ordering::Relaxed) {
+ eprintln!("epoll_wait failed: {e}");
+ }
+ }
+}
+
+fn percentile(hist: &[u64], over: u64, p: f64) -> f64 {
+ let total: u64 = hist.iter().sum::() + over;
+ if total == 0 {
+ return 0.0;
+ }
+ let target = (total as f64 * p).ceil() as u64;
+ let mut seen = 0;
+ for (i, c) in hist.iter().enumerate() {
+ seen += c;
+ if seen >= target {
+ return i as f64 + 1.0; // upper edge of the bucket, in microseconds
+ }
+ }
+ f64::INFINITY
+}
+
+fn main() {
+ let args: Vec = std::env::args().collect();
+ let get = |name: &str, default: &str| -> String {
+ args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned().unwrap_or_else(|| default.to_string())
+ };
+ let links_arg = get("--links", "");
+ let threads: usize = get("--threads", "1").parse().unwrap();
+ let vnet = args.iter().any(|a| a == "--vnet-hdr");
+ let timer = match get("--timer", "epoll_ns").as_str() {
+ "epoll_ms" => Timer::EpollMs,
+ "epoll_ns" => Timer::EpollNs,
+ "timerfd" => Timer::Timerfd,
+ "spin" => Timer::Spin,
+ "spinsleep" => Timer::SpinSleep,
+ other => panic!("unknown timer {other}"),
+ };
+ let config = Config {
+ delay_ns: get("--delay-us", "0").parse::().unwrap() * 1000,
+ jitter_ns: get("--jitter-us", "0").parse::().unwrap() * 1000,
+ loss: get("--loss", "0").parse::().unwrap() / 100.0,
+ seed: get("--seed", "1").parse().unwrap(),
+ timer,
+ };
+ let stop = Arc::new(AtomicBool::new(false));
+ for signal in [signal_hook::consts::SIGTERM, signal_hook::consts::SIGINT] {
+ signal_hook::flag::register(signal, Arc::clone(&stop)).expect("signal");
+ }
+ let mut groups: Vec> = (0..threads).map(|_| Vec::new()).collect();
+ for (i, pair) in links_arg.split(',').filter(|s| !s.is_empty()).enumerate() {
+ let (a, b) = pair.split_once(':').expect("links are A:B");
+ groups[i % threads].push((sys::open_packet(a, vnet), sys::open_packet(b, vnet)));
+ }
+ let mut handles = Vec::new();
+ for (i, links) in groups.into_iter().enumerate() {
+ let mut c = config.clone();
+ c.seed = config.seed.wrapping_add(i as u64);
+ let stop = Arc::clone(&stop);
+ handles.push(std::thread::spawn(move || run_thread(links, c, stop)));
+ }
+ let started = now_ns();
+ let all: Vec = handles.into_iter().map(|h| h.join().unwrap()).collect();
+ let wall = (now_ns() - started) as f64 / 1e9;
+ let mut total = Stats { hist: vec![0; BUCKETS], ..Stats::default() };
+ for s in &all {
+ total.forwarded += s.forwarded;
+ total.dropped += s.dropped;
+ total.send_errors += s.send_errors;
+ total.reordered += s.reordered;
+ total.max_queue = total.max_queue.max(s.max_queue);
+ total.wakeups += s.wakeups;
+ total.late_over += s.late_over;
+ total.late_max_ns = total.late_max_ns.max(s.late_max_ns);
+ total.late_sum_ns += s.late_sum_ns;
+ total.cpu_seconds += s.cpu_seconds;
+ for (i, c) in s.hist.iter().enumerate() {
+ total.hist[i] += c;
+ }
+ }
+ let n = total.forwarded.max(1);
+ println!(
+ "{{\"role\":\"engine\",\"timer\":\"{:?}\",\"threads\":{threads},\"forwarded\":{},\"dropped\":{},\"send_errors\":{},\"reordered\":{},\"max_queue\":{},\"wakeups\":{},\"cpu_seconds\":{:.2},\"wall_seconds\":{wall:.2},\"late_mean_us\":{:.1},\"late_p50_us\":{},\"late_p99_us\":{},\"late_p999_us\":{},\"late_max_us\":{:.0},\"late_over_20ms\":{}}}",
+ config.timer,
+ total.forwarded,
+ total.dropped,
+ total.send_errors,
+ total.reordered,
+ total.max_queue,
+ total.wakeups,
+ total.cpu_seconds,
+ total.late_sum_ns as f64 / n as f64 / 1000.0,
+ percentile(&total.hist, total.late_over, 0.5),
+ percentile(&total.hist, total.late_over, 0.99),
+ percentile(&total.hist, total.late_over, 0.999),
+ total.late_max_ns as f64 / 1000.0,
+ total.late_over,
+ );
+ let _ = Duration::from_secs(0);
+}
diff --git a/spikes/rust-engine-safe/src/sys.rs b/spikes/rust-engine-safe/src/sys.rs
new file mode 100644
index 0000000..9678a46
--- /dev/null
+++ b/spikes/rust-engine-safe/src/sys.rs
@@ -0,0 +1,48 @@
+//! The one place with `unsafe`: a packet socket bound to an interface, which does not receive what it sends itself.
+//!
+//! `nix` 0.31 cannot make the address that `bind` and `send` on a packet socket need (the field of `LinkAddr` is private and
+//! there is no constructor), and neither `nix` nor `rustix` knows `PACKET_IGNORE_OUTGOING`. So these two calls are made here.
+#![allow(unsafe_code)]
+
+use std::os::fd::{AsRawFd, OwnedFd};
+
+use nix::sys::socket::{setsockopt, socket, sockopt, AddressFamily, SockFlag, SockProtocol, SockType};
+
+const PACKET_IGNORE_OUTGOING: libc::c_int = 23;
+const PACKET_VNET_HDR: libc::c_int = 15;
+
+pub fn open_packet(interface: &str, vnet_hdr: bool) -> OwnedFd {
+ let fd = socket(
+ AddressFamily::Packet,
+ SockType::Raw,
+ SockFlag::SOCK_NONBLOCK | SockFlag::SOCK_CLOEXEC,
+ SockProtocol::EthAll,
+ )
+ .unwrap_or_else(|e| panic!("socket(AF_PACKET): {e}"));
+ let index = nix::net::if_::if_nametoindex(interface).unwrap_or_else(|e| panic!("no interface {interface}: {e}"));
+ // SAFETY: `sll` is a plain C struct, filled completely before the call, and the length is its size.
+ let bound = unsafe {
+ let mut sll: libc::sockaddr_ll = std::mem::zeroed();
+ sll.sll_family = libc::AF_PACKET as u16;
+ sll.sll_protocol = (libc::ETH_P_ALL as u16).to_be();
+ sll.sll_ifindex = index as i32;
+ libc::bind(fd.as_raw_fd(), (&raw const sll).cast(), std::mem::size_of::() as u32)
+ };
+ assert!(bound == 0, "bind {interface}: {}", std::io::Error::last_os_error());
+ let one: libc::c_int = 1;
+ // SAFETY: a pointer to a live `c_int` and its size.
+ let set = unsafe {
+ libc::setsockopt(fd.as_raw_fd(), libc::SOL_PACKET, PACKET_IGNORE_OUTGOING, (&raw const one).cast(), 4)
+ };
+ assert!(set == 0, "PACKET_IGNORE_OUTGOING: {}", std::io::Error::last_os_error());
+ if vnet_hdr {
+ // Every frame is read and written with a 10 byte `virtio_net_hdr` in front, which keeps an unfinished checksum unfinished
+ // (the sender left it to the device) so that the interface that receives the frame finishes it, as it would from a real wire
+ // SAFETY: a pointer to a live `c_int` and its size.
+ let set = unsafe { libc::setsockopt(fd.as_raw_fd(), libc::SOL_PACKET, PACKET_VNET_HDR, (&raw const one).cast(), 4) };
+ assert!(set == 0, "PACKET_VNET_HDR: {}", std::io::Error::last_os_error());
+ }
+ let _ = setsockopt(&fd, sockopt::RcvBuf, &(4 * 1024 * 1024));
+ let _ = setsockopt(&fd, sockopt::SndBuf, &(4 * 1024 * 1024));
+ fd
+}
diff --git a/spikes/rust-netlink-own/Cargo.lock b/spikes/rust-netlink-own/Cargo.lock
new file mode 100644
index 0000000..5433af7
--- /dev/null
+++ b/spikes/rust-netlink-own/Cargo.lock
@@ -0,0 +1,121 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "bitflags"
+version = "2.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+
+[[package]]
+name = "bytes"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+
+[[package]]
+name = "libc"
+version = "0.2.191"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
+
+[[package]]
+name = "log"
+version = "0.4.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"
+
+[[package]]
+name = "netlink-packet-core"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d6e2daf9a8c2e21302714706860a0e6be02e03d17294ecc66b354ea7d4059dd"
+
+[[package]]
+name = "netlink-packet-route"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3c407d8e95c0128ee5729728482a369f3ba9a0d9ba8335c20949a8d8989cf6b"
+dependencies = [
+ "bitflags",
+ "libc",
+ "log",
+ "netlink-packet-core",
+ "zerocopy",
+]
+
+[[package]]
+name = "netlink-sys"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c99d38e00b420df49e940fe0826e667c3dad82ac68eb4c2bbf754c1c14a83a10"
+dependencies = [
+ "bytes",
+ "libc",
+ "log",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "spike-netlink-own"
+version = "0.0.0"
+dependencies = [
+ "netlink-packet-core",
+ "netlink-packet-route",
+ "netlink-sys",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
+
+[[package]]
+name = "zerocopy"
+version = "0.8.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
diff --git a/spikes/rust-netlink-own/Cargo.toml b/spikes/rust-netlink-own/Cargo.toml
new file mode 100644
index 0000000..6c3a5fa
--- /dev/null
+++ b/spikes/rust-netlink-own/Cargo.toml
@@ -0,0 +1,17 @@
+# Spike S22: netlink by a thin layer of our own over `netlink-packet-route` (no `unsafe`) and `netlink-sys`. A scratch crate, not part of any workspace.
+[package]
+name = "spike-netlink-own"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+netlink-packet-core = "0.9"
+netlink-packet-route = "0.34"
+netlink-sys = "0.9"
+
+[profile.release]
+debug = false
+opt-level = 3
+
+[workspace]
diff --git a/spikes/rust-netlink-own/run.sh b/spikes/rust-netlink-own/run.sh
new file mode 100755
index 0000000..1ea2b5c
--- /dev/null
+++ b/spikes/rust-netlink-own/run.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# S22: netlink by a thin layer of our own over `netlink-packet-route` (no `unsafe`) and `netlink-sys`: the messages for netem, the
+# fifos, codel, pie, fq_codel, tbf and htb are encoded here, applied in a user namespace, and the kernel's dump is compared with
+# what the `tc` command makes of the same settings. Also the time of a change of a running netem.
+# bash spikes/rust-netlink-own/run.sh
+set -u
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export PATH="$PATH:/usr/sbin:/sbin"
+cargo build --release --quiet --manifest-path "$here/Cargo.toml" || exit 1
+if [ "$(id -u)" -eq 0 ]; then exec unshare -n "$here/target/release/spike-netlink-own"; fi
+exec unshare -Urn "$here/target/release/spike-netlink-own"
diff --git a/spikes/rust-netlink-own/src/main.rs b/spikes/rust-netlink-own/src/main.rs
new file mode 100644
index 0000000..5fdede9
--- /dev/null
+++ b/spikes/rust-netlink-own/src/main.rs
@@ -0,0 +1,449 @@
+//! Spike S22: netlink by a thin layer of our own.
+//!
+//! `netlink-packet-route` 0.34 has no `unsafe` and knows links, addresses, routes and a few qdiscs, but not `netem`, `htb`, `tbf`,
+//! `codel`, `pie` or the fifos. Its `TcAttribute::Other` carries bytes that we encode ourselves, so this program builds the
+//! messages for those qdiscs from the structures of `linux/pkt_sched.h`, applies them in a namespace, and compares what the kernel
+//! reports (`tc -d qdisc show`) with what the `tc` command makes of the same settings. It runs inside `unshare -Urn` (`run.sh`).
+//!
+//! Each check prints PASS, FAIL or INFO. The last part times a change of a running `netem`.
+
+use std::process::Command;
+use std::time::Instant;
+
+use netlink_packet_core::{
+ DefaultNla, NetlinkHeader, NetlinkMessage, NetlinkPayload, NLM_F_ACK, NLM_F_CREATE, NLM_F_EXCL, NLM_F_REPLACE, NLM_F_REQUEST,
+};
+use netlink_packet_route::link::{InfoKind, LinkAttribute, LinkFlags, LinkInfo, LinkMessage};
+use netlink_packet_route::tc::{TcAttribute, TcHandle, TcHeader, TcMessage, TcOption};
+use netlink_packet_route::{AddressFamily, RouteNetlinkMessage};
+use netlink_sys::{protocols::NETLINK_ROUTE, Socket, SocketAddr};
+
+const TCA_OPTIONS: u16 = 2;
+const NLA_F_NESTED: u16 = 1 << 15;
+
+// ------------------------------------------------------------------------------------------------ the socket
+
+struct Netlink {
+ socket: Socket,
+ sequence: u32,
+}
+
+impl Netlink {
+ fn new() -> Self {
+ let mut socket = Socket::new(NETLINK_ROUTE).expect("netlink socket");
+ socket.bind_auto().expect("bind");
+ socket.connect(&SocketAddr::new(0, 0)).expect("connect");
+ Netlink { socket, sequence: 1 }
+ }
+
+ /// One request, one acknowledgement.
+ fn request(&mut self, payload: RouteNetlinkMessage, flags: u16) -> Result<(), String> {
+ let mut header = NetlinkHeader::default();
+ header.flags = NLM_F_REQUEST | NLM_F_ACK | flags;
+ header.sequence_number = self.sequence;
+ self.sequence += 1;
+ let mut message = NetlinkMessage::new(header, NetlinkPayload::from(payload));
+ message.finalize();
+ let mut buffer = vec![0u8; message.buffer_len()];
+ message.serialize(&mut buffer);
+ self.socket.send(&buffer, 0).map_err(|e| format!("send: {e}"))?;
+ let mut receive = vec![0u8; 8192];
+ let size = self.socket.recv(&mut &mut receive[..], 0).map_err(|e| format!("recv: {e}"))?;
+ let reply = NetlinkMessage::::deserialize(&receive[..size]).map_err(|e| format!("decode: {e}"))?;
+ match reply.payload {
+ NetlinkPayload::Error(error) => match error.code {
+ None => Ok(()),
+ Some(code) => Err(format!("kernel: {}", std::io::Error::from_raw_os_error(-code.get()))),
+ },
+ other => Err(format!("unexpected reply {other:?}")),
+ }
+ }
+
+ fn add_dummy(&mut self, name: &str) -> Result<(), String> {
+ let mut link = LinkMessage::default();
+ link.attributes = vec![LinkAttribute::IfName(name.to_string()), LinkAttribute::LinkInfo(vec![LinkInfo::Kind(InfoKind::Dummy)])];
+ self.request(RouteNetlinkMessage::NewLink(link), NLM_F_CREATE | NLM_F_EXCL)
+ }
+
+ fn set_up(&mut self, index: u32) -> Result<(), String> {
+ let mut link = LinkMessage::default();
+ link.header.index = index;
+ link.header.flags = LinkFlags::Up;
+ link.header.change_mask = LinkFlags::Up;
+ self.request(RouteNetlinkMessage::SetLink(link), 0)
+ }
+
+ fn qdisc(&mut self, index: u32, parent: TcHandle, handle: TcHandle, kind: &str, options: Option>, replace: bool) -> Result<(), String> {
+ let mut header = TcHeader::default();
+ header.index = index as i32;
+ header.parent = parent;
+ header.handle = handle;
+ let mut attributes = vec![TcAttribute::Kind(kind.to_string())];
+ if let Some(bytes) = options {
+ attributes.push(TcAttribute::Other(DefaultNla::new(TCA_OPTIONS, bytes)));
+ }
+ let message = TcMessage::from_parts(header, attributes);
+ let flags = if replace { NLM_F_CREATE | NLM_F_REPLACE } else { NLM_F_CREATE | NLM_F_EXCL };
+ self.request(RouteNetlinkMessage::NewQueueDiscipline(message), flags)
+ }
+
+ fn class(&mut self, index: u32, parent: TcHandle, handle: TcHandle, kind: &str, options: Vec) -> Result<(), String> {
+ let mut header = TcHeader::default();
+ header.index = index as i32;
+ header.parent = parent;
+ header.handle = handle;
+ let attributes = vec![TcAttribute::Kind(kind.to_string()), TcAttribute::Other(DefaultNla::new(TCA_OPTIONS, options))];
+ let message = TcMessage::from_parts(header, attributes);
+ self.request(RouteNetlinkMessage::NewTrafficClass(message), NLM_F_CREATE | NLM_F_EXCL)
+ }
+}
+
+// ----------------------------------------------------------------------------------------------- the encoders
+
+fn nla(kind: u16, value: &[u8]) -> Vec {
+ let length = 4 + value.len();
+ let mut out = Vec::with_capacity(length + 3);
+ out.extend_from_slice(&(length as u16).to_ne_bytes());
+ out.extend_from_slice(&kind.to_ne_bytes());
+ out.extend_from_slice(value);
+ while out.len() % 4 != 0 {
+ out.push(0);
+ }
+ out
+}
+fn nested(kind: u16, inner: &[u8]) -> Vec {
+ nla(kind | NLA_F_NESTED, inner)
+}
+fn u32s(values: &[u32]) -> Vec {
+ values.iter().flat_map(|v| v.to_ne_bytes()).collect()
+}
+
+/// A probability in percent as the kernel keeps it: a fraction of 2^32 - 1, rounded as `tc` does.
+fn probability(percent: f64) -> u32 {
+ (percent / 100.0 * 4_294_967_295.0).round() as u32
+}
+
+#[derive(Default, Clone)]
+struct Netem {
+ delay_us: u64,
+ jitter_us: u64,
+ limit: u32,
+ loss_percent: f64,
+ loss_correlation: f64,
+ gemodel: Option<(f64, f64, f64, f64)>, // p r 1-h 1-k in percent, as `tc` prints them
+ duplicate_percent: f64,
+ corrupt_percent: f64,
+ reorder: Option<(f64, f64, u32)>, // percent, correlation, gap
+ rate_bps: u64,
+ ecn: bool,
+ seed: Option,
+}
+
+impl Netem {
+ /// The options of a `netem` qdisc: `tc_netem_qopt`, then attributes (the one qdisc whose options begin with a plain struct).
+ fn encode(&self) -> Vec {
+ let mut out = u32s(&[
+ 0, // latency in psched ticks: the 64-bit attribute below carries it (the kernel reads that one when it is there)
+ if self.limit == 0 { 1000 } else { self.limit },
+ probability(self.loss_percent),
+ self.reorder.map_or(0, |r| r.2),
+ probability(self.duplicate_percent),
+ 0, // jitter in ticks, as the latency
+ ]);
+ if self.delay_us > 0 || self.jitter_us > 0 {
+ out.extend(nla(10, &(self.delay_us * 1000).to_ne_bytes())); // TCA_NETEM_LATENCY64
+ out.extend(nla(11, &(self.jitter_us * 1000).to_ne_bytes())); // TCA_NETEM_JITTER64
+ }
+ if self.loss_correlation > 0.0 {
+ out.extend(nla(1, &u32s(&[0, probability(self.loss_correlation), 0]))); // TCA_NETEM_CORR (delay, loss, duplicate)
+ }
+ if let Some((p, c, _)) = self.reorder {
+ out.extend(nla(3, &u32s(&[probability(p), probability(c)]))); // TCA_NETEM_REORDER
+ }
+ if self.corrupt_percent > 0.0 {
+ out.extend(nla(4, &u32s(&[probability(self.corrupt_percent), 0]))); // TCA_NETEM_CORRUPT
+ }
+ if let Some((p, r, one_h, one_k)) = self.gemodel {
+ // TCA_NETEM_LOSS { TCA_NETEM_LOSS_GE = 2: tc_netem_gemodel { p, r, h, k1 } }
+ let ge = u32s(&[probability(p), probability(r), probability(100.0 - one_h), probability(one_k)]);
+ out.extend(nested(5, &nla(2, &ge)));
+ }
+ if self.rate_bps > 0 {
+ let bytes_per_second = self.rate_bps / 8;
+ let mut rate = (bytes_per_second.min(u64::from(u32::MAX)) as u32).to_ne_bytes().to_vec();
+ rate.extend(0i32.to_ne_bytes()); // packet_overhead
+ rate.extend(0u32.to_ne_bytes()); // cell_size
+ rate.extend(0i32.to_ne_bytes()); // cell_overhead
+ out.extend(nla(6, &rate)); // TCA_NETEM_RATE
+ if bytes_per_second > u64::from(u32::MAX) {
+ out.extend(nla(8, &bytes_per_second.to_ne_bytes())); // TCA_NETEM_RATE64
+ }
+ }
+ if self.ecn {
+ out.extend(nla(7, &1u32.to_ne_bytes())); // TCA_NETEM_ECN
+ }
+ if let Some(seed) = self.seed {
+ out.extend(nla(14, &seed.to_ne_bytes())); // TCA_NETEM_PRNG_SEED
+ }
+ out
+ }
+
+ /// The same settings as arguments of `tc`.
+ fn tc_args(&self) -> Vec {
+ let mut args: Vec = vec!["netem".into()];
+ if self.limit > 0 {
+ args.extend(["limit".into(), self.limit.to_string()]);
+ }
+ if self.delay_us > 0 || self.jitter_us > 0 {
+ args.extend(["delay".into(), format!("{}us", self.delay_us)]);
+ if self.jitter_us > 0 {
+ args.push(format!("{}us", self.jitter_us));
+ }
+ }
+ if let Some((p, r, h, k)) = self.gemodel {
+ args.extend(["loss".into(), "gemodel".into(), format!("{p}%"), format!("{r}%"), format!("{h}%"), format!("{k}%")]);
+ } else if self.loss_percent > 0.0 {
+ args.extend(["loss".into(), format!("{}%", self.loss_percent)]);
+ if self.loss_correlation > 0.0 {
+ args.push(format!("{}%", self.loss_correlation));
+ }
+ }
+ if self.duplicate_percent > 0.0 {
+ args.extend(["duplicate".into(), format!("{}%", self.duplicate_percent)]);
+ }
+ if self.corrupt_percent > 0.0 {
+ args.extend(["corrupt".into(), format!("{}%", self.corrupt_percent)]);
+ }
+ if let Some((p, c, gap)) = self.reorder {
+ args.extend(["reorder".into(), format!("{p}%"), format!("{c}%")]);
+ if gap > 0 {
+ args.extend(["gap".into(), gap.to_string()]);
+ }
+ }
+ if self.rate_bps > 0 {
+ args.extend(["rate".into(), format!("{}bit", self.rate_bps)]);
+ }
+ if self.ecn {
+ args.push("ecn".into());
+ }
+ if let Some(seed) = self.seed {
+ args.extend(["seed".into(), seed.to_string()]);
+ }
+ args
+ }
+}
+
+/// `tc_ratespec`: cell_log, linklayer (1 = ethernet), overhead, cell_align, mpu, rate in bytes per second.
+fn ratespec(bytes_per_second: u32) -> Vec {
+ let mut out = vec![0u8, 1];
+ out.extend(0u16.to_ne_bytes()); // overhead
+ out.extend(0i16.to_ne_bytes()); // cell_align
+ out.extend(0u16.to_ne_bytes()); // mpu
+ out.extend(bytes_per_second.to_ne_bytes());
+ out
+}
+
+/// The time to send `size` bytes at `bytes_per_second`, in the 64 ns ticks the scheduler counts in.
+fn ticks(size: u32, bytes_per_second: u32) -> u32 {
+ ((f64::from(size) / f64::from(bytes_per_second)) * 1e9 / 64.0) as u32
+}
+
+// --------------------------------------------------------------------------------------------------- the checks
+
+fn tc(args: &[&str]) -> Result {
+ let path = format!("{}:/usr/sbin:/sbin", std::env::var("PATH").unwrap_or_default());
+ let out = Command::new("tc").args(args).env("PATH", path).output().map_err(|e| e.to_string())?;
+ if out.status.success() {
+ Ok(String::from_utf8_lossy(&out.stdout).into_owned())
+ } else {
+ Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
+ }
+}
+
+fn verdict(id: &str, text: &str, ok: bool, detail: &str) {
+ println!("{} {id:<8} {text}{}", if ok { "PASS" } else { "FAIL" }, if ok { String::new() } else { format!("\n {detail}") });
+}
+
+fn index_of(name: &str) -> u32 {
+ // /sys shows the devices of the namespace it was mounted in, not of this one: ask `ip`
+ let path = format!("{}:/usr/sbin:/sbin", std::env::var("PATH").unwrap_or_default());
+ let out = Command::new("ip").args(["-o", "link", "show", name]).env("PATH", path).output().expect("ip");
+ String::from_utf8_lossy(&out.stdout).split(':').next().unwrap().trim().parse().expect("ifindex")
+}
+
+/// Set up two dummy devices, `ta` (settings by `tc`) and `na` (the same by netlink), and compare the dumps of the kernel.
+fn compare(nl: &mut Netlink, id: &str, text: &str, by_tc: &[&str], by_netlink: impl FnOnce(&mut Netlink, u32) -> Result<(), String>) {
+ let _ = tc(&["qdisc", "del", "dev", "ta", "root"]);
+ let _ = tc(&["qdisc", "del", "dev", "na", "root"]);
+ let mut full = vec!["qdisc", "add", "dev", "ta"];
+ full.extend_from_slice(by_tc);
+ if let Err(e) = tc(&full) {
+ println!("INFO {id:<8} tc itself refuses this here: {e}");
+ return;
+ }
+ if let Err(e) = by_netlink(nl, index_of("na")) {
+ verdict(id, text, false, &format!("netlink: {e}"));
+ return;
+ }
+ let (a, b) = (tc_dump("ta"), tc_dump("na"));
+ verdict(id, text, a == b, &format!("tc: {a}\n netlink: {b}"));
+}
+
+/// The dump of the kernel as one line, without what differs by chance: the handle that the kernel hands out when none was asked for
+/// (four or more hex digits) and a seed that was drawn (a number above a million; the seeds asked for in the checks are small).
+fn normalise(text: &str) -> String {
+ let words: Vec<&str> = text.split_whitespace().collect();
+ let mut out: Vec = Vec::new();
+ let mut i = 0;
+ while i < words.len() {
+ let w = words[i];
+ if w == "seed" && words.get(i + 1).and_then(|n| n.parse::().ok()).is_some_and(|n| n > 1_000_000) {
+ i += 2;
+ continue;
+ }
+ if w.ends_with(':') && w.len() >= 5 {
+ out.push("AUTO:".to_string());
+ } else {
+ out.push(w.to_string());
+ }
+ i += 1;
+ }
+ out.join(" ")
+}
+
+fn tc_dump(device: &str) -> String {
+ normalise(&tc(&["-d", "qdisc", "show", "dev", device]).unwrap_or_else(|e| format!("error: {e}")))
+}
+
+fn tc_dump_class(device: &str) -> String {
+ normalise(&tc(&["-d", "class", "show", "dev", device]).unwrap_or_else(|e| format!("error: {e}")))
+}
+
+fn handle(major: u16, minor: u16) -> TcHandle {
+ TcHandle { major, minor }
+}
+
+fn main() {
+ let mut nl = Netlink::new();
+ for name in ["ta", "na"] {
+ nl.add_dummy(name).expect("dummy link");
+ nl.set_up(index_of(name)).expect("up");
+ }
+ println!("INFO S22 netlink-packet-route 0.34.0, netlink-sys 0.9.0, no `unsafe` in the layer; tc: {}", tc(&["-V"]).unwrap_or_default().trim());
+
+ // ---- netem
+ let cases: Vec<(&str, &str, Netem)> = vec![
+ ("N1", "delay 20 ms, jitter 5 ms, limit 1000", Netem { delay_us: 20_000, jitter_us: 5_000, limit: 1000, ..Netem::default() }),
+ ("N2", "loss 10 % with seed 7", Netem { loss_percent: 10.0, seed: Some(7), ..Netem::default() }),
+ ("N3", "loss 10 % with a correlation of 25 %", Netem { loss_percent: 10.0, loss_correlation: 25.0, ..Netem::default() }),
+ ("N4", "Gilbert-Elliott loss", Netem { gemodel: Some((5.0, 20.0, 10.0, 0.5)), ..Netem::default() }),
+ ("N5", "duplicate 5 %, corrupt 2 %", Netem { duplicate_percent: 5.0, corrupt_percent: 2.0, ..Netem::default() }),
+ ("N6", "reorder 25 % 50 % gap 5 with a delay", Netem { delay_us: 10_000, reorder: Some((25.0, 50.0, 5)), ..Netem::default() }),
+ ("N7", "rate 8 mbit with a delay", Netem { delay_us: 1_000, rate_bps: 8_000_000, ..Netem::default() }),
+ ("N8", "ecn and a loss", Netem { loss_percent: 1.0, ecn: true, ..Netem::default() }),
+ ("N9", "everything at once", Netem { delay_us: 30_000, jitter_us: 2_000, limit: 500, loss_percent: 3.0, duplicate_percent: 1.0, corrupt_percent: 0.5, rate_bps: 100_000_000, seed: Some(99), ..Netem::default() }),
+ ];
+ for (id, text, config) in cases {
+ let args = config.tc_args();
+ let args: Vec<&str> = args.iter().map(String::as_str).collect();
+ let mut by_tc = vec!["root"];
+ by_tc.extend(args);
+ let encoded = config.encode();
+ compare(&mut nl, id, text, &by_tc, |nl, index| nl.qdisc(index, TcHandle::ROOT, handle(0, 0), "netem", Some(encoded), false));
+ }
+
+ // ---- the fifos, codel, pie, fq_codel
+ compare(&mut nl, "F1", "pfifo limit 77", &["root", "pfifo", "limit", "77"], |nl, i| {
+ nl.qdisc(i, TcHandle::ROOT, handle(0, 0), "pfifo", Some(u32s(&[77])), false)
+ });
+ compare(&mut nl, "F2", "bfifo limit 30000", &["root", "bfifo", "limit", "30000"], |nl, i| {
+ nl.qdisc(i, TcHandle::ROOT, handle(0, 0), "bfifo", Some(u32s(&[30_000])), false)
+ });
+ compare(&mut nl, "C1", "codel target 5 ms interval 100 ms limit 500 ecn", &["root", "codel", "target", "5ms", "interval", "100ms", "limit", "500", "ecn"], |nl, i| {
+ let mut o = nla(1, &5000u32.to_ne_bytes()); // TARGET in microseconds
+ o.extend(nla(2, &500u32.to_ne_bytes())); // LIMIT
+ o.extend(nla(3, &100_000u32.to_ne_bytes())); // INTERVAL in microseconds
+ o.extend(nla(4, &1u32.to_ne_bytes())); // ECN
+ nl.qdisc(i, TcHandle::ROOT, handle(0, 0), "codel", Some(o), false)
+ });
+ compare(&mut nl, "P1", "pie target 15 ms limit 200 tupdate 16 ms ecn", &["root", "pie", "target", "15ms", "limit", "200", "tupdate", "16ms", "ecn"], |nl, i| {
+ let mut o = nla(1, &15_000u32.to_ne_bytes()); // TARGET in microseconds
+ o.extend(nla(2, &200u32.to_ne_bytes())); // LIMIT
+ o.extend(nla(3, &16_000u32.to_ne_bytes())); // TUPDATE in microseconds
+ o.extend(nla(6, &1u32.to_ne_bytes())); // ECN
+ nl.qdisc(i, TcHandle::ROOT, handle(0, 0), "pie", Some(o), false)
+ });
+ compare(&mut nl, "Q1", "fq_codel limit 1000 flows 256 target 5 ms interval 100 ms ecn", &["root", "fq_codel", "limit", "1000", "flows", "256", "target", "5ms", "interval", "100ms", "ecn"], |nl, i| {
+ let mut o = nla(2, &1000u32.to_ne_bytes()); // LIMIT
+ o.extend(nla(1, &5000u32.to_ne_bytes())); // TARGET
+ o.extend(nla(3, &100_000u32.to_ne_bytes())); // INTERVAL
+ o.extend(nla(4, &1u32.to_ne_bytes())); // ECN
+ o.extend(nla(5, &256u32.to_ne_bytes())); // FLOWS
+ nl.qdisc(i, TcHandle::ROOT, handle(0, 0), "fq_codel", Some(o), false)
+ });
+
+ // ---- tbf
+ compare(&mut nl, "T1", "tbf rate 8 mbit burst 5000 limit 20000", &["root", "tbf", "rate", "8mbit", "burst", "5000", "limit", "20000"], |nl, i| {
+ let rate = 1_000_000u32; // bytes per second
+ let mut parms = ratespec(rate);
+ parms.extend(ratespec(0)); // peakrate
+ parms.extend(u32s(&[20_000, ticks(5000, rate), 0])); // limit, buffer in ticks, mtu
+ let mut o = nla(1, &parms); // TCA_TBF_PARMS
+ o.extend(nla(6, &5000u32.to_ne_bytes())); // TCA_TBF_BURST
+ nl.qdisc(i, TcHandle::ROOT, handle(0, 0), "tbf", Some(o), false)
+ });
+
+ // ---- htb: a root, a class, a leaf
+ {
+ let _ = tc(&["qdisc", "del", "dev", "ta", "root"]);
+ let _ = tc(&["qdisc", "del", "dev", "na", "root"]);
+ let by_tc = [
+ vec!["qdisc", "add", "dev", "ta", "root", "handle", "1:", "htb", "default", "1"],
+ vec!["class", "add", "dev", "ta", "parent", "1:", "classid", "1:1", "htb", "rate", "8mbit", "ceil", "8mbit", "burst", "1600", "cburst", "1600"],
+ vec!["qdisc", "add", "dev", "ta", "parent", "1:1", "handle", "10:", "pfifo", "limit", "50"],
+ ];
+ for args in &by_tc {
+ tc(args).expect("tc htb");
+ }
+ let index = index_of("na");
+ let rate = 1_000_000u32;
+ let result = (|| -> Result<(), String> {
+ let glob = u32s(&[3, 10, 1, 0, 0]); // tc_htb_glob: version, rate2quantum, defcls, debug, direct_pkts
+ nl.qdisc(index, TcHandle::ROOT, handle(1, 0), "htb", Some(nla(2, &glob)), false)?; // TCA_HTB_INIT
+ let mut opt = ratespec(rate);
+ opt.extend(ratespec(rate));
+ opt.extend(u32s(&[ticks(1600, rate), ticks(1600, rate), 0, 0, 0])); // buffer, cbuffer, quantum, level, prio
+ nl.class(index, handle(1, 0), handle(1, 1), "htb", nla(1, &opt))?; // TCA_HTB_PARMS
+ nl.qdisc(index, handle(1, 1), handle(0x10, 0), "pfifo", Some(u32s(&[50])), false)
+ })();
+ match result {
+ Err(e) => verdict("H1", "htb root, class 8 mbit, pfifo leaf", false, &format!("netlink: {e}")),
+ Ok(()) => {
+ let (a, b) = (tc_dump("ta"), tc_dump("na"));
+ let (c, d) = (tc_dump_class("ta"), tc_dump_class("na"));
+ verdict("H1", "htb root, class 8 mbit, pfifo leaf", a == b && c == d, &format!("tc: {a} / {c}\n netlink: {b} / {d}"));
+ }
+ }
+ }
+
+ // ---- a change of a running netem, and its cost
+ {
+ let _ = tc(&["qdisc", "del", "dev", "na", "root"]);
+ let index = index_of("na");
+ let base = Netem { delay_us: 20_000, jitter_us: 1_000, limit: 1000, loss_percent: 1.0, seed: Some(5), ..Netem::default() };
+ nl.qdisc(index, TcHandle::ROOT, handle(0, 0), "netem", Some(base.encode()), false).unwrap();
+ let rounds = 20_000u32;
+ let started = Instant::now();
+ let mut failed = 0;
+ for round in 0..rounds {
+ let config = Netem { delay_us: 20_000 + u64::from(round % 100), ..base.clone() };
+ if nl.qdisc(index, TcHandle::ROOT, handle(0, 0), "netem", Some(config.encode()), true).is_err() {
+ failed += 1;
+ }
+ }
+ let each = started.elapsed().as_secs_f64() / f64::from(rounds) * 1e6;
+ verdict("L1", &format!("{rounds} changes of a running netem took {each:.1} microseconds each"), failed == 0, &format!("{failed} changes failed"));
+ }
+ let _ = TcOption::Other(DefaultNla::new(0, vec![]));
+ let _ = AddressFamily::Inet;
+}
diff --git a/spikes/s24-ecn.sh b/spikes/s24-ecn.sh
new file mode 100755
index 0000000..80225be
--- /dev/null
+++ b/spikes/s24-ecn.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# S24: does the kernel mark instead of drop, and does TCP in a namespace use the mark?
+#
+# client namespace --veth-- server namespace, a shaper with a queue discipline on the client's interface
+#
+# For codel, fq_codel and pie with `ecn`, and for netem with `ecn`: one TCP flow with net.ipv4.tcp_ecn=1 (and a run with 0 for
+# the contrast); the counters of the queue (`tc -s`: ecn_mark, drops) and of the receiver (CE-marked packets in /proc/net/netstat).
+# bash spikes/s24-ecn.sh
+set -u
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+export PATH="$PATH:/usr/sbin:/sbin"
+if [ "${1:-outer}" = outer ]; then
+ if [ "$(id -u)" -eq 0 ]; then exec unshare -n bash "$0" inner; fi
+ exec unshare -Urn bash "$0" inner
+fi
+secs="${SECS:-5}"
+pids=()
+cleanup() { for p in "${pids[@]}"; do kill "$p" 2>/dev/null; done; wait 2>/dev/null; }
+trap cleanup EXIT
+ns_new() { unshare -n sleep 3600 >/dev/null 2>&1 & pids+=($!); printf -v "$1" '%s' "$!"; for _ in $(seq 50); do nsenter -t "$!" -n true 2>/dev/null && return 0; sleep 0.1; done; return 1; }
+nsmac() { nsenter -t "$1" -n ip -br link show dev "$2" | awk '{print $3}'; }
+ns_new PC; ns_new PR; ns_new PS
+for p in "$PC" "$PR" "$PS"; do nsenter -t "$p" -n sysctl -qw net.ipv6.conf.all.disable_ipv6=1; done
+nsenter -t "$PR" -n sysctl -qw net.ipv4.ip_forward=1
+
+run_case() { # LABEL ECN(0|1) QDISC_ARGS... the discipline sits under an htb class of 20 mbit on the router's interface to the server
+ local label="$1" ecn="$2"; shift 2
+ for n in "$PC:c0" "$PR:r0" "$PR:r1" "$PS:s0"; do nsenter -t "${n%%:*}" -n ip link del "${n##*:}" 2>/dev/null; done
+ ip link add c0 type veth peer name r0 && ip link set c0 netns "$PC" && ip link set r0 netns "$PR"
+ ip link add r1 type veth peer name s0 && ip link set r1 netns "$PR" && ip link set s0 netns "$PS"
+ nsenter -t "$PC" -n ip addr add 10.9.0.1/24 dev c0; nsenter -t "$PR" -n ip addr add 10.9.0.2/24 dev r0
+ nsenter -t "$PR" -n ip addr add 10.9.1.1/24 dev r1; nsenter -t "$PS" -n ip addr add 10.9.1.2/24 dev s0
+ for x in "$PC:c0" "$PR:r0" "$PR:r1" "$PS:s0"; do nsenter -t "${x%%:*}" -n ip link set "${x##*:}" up gso_max_size 1514 gso_max_segs 1; done
+ nsenter -t "$PC" -n ip route add 10.9.1.0/24 via 10.9.0.2; nsenter -t "$PS" -n ip route add 10.9.0.0/24 via 10.9.1.1
+ for p in "$PC" "$PR" "$PS"; do nsenter -t "$p" -n sysctl -qw net.ipv4.tcp_ecn="$ecn"; done
+ nsenter -t "$PR" -n tc qdisc add dev r1 root handle 1: htb default 1 r2q 1000
+ nsenter -t "$PR" -n tc class add dev r1 parent 1: classid 1:1 htb rate 20mbit ceil 20mbit burst 3000
+ nsenter -t "$PR" -n tc qdisc add dev r1 parent 1:1 handle 10: "$@" || { echo " $label: tc refused: $*"; return; }
+ nsenter -t "$PS" -n python3 "$here/tcpflow.py" server 10.9.1.2 9100 1 "$secs" >"$SCRATCH/server.json" &
+ local sp=$!; sleep 0.5
+ nsenter -t "$PC" -n python3 "$here/tcpflow.py" client 10.9.1.2 9100 "$((secs + 1))" cubic >"$SCRATCH/client.json" 2>&1
+ wait "$sp"
+ echo "=== $label (tcp_ecn=$ecn, qdisc: $*)"
+ python3 - "$SCRATCH/server.json" "$SCRATCH/client.json" <<'PY'
+import json, sys
+def last(path):
+ lines = [l for l in open(path).read().strip().splitlines() if l.startswith("{")]
+ return json.loads(lines[-1]) if lines else {}
+server, client = last(sys.argv[1]), last(sys.argv[2])
+rates = [x for r in server.get("bytes_per_second", []) for x in (r if isinstance(r, list) else [r])]
+print(" goodput mbit/s:", round(sum(rates) * 8 / 1e6 / max(len(rates), 1), 1) if rates else None, " client info:", {k: client.get(k) for k in ("retrans", "total_retrans", "snd_cwnd", "rtt") if k in client})
+PY
+ nsenter -t "$PR" -n tc -s qdisc show dev r1 | tr '\n' ' ' | sed 's/ */ /g' | grep -o 'qdisc \(codel\|fq_codel\|pie\|netem\) .*' | cut -c1-420
+ echo " server side, packets with ECT(0) and with CE (IpExt):"
+ nsenter -t "$PS" -n awk '/^IpExt:/ { if (!h) { for (i=1;i<=NF;i++) n[i]=$i; h=1 } else { for (i=2;i<=NF;i++) if (n[i]=="InCEPkts"||n[i]=="InECT0Pkts") printf " %s %s\n", n[i], $i } }' /proc/net/netstat
+ echo " client side, TcpExt ECN counters:"
+ nsenter -t "$PC" -n awk '/^TcpExt:/ { if (!h) { for (i=1;i<=NF;i++) n[i]=$i; h=1 } else { for (i=2;i<=NF;i++) if (n[i] ~ /ECN|CWR|Retrans$|TCPLostRetransmit/ && $i>0) printf " %s %s\n", n[i], $i } }' /proc/net/netstat
+}
+SCRATCH="$(mktemp -d)"; trap 'cleanup; rm -rf "$SCRATCH"' EXIT
+echo "# S24: kernel $(uname -r)"
+run_case "codel with ecn, TCP uses ECN" 1 codel ecn
+run_case "codel with ecn, TCP does not" 0 codel ecn
+run_case "codel without ecn, TCP uses ECN" 1 codel noecn
+run_case "fq_codel with ecn, TCP uses ECN" 1 fq_codel ecn
+run_case "pie with ecn, TCP uses ECN" 1 pie ecn
+run_case "pie without ecn, TCP uses ECN" 1 pie noecn
+run_case "netem loss 1% with ecn, TCP uses ECN" 1 netem loss 1% ecn limit 1000
+run_case "netem loss 1% without ecn, TCP uses ECN" 1 netem loss 1% limit 1000