Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 48 additions & 18 deletions .github/workflows/spikes.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ on:
description: "Run only S20 (the fabric link of release 0.3)"
type: boolean
default: false
only_engine:
description: "Run only S21 to S24 (release 0.4: the engine, netlink, the comparison, ECN)"
type: boolean
default: false

permissions:
contents: read
Expand All @@ -39,15 +43,15 @@ jobs:
# Three cases: as the runner user with the defaults of the image, as root, and as the runner
# user after relaxing the AppArmor rule that blocks unprivileged user namespaces on Ubuntu 24.04.
- name: Run the spikes as an unprivileged user
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: bash spikes/run-all.sh
continue-on-error: true
- name: Run the spikes as root
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: sudo SPIKE_OUT="$PWD/spikes/out-root" bash spikes/run-all.sh
continue-on-error: true
- name: Run the spikes as an unprivileged user, AppArmor restriction off
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
if sysctl -n kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
Expand All @@ -72,7 +76,7 @@ jobs:
sudo env PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SPIKE_OUT="$PWD/spikes/out-newtc" bash spikes/s2-netem-seed.sh
continue-on-error: true
- name: Which options does tc know?
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
tc -V || true
ip -V || true
Expand All @@ -82,7 +86,7 @@ jobs:
# The Rust and Python experiments (S7 to S10) run as the runner user. On Ubuntu 24.04 the AppArmor
# restriction was relaxed by the step above, and the setting stays for the rest of the job.
- name: S10 Toolchain of this image
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
mkdir -p spikes/out-rust
{
Expand All @@ -102,21 +106,21 @@ jobs:
} 2>&1 | tee spikes/out-rust/s10.txt
continue-on-error: true
- name: S7 Netlink (nlink) against tc
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
mkdir -p spikes/out-rust
bash spikes/rust-netlink/run.sh 2>&1 | tee spikes/out-rust/s7.txt
continue-on-error: true
- name: S9 PyO3 and maturin
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
mkdir -p spikes/out-rust
bash spikes/rust-pyo3/run.sh 2>&1 | tee spikes/out-rust/s9.txt
continue-on-error: true
# The engine series pins three cores; on a runner with fewer than four it is skipped. A shared virtual
# machine is noisier than a development machine, which is the point of running it here as well.
- name: S8 Packet engine
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
run: |
mkdir -p spikes/out-rust
if [ "$(nproc)" -lt 4 ]; then
Expand All @@ -128,21 +132,21 @@ jobs:
# S11 to S14 (release 0.2). They use `netem ... seed`, which the tc of the images does not know, so they run
# with the tc built above, as the runner user (the AppArmor restriction was relaxed by an earlier step).
- name: S11 Timeline precision
if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 30
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SPIKE_OUT="$PWD/spikes/out-rust/s11" bash spikes/rust-timeline/run.sh 2>&1 | tee spikes/out-rust/s11.txt
continue-on-error: true
- name: S12 The netem models against their theory
if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 20
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SPIKE_OUT="$PWD/spikes/out-rust/s12" bash spikes/s12-netem-models.sh 2>&1 | tee spikes/out-rust/s12.txt
continue-on-error: true
- name: S13 Capture
if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 10
run: |
mkdir -p spikes/out-rust
Expand All @@ -151,6 +155,7 @@ jobs:
continue-on-error: true
# tcpdump of Ubuntu 22.04 cannot run in a user namespace; then S13 keeps the pcap and it is read here
- name: S13 Read the pcap outside the namespace
if: ${{ !inputs.only_engine }}
timeout-minutes: 2
run: |
f="$PWD/spikes/out-rust/s13-ours.pcap"
Expand All @@ -166,62 +171,87 @@ jobs:
rm -f "$f"
continue-on-error: true
- name: S13b Corruption accounting
if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 5
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SPIKE_OUT="$PWD/spikes/out-rust/s13b" bash spikes/s13b-corruption-accounting.sh > spikes/out-rust/s13b.txt 2>&1
cat spikes/out-rust/s13b.txt
continue-on-error: true
- name: S15 A duplicating netem in a tree with another netem
if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 3
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SPIKE_OUT="$PWD/spikes/out-rust/s15" bash spikes/s15-duplicate-in-tree.sh 2>&1 | tee spikes/out-rust/s15.txt
continue-on-error: true
- name: S14 A program as a node
if: ${{ !inputs.only_shapes && !inputs.only_fabric }}
if: ${{ !inputs.only_shapes && !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 5
run: |
mkdir -p spikes/out-rust
SPIKE_OUT="$PWD/spikes/out-rust/s14" bash spikes/s14-program-node.sh 2>&1 | tee spikes/out-rust/s14.txt
continue-on-error: true
# S16 to S19 (release 0.3): many nodes, queue disciplines, TCP, the steps of a trace. They use the tc built above.
- name: S16 Many nodes
if: ${{ !inputs.only_fabric }}
if: ${{ !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 10
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s16" bash spikes/s16-many-nodes.sh 2>&1 | tee spikes/out-rust/s16.txt
continue-on-error: true
- name: S17 The queue lab
if: ${{ !inputs.only_fabric }}
if: ${{ !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 15
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s17" bash spikes/s17-queue-lab.sh 2>&1 | tee spikes/out-rust/s17.txt
continue-on-error: true
- name: S18 TCP
if: ${{ !inputs.only_fabric }}
if: ${{ !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 20
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s18" bash spikes/s18-tcp.sh 2>&1 | tee spikes/out-rust/s18.txt
continue-on-error: true
- name: S19 The steps of a trace
if: ${{ !inputs.only_fabric }}
if: ${{ !inputs.only_fabric && !inputs.only_engine }}
timeout-minutes: 15
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s19" bash spikes/s19-trace-steps.sh 2>&1 | tee spikes/out-rust/s19.txt
continue-on-error: true
- name: S20 A link through a fabric namespace
if: ${{ !inputs.only_engine }}
timeout-minutes: 15
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" TC_LIB_DIR="$RUNNER_TEMP/iproute2/netem" SPIKE_OUT="$PWD/spikes/out-rust/s20" bash spikes/s20-fabric.sh 2>&1 | tee spikes/out-rust/s20.txt
continue-on-error: true
# S21 to S24 (release 0.4): the data path of the engine, netlink by a layer of our own, the same link on the kernel and on the
# engine, and ECN. They use the tc built above. The runners have 4 CPUs: the cores are set for that.
- name: S21 The data path of the engine (timers, a jitter, the same seed twice, TCP), and S23 the same link on the kernel
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
timeout-minutes: 30
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SECS=4 CORE_ENGINE=1 CORE_RECV=2 CORE_SEND=3 bash spikes/rust-engine-safe/run.sh 2>&1 | tee spikes/out-rust/s21.txt
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" SECTION=compare SECS=4 CORE_ENGINE=1 CORE_RECV=2 CORE_SEND=3 bash spikes/rust-engine-safe/run.sh 2>&1 | tee spikes/out-rust/s23.txt
continue-on-error: true
- name: S22 Netlink by a layer of our own against tc
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
timeout-minutes: 10
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" bash spikes/rust-netlink-own/run.sh 2>&1 | tee spikes/out-rust/s22.txt
continue-on-error: true
- name: S24 ECN
if: ${{ !inputs.only_new && !inputs.only_shapes && !inputs.only_fabric }}
timeout-minutes: 10
run: |
mkdir -p spikes/out-rust
PATH="$RUNNER_TEMP/iproute2/tc:$PATH" bash spikes/s24-ecn.sh 2>&1 | tee spikes/out-rust/s24.txt
continue-on-error: true
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always()
with:
Expand Down
10 changes: 6 additions & 4 deletions FEATURES.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ A line moves to `in progress` when the first code of the feature exists, and to
| LNK-10 | MTU and fragmentation | S | 0.3 | done (`mtu` on a link, the sender cuts a packet that does not fit, a router never cuts, the account is by frames, `dont-fragment` makes the sender refuse, [D-081](docs/decisions/D-081.md); `tests/network.rs`, `tests/fidelity.rs`: a lost fragment loses the packet) |
| LNK-11 | Different values per direction | M | 0.1 | done (`tests/provision.rs`: `the_impairments_are_in_the_queues_of_the_right_interfaces`; `tests/agent.rs`) |
| LNK-12 | Outages (windows without any delivery, also of the packets on their way) | M | 0.2 | done (format with D-050; the Kernel backend since D-064; `tests/timeline.rs`: nothing arrives in the window and the packets in flight are dropped; the edges are placed by the clock) |
| LNK-13 | ECN marking | C | 0.4 | idea |
| LNK-13 | ECN marking: a queue marks an ECN-capable packet instead of dropping it, and TCP in the nodes uses it (`tcp_ecn`) ([D-100](docs/decisions/D-100.md); spike S24: `codel`, `fq_codel`, `pie` and `netem ... ecn` mark, TCP needs no retransmission for a mark) | S | 0.4 | planned |

## Link models over time (LMD)

Expand Down Expand Up @@ -165,15 +165,16 @@ A line moves to `in progress` when the first code of the feature exists, and to
| BKD-02 | Kernel backend in an unprivileged user namespace | M | 0.1 | done (the CI runs the tests as an ordinary user on `ubuntu-22.04` and `ubuntu-24.04`; tests: [`doctor.rs`](crates/netscenario-backend-kernel/tests/doctor.rs), [`run.rs`](crates/netscenario-backend-kernel/tests/run.rs)) |
| BKD-03 | Every backend declares what it reproduces faithfully, and refuses the rest with an error | M | 0.1 | done (`tests/refusal.rs`, `tests/coverage.rs`, `capability.rs`; D-023; a restriction that a scenario accepts with `allow-limited` is recorded in the result, D-092) |
| BKD-04 | Cleanup guarantee: no leftovers after a crash (PID namespace, D-022), and `netscenario clean` as a safety net | M | 0.1 | done (the guarantee is tested at 18 steps in `tests/crash.rs`; `clean` in `tests/clean.rs`, D-024) |
| BKD-05 | Netlink instead of calls to `ip` and `tc` | S | 0.4 | planned |
| BKD-06 | Packet engine in userspace (deterministic, no root) | M | 0.4? | planned |
| BKD-07 | Kernel and engine compared by differential tests | M | 0.4 | planned |
| BKD-05 | Netlink instead of calls to `ip` and `tc`: a thin layer of our own over `netlink-packet-route`, the options of the qdiscs encoded by us ([D-101](docs/decisions/D-101.md); spike S22: 17 of 17 dumps of the kernel equal those of `tc`) | S | 0.4 | designed |
| BKD-06 | Packet engine in userspace (deterministic, no root): a process in the fabric namespace with packet sockets on the veth ends, safe Rust but for one small module ([D-098](docs/decisions/D-098.md), [D-099](docs/decisions/D-099.md), [D-100](docs/decisions/D-100.md); spikes S21 and S23) | M | 0.4 | designed |
| BKD-07 | Kernel and engine compared by differential tests: equivalence inside a margin derived from what each backend measures about itself ([D-102](docs/decisions/D-102.md)) | M | 0.4 | designed |
| BKD-08 | Portable proxy backend (UDP exact, TCP at stream level) | M | 0.5 | planned |
| BKD-09 | Docker backend (containers as nodes) | S | 0.5 | planned |
| BKD-10 | Nodes on several machines | C | 1.x | idea |
| BKD-11 | Hardware in the loop (a real device behind a physical or virtual port) | C | 1.x | idea |
| BKD-12 | Exchange of traces with ns-3 | C | 1.x | idea |
| BKD-13 | An engine based on eBPF/XDP | C | 1.x | idea |
| BKD-14 | The run chooses the backend (`--backend kernel\|engine\|auto`, `backend=`, a plugin option); the default is the kernel and `auto` says what it chose ([D-103](docs/decisions/D-103.md)) | S | 0.4 | designed |

## Reproducibility and environment (ENV)

Expand All @@ -186,6 +187,7 @@ A line moves to `in progress` when the first code of the feature exists, and to
| ENV-05 | Report on timer and clock precision | S | 0.3 | done (`provenance.timers` of every result, `netscenario doctor --timers`, `ns.doctor(timers=True)`: [DOCTOR.md](docs/DOCTOR.md), D-097; tests in [`timers.rs`](crates/netscenario-core/src/timers.rs) and [`timers.rs`](crates/netscenario-backend-kernel/src/timers.rs)) |
| ENV-06 | Result format with a version | M | 0.1 | done (`format` and `version`, 0 until 1.0; tests: [`test_results.py`](python/netscenario/tests/test_results.py), [`run.rs`](crates/netscenario-backend-kernel/tests/run.rs)) |
| ENV-07 | Reproducibility check: the run compares the packet counter of each queue with the packets of the workload, and marks a run in which something else entered the queue as not reproducible; `doctor` checks that `tc` knows `seed` | M | 0.1 | done (`doctor` checks that `tc` knows `seed`; a run counts the packets of the queue of each direction with loss and compares them with the packets the workloads sent: `check_queues` in `run.rs`, `tests/run.rs` with a canary that sends one extra packet, D-036; two workloads in one direction whose queue draws are said not to be reproducible, D-065, because the count cannot see the order of their packets) |
| ENV-08 | The engine reports itself: the lateness of its releases as a distribution, an overload flag, its timer and the reason for it, and a log of its decisions ([D-098](docs/decisions/D-098.md), [D-099](docs/decisions/D-099.md)) | M | 0.4 | planned |

## Safety (SAF)

Expand Down
2 changes: 1 addition & 1 deletion docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ The layout of the repository: a Cargo workspace plus Python packages. The two ba
A backend implements a small interface: report capabilities, create a node, create a link, apply impairments to a link direction, start a program in a node, capture, tear down. The engine uses nothing else, so a new backend means implementing that interface and filling its column in the coverage matrix.

- **Kernel** (reference). One network namespace per node, virtual Ethernet pairs for links, `tc` with `netem` on each direction; a direction with a `queue` goes through a fabric namespace with an `htb` shaper and the discipline of the queue (D-079, D-082). Time-varying behaviour is done by changing the parameters at the steps of a trace. It is real, and its limits are the kernel's: a change of a running `netem` takes about 12 microseconds in a batch (spike S5, and through netlink in S7), so the resolution is set by the scheduler and not by `tc`, and the seed of `netem` guides only the loss and corruption events and needs a quiet namespace (see [COVERAGE.md](COVERAGE.md)).
- **Engine** (from 0.4). A Rust process that forwards packets between virtual devices inside a user namespace, with its own queues and impairment code. It needs no root, can be seeded exactly, can log every packet, and can implement behaviour that `netem` lacks (traces per packet, any loss model). Its correctness is not assumed: it is compared with the Kernel backend by differential tests. A prototype (spike S8) forwarded 100,000 packets per second on one thread with an exact delay and a deterministic seeded loss; the timer, the completion of checksums for TCP and the queue for jitter are still to be designed (see D-014 in [DECISIONS.md](DECISIONS.md)).
- **Engine** (from 0.4). A Rust process that forwards packets between virtual devices inside a user namespace, with its own queues and impairment code. It needs no root, can be seeded exactly, can log every packet, and can implement behaviour that `netem` lacks (traces per packet, any loss model). Its correctness is not assumed: it is compared with the Kernel backend by differential tests. It is a process in the fabric namespace of a run, with packet sockets on the veth ends, in safe Rust but for one small module that binds a packet socket ([D-098](decisions/D-098.md)); its decisions depend on the seed and the order of the packets only, and it writes a log of them ([D-099](decisions/D-099.md)); it is compared with the Kernel backend by equivalence inside a margin ([D-102](decisions/D-102.md)). The spikes S8, S21 and S23 measured it: 100,000 packets per second on one thread with an exact delay, and a lateness that depends on the timer and that every run reports.
- **Proxy** (from 0.5). Your program talks to a proxy process. Exact for UDP, stream-level for TCP. It exists so that people on Windows and macOS, or in containers without rights, can run the scenarios that are meaningful there. Everything it cannot do is refused.

**Rule:** a backend never approximates silently. If it can do something only in part, the scenario must say `allow-limited` for that feature, and the result records that it did.
Expand Down
Loading
Loading