From dcba2ba1928e49decc80ae8322fc79f54254bd37 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Tue, 21 Jul 2026 07:51:12 +0100 Subject: [PATCH] Add local vhost loopback example for single-machine development Machnet previously required a physical DPDK NIC (and typically a pair of cloud VMs) to try anything end-to-end. This adds a self-contained local test that runs the full datapath on one Linux machine using DPDK virtual devices: a net_vhost port and a virtio_user port wired back-to-back over a unix socket, driven by the existing hello_world app. - examples/local_vhost_test.sh: starts the daemon on the vhost pair and verifies a hello_world message crosses the loop; heavily commented so it doubles as local-development documentation. - examples/local_vhost_config.json: two-interface config for the loop. - machnet_config: new optional per-interface "vdev" key, passed to DPDK as --vdev (plus --no-pci when no physical ports are configured). vdev arguments are sorted so DPDK port ids -- and thus the net_vhost MAC, which is derived from the port id -- are deterministic. - pmd.cc: fall back to a basic ethernet configuration with a warning, instead of LOG(FATAL), when a port lacks TX checksum offloads; virtual devices have none, and nothing on a local loop verifies checksums. - README.md / src/apps/machnet/README.md: document the new field and the local workflow. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EFQ6Rh1j4r3i9QURDaUyXm --- README.md | 21 +++++ examples/local_vhost_config.json | 14 +++ examples/local_vhost_test.sh | 151 +++++++++++++++++++++++++++++++ src/apps/machnet/README.md | 2 + src/core/drivers/dpdk/pmd.cc | 11 ++- src/core/machnet_config.cc | 35 ++++++- src/include/machnet_config.h | 13 ++- 7 files changed, 237 insertions(+), 10 deletions(-) create mode 100644 examples/local_vhost_config.json create mode 100755 examples/local_vhost_test.sh diff --git a/README.md b/README.md index 42d18e5c..7962c423 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,27 @@ MSG_GEN="~/machnet/build/src/apps/msg_gen/msg_gen" ``` +## Local development without a NIC + +You can develop and test Machnet applications on a single Linux machine with +no dedicated NIC, VM, or cloud setup. The trick is to run the Machnet stack on +a pair of DPDK virtual devices (a `vhost` port and a `virtio-user` port) wired +back-to-back over a unix socket, which behaves like two machines on the same +L2 segment: + +```bash +# Build Machnet and the examples first (see CONTRIBUTING.md and ./build_shim.sh) +cd examples +sudo ./local_vhost_test.sh +``` + +The script starts the Machnet daemon with +[local_vhost_config.json](examples/local_vhost_config.json), then runs the +`hello_world` server and client against the two local IPs (`10.66.0.1` and +`10.66.0.2`). See the comments in +[local_vhost_test.sh](examples/local_vhost_test.sh) for details. + + ## Machnet API See [machnet.h](src/ext/machnet.h) for the full API documentation. Applications use the following steps to interact with the Machnet service: diff --git a/examples/local_vhost_config.json b/examples/local_vhost_config.json new file mode 100644 index 00000000..6239e8d0 --- /dev/null +++ b/examples/local_vhost_config.json @@ -0,0 +1,14 @@ +{ + "machnet_config": { + "56:48:4f:53:54:00": { + "ip": "10.66.0.1", + "engine_threads": 1, + "vdev": "net_vhost0,iface=/var/run/machnet/local_vhost.sock,queues=1" + }, + "aa:19:be:ef:00:02": { + "ip": "10.66.0.2", + "engine_threads": 1, + "vdev": "virtio_user1,path=/var/run/machnet/local_vhost.sock,queues=1,queue_size=1024,mac=aa:19:be:ef:00:02" + } + } +} diff --git a/examples/local_vhost_test.sh b/examples/local_vhost_test.sh new file mode 100755 index 00000000..c13631ee --- /dev/null +++ b/examples/local_vhost_test.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# +# Local Machnet test over a vhost port -- no physical NIC, VM, or cloud +# instance required. This is the quickest way to develop and test Machnet +# applications on a single Linux machine. +# +# How it works +# ------------ +# A single Machnet daemon is started with two DPDK *virtual* devices instead +# of a physical NIC (see local_vhost_config.json): +# +# +-------------------- machnet daemon ---------------------+ +# | engine 0 (10.66.0.1) engine 1 (10.66.0.2) | +# | | | | +# | net_vhost0 <-- unix socket --> virtio_user1 | +# +---------------------------------------------------------+ +# ^ ^ +# | | +# hello_world server hello_world client +# (--local 10.66.0.1) (--local 10.66.0.2) +# +# `net_vhost0' is a vhost-user server backed by a unix socket; +# `virtio_user1' connects to that socket. Every packet transmitted by one +# port is received by the other, so the two Machnet engines behave like two +# machines on the same L2 segment. The hello_world client then sends a +# message from 10.66.0.2 to the server listening on 10.66.0.1, exercising +# the full Machnet datapath (ARP, handshake, data transfer). +# +# Notes +# ----- +# * The net_vhost PMD derives its MAC address from its DPDK port id +# (56:48:4f:53:54:). If machnet logs "port not found"-style +# errors, compare the MAC keys in local_vhost_config.json with the +# [PMDPORT] lines in the machnet log printed by this script. +# * Checksum offloads are unavailable on virtual devices, so packets carry +# unfilled checksums. That is fine here: nothing on the vhost loop +# verifies them. Do not use vhost ports to talk to a real network. +# +# Prerequisites: Linux, a Machnet build (see main README), hugepages. +# +# Usage: sudo ./local_vhost_test.sh +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +BUILD_DIR="${BUILD_DIR:-${REPO_ROOT}/build}" + +MACHNET_BIN="${BUILD_DIR}/src/apps/machnet/machnet" +CONFIG_JSON="${SCRIPT_DIR}/local_vhost_config.json" +HELLO_WORLD="${SCRIPT_DIR}/hello_world" +VHOST_SOCK="/var/run/machnet/local_vhost.sock" +CTRL_SOCK="/var/run/machnet/machnet_ctrl.sock" + +SERVER_IP="10.66.0.1" +CLIENT_IP="10.66.0.2" + +LOG_DIR="$(mktemp -d /tmp/machnet_local_vhost.XXXXXX)" +MACHNET_LOG="${LOG_DIR}/machnet.log" +SERVER_LOG="${LOG_DIR}/hello_server.log" +CLIENT_LOG="${LOG_DIR}/hello_client.log" + +die() { echo "ERROR: $*" >&2; exit 1; } + +cleanup() { + # Kill the hello_world server and the machnet daemon, if still running. + [[ -n "${SERVER_PID:-}" ]] && kill "${SERVER_PID}" 2>/dev/null || true + [[ -n "${MACHNET_PID:-}" ]] && kill "${MACHNET_PID}" 2>/dev/null || true + # Give the daemon a moment to shut down and release hugepages. + sleep 1 + rm -f "${VHOST_SOCK}" + echo "Logs kept in ${LOG_DIR}" +} +trap cleanup EXIT + +# ---------------------------------------------------------------- preflight +[[ "$(uname -s)" == "Linux" ]] || die "This test requires Linux (DPDK)." +[[ "$(id -u)" -eq 0 ]] || die "Please run as root: sudo $0" +[[ -x "${MACHNET_BIN}" ]] || \ + die "machnet binary not found at ${MACHNET_BIN}. Build the project first (see main README), or set BUILD_DIR." + +if [[ ! -x "${HELLO_WORLD}" ]]; then + echo "hello_world not built; building it now..." + [[ -f "${REPO_ROOT}/libmachnet_shim.so" ]] || \ + die "libmachnet_shim.so not found in ${REPO_ROOT}. Run ${REPO_ROOT}/build_shim.sh first." + make -C "${SCRIPT_DIR}" hello_world +fi + +# Hugepages: DPDK needs some. Allocate 512 x 2MB pages if none are free. +free_hugepages=$(awk '/HugePages_Free/ {print $2}' /proc/meminfo) +if [[ "${free_hugepages:-0}" -eq 0 ]]; then + echo "No free hugepages; allocating 512 x 2MB hugepages..." + echo 512 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + free_hugepages=$(awk '/HugePages_Free/ {print $2}' /proc/meminfo) + [[ "${free_hugepages}" -gt 0 ]] || die "Failed to allocate hugepages." +fi + +if pgrep -x machnet > /dev/null; then + die "Another machnet daemon is already running; stop it first." +fi + +mkdir -p /var/run/machnet +rm -f "${VHOST_SOCK}" "${CTRL_SOCK}" + +# ---------------------------------------------------- start the machnet daemon +echo "Starting machnet daemon (log: ${MACHNET_LOG})..." +GLOG_logtostderr=1 "${MACHNET_BIN}" --config_json "${CONFIG_JSON}" \ + &> "${MACHNET_LOG}" & +MACHNET_PID=$! + +# Wait for the controller socket and for both ports to come up. +for _ in $(seq 1 60); do + kill -0 "${MACHNET_PID}" 2>/dev/null || \ + { tail -n 30 "${MACHNET_LOG}"; die "machnet daemon exited early. Full log: ${MACHNET_LOG}"; } + if [[ -S "${CTRL_SOCK}" ]] && \ + [[ "$(grep -c "Link is UP" "${MACHNET_LOG}" || true)" -ge 2 ]]; then + break + fi + sleep 1 +done +[[ -S "${CTRL_SOCK}" ]] || \ + { tail -n 30 "${MACHNET_LOG}"; die "machnet daemon did not come up. Full log: ${MACHNET_LOG}"; } +echo "machnet daemon is up (pid ${MACHNET_PID})." + +# --------------------------------------------------------- run the experiment +echo "Starting hello_world server on ${SERVER_IP}..." +"${HELLO_WORLD}" --local "${SERVER_IP}" &> "${SERVER_LOG}" & +SERVER_PID=$! +sleep 2 +kill -0 "${SERVER_PID}" 2>/dev/null || \ + { cat "${SERVER_LOG}"; die "hello_world server exited early."; } + +echo "Sending a message from ${CLIENT_IP} to ${SERVER_IP}..." +"${HELLO_WORLD}" --local "${CLIENT_IP}" --remote "${SERVER_IP}" \ + &> "${CLIENT_LOG}" || { cat "${CLIENT_LOG}"; die "hello_world client failed."; } + +# The server prints "Received message: Hello World!" when the message lands. +for _ in $(seq 1 10); do + if grep -q "Received message" "${SERVER_LOG}"; then + echo + echo "PASS: message delivered over the local vhost loop:" + grep "Received message" "${SERVER_LOG}" + exit 0 + fi + sleep 1 +done + +echo "---- machnet log (tail) ----"; tail -n 30 "${MACHNET_LOG}" +echo "---- server log ----"; cat "${SERVER_LOG}" +echo "---- client log ----"; cat "${CLIENT_LOG}" +die "Message was not received within 10 seconds." diff --git a/src/apps/machnet/README.md b/src/apps/machnet/README.md index b6b7d06d..fa5e3409 100644 --- a/src/apps/machnet/README.md +++ b/src/apps/machnet/README.md @@ -20,6 +20,8 @@ the interface, and the value is a dictionary with the following fields: * `ip`: the IP address of the interface. * `engine_threads`: The number of threads (and NIC HW queues) to use for this interface. * `cpu_mask`: The CPU mask to use to affine all engine threads. If not specified, the default is to use all available cores. + * `pcie`: (optional) The PCIe address of the interface. If not specified, Machnet discovers it from sysfs using the MAC address. + * `vdev`: (optional) A DPDK virtual device specification (e.g., `net_vhost0,iface=/var/run/machnet/local_vhost.sock,queues=1`) to use instead of a physical NIC. Passed to DPDK as `--vdev`; when all configured interfaces are virtual devices, PCI probing is disabled (`--no-pci`). Virtual devices lack checksum offloads, so use them only for local development and testing — see [examples/local_vhost_test.sh](../../../examples/local_vhost_test.sh). **Example [config.json](config.json):** ```json diff --git a/src/core/drivers/dpdk/pmd.cc b/src/core/drivers/dpdk/pmd.cc index 216f646a..d3267c75 100644 --- a/src/core/drivers/dpdk/pmd.cc +++ b/src/core/drivers/dpdk/pmd.cc @@ -63,8 +63,15 @@ static rte_eth_conf DefaultEthConf(const rte_eth_dev_info *devinfo) { const auto tx_offload_capa = devinfo->tx_offload_capa; if (!(tx_offload_capa & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM) || !(tx_offload_capa & RTE_ETH_TX_OFFLOAD_UDP_CKSUM)) { - // Making this fatal; not sure what NIC does not support checksum offloads. - LOG(FATAL) << "Hardware does not support checksum offloads."; + // Virtual devices (e.g., `net_vhost', `net_virtio_user') do not support + // checksum offloads. Machnet leaves checksum fields unfilled and relies on + // the offloads, so a port without them is only usable on a local loop + // where nothing verifies checksums (see examples/local_vhost_test.sh). + LOG(WARNING) << "Driver " << devinfo->driver_name + << " does not support TX checksum offloads. Falling back to " + "a basic ethernet configuration; outgoing checksums are " + "NOT computed. Use this port for local testing only."; + return rte_eth_conf(); } port_conf.txmode.mq_mode = RTE_ETH_MQ_TX_NONE; diff --git a/src/core/machnet_config.cc b/src/core/machnet_config.cc index c891cd3e..f13df9d5 100644 --- a/src/core/machnet_config.cc +++ b/src/core/machnet_config.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include "dpdk.h" #include "ether.h" @@ -84,7 +85,7 @@ void MachnetConfigProcessor::AssertJsonValidMachnetConfig() { } for (const auto &[key, _] : interface.items()) { if (key != "ip" && key != "engine_threads" && key != "cpu_mask" && - key != "pcie") { + key != "pcie" && key != "vdev") { LOG(FATAL) << "Invalid key " << key << " in " << interface << " in " << config_json_filename_; } @@ -121,12 +122,19 @@ void MachnetConfigProcessor::DiscoverInterfaceConfiguration() { LOG(INFO) << "Using default CPU mask for " << l2_addr.ToString(); } + std::string vdev = ""; + if (json_val.find("vdev") != json_val.end()) { + vdev = json_val.at("vdev"); + LOG(INFO) << "Using DPDK virtual device " << vdev << " for " + << l2_addr.ToString(); + } + std::string pci_addr = ""; if (json_val.find("pcie") != json_val.end()) { pci_addr = json_val.at("pcie"); LOG(INFO) << "Using config file PCIe address " << pci_addr << " for " << l2_addr.ToString(); - } else { + } else if (vdev.empty()) { const std::optional ret = GetPCIeAddressSysfs(l2_addr); if (ret.has_value()) { pci_addr = ret.value(); @@ -138,7 +146,7 @@ void MachnetConfigProcessor::DiscoverInterfaceConfiguration() { } interfaces_config_.emplace(pci_addr, l2_addr, ip_addr, engine_threads, - cpu_mask); + cpu_mask, vdev); } for (const auto &interface : interfaces_config_) { interface.Dump(); @@ -151,15 +159,34 @@ utils::CmdLineOpts MachnetConfigProcessor::GetEalOpts() const { eal_opts.Append({"-c", "0x1"}); eal_opts.Append({"-n", "4"}); eal_opts.Append({"--telemetry"}); + bool have_pci_interface = false; + std::vector vdevs; for (const auto &interface : interfaces_config_) { - if (interface.pcie_addr() != "") { + if (interface.vdev() != "") { + vdevs.push_back(interface.vdev()); + } else if (interface.pcie_addr() != "") { eal_opts.Append({"-a", interface.pcie_addr()}); + have_pci_interface = true; } else { LOG(WARNING) << "Not passing PCIe allowlist for interface " << interface.l2_addr().ToString(); + have_pci_interface = true; } } + // `interfaces_config_` is an unordered set, but DPDK assigns port ids in + // probe order, and some virtual drivers (e.g., `net_vhost') derive their MAC + // address from the port id. Sort the vdev arguments so that port ids are + // deterministic across runs. + std::sort(vdevs.begin(), vdevs.end()); + for (const auto &vdev : vdevs) { + eal_opts.Append({"--vdev", vdev}); + } + if (!vdevs.empty() && !have_pci_interface) { + // All interfaces are virtual devices; skip PCI bus probing altogether. + eal_opts.Append({"--no-pci"}); + } + return eal_opts; } diff --git a/src/include/machnet_config.h b/src/include/machnet_config.h index bfb0c506..a28fb37d 100644 --- a/src/include/machnet_config.h +++ b/src/include/machnet_config.h @@ -24,8 +24,10 @@ class NetworkInterfaceConfig { const net::Ethernet::Address &l2_addr, const net::Ipv4::Address &ip_addr, size_t engine_threads = 1, - cpu_set_t cpu_mask = kDefaultCpuMask) + cpu_set_t cpu_mask = kDefaultCpuMask, + const std::string vdev = "") : pcie_addr_(pcie_addr), + vdev_(vdev), l2_addr_(l2_addr), ip_addr_(ip_addr), engine_threads_(engine_threads), @@ -36,6 +38,7 @@ class NetworkInterfaceConfig { } const std::string &pcie_addr() const { return pcie_addr_; } + const std::string &vdev() const { return vdev_; } const net::Ethernet::Address &l2_addr() const { return l2_addr_; } const net::Ipv4::Address &ip_addr() const { return ip_addr_; } size_t engine_threads() const { return engine_threads_; } @@ -44,9 +47,10 @@ class NetworkInterfaceConfig { void Dump() const { LOG(INFO) << "NetworkInterfaceConfig: " << utils::Format( - "[PCIe: %s, L2: %s, IP: %s, engine_threads: %zu, " - "cpu_mask: %lu, dpdk_port_id: %d]", - pcie_addr_.c_str(), l2_addr_.ToString().c_str(), + "[PCIe: %s, vdev: %s, L2: %s, IP: %s, engine_threads: " + "%zu, cpu_mask: %lu, dpdk_port_id: %d]", + pcie_addr_.c_str(), vdev_.c_str(), + l2_addr_.ToString().c_str(), ip_addr_.ToString().c_str(), engine_threads_, utils::cpuset_to_sizet(cpu_mask_), dpdk_port_id_.value_or(-1)); @@ -58,6 +62,7 @@ class NetworkInterfaceConfig { private: const std::string pcie_addr_; + const std::string vdev_; const net::Ethernet::Address l2_addr_; const net::Ipv4::Address ip_addr_; const size_t engine_threads_;