From a542fc8da8f2f7a0c7d98ce9e5571d69bd2f5094 Mon Sep 17 00:00:00 2001 From: brunota20 Date: Thu, 18 Jun 2026 12:50:15 -0300 Subject: [PATCH] test(supervisor): multi-chain isolation regression tests (COW-1073) The supervisor's dispatch path is per-chain by construction (`dispatch_block(block)` filters modules by `block.chain_id` matching their `[[subscription]]` table), and the COW-1071 WS reconnect tasks own one per-chain backoff timer each. Multi-chain isolation is therefore structural, not derived. This PR locks the guarantee into the test suite with two new integration tests + a supervisor.rs docstring stating the invariant explicitly. ## New tests `multi_chain_dispatch_isolates_modules_by_chain`: - Boot two `example` modules with different `[[subscription]]` chain_ids (1 + 100). - Dispatch a block on chain 1 -> only module-a receives it (dispatched=1, alive_count=2 unchanged). - Dispatch a block on chain 100 -> only module-b receives it. - Validates: subscription filter is per-chain; a block on one chain does not even enter modules subscribed to a different chain. `multi_chain_poisoned_module_does_not_affect_other_chains`: - Boot fuel-bomb (always-traps) on chain 1 + example (healthy) on chain 100, with `PoisonPolicy::new(2, 60s)`. - Trap bomb #1 on chain 1 -> bomb dies, poisoned=0, example untouched. - Dispatch on chain 100 -> example receives (1/1). - Wait 1.1 s (bomb backoff window), trap bomb #2 -> poisoned=1. - Dispatch on chain 100 again -> example STILL receives. - Validates: a permanently-poisoned module on one chain does not consume restart slots, fuel, or scheduling attention from modules on any other chain. Total wall-clock ~1.2 s for the second test (one backoff window). ## supervisor.rs docstring The module-level comment now articulates the multi-chain isolation invariant explicitly so a future reader of the dispatch path knows the property is load-bearing. ## What this proves Supervisor side (dispatch fast-path): - Per-module `alive`, `failure_count`, `next_attempt`, `poisoned` are independent of which chain triggered the event. - Subscription filter excludes mismatched modules before any dispatch / restart logic runs. Upstream side (already proven by COW-1071's architecture): - `open_block_streams` spawns one task per chain; tasks share no state. A chain-A WS drop changes only chain-A's task state. - `open_log_streams` is per-(module, chain) -> even tighter isolation than block streams. ## Out of scope - A unit test that "fakes a WS drop" on chain A while chain B keeps yielding. Requires mocking `ProviderPool::subscribe_blocks` which today goes through real alloy / tokio infrastructure. The COW-1064 (E2E 4-6h testnet) and COW-1031 (7-day soak) will exercise this path against live RPCs. - Per-chain configurable backoff / health-window. Today the reconnect policy is workspace-wide; per-chain tuning is a 0.3 follow-up. ## Workspace impact - `cargo test --workspace` -> 163 host tests + 6 doctests passing (was 161 + 6; +2 from the new integration tests). - `cargo clippy --all-targets --workspace -- -D warnings` clean. - `cargo fmt --all --check` clean. Linear: COW-1073. Ninth M4 issue landed; stacks on #42 (COW-1072). --- crates/nexum-engine/src/supervisor.rs | 8 + crates/nexum-engine/src/supervisor/tests.rs | 234 ++++++++++++++++++++ 2 files changed, 242 insertions(+) diff --git a/crates/nexum-engine/src/supervisor.rs b/crates/nexum-engine/src/supervisor.rs index 0be581d5..da265a46 100644 --- a/crates/nexum-engine/src/supervisor.rs +++ b/crates/nexum-engine/src/supervisor.rs @@ -17,6 +17,14 @@ //! Modules whose `init` returned `Err(HostError)` are dead with //! `next_attempt = None` and never get scheduled - the init failure //! is treated as a manifest / config bug, not a transient (COW-1070). +//! +//! Multi-chain isolation (COW-1073): `dispatch_block(block)` walks +//! every module but only enters those whose subscriptions match +//! `block.chain_id`. Per-module restart / poison / fuel limits are +//! independent across chains, so a poisoned module on chain A +//! cannot starve modules on chain B. The upstream WS reconnect +//! tasks (COW-1071) own one per-chain backoff timer each, so a +//! chain-A connection drop does not block chain-B events. use std::collections::BTreeSet; use std::path::Path; diff --git a/crates/nexum-engine/src/supervisor/tests.rs b/crates/nexum-engine/src/supervisor/tests.rs index bb15ba16..4e3103c4 100644 --- a/crates/nexum-engine/src/supervisor/tests.rs +++ b/crates/nexum-engine/src/supervisor/tests.rs @@ -852,6 +852,240 @@ async fn poison_pill_quarantines_module_after_threshold() { assert_eq!(supervisor.poisoned_count(), 1); } +// ── COW-1073: multi-chain isolation ─────────────────────────────────── +// +// The supervisor's dispatch path is per-chain: `dispatch_block(block)` +// walks every module but only invokes those whose +// `[[subscription]] kind = "block"` matches `block.chain_id`. A +// module on chain A receives nothing when a chain-B block arrives, +// and vice versa. Combined with the per-module restart / poison +// state, this gives the engine multi-chain isolation by +// construction: a poisoned module on one chain cannot starve +// modules on any other chain. +// +// The COW-1071 WS reconnect tasks add the upstream symmetry: each +// chain owns its own subscription task + backoff timer, so a chain-A +// WS drop never blocks chain-B events. + +#[tokio::test] +async fn multi_chain_dispatch_isolates_modules_by_chain() { + // Two example modules on two different chains. Confirm dispatch + // on chain A reaches only the chain-A module and vice versa. + let Some(wasm) = example_wasm_or_skip() else { + return; + }; + + let dir = tempfile::tempdir().unwrap(); + let chain_a_manifest = dir.path().join("a.toml"); + let chain_b_manifest = dir.path().join("b.toml"); + std::fs::write( + &chain_a_manifest, + r#" +[module] +name = "module-a" + +[capabilities] +required = ["logging"] + +[[subscription]] +kind = "block" +chain_id = 1 +"#, + ) + .unwrap(); + std::fs::write( + &chain_b_manifest, + r#" +[module] +name = "module-b" + +[capabilities] +required = ["logging"] + +[[subscription]] +kind = "block" +chain_id = 100 +"#, + ) + .unwrap(); + + let engine = make_wasmtime_engine(); + let linker = make_linker(&engine); + let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); + let provider_pool = crate::host::provider_pool::ProviderPool::empty(); + let (_dir, local_store) = temp_local_store(); + + let engine_cfg = crate::engine_config::EngineConfig { + engine: crate::engine_config::EngineSection { + state_dir: dir.path().to_path_buf(), + log_level: "info".into(), + metrics: crate::engine_config::MetricsSection::default(), + }, + chains: std::collections::BTreeMap::new(), + modules: vec![ + crate::engine_config::ModuleEntry { + path: wasm.clone(), + manifest: Some(chain_a_manifest), + }, + crate::engine_config::ModuleEntry { + path: wasm, + manifest: Some(chain_b_manifest), + }, + ], + }; + + let mut supervisor = Supervisor::boot( + &engine, + &linker, + &engine_cfg, + &cow_pool, + &provider_pool, + &local_store, + ) + .await + .expect("boot"); + assert_eq!(supervisor.module_count(), 2); + assert_eq!(supervisor.alive_count(), 2); + + let block_a = nexum::host::types::Block { + chain_id: 1, + number: 1, + hash: vec![0; 32], + timestamp: 1_700_000_000_000, + }; + let block_b = nexum::host::types::Block { + chain_id: 100, + number: 1, + hash: vec![0; 32], + timestamp: 1_700_000_000_000, + }; + + // Chain A block reaches only module-a. + let dispatched = supervisor.dispatch_block(block_a).await; + assert_eq!(dispatched, 1, "only module-a subscribed to chain 1"); + assert_eq!(supervisor.alive_count(), 2); + + // Chain B block reaches only module-b. + let dispatched = supervisor.dispatch_block(block_b).await; + assert_eq!(dispatched, 1, "only module-b subscribed to chain 100"); + assert_eq!(supervisor.alive_count(), 2); +} + +#[tokio::test] +async fn multi_chain_poisoned_module_does_not_affect_other_chains() { + // fuel-bomb (always-traps) on chain 1, example (healthy) on + // chain 100. Trap the bomb a few times with a tight poison + // policy so it gets quarantined; verify the example keeps + // dispatching on chain 100 throughout. + let Some(bomb_wasm) = module_wasm_or_skip("fuel-bomb") else { + return; + }; + let Some(example_wasm) = example_wasm_or_skip() else { + return; + }; + + let dir = tempfile::tempdir().unwrap(); + let example_manifest = dir.path().join("example.toml"); + std::fs::write( + &example_manifest, + r#" +[module] +name = "example" + +[capabilities] +required = ["logging"] + +[[subscription]] +kind = "block" +chain_id = 100 +"#, + ) + .unwrap(); + + let engine = make_wasmtime_engine(); + let linker = make_linker(&engine); + let cow_pool = crate::host::cow_orderbook::OrderBookPool::default(); + let provider_pool = crate::host::provider_pool::ProviderPool::empty(); + let (_dir, local_store) = temp_local_store(); + + let engine_cfg = crate::engine_config::EngineConfig { + engine: crate::engine_config::EngineSection { + state_dir: dir.path().to_path_buf(), + log_level: "info".into(), + metrics: crate::engine_config::MetricsSection::default(), + }, + chains: std::collections::BTreeMap::new(), + modules: vec![ + crate::engine_config::ModuleEntry { + path: bomb_wasm, + manifest: Some(fixture_module_toml( + "modules/fixtures/fuel-bomb/module.toml", + )), + }, + crate::engine_config::ModuleEntry { + path: example_wasm, + manifest: Some(example_manifest), + }, + ], + }; + + let policy = + crate::runtime::poison_policy::PoisonPolicy::new(2, std::time::Duration::from_secs(60)); + let mut supervisor = Supervisor::boot( + &engine, + &linker, + &engine_cfg, + &cow_pool, + &provider_pool, + &local_store, + ) + .await + .expect("boot") + .with_poison_policy(policy); + assert_eq!(supervisor.module_count(), 2); + assert_eq!(supervisor.alive_count(), 2); + + let block_bomb_chain = nexum::host::types::Block { + chain_id: 1, // fuel-bomb's manifest declares chain 1 + number: 1, + hash: vec![0; 32], + timestamp: 1_700_000_000_000, + }; + let block_healthy_chain = nexum::host::types::Block { + chain_id: 100, + number: 1, + hash: vec![0; 32], + timestamp: 1_700_000_000_000, + }; + + // Trap #1 on the bomb's chain: bomb dies, example untouched. + supervisor.dispatch_block(block_bomb_chain.clone()).await; + assert_eq!(supervisor.poisoned_count(), 0); + + // Example keeps dispatching on its own chain - confirm before + // the bomb hits the poison threshold. + let dispatched_b = supervisor.dispatch_block(block_healthy_chain.clone()).await; + assert_eq!(dispatched_b, 1, "module-b receives chain-100 blocks"); + + // Wait out the bomb's backoff so trap #2 can land. + tokio::time::sleep(std::time::Duration::from_millis(1_100)).await; + supervisor.dispatch_block(block_bomb_chain).await; + assert_eq!( + supervisor.poisoned_count(), + 1, + "bomb quarantined at 2 failures", + ); + + // POST-poison: bomb stays dead, example still healthy. + let dispatched_after = supervisor.dispatch_block(block_healthy_chain).await; + assert_eq!( + dispatched_after, 1, + "chain-100 module unaffected by chain-1 poison", + ); + assert_eq!(supervisor.alive_count(), 1, "only example is alive"); + assert_eq!(supervisor.poisoned_count(), 1); +} + // ── build_alloy_filter ──────────────────────────────────────────────── #[test]