diff --git a/docs/operations/e2e-reports/e2e-report.template.md b/docs/operations/e2e-reports/e2e-report.template.md new file mode 100644 index 00000000..fec457da --- /dev/null +++ b/docs/operations/e2e-reports/e2e-report.template.md @@ -0,0 +1,132 @@ +# E2E testnet integration report — YYYY-MM-DD + +> Copy this file to `e2e-report-YYYY-MM-DD.md` in the same directory +> at the start of the run and fill it in as the run progresses. +> Sections marked **(operator)** must be filled in manually; the rest +> are derived from logs and `/metrics` snapshots. + +## 1. Run metadata + +| Field | Value | +|---|---| +| Operator | (operator) | +| Start (UTC) | YYYY-MM-DDTHH:MM:SSZ | +| End (UTC) | YYYY-MM-DDTHH:MM:SSZ | +| Wall clock | Hh Mm | +| Engine commit | (`git rev-parse HEAD`) | +| Engine config | `engine.e2e.toml` | +| Run host | (e.g. `bruno@bleu-mbp-m1`, `ec2-...`) | +| RPC provider | (alchemy / infura / publicnode / ...) | + +## 2. Chain coverage + +| Chain | First block | Last block | Block delta | Notes | +|---|---|---|---|---| +| Sepolia (11155111) | | | | | + +Target: `block delta >= 1500` to clear the COW-1064 acceptance bar +(>= 1500 Sepolia blocks ≈ 5 h at 12 s block time). + +## 3. On-chain actions submitted by operator + +### 3.1 TWAP conditional order (operator) + +| Field | Value | +|---|---| +| Tx hash | 0x... | +| Block | | +| Safe / EOA | 0x... | +| ComposableCoW order hash | 0x... | +| Expected detection | twap-monitor logs `watch:{orderHash}` | + +### 3.2 EthFlow swap (operator) + +| Field | Value | +|---|---| +| Tx hash | 0x... | +| Block | | +| Sender EOA | 0x... | +| Sell amount (ETH wei) | | +| Expected detection | ethflow-watcher logs `submitted:{uid}` | + +### 3.3 stop-loss pre-signature (operator) + +| Field | Value | +|---|---| +| `setPreSignature` tx hash | 0x... | +| `sell_token` allowance tx hash | 0x... | +| Owner EOA | 0x... | +| Expected UID | 0x... | +| Expected detection | stop-loss logs `submitted:{uid}` once oracle trips | + +## 4. Per-module terminal-state markers + +> Pull from the engine log with the JSON filter +> `jq 'select(.fields.message | test("submitted:|dropped:|backoff:|TRIGGERED|trapped"))'`. +> Each module must show at least ONE marker for the acceptance bar. + +| Module | First marker timestamp | Marker | Sample line | +|---|---|---|---| +| twap-monitor | | `watch:` / `submitted:` / `dropped:` | | +| ethflow-watcher | | `submitted:` / `dropped:` | | +| price-alert | | `TRIGGERED` (Warn) | | +| balance-tracker | | `last:` write on first dispatch | | +| stop-loss | | `TRIGGERED` / `submitted:` / `dropped:` | | + +## 5. Error counts (from `/metrics` delta) + +> Capture two snapshots: at boot (`/metrics > metrics-start.txt`) and +> immediately before shutdown (`/metrics > metrics-end.txt`). Fill in +> the delta column. + +| Metric | Start | End | Delta | +|---|---|---|---| +| `shepherd_module_errors_total{module="...",reason="trap"}` (per module) | | | | +| `shepherd_module_restarts_total{module="..."}` (per module) | | | | +| `shepherd_module_poisoned{module="..."}` (gauge, end-state per module) | n/a | | n/a | +| `shepherd_cow_api_submit_total{result="ok"}` | | | | +| `shepherd_cow_api_submit_total{result="err"}` | | | | +| `shepherd_chain_request_total{result="ok"}` | | | | +| `shepherd_chain_request_total{result="err"}` | | | | +| `shepherd_stream_reconnects_total{kind="block"}` | | | | +| `shepherd_stream_reconnects_total{kind="log"}` | | | | +| `shepherd_event_latency_seconds` (p50 / p95 / p99) | | | | + +## 6. Anomalies + defects + +> Anything outside the expected log shape. Each anomaly that is +> reproducible OR has an unclear root cause must be filed as a +> separate Linear issue and linked here. + +| # | Time (UTC) | Module | Summary | Linear | +|---|---|---|---|---| +| 1 | | | | COW-... | + +## 7. Acceptance checklist (COW-1064) + +- [ ] `block delta >= 1500` (≥ 5 h coverage) +- [ ] All 5 modules have ≥ 1 terminal-state marker in section 4 +- [ ] `shepherd_module_errors_total{reason="trap"}` for well-behaved modules == 0 +- [ ] No `[[modules]]`-listed module is `shepherd_module_poisoned == 1` at end +- [ ] No `ERROR` lines from `nexum_engine` in the supervisor log +- [ ] At least one orderbook submit attempt landed (`ok` or typed + `err` with retry/drop classification) on twap-monitor, + ethflow-watcher, AND stop-loss +- [ ] Report committed in this directory +- [ ] Defects filed in Linear and linked in section 6 + +## 8. Sign-off (operator) + +> Brief paragraph: ran clean / found N defects / blocking issues for +> COW-1031 soak Y/N. The COW-1031 soak MUST NOT start until this +> section says "no blocking issues". + +… + +## 9. Attachments + +- `engine.log` (full supervisor JSON log; ≥ 4 h) +- `metrics-start.txt` +- `metrics-end.txt` +- (optional) `metrics-snapshots/` — every 60 s scrape if a soak-style + Prometheus pull was not running diff --git a/docs/operations/e2e-testnet-runbook.md b/docs/operations/e2e-testnet-runbook.md new file mode 100644 index 00000000..0a783452 --- /dev/null +++ b/docs/operations/e2e-testnet-runbook.md @@ -0,0 +1,300 @@ +# E2E testnet runbook (COW-1064) + +How to exercise **all 5 modules** — twap-monitor, ethflow-watcher, +price-alert, balance-tracker, stop-loss — on a real Sepolia host +**simultaneously for 4-6 hours**. Same shape as the M2 + M3 +runbooks, but this one runs the full production module suite and +captures a structured report (`docs/operations/e2e-reports/`). + +The E2E run is the integration step between unit-test coverage +(MockHost, per-module strategy tests) and the COW-1031 7-day soak. +The soak validates *stability*; this validates *correctness in a +live dispatch context* and surfaces cross-module bugs the soak +should not be discovering. + +The acceptance bar (from COW-1064) is: + +- ≥ 1500 Sepolia blocks (≈ 5 h at 12 s block time). +- Each of the 5 modules writes at least one terminal-state marker + (`submitted:` / `dropped:` / `backoff:` / `TRIGGERED` / `last:`). +- 0 unexpected errors in the supervisor log. +- 0 well-behaved modules trapped or poisoned at end of run. +- A committed report + filed defects. + +--- + +## 0. Prerequisites + +### Toolchain + +Same as the M2 + M3 runbooks (`rustup target add wasm32-wasip2`, +optionally `just`, a Sepolia WS RPC). + +### RPC + +The public Sepolia node (`wss://ethereum-sepolia-rpc.publicnode.com`) +throttles `eth_subscribe` and `eth_call` under sustained load. The +E2E run does at minimum: + +- 1 block subscription (shared across 4 modules — price-alert, + balance-tracker, stop-loss, twap-monitor block-tick). +- 2 log subscriptions (twap-monitor's + `ComposableCoW.ConditionalOrderCreated` + ethflow-watcher's + `CoWSwapEthFlow.OrderPlacement`). +- ≥ 4 `eth_call` per block from price-alert + balance-tracker + (×2 addresses) + stop-loss, + 1 per registered TWAP order + per block. + +Override the `[chains.11155111] rpc_url` in `engine.e2e.toml` +with an Alchemy / Infura WS for the run: + +```toml +[chains.11155111] +rpc_url = "wss://eth-sepolia.g.alchemy.com/v2/" +``` + +### On-chain prep (operator) + +The acceptance bar requires real on-chain submissions. Before +launching the run, prepare: + +1. **A funded test EOA on Sepolia** (≥ 0.05 ETH for gas; the same + EOA can satisfy the EthFlow swap + stop-loss `setPreSignature` + sub-tasks). +2. **A Safe (or direct caller) that can call ComposableCoW** on + Sepolia — for the TWAP conditional-order submission. +3. **stop-loss config aligned with that EOA**: update + `modules/examples/stop-loss/module.toml::[config].owner` to the + EOA address you control, and pick a `sell_token` / `buy_token` + pair the EOA holds + has approved to the GPv2VaultRelayer. + See `docs/operations/m3-testnet-runbook.md` section 2 for the + full pre-sign + allowance recipe. + +The E2E run will start cleanly without (1)/(2)/(3), but the +acceptance bar requires at least one `submitted:` marker on each +of twap-monitor / ethflow-watcher / stop-loss, and you only get +those by triggering each path on-chain. + +--- + +## 1. Boot + +The engine + all 5 modules + Prometheus `/metrics` endpoint: + +```bash +just run-e2e +``` + +Equivalent long form: + +```bash +just build-e2e # builds the 5 module .wasm artefacts +cargo build -p nexum-engine +cargo run -p nexum-engine -- --engine-config engine.e2e.toml +``` + +### Expected boot sequence (~5 s) + +``` +INFO nexum-engine starting +INFO opening chain RPC provider chain_id=11155111 url="wss://..." +INFO metrics exporter listening at /metrics addr=127.0.0.1:9100 +INFO loading module manifest manifest=modules/twap-monitor/module.toml +INFO compiling component component=...twap_monitor.wasm +INFO init succeeded module=twap-monitor +INFO loading module manifest manifest=modules/ethflow-watcher/module.toml +INFO init succeeded module=ethflow-watcher +INFO loading module manifest manifest=modules/examples/price-alert/module.toml +INFO init succeeded module=price-alert +INFO loading module manifest manifest=modules/examples/balance-tracker/module.toml +INFO init succeeded module=balance-tracker +INFO loading module manifest manifest=modules/examples/stop-loss/module.toml +INFO init succeeded module=stop-loss +INFO supervisor up count=5 +INFO supervisor ready modules=5 chains=1 +INFO block subscription open chain_id=11155111 +INFO log subscription open chain_id=11155111 module=twap-monitor +INFO log subscription open chain_id=11155111 module=ethflow-watcher +``` + +If any of `count=5`, `modules=5`, or both log subscriptions are +missing, **stop the run and triage** — running 4-6 h on a +degraded engine wastes time the operator does not get back. + +### Smoke at first block (~12 s after boot) + +Within the first Sepolia block dispatched: + +``` +DEBUG dispatch block chain_id=11155111 number=N +DEBUG chain::request method=eth_call # price-alert oracle read +DEBUG chain::request method=eth_getBalance # balance-tracker addr 1 +DEBUG chain::request method=eth_getBalance # balance-tracker addr 2 +DEBUG chain::request method=eth_call # stop-loss oracle read +WARN price-alert: TRIGGERED answer=... threshold=... +``` + +(See `docs/operations/m3-testnet-runbook.md` for the per-module +single-block expectations — the E2E run reproduces those plus +twap-monitor's empty poll loop until a `watch:` is registered.) + +--- + +## 2. The 4-6 h run + +### 2.1 Start the clock + +Pipe the engine output to a JSON log file the operator can mine +with `jq` after the run: + +```bash +just run-e2e 2>&1 | tee -a docs/operations/e2e-reports/engine-$(date -u +%Y%m%dT%H%M%SZ).log +``` + +Record `date -u --iso-8601=seconds` and `git rev-parse HEAD` in +section 1 of the report template. + +### 2.2 Capture the metrics baseline + +```bash +curl -s http://127.0.0.1:9100/metrics > docs/operations/e2e-reports/metrics-start.txt +``` + +### 2.3 Trigger each on-chain action + +Run these as soon as the supervisor is `ready`: + +1. **TWAP order** — call ComposableCoW from your Safe (or directly + if you control the user). Within 1-2 blocks, twap-monitor logs: + ``` + INFO twap-monitor watch:{orderHash} chain_id=11155111 + ``` +2. **EthFlow swap** — execute a small ETH-flow swap from your EOA + via the cow-swap front-end pointed at Sepolia. Within 1-2 blocks + ethflow-watcher logs: + ``` + INFO ethflow-watcher submitted:{uid} + ``` + (or a typed `dropped:{uid}` if the orderbook rejected — both + count as a terminal-state marker for section 4.) +3. **stop-loss trigger** — once your owner EOA has called + `setPreSignature` and approved the sell token, lower + `trigger_price` in `modules/examples/stop-loss/module.toml` to + ≤ the current Sepolia Chainlink ETH/USD answer and reload the + engine (or set it pre-boot if you already know the feed value). + Within 1 block stop-loss logs: + ``` + INFO stop-loss TRIGGERED price=... trigger=... + INFO stop-loss submitted:{uid} + ``` + +### 2.4 Idle until end of run + +Once all three terminal markers are observed and the report's +section 4 has at least one entry per module, leave the engine +running undisturbed for the remainder of the 4-6 h window. + +The operator should watch for these red flags (if any appears, +the run is a defect and section 6 must capture it): + +| Red flag | Why it matters | +|---|---| +| `ERROR` from `nexum_engine::*` | Acceptance #5: zero ERROR lines. | +| `module ... trapped:` for a non-fixture module | Trapping production-side modules is a defect. | +| `module ... poisoned` | Quarantine of a real module is a defect. | +| `stream reconnect attempt=N` with N rising | The WS is flapping (RPC issue or bug). One reconnect per chain is fine. | +| `chain::request` `err` rate > 5% | The RPC is degraded. Switch keys / providers. | + +### 2.5 Capture metrics deltas + shutdown + +At the end of the run window: + +```bash +curl -s http://127.0.0.1:9100/metrics > docs/operations/e2e-reports/metrics-end.txt +# Ctrl-C the engine — graceful shutdown writes last_dispatched_block (COW-1072): +# > INFO graceful shutdown complete dispatched_blocks=N dispatched_logs=M uptime_secs=K +``` + +Diff the two snapshots to fill in the report's section 5: + +```bash +diff <(grep '^shepherd_' docs/operations/e2e-reports/metrics-start.txt) \ + <(grep '^shepherd_' docs/operations/e2e-reports/metrics-end.txt) +``` + +--- + +## 3. Filling in the report + +Copy the template at the start of the run: + +```bash +DATE=$(date -u +%Y-%m-%d) +cp docs/operations/e2e-reports/e2e-report.template.md \ + docs/operations/e2e-reports/e2e-report-${DATE}.md +$EDITOR docs/operations/e2e-reports/e2e-report-${DATE}.md +``` + +Fill sections in this order: + +1. **Section 1 (run metadata)** at boot. +2. **Section 3 (on-chain actions)** as you submit each one. +3. **Section 4 (terminal markers)** as each first marker fires. +4. **Section 5 (metrics)** once `metrics-end.txt` is captured. +5. **Section 6 (anomalies)** continuously — anything unexpected + gets a row + a Linear issue. +6. **Section 7 (acceptance checklist)** at the end — every box + must be `[x]` for COW-1064 to close. +7. **Section 8 (sign-off)** is the gating decision for the + COW-1031 7-day soak. + +Commit the filled-in report on the same branch as this runbook: + +```bash +git add docs/operations/e2e-reports/e2e-report-${DATE}.md +git commit -m "ops(e2e): report from ${DATE} run (COW-1064)" +git push +``` + +--- + +## 4. What this does NOT prove + +- **Stability beyond ~5 h** → COW-1031 (7-day soak, + Sepolia + Arb Sepolia). +- **Adversarial resource exhaustion** → COW-1036 (fuel / + memory bombs as fixtures). +- **Security review** → COW-1065. +- **Production deployment story** → COW-1030. +- **Multi-chain isolation under live WS drops** → partially + proven by the COW-1073 integration tests; full validation + requires Arb Sepolia + Sepolia simultaneously, which the soak + exercises. + +--- + +## 5. Troubleshooting + +Inherits the M2 + M3 runbook tables. E2E-specific: + +| Symptom | Likely cause | Fix | +|---|---|---| +| `supervisor ready modules=4 chains=1` (or less) at boot | One of the 5 module manifests failed to load — likely a missing wasm artefact under `target/wasm32-wasip2/release/` | Re-run `just build-e2e` and verify all 5 `.wasm` files are present. | +| `INFO log subscription open chain_id=11155111` appears only once | One of the two log-subscribing modules failed init | Check the immediately preceding `init failed module=...` line; the failing module's `[capabilities]` or subscription `address` is the usual culprit. | +| RPC drops every ~30 min on `publicnode.com` | Public node rate limits | Switch to Alchemy / Infura per section 0. | +| `stop-loss TRIGGERED` fires immediately on default config | Default `trigger_price = 2500.00` is above Sepolia Chainlink ETH/USD (~$1745) and `direction = "below"`. See M3 runbook §1. | Tune `trigger_price` lower to test the "silent until trigger" path. | +| `twap-monitor` never logs `watch:` | No `ConditionalOrderCreated` event observed on Sepolia during the window | Submit the TWAP order from section 2.3 step 1. | +| `ethflow-watcher` never logs `submitted:` | No `OrderPlacement` event observed on Sepolia during the window | Execute the EthFlow swap from section 2.3 step 2. | + +--- + +## 6. References + +- M2 runbook (sister doc): `docs/operations/m2-testnet-runbook.md` +- M3 runbook (sister doc): `docs/operations/m3-testnet-runbook.md` +- Engine config: `engine.e2e.toml` +- Report template: `docs/operations/e2e-reports/e2e-report.template.md` +- Linear COW-1064 (this runbook's issue): + https://linear.app/bleu-builders/issue/COW-1064 +- COW-1031 (downstream soak; do not start until COW-1064 closes): + https://linear.app/bleu-builders/issue/COW-1031 diff --git a/engine.e2e.toml b/engine.e2e.toml new file mode 100644 index 00000000..96c6e59e --- /dev/null +++ b/engine.e2e.toml @@ -0,0 +1,67 @@ +# E2E testnet integration config for nexum-engine (COW-1064). +# +# Boots all 5 production + example modules on Sepolia simultaneously +# for the 4-6 h E2E run: +# +# - twap-monitor (modules/twap-monitor) +# - ethflow-watcher (modules/ethflow-watcher) +# - price-alert (modules/examples/price-alert) +# - balance-tracker (modules/examples/balance-tracker) +# - stop-loss (modules/examples/stop-loss) +# +# This is the integration step between the M3 single-chain runbook +# (`engine.m3.toml`, 3 modules) and the COW-1031 7-day soak +# (Sepolia + Arb Sepolia, all modules, no human-in-the-loop). The +# E2E run validates correctness in a real-chain dispatch context; +# the soak validates stability afterwards. +# +# Usage: +# just run-e2e +# # or: +# just build-e2e +# cargo run -p nexum-engine -- --engine-config engine.e2e.toml +# +# Operator runbook: docs/operations/e2e-testnet-runbook.md + +[engine] +# Separate from data/m2 and data/m3 so the run starts on a clean +# local-store and the report's UID round-trip section is uncluttered. +state_dir = "./data/e2e" +log_level = "info,nexum_engine=debug" + +# COW-1034: bind /metrics so the operator can scrape Prometheus at +# 60 s intervals during the run and check the e2e report's metrics +# delta section. 127.0.0.1 is intentional — do not expose a metrics +# port on a public interface. +[engine.metrics] +enabled = true +bind_addr = "127.0.0.1:9100" + +# Sepolia. Override with an Alchemy / Infura WS for the run; the +# public node throttles `eth_subscribe` under sustained load (>1 +# `eth_call` per module per block = 4 calls/12 s window minimum, +# more under bursts of EthFlow / TWAP activity). +[chains.11155111] +rpc_url = "wss://ethereum-sepolia-rpc.publicnode.com" + +# --- modules ---------------------------------------------------------- + +[[modules]] +path = "target/wasm32-wasip2/release/twap_monitor.wasm" +manifest = "modules/twap-monitor/module.toml" + +[[modules]] +path = "target/wasm32-wasip2/release/ethflow_watcher.wasm" +manifest = "modules/ethflow-watcher/module.toml" + +[[modules]] +path = "target/wasm32-wasip2/release/price_alert.wasm" +manifest = "modules/examples/price-alert/module.toml" + +[[modules]] +path = "target/wasm32-wasip2/release/balance_tracker.wasm" +manifest = "modules/examples/balance-tracker/module.toml" + +[[modules]] +path = "target/wasm32-wasip2/release/stop_loss.wasm" +manifest = "modules/examples/stop-loss/module.toml" diff --git a/justfile b/justfile index 89e35451..21dac0a6 100644 --- a/justfile +++ b/justfile @@ -49,6 +49,18 @@ build-m3: run-m3: build-m3 build-engine cargo run -p nexum-engine -- --engine-config engine.m3.toml --pretty-logs +# Build all 5 modules required by the E2E run (twap-monitor + +# ethflow-watcher + price-alert + balance-tracker + stop-loss). +build-e2e: build-m2 build-m3 + +# Run the 4-6 h E2E integration scenario on Sepolia. All 5 modules +# dispatched simultaneously against a live RPC; metrics scraped at +# 127.0.0.1:9100/metrics. JSON logs (no --pretty-logs) so a +# downstream `jq` filter can mine submitted/dropped/backoff markers +# for the e2e report. See `docs/operations/e2e-testnet-runbook.md`. +run-e2e: build-e2e build-engine + cargo run -p nexum-engine -- --engine-config engine.e2e.toml + # Check the entire workspace check: cargo check --target wasm32-wasip2 -p example