diff --git a/Cargo.lock b/Cargo.lock index b53a2f58f0..fe7cb32148 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,10 +899,12 @@ dependencies = [ "rustls-pemfile", "serde", "serde_json", + "sse-stream", "tar", "tempfile", "thiserror 2.0.18", "tokio", + "tokio-stream", "tower", "tower-http 0.7.0", "tracing", @@ -2122,6 +2124,7 @@ dependencies = [ "tonic", "tonic-prost", "tonic-prost-build", + "tower", "tracing", "tracing-subscriber", "url", diff --git a/Cargo.toml b/Cargo.toml index 9f2b9be203..0ae902b62b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -215,6 +215,7 @@ rustls-pemfile = "2.2.0" serde = "1.0.228" serde_derive = "1" serde_json = "1.0" +sse-stream = "0.2.4" serde_regex = "1.1.0" serde_with = "3.12.0" serde_yaml = "0.9" diff --git a/crates/bmc-mock/Cargo.toml b/crates/bmc-mock/Cargo.toml index a4a4a82df8..453aee86c5 100644 --- a/crates/bmc-mock/Cargo.toml +++ b/crates/bmc-mock/Cargo.toml @@ -37,6 +37,7 @@ flate2 = { workspace = true } form_urlencoded = { workspace = true } futures = { workspace = true } http-body-util = { workspace = true } +tokio-stream = { workspace = true } hyper = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } @@ -51,6 +52,7 @@ rustls = { workspace = true } rustls-pemfile = { workspace = true } serde = { features = ["derive"], workspace = true } serde_json = { workspace = true } +sse-stream = { workspace = true } tar = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } diff --git a/crates/bmc-mock/src/bmc_state.rs b/crates/bmc-mock/src/bmc_state.rs index cf1ac41b72..6bd11dd925 100644 --- a/crates/bmc-mock/src/bmc_state.rs +++ b/crates/bmc-mock/src/bmc_state.rs @@ -21,6 +21,7 @@ use crate::redfish; use crate::redfish::account_service::AccountServiceState; use crate::redfish::chassis::ChassisState; use crate::redfish::computer_system::SystemState; +use crate::redfish::event_service::EventServiceState; use crate::redfish::manager::ManagerState; use crate::redfish::session_service::SessionServiceState; use crate::redfish::update_service::UpdateServiceState; @@ -37,6 +38,7 @@ pub struct BmcState { pub update_service_state: Arc, pub account_service_state: Arc, pub(crate) session_service_state: Arc, + pub(crate) event_service_state: Option>, pub injection: Arc, pub(crate) callbacks: Option>, /// Whether this BMC advertises and serves the `/redfish/v1/Systems` diff --git a/crates/bmc-mock/src/lib.rs b/crates/bmc-mock/src/lib.rs index 7b72a93a6f..8411695261 100644 --- a/crates/bmc-mock/src/lib.rs +++ b/crates/bmc-mock/src/lib.rs @@ -53,9 +53,13 @@ pub use machine_info::{ }; pub use mock_machine_router::{ BmcCommand, MachineRouterOptions, SetSystemPowerError, SetSystemPowerResult, machine_router, - machine_router_with_injection_store, + machine_router_with_injection_store, machine_router_with_options_and_injection_store, }; pub use rack_info::RackInfo; +pub use redfish::event_service::{ + EventScenario, EventServiceConfig, LinkedResourceResponse, SERVICE_PATH as EVENT_SERVICE_PATH, + SSE_PATH as EVENT_SERVICE_SSE_PATH, TRIGGER_PATH as EVENT_SERVICE_TRIGGER_PATH, +}; pub use redfish::virtual_media::DeviceConfig as VirtualMediaDeviceConfig; pub const DUMMY_FACTORY_USERNAME: &str = "root"; diff --git a/crates/bmc-mock/src/libvirt.rs b/crates/bmc-mock/src/libvirt.rs index 0e490f733e..a7f658fc1b 100644 --- a/crates/bmc-mock/src/libvirt.rs +++ b/crates/bmc-mock/src/libvirt.rs @@ -655,6 +655,7 @@ esac name: Cow::Borrowed("Operating System Virtual CD"), media_types: vec![Cow::Borrowed("CD"), Cow::Borrowed("DVD")], }]), + ..Default::default() }, ); callbacks.bind_state(&state).unwrap(); diff --git a/crates/bmc-mock/src/main.rs b/crates/bmc-mock/src/main.rs index 6efef3fad7..050854cbe4 100644 --- a/crates/bmc-mock/src/main.rs +++ b/crates/bmc-mock/src/main.rs @@ -373,6 +373,7 @@ fn generated_mock(config: GeneratedMockConfig) -> (Router, BmcState) { media_types: vec![Cow::Borrowed("CD"), Cow::Borrowed("DVD")], }, ]), + ..Default::default() }, ) } else { diff --git a/crates/bmc-mock/src/mock_machine_router.rs b/crates/bmc-mock/src/mock_machine_router.rs index aef9941447..8cf57eb687 100644 --- a/crates/bmc-mock/src/mock_machine_router.rs +++ b/crates/bmc-mock/src/mock_machine_router.rs @@ -31,6 +31,7 @@ use crate::{ #[derive(Debug, Default)] pub struct MachineRouterOptions { pub virtual_media_devices: Option>, + pub event_service: Option, } #[derive(Debug)] @@ -91,7 +92,7 @@ pub fn machine_router_with_injection_store( redfish_auth: bool, injection: Arc, ) -> (Router, BmcState) { - machine_router_inner( + machine_router_with_options_and_injection_store( machine_info, callbacks, mat_host_id, @@ -101,6 +102,24 @@ pub fn machine_router_with_injection_store( ) } +pub fn machine_router_with_options_and_injection_store( + machine_info: &MachineInfo, + callbacks: Arc, + mat_host_id: String, + redfish_auth: bool, + injection: Arc, + options: MachineRouterOptions, +) -> (Router, BmcState) { + machine_router_inner( + machine_info, + callbacks, + mat_host_id, + redfish_auth, + injection, + options, + ) +} + fn machine_router_inner( machine_info: &MachineInfo, callbacks: Arc, @@ -124,6 +143,7 @@ fn machine_router_inner( .add_routes(crate::redfish::update_service::add_routes) .add_routes(crate::redfish::task_service::add_routes) .add_routes(crate::redfish::telemetry_service::add_routes) + .add_routes(crate::redfish::event_service::add_routes) .add_routes(crate::redfish::account_service::add_routes) .add_routes(crate::redfish::session_service::add_routes) .add_routes(|routes| crate::redfish::computer_system::add_routes(routes, bmc_vendor)) @@ -156,6 +176,11 @@ fn machine_router_inner( ); let session_service_state = Arc::new(crate::redfish::session_service::SessionServiceState::new()); + let event_service_state = options.event_service.map(|config| { + Arc::new(crate::redfish::event_service::EventServiceState::new( + config, + )) + }); let state = BmcState { bmc_vendor, bmc_product, @@ -167,6 +192,7 @@ fn machine_router_inner( update_service_state, account_service_state, session_service_state, + event_service_state, injection: injection.clone(), callbacks: Some(callbacks.clone()), exposes_computer_systems: machine_info.exposes_computer_systems(), diff --git a/crates/bmc-mock/src/redfish/event_service.rs b/crates/bmc-mock/src/redfish/event_service.rs new file mode 100644 index 0000000000..19bec6c9c1 --- /dev/null +++ b/crates/bmc-mock/src/redfish/event_service.rs @@ -0,0 +1,214 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use std::collections::{BTreeMap, VecDeque}; +use std::convert::Infallible; +use std::sync::Mutex; + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use futures::StreamExt; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use tokio::sync::broadcast; +use tokio_stream::wrappers::BroadcastStream; + +use crate::bmc_state::BmcState; +use crate::json::JsonExt; + +pub const SERVICE_PATH: &str = "/redfish/v1/EventService"; +pub const SSE_PATH: &str = "/redfish/v1/EventService/SSE"; +pub const TRIGGER_PATH: &str = "/redfish/v1/EventService/Actions/EventService.TriggerScenario"; + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct EventServiceConfig { + #[serde(default)] + pub scenarios: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct EventScenario { + pub payload: Value, + #[serde(default)] + pub linked_resources: BTreeMap>, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct LinkedResourceResponse { + pub status: u16, + #[serde(default)] + pub body: Value, +} + +pub(crate) struct EventServiceState { + config: EventServiceConfig, + linked_responses: Mutex>>, + events: broadcast::Sender, +} + +impl EventServiceState { + pub(crate) fn new(config: EventServiceConfig) -> Self { + let (events, _) = broadcast::channel(16); + Self { + config, + linked_responses: Mutex::new(BTreeMap::new()), + events, + } + } + + fn trigger(&self, name: &str) -> Result<(), TriggerError> { + let scenario = self + .config + .scenarios + .get(name) + .ok_or(TriggerError::UnknownScenario)?; + let mut linked_responses = self.linked_responses.lock().unwrap(); + linked_responses.clear(); + for (path, responses) in &scenario.linked_resources { + linked_responses.insert(path.clone(), responses.clone().into()); + } + drop(linked_responses); + self.events + .send(scenario.payload.clone()) + .map(|_| ()) + .map_err(|_| TriggerError::NoSubscriber) + } + + fn linked_response(&self, path: &str) -> Option { + let mut responses = self.linked_responses.lock().unwrap(); + let responses = responses.get_mut(path)?; + if responses.len() > 1 { + responses.pop_front() + } else { + responses.front().cloned() + } + } +} + +enum TriggerError { + UnknownScenario, + NoSubscriber, +} + +#[derive(Deserialize)] +struct TriggerRequest { + scenario: String, +} + +pub(crate) fn add_routes(router: Router) -> Router { + router + .route(SERVICE_PATH, get(get_service)) + .route(SSE_PATH, get(sse)) + .route(TRIGGER_PATH, post(trigger)) + .route("/redfish/v1/{*resource}", get(get_linked_resource)) +} + +async fn get_service(State(state): State) -> Response { + let Some(_) = state.event_service_state else { + return StatusCode::NOT_FOUND.into_response(); + }; + json!({ + "@odata.id": SERVICE_PATH, + "@odata.type": "#EventService.v1_8_0.EventService", + "Id": "EventService", + "Name": "Event Service", + "ServiceEnabled": true, + "ServerSentEventUri": SSE_PATH, + "Actions": { + "#EventService.TriggerScenario": { "target": TRIGGER_PATH } + } + }) + .into_ok_response() +} + +async fn sse(State(state): State) -> Response { + let Some(event_service) = state.event_service_state else { + return StatusCode::NOT_FOUND.into_response(); + }; + let stream = BroadcastStream::new(event_service.events.subscribe()).filter_map(|event| async { + event + .ok() + .map(|payload| Ok::<_, Infallible>(Event::default().data(payload.to_string()))) + }); + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() +} + +async fn trigger(State(state): State, Json(request): Json) -> Response { + let Some(event_service) = state.event_service_state else { + return StatusCode::NOT_FOUND.into_response(); + }; + match event_service.trigger(&request.scenario) { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(TriggerError::UnknownScenario) => { + (StatusCode::NOT_FOUND, "unknown event scenario").into_response() + } + Err(TriggerError::NoSubscriber) => { + (StatusCode::CONFLICT, "no active SSE subscriber").into_response() + } + } +} + +async fn get_linked_resource( + State(state): State, + Path(resource): Path, +) -> Response { + let Some(event_service) = state.event_service_state else { + return StatusCode::NOT_FOUND.into_response(); + }; + let path = format!("/redfish/v1/{resource}"); + let Some(response) = event_service.linked_response(&path) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let status = StatusCode::from_u16(response.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + (status, Json(response.body)).into_response() +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use axum::body::Body; + use axum::http::Request; + use tower::ServiceExt; + + use super::*; + use crate::test_support::{NoopCallbacks, host_info}; + use crate::{HardwareType, MachineRouterOptions, machine_router}; + + #[tokio::test] + async fn event_service_is_opt_in() { + let (router, _) = machine_router( + &host_info(HardwareType::GenericAmi), + Arc::new(NoopCallbacks), + "test-host-id".to_string(), + false, + MachineRouterOptions::default(), + ); + + let root = router + .clone() + .oneshot(Request::get("/redfish/v1").body(Body::empty()).unwrap()) + .await + .unwrap(); + let root = http_body_util::BodyExt::collect(root.into_body()) + .await + .unwrap() + .to_bytes(); + let root: Value = serde_json::from_slice(&root).unwrap(); + assert!(root.get("EventService").is_none()); + + let response = router + .oneshot(Request::get(SERVICE_PATH).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } +} diff --git a/crates/bmc-mock/src/redfish/mod.rs b/crates/bmc-mock/src/redfish/mod.rs index 4d34266b0f..58ff5d8744 100644 --- a/crates/bmc-mock/src/redfish/mod.rs +++ b/crates/bmc-mock/src/redfish/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod chassis; mod collection; pub(crate) mod computer_system; pub(crate) mod ethernet_interface; +pub(crate) mod event_service; pub(crate) mod host_interface; pub(crate) mod leak_detector; pub(crate) mod log_service; diff --git a/crates/bmc-mock/src/redfish/service_root.rs b/crates/bmc-mock/src/redfish/service_root.rs index ccc1220611..7ab6bcf847 100644 --- a/crates/bmc-mock/src/redfish/service_root.rs +++ b/crates/bmc-mock/src/redfish/service_root.rs @@ -74,6 +74,7 @@ async fn get_service_root(State(state): State) -> Response { .manager_collection(&redfish::manager::collection()) .update_service(&redfish::update_service::resource()) .telemetry_service(&redfish::telemetry_service::resource()) + .maybe_event_service(state.event_service_state.is_some()) .build() .into_ok_response() } @@ -134,4 +135,14 @@ impl ServiceRootBuilder { fn telemetry_service(self, v: &redfish::Resource<'_>) -> Self { self.apply_patch(v.nav_property("TelemetryService")) } + + fn maybe_event_service(self, enabled: bool) -> Self { + if enabled { + self.apply_patch(json!({ + "EventService": { "@odata.id": redfish::event_service::SERVICE_PATH } + })) + } else { + self + } + } } diff --git a/crates/bmc-mock/src/redfish/virtual_media.rs b/crates/bmc-mock/src/redfish/virtual_media.rs index c028bba81d..32d9e7f00a 100644 --- a/crates/bmc-mock/src/redfish/virtual_media.rs +++ b/crates/bmc-mock/src/redfish/virtual_media.rs @@ -308,6 +308,7 @@ mod tests { media_types: vec!["CD".into(), "DVD".into()], }, ]), + ..Default::default() }, ) .0; diff --git a/crates/bmc-mock/src/test_support/axum_http_client.rs b/crates/bmc-mock/src/test_support/axum_http_client.rs index a6e9429d95..10698ad957 100644 --- a/crates/bmc-mock/src/test_support/axum_http_client.rs +++ b/crates/bmc-mock/src/test_support/axum_http_client.rs @@ -20,6 +20,7 @@ use std::fmt; use axum::Router; use axum::body::Body; use axum::http::{HeaderMap, Method, Request, StatusCode}; +use futures::StreamExt; use http_body_util::BodyExt; use nv_redfish::bmc_http::{ BmcCredentials, CacheableError, HttpClient, RejectedUriReferenceError, RequestError, @@ -42,6 +43,7 @@ pub enum Error { Http(axum::http::Error), Cache(String), RejectedUriReference(String), + Sse(String), NotSupported(&'static str), } @@ -57,6 +59,7 @@ impl fmt::Display for Error { Self::RejectedUriReference(reason) => { write!(f, "rejected URI reference: {reason}") } + Self::Sse(reason) => write!(f, "SSE error: {reason}"), Self::NotSupported(what) => write!(f, "not supported in test client: {what}"), } } @@ -208,11 +211,30 @@ impl HttpClient for AxumRouterHttpClient { async fn sse serde::Deserialize<'a>>( &self, - _url: Url, - _credentials: &BmcCredentials, - _custom_headers: &HeaderMap, + url: Url, + credentials: &BmcCredentials, + custom_headers: &HeaderMap, ) -> Result, Self::Error> { - Err(Error::NotSupported("SSE stream is not supported yet")) + let builder = Self::request_builder(Method::GET, &url, credentials, custom_headers); + let request = builder.body(Body::empty()).map_err(Error::Http)?; + let response = self.call(request).await?; + if !response.status().is_success() { + let (status, _, bytes) = Self::response_bytes(response).await?; + return Err(Error::InvalidResponse { + url, + status, + text: String::from_utf8_lossy(&bytes).to_string(), + }); + } + let stream = sse_stream::SseStream::new(response.into_body()).filter_map(|event| async { + match event { + Ok(event) => event + .data + .map(|data| serde_json::from_str(&data).map_err(Error::Json)), + Err(error) => Some(Err(Error::Sse(error.to_string()))), + } + }); + Ok(Box::pin(stream)) } async fn post_session( diff --git a/crates/bmc-mock/src/test_support/mod.rs b/crates/bmc-mock/src/test_support/mod.rs index 64eb068ac9..125916c85a 100644 --- a/crates/bmc-mock/src/test_support/mod.rs +++ b/crates/bmc-mock/src/test_support/mod.rs @@ -38,7 +38,7 @@ use crate::{ pub mod axum_http_client; #[derive(Debug)] -pub(super) struct NoopCallbacks; +pub(crate) struct NoopCallbacks; impl Callbacks for NoopCallbacks { fn get_power_state(&self) -> MockPowerState { @@ -102,7 +102,20 @@ pub async fn bmc_for_machine(machine_info: MachineInfo) -> TestBmcHandle { .await } -pub(super) fn host_info(hw_type: HardwareType) -> MachineInfo { +pub fn router_for_machine_with_options( + machine_info: &MachineInfo, + options: MachineRouterOptions, +) -> (Router, BmcState) { + machine_router( + machine_info, + Arc::new(NoopCallbacks), + "test-host-id".to_string(), + false, + options, + ) +} + +pub fn host_info(hw_type: HardwareType) -> MachineInfo { let ndpu = hw_type.fixed_number_of_dpu().unwrap_or(0); let mut pool = TEST_MAC_POOL.lock().unwrap(); let ranges_config = pool.allocate_range_config().unwrap(); diff --git a/crates/health/Cargo.toml b/crates/health/Cargo.toml index 64e9f7fddf..e60ef30728 100644 --- a/crates/health/Cargo.toml +++ b/crates/health/Cargo.toml @@ -126,6 +126,7 @@ criterion = { workspace = true } rcgen = { workspace = true } tempfile = { workspace = true } time = { workspace = true } +tower = { workspace = true } [[bench]] name = "sink_pipeline" diff --git a/crates/health/src/collectors/logs/sse.rs b/crates/health/src/collectors/logs/sse.rs index dbb4bff769..0420efdb30 100644 --- a/crates/health/src/collectors/logs/sse.rs +++ b/crates/health/src/collectors/logs/sse.rs @@ -363,9 +363,10 @@ fn record_to_log( #[cfg(test)] mod tests { + use std::collections::BTreeMap; use std::sync::atomic::{AtomicUsize, Ordering}; - use axum::http::StatusCode; + use axum::http::{Request, StatusCode}; use axum::response::IntoResponse; use axum::routing::get; use axum::{Json, Router}; @@ -373,6 +374,7 @@ mod tests { use futures::FutureExt; use nv_redfish::bmc_http::{BmcCredentials, CacheSettings, HttpBmc}; use serde_json::{Value, json}; + use tower::ServiceExt; use url::Url; use super::*; @@ -580,6 +582,73 @@ mod tests { ); } + #[tokio::test] + async fn configured_bmc_event_is_streamed_and_correlated_after_retry() { + let path = "/redfish/v1/Chassis/chassis_1/Oem/Nvidia/Faults/delayed"; + let event_config = bmc_mock::EventServiceConfig { + scenarios: BTreeMap::from([( + "platform-fault".to_string(), + bmc_mock::EventScenario { + payload: json!({ + "@odata.id": "/redfish/v1/EventService/Events/1", + "@odata.type": "#Event.v1_0_0.Event", + "Id": "1", + "Name": "Test event", + "Events": [{ "@odata.id": path }] + }), + linked_resources: BTreeMap::from([( + path.to_string(), + vec![ + bmc_mock::LinkedResourceResponse { + status: 404, + body: Value::Null, + }, + bmc_mock::LinkedResourceResponse { + status: 200, + body: c12_platform_record(path), + }, + ], + )]), + }, + )]), + }; + let (router, _) = bmc_mock::test_support::router_for_machine_with_options( + &bmc_mock::test_support::host_info(bmc_mock::HardwareType::GenericAmi), + bmc_mock::MachineRouterOptions { + event_service: Some(event_config), + ..Default::default() + }, + ); + let bmc = Arc::new(test_bmc(router.clone())); + let sse_stream = open_sse_stream(Arc::clone(&bmc)).await.unwrap(); + let mut event_stream = map_event_stream(sse_stream, bmc, false, 1); + let event_task = tokio::spawn(async move { event_stream.next().await }); + + let trigger = Request::post(bmc_mock::EVENT_SERVICE_TRIGGER_PATH) + .header("content-type", "application/json") + .body(axum::body::Body::from(r#"{"scenario":"platform-fault"}"#)) + .unwrap(); + let response = router.oneshot(trigger).await.unwrap(); + assert_eq!(response.status(), StatusCode::NO_CONTENT); + + let event = tokio::time::timeout(Duration::from_secs(2), event_task) + .await + .expect("collector should receive the configured event") + .unwrap() + .expect("collector stream should return one item") + .expect("configured event should produce a log"); + let record = log_record(&event); + assert_eq!(record.body, ""); + assert_eq!( + attribute(record, "message_id"), + Some("IANA.0.1.CPLD-PSEQ-FAULT") + ); + assert_eq!( + attribute(record, "oem.nvidia.error_id"), + Some("CPLD-PSEQ-FAULT") + ); + } + #[tokio::test] async fn failed_reference_does_not_drop_sibling_record() { let missing_path = "/redfish/v1/EventService/Events/records/missing"; diff --git a/crates/machine-a-tron/src/bmc_mock_wrapper.rs b/crates/machine-a-tron/src/bmc_mock_wrapper.rs index d17323c686..7ebc2b27a7 100644 --- a/crates/machine-a-tron/src/bmc_mock_wrapper.rs +++ b/crates/machine-a-tron/src/bmc_mock_wrapper.rs @@ -23,7 +23,8 @@ use axum::Router; use bmc_mock::injection::InjectionStore; use bmc_mock::ipmi_sim::{IpmiEndpoint, IpmiSimConfig, IpmiSimHandle}; use bmc_mock::{ - BmcState, Callbacks, CombinedServer, HostnameQuerying, ListenerOrAddress, MachineInfo, + BmcState, Callbacks, CombinedServer, EventServiceConfig, HostnameQuerying, ListenerOrAddress, + MachineInfo, MachineRouterOptions, }; use carbide_ipmi::DEFAULT_IPMI_PORT; use tokio::sync::RwLock; @@ -56,14 +57,20 @@ impl BmcMockWrapper { hostname: Arc, host_id: Uuid, injection: Arc, + event_service: Option, ) -> Self { - let (bmc_mock_router, bmc_mock_state) = bmc_mock::machine_router_with_injection_store( - machine_info, - callbacks, - host_id.to_string(), - true, - injection, - ); + let (bmc_mock_router, bmc_mock_state) = + bmc_mock::machine_router_with_options_and_injection_store( + machine_info, + callbacks, + host_id.to_string(), + true, + injection, + MachineRouterOptions { + event_service, + ..Default::default() + }, + ); BmcMockWrapper { ssh_prompt_behavior: match machine_info { diff --git a/crates/machine-a-tron/src/config.rs b/crates/machine-a-tron/src/config.rs index 40b62095da..e803717c28 100644 --- a/crates/machine-a-tron/src/config.rs +++ b/crates/machine-a-tron/src/config.rs @@ -138,6 +138,10 @@ pub struct MachineConfig { #[serde(default)] pub host_firmware_versions: Option, + /// Optional deterministic Redfish EventService scenarios for each simulated host BMC. + #[serde(default)] + pub redfish_event_service: Option, + #[serde(default)] pub dpu_agent_version: Option, } @@ -218,6 +222,7 @@ impl WiwynnGb200RackConfig { dpus_in_nic_mode: self.dpus_in_nic_mode, dpu_firmware_versions: self.dpu_firmware_versions.clone(), host_firmware_versions: None, + redfish_event_service: None, dpu_agent_version: self.dpu_agent_version.clone(), } } @@ -292,6 +297,7 @@ impl LenovoGb300RackConfig { dpus_in_nic_mode: self.dpus_in_nic_mode, dpu_firmware_versions: self.dpu_firmware_versions.clone(), host_firmware_versions: None, + redfish_event_service: None, dpu_agent_version: self.dpu_agent_version.clone(), } } @@ -1042,7 +1048,17 @@ scout_run_interval = "5s" #[test] fn test_serialize_config() { - let cfg = rack_config(); + let mut cfg = rack_config(); + Arc::make_mut(cfg.machines.get_mut("config").unwrap()).redfish_event_service = + Some(bmc_mock::EventServiceConfig { + scenarios: BTreeMap::from([( + "platform-fault".to_string(), + bmc_mock::EventScenario { + payload: serde_json::json!({ "Events": [] }), + linked_resources: BTreeMap::new(), + }, + )]), + }); cfg.validate().expect("Could not validate config"); let serialized = toml::to_string(&cfg).expect("Could not serialize config"); let round_tripped = toml::from_str::(&serialized) diff --git a/crates/machine-a-tron/src/machine_state_machine.rs b/crates/machine-a-tron/src/machine_state_machine.rs index 853e438bf2..6a60d5f089 100644 --- a/crates/machine-a-tron/src/machine_state_machine.rs +++ b/crates/machine-a-tron/src/machine_state_machine.rs @@ -1266,6 +1266,7 @@ impl MachineStateMachine { Arc::new(LiveStateHostnameQuery(self.live_state.clone())), self.mat_host_id, self.bmc_injection.clone(), + self.config.redfish_event_service.clone(), ); let pw_override = match &self.machine_info { diff --git a/crates/machine-a-tron/src/power_shelf_simulator.rs b/crates/machine-a-tron/src/power_shelf_simulator.rs index 5f495803db..3ead2faa60 100644 --- a/crates/machine-a-tron/src/power_shelf_simulator.rs +++ b/crates/machine-a-tron/src/power_shelf_simulator.rs @@ -326,6 +326,7 @@ impl PowerShelfActor { Arc::new(PowerShelfHostname), self.mat_id, self.bmc_injection.clone(), + None, ); if self.host_info.hw_type != HardwareType::LiteOnPowerShelf && let Some(password) = self.app_context.app_config.host_bmc_password.as_deref() diff --git a/crates/machine-a-tron/src/switch_simulator.rs b/crates/machine-a-tron/src/switch_simulator.rs index 63ded7bcff..0571ab9184 100644 --- a/crates/machine-a-tron/src/switch_simulator.rs +++ b/crates/machine-a-tron/src/switch_simulator.rs @@ -364,6 +364,7 @@ impl SwitchActor { Arc::new(SwitchHostname), self.mat_id, self.bmc_injection.clone(), + None, ); if let Some(password) = self.app_context.app_config.host_bmc_password.as_deref() { bmc_mock