diff --git a/crates/api-core/tests/integration/connected_device.rs b/crates/api-core/tests/integration/connected_device.rs index d61713ba3b..ddc0102902 100644 --- a/crates/api-core/tests/integration/connected_device.rs +++ b/crates/api-core/tests/integration/connected_device.rs @@ -31,14 +31,14 @@ async fn init(pool: PgPool) -> (TestHarness, TestManagedHost) { let network_controller = env.network_controller(); let domain = env.test_domain().await; let underlay_segment = network_controller.create_underlay_segment(&domain).await; - let admin_segment = network_controller.create_admin_segment(&domain).await; + network_controller.create_admin_segment(&domain).await; let site_explorer = env.default_test_site_explorer(); let (mh, _) = env .managed_host_builder(&site_explorer, underlay_segment) .with_config(ManagedHostConfig::default().with_dpu_count(1)) .build() .await; - mh.first_dpu().discover_oob_iface(admin_segment).await; + mh.first_dpu().discover_oob_iface(underlay_segment).await; (env, mh) } diff --git a/crates/api-core/tests/integration/network_device.rs b/crates/api-core/tests/integration/network_device.rs index f88577e944..8bfdebee5e 100644 --- a/crates/api-core/tests/integration/network_device.rs +++ b/crates/api-core/tests/integration/network_device.rs @@ -30,7 +30,7 @@ async fn init(pool: PgPool) -> TestHarness { let network_controller = env.network_controller(); let domain = env.test_domain().await; let underlay_segment = network_controller.create_underlay_segment(&domain).await; - let admin_segment = network_controller.create_admin_segment(&domain).await; + network_controller.create_admin_segment(&domain).await; let site_explorer = env.default_test_site_explorer(); let (managed_host, _) = env .managed_host_builder(&site_explorer, underlay_segment) @@ -39,7 +39,7 @@ async fn init(pool: PgPool) -> TestHarness { .await; managed_host .first_dpu() - .discover_oob_iface(admin_segment) + .discover_oob_iface(underlay_segment) .await; env } diff --git a/crates/api-db/src/machine_interface.rs b/crates/api-db/src/machine_interface.rs index 6141c29bef..0f7f223571 100644 --- a/crates/api-db/src/machine_interface.rs +++ b/crates/api-db/src/machine_interface.rs @@ -2584,7 +2584,7 @@ pub async fn move_predicted_machine_interface_to_machine( != predicted_machine_interface.expected_network_segment_type { return Err(DatabaseError::internal(format!( - "Got DHCP for predicted host with MAC address {0} on network segment {1}, which is not of the expected type {2}", + "Got DHCP for predicted interface with MAC address {0} on network segment {1}, which is not of the expected type {2}", predicted_machine_interface.mac_address, network_segment.id, predicted_machine_interface.expected_network_segment_type, @@ -2684,6 +2684,22 @@ pub async fn move_predicted_machine_interface_to_machine( ) .await?; + if predicted_machine_interface + .machine_id + .machine_type() + .is_dpu() + { + // Site Explorer is the trusted source for a DPU's OOB MAC. Preserve that trust when DHCP + // materializes the predicted row so anonymous DiscoverMachine can authenticate the DPU on + // its first attempt without being allowed to claim an arbitrary existing machine. + associate_interface_with_dpu_machine( + &machine_interface_id, + &predicted_machine_interface.machine_id, + txn, + ) + .await?; + } + // Resolve the promoted row's boot interface id. The prediction's value // comes from the live report and outranks an existing row value: that // row may have been created from a static preallocation (an diff --git a/crates/api-integration-tests/tests/lib.rs b/crates/api-integration-tests/tests/lib.rs index 0731000c5a..df7bdf4a99 100644 --- a/crates/api-integration-tests/tests/lib.rs +++ b/crates/api-integration-tests/tests/lib.rs @@ -40,6 +40,8 @@ use sqlx::{Postgres, Row}; use tokio::time::sleep; use tokio_util::sync::CancellationToken; +const DPU_UNDERLAY_DHCP_RELAY_ADDRESS: Ipv4Addr = Ipv4Addr::new(172, 20, 1, 1); + #[ctor::ctor(unsafe)] fn setup() { api_test_helper::setup_logging() @@ -149,8 +151,7 @@ async fn test_integration() -> eyre::Result<()> { &test_env, &bmc_address_registry, &managed_segment_id, - // Relay IP in admin net - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, ) .boxed(), test_machine_a_tron_multidpu( @@ -158,8 +159,7 @@ async fn test_integration() -> eyre::Result<()> { &test_env, &bmc_address_registry, &managed_segment_id, - // Relay IP in admin net - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, ) .boxed(), test_machine_a_tron_multidpu( @@ -167,8 +167,7 @@ async fn test_integration() -> eyre::Result<()> { &test_env, &bmc_address_registry, &managed_segment_id, - // Relay IP in admin net - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, ) .boxed(), test_machine_a_tron_multidpu( @@ -176,8 +175,7 @@ async fn test_integration() -> eyre::Result<()> { &test_env, &bmc_address_registry, &managed_segment_id, - // Relay IP in admin net - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, ) .boxed(), test_machine_a_tron_multidpu( @@ -185,8 +183,7 @@ async fn test_integration() -> eyre::Result<()> { &test_env, &bmc_address_registry, &managed_segment_id, - // Relay IP in admin net - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, ) .boxed(), test_machine_a_tron_multidpu( @@ -194,8 +191,7 @@ async fn test_integration() -> eyre::Result<()> { &test_env, &bmc_address_registry, &managed_segment_id, - // Relay IP in admin net - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, ) .boxed(), test_machine_a_tron_zerodpu( @@ -249,8 +245,7 @@ async fn test_integration() -> eyre::Result<()> { tenant_org_id, &v4_vpc_prefix_id, &v6_vpc_prefix_id, - // Relay IP in admin net - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, ) .boxed(), test_machine_a_tron_dual_stack_l2( @@ -258,8 +253,7 @@ async fn test_integration() -> eyre::Result<()> { &test_env, &bmc_address_registry, &dual_stack_l2_segment_id, - // Relay IP in admin net - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, ) .boxed(), ]); @@ -473,7 +467,7 @@ async fn test_metrics_integration() -> eyre::Result<()> { false, &test_env, &bmc_address_registry, - Ipv4Addr::new(172, 20, 0, 1), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, |machine_handle| { let db_pool = db_pool.clone(); let carbide_api_addrs = carbide_api_addrs.to_vec(); @@ -709,7 +703,7 @@ async fn test_machine_a_tron_zerodpu( false, test_env, bmc_mock_registry, - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, |machine_handle| { let carbide_api_addrs = &test_env.carbide_api_addrs; let flat_vpc_id = flat_vpc_id.to_string(); @@ -771,7 +765,7 @@ async fn test_machine_a_tron_nic_mode( true, test_env, bmc_mock_registry, - Ipv4Addr::new(172, 20, 0, 2), + DPU_UNDERLAY_DHCP_RELAY_ADDRESS, |machine_handle| { let carbide_api_addrs = &test_env.carbide_api_addrs; let flat_vpc_id = flat_vpc_id.to_string(); @@ -1382,9 +1376,11 @@ where dpu_per_host_count, dpu_reboot_delay: 1, host_reboot_delay: 1, + // MAT currently uses this legacy-named field for DPU OS DHCP. Route that + // request through Underlay so it matches the predicted DPU interface. admin_dhcp_relay_address, - // Keep this distinct from the Admin relay so NIC-mode tests - // fail if machine-a-tron sends host DHCP through Admin. + // Keep this distinct from the DPU Underlay relay so NIC-mode tests fail if + // machine-a-tron sends direct host DHCP through the DPU network. host_inband_dhcp_relay_address: Some(Ipv4Addr::new(10, 10, 11, 2)), oob_dhcp_relay_address: Ipv4Addr::new(172, 20, 1, 1), vpc_count: 0, diff --git a/crates/api-integration-tests/tests/rack.rs b/crates/api-integration-tests/tests/rack.rs index 71592953f4..9d93e45b3b 100644 --- a/crates/api-integration-tests/tests/rack.rs +++ b/crates/api-integration-tests/tests/rack.rs @@ -81,7 +81,8 @@ async fn test_machine_a_tron_racks_integration() -> eyre::Result<()> { run_machine_a_tron_racks_test( &test_env, &bmc_address_registry, - Ipv4Addr::new(172, 20, 0, 2), + // MAT currently uses admin_dhcp_relay_address for DPU OS DHCP. + Ipv4Addr::new(172, 20, 1, 1), ) .await?; diff --git a/crates/api-model/src/predicted_machine_interface.rs b/crates/api-model/src/predicted_machine_interface.rs index c20ecf5d48..66fed709fd 100644 --- a/crates/api-model/src/predicted_machine_interface.rs +++ b/crates/api-model/src/predicted_machine_interface.rs @@ -32,11 +32,10 @@ pub struct PredictedMachineInterface { /// MAC, handed to the `machine_interfaces` row at DHCP promotion so /// the host's boot target is a full pair from its first owned interface. pub boot_interface_id: Option, - /// The declared `ExpectedInterface.primary` intent, stored so promotion - /// into `machine_interfaces` lands the operator's chosen boot interface as - /// `primary_interface`. `false` when nothing is declared -- promotion then - /// leaves the row non-primary and the boot interface falls to the - /// `pick_boot_interface` automation. + /// Whether promotion should make this the machine's primary interface. + /// + /// For hosts this carries the declared `ExpectedInterface.primary` intent. For DPUs the + /// trusted OOB prediction is always primary because it is the DPU OS data interface. pub primary_interface: bool, } diff --git a/crates/api-web/src/tests/env.rs b/crates/api-web/src/tests/env.rs index 6216795d70..82e62606ad 100644 --- a/crates/api-web/src/tests/env.rs +++ b/crates/api-web/src/tests/env.rs @@ -85,7 +85,7 @@ impl TestEnv { host.host.discover_primary_iface(self.admin_segment).await; for dpu in &host.dpus { - dpu.discover_oob_iface(self.admin_segment).await; + dpu.discover_oob_iface(self.underlay_segment).await; } host.report_dpu_network_status().await; host.insert_empty_host_health_report("test-harness-health") diff --git a/crates/site-explorer/src/machine_creator.rs b/crates/site-explorer/src/machine_creator.rs index e3c24ddb31..25df0ab137 100644 --- a/crates/site-explorer/src/machine_creator.rs +++ b/crates/site-explorer/src/machine_creator.rs @@ -1025,9 +1025,10 @@ impl MachineCreator { .await } - // configure_dpu_interface checks the machine_interfaces table to see if the DPU's machine interface has its machine id set. - // If the machine ID is already configured appropriately for the DPU's machine interface, configure_dpu_interface will return false - // If the DPU's machine interface was missing the machine ID in the table, configure_dpu_interface will set the machine ID and return true. + // Ensure the DPU's OOB interface is owned as soon as it appears. If DHCP has already created + // the interface, associate it directly. Otherwise record a trusted prediction that DHCP can + // promote atomically. DiscoverMachine requires this ownership before returning credentials, so + // deferring the association to a later Site Explorer sweep would strand the DPU in discovery. async fn configure_dpu_interface( &self, txn: &mut PgConnection, @@ -1073,6 +1074,39 @@ impl MachineCreator { .await?; return Ok(true); } + + if mi.is_empty() { + if let Some(prediction) = + db::predicted_machine_interface::find_by_mac_address(&mut *txn, oob_net0_mac) + .await? + { + if prediction.machine_id != *dpu_machine_id { + return Err(SiteExplorerError::AlreadyFoundError { + kind: "PredictedMachineInterface", + id: oob_net0_mac.to_string(), + }); + } + return Ok(false); + } + + db::predicted_machine_interface::create( + NewPredictedMachineInterface { + machine_id: dpu_machine_id, + mac_address: oob_net0_mac, + expected_network_segment_type: NetworkSegmentType::Underlay, + boot_interface_id: None, + primary_interface: true, + }, + txn, + ) + .await?; + tracing::info!( + machine_id = %dpu_machine_id, + mac_address = %oob_net0_mac, + "Created predicted DPU OOB interface" + ); + return Ok(true); + } } Ok(false) diff --git a/crates/site-explorer/tests/integration/machine_creator.rs b/crates/site-explorer/tests/integration/machine_creator.rs index 865719913f..70a4f56fd1 100644 --- a/crates/site-explorer/tests/integration/machine_creator.rs +++ b/crates/site-explorer/tests/integration/machine_creator.rs @@ -32,7 +32,6 @@ use carbide_utils::arch::CpuArchitecture; use carbide_uuid::machine::MachineId; use carbide_uuid::rack::{RackId, RackProfileId}; use db::ObjectFilter; -use itertools::Itertools; use librms::protos::rack_manager as rms; use mac_address::MacAddress; use model::expected_machine::{ExpectedMachine, ExpectedMachineData}; @@ -909,7 +908,7 @@ async fn test_mi_attach_dpu_if_mi_exists_during_machine_creation( } #[sqlx_test] -async fn test_mi_attach_dpu_if_mi_created_after_machine_creation( +async fn test_dpu_interface_predictions_apply_when_dhcp_follows_machine_creation( pool: PgPool, ) -> Result<(), Box> { let env = Env::new(pool).await; @@ -919,8 +918,7 @@ async fn test_mi_attach_dpu_if_mi_created_after_machine_creation( let mut fixture = explored_host_fixture(&env, &mock_host).await; let dpu_machine_id = fixture.dpu_machine_ids[&0]; - // No way to find a machine_interface using machine id as machine id is not yet associated with - // interface (right now no machine interface is created yet). + // The DPU machine exists before its first DHCP request, so there is no real interface yet. let mut txn = env.pool.begin().await?; let mi = db::machine_interface::find_by_machine_ids(&mut txn, &[dpu_machine_id]).await?; assert!(mi.is_empty()); @@ -937,8 +935,8 @@ async fn test_mi_attach_dpu_if_mi_created_after_machine_creation( .await? ); - // At this point, create_managed_host must have created machine but can not associate it with to - // any interface as interface does not exist. + // Site Explorer cannot associate an interface that does not exist yet, but it must leave a + // trusted MAC claim for DHCP to promote without waiting for another exploration sweep. let mut txn = env.pool.begin().await?; let machine = db::machine::find_one( txn.as_mut(), @@ -951,42 +949,61 @@ async fn test_mi_attach_dpu_if_mi_created_after_machine_creation( .await?; assert!(machine.is_some()); - // No way to find a machine_interface using machine id as machine id is not yet associated with - // interface (right now no machine interface is created yet). let mi = db::machine_interface::find_by_machine_ids(&mut txn, &[dpu_machine_id]).await?; assert!(mi.is_empty()); + let prediction = + db::predicted_machine_interface::find_by_mac_address(&mut txn, mock_dpu.oob_mac_address) + .await? + .expect("DPU OOB prediction should exist"); + assert_eq!(prediction.machine_id, dpu_machine_id); + assert_eq!( + prediction.expected_network_segment_type, + model::network_segment::NetworkSegmentType::Underlay + ); + assert!(prediction.primary_interface); txn.rollback().await?; - // Create MI now. + // DHCP must consume the trusted prediction and create an already-owned interface. dhcp_discover_dpu_oob_iface(env.api(), env.underlay_segment, mock_dpu.oob_mac_address).await; - // Machine is already created, create_managed_host should return false. + let mut txn = env.pool.begin().await?; + let interfaces = + db::machine_interface::find_by_mac_address(txn.as_mut(), mock_dpu.oob_mac_address).await?; + let [interface] = interfaces.as_slice() else { + panic!("expected one promoted DPU OOB interface, got {interfaces:#?}"); + }; + assert_eq!(interface.machine_id, Some(dpu_machine_id)); + assert_eq!(interface.attached_dpu_machine_id, Some(dpu_machine_id)); + assert!(interface.primary_interface); assert!( - !creator - .create_managed_host( - &fixture.host, - &mut EndpointExplorationReport::default(), - Some(&expected_machine(&mock_host)), - &env.pool, - ) + db::predicted_machine_interface::find_by_mac_address(&mut txn, mock_dpu.oob_mac_address,) .await? + .is_none() ); - // At this point, create_managed_host must have updated the associated machine id in - // machine_interfaces table. - let mut txn = env.pool.begin().await?; - let mi = db::machine_interface::find_by_machine_ids(&mut txn, &[dpu_machine_id]).await?; - assert!(!mi.is_empty()); - let value = mi.values().collect_vec()[0].clone()[0].clone(); - assert_eq!(value.attached_dpu_machine_id.unwrap(), dpu_machine_id); - assert_eq!(value.machine_id.unwrap(), dpu_machine_id); - txn.rollback().await?; + let topologies = db::machine_topology::find_by_machine_ids(&mut txn, &[dpu_machine_id]).await?; + let topology = &topologies[&dpu_machine_id][0]; + let discovery_info = DiscoveryInfo::try_from(topology.topology().discovery_data.info.clone())?; + let interface_id = interface.id; + txn.commit().await?; + + let response = env + .api() + .discover_machine(Request::new(MachineDiscoveryInfo { + machine_interface_id: Some(interface_id), + discovery_data: Some(DiscoveryData::Info(discovery_info)), + create_machine: true, + ..Default::default() + })) + .await? + .into_inner(); + assert_eq!(response.machine_id, Some(dpu_machine_id)); Ok(()) } #[sqlx_test] -async fn test_all_dpu_interfaces_attach_if_created_after_multi_dpu_machine_creation( +async fn test_dpu_interface_predictions_apply_when_dhcp_follows_multi_dpu_machine_creation( pool: PgPool, ) -> Result<(), Box> { const NUM_DPUS: usize = 2; @@ -1011,17 +1028,6 @@ async fn test_all_dpu_interfaces_attach_if_created_after_multi_dpu_machine_creat dhcp_discover_dpu_oob_iface(env.api(), env.underlay_segment, dpu.oob_mac_address).await; } - assert!( - !creator - .create_managed_host( - &fixture.host, - &mut EndpointExplorationReport::default(), - Some(&expected_machine(&mock_host)), - &env.pool, - ) - .await? - ); - let mut txn = env.pool.begin().await?; for (dpu_index, dpu) in mock_host.dpus.iter().enumerate() { let dpu_index = dpu_index.try_into().expect("DPU index should fit into u8");