diff --git a/crates/api-integration-tests/tests/lib.rs b/crates/api-integration-tests/tests/lib.rs index 267dfa947a..0a252050f8 100644 --- a/crates/api-integration-tests/tests/lib.rs +++ b/crates/api-integration-tests/tests/lib.rs @@ -1394,6 +1394,7 @@ where run_interval_working: Duration::from_millis(100), network_status_run_interval: Duration::from_secs(1), scout_run_interval: Duration::from_secs(1), + discovery_retry_interval: Duration::from_millis(100), network_virtualization_type: None, dpus_in_nic_mode, dpu_firmware_versions: None, diff --git a/crates/api-integration-tests/tests/rack.rs b/crates/api-integration-tests/tests/rack.rs index 7b591886a3..2eafe9477d 100644 --- a/crates/api-integration-tests/tests/rack.rs +++ b/crates/api-integration-tests/tests/rack.rs @@ -120,6 +120,7 @@ async fn run_machine_a_tron_racks_test( dpu_reboot_delay: 1, host_reboot_delay: 1, scout_run_interval: Duration::from_secs(1), + discovery_retry_interval: Duration::from_millis(100), oob_dhcp_relay_address: Ipv4Addr::new(172, 20, 1, 1), admin_dhcp_relay_address, host_inband_dhcp_relay_address: Some(Ipv4Addr::new(10, 10, 11, 2)), @@ -144,6 +145,7 @@ async fn run_machine_a_tron_racks_test( dpu_reboot_delay: 1, host_reboot_delay: 1, scout_run_interval: Duration::from_secs(1), + discovery_retry_interval: Duration::from_millis(100), oob_dhcp_relay_address: Ipv4Addr::new(172, 20, 1, 1), admin_dhcp_relay_address, host_inband_dhcp_relay_address: Some(Ipv4Addr::new(10, 10, 11, 2)), diff --git a/crates/machine-a-tron/src/config.rs b/crates/machine-a-tron/src/config.rs index fb15c411ef..84f6775e4c 100644 --- a/crates/machine-a-tron/src/config.rs +++ b/crates/machine-a-tron/src/config.rs @@ -93,6 +93,14 @@ pub struct MachineConfig { serialize_with = "as_std_duration" )] pub scout_run_interval: Duration, + /// Delay before retrying a failed DiscoverMachine request. The default matches the + /// production DPU agent; local development can override it independently of the MAT work loop. + #[serde( + default = "default_discovery_retry_interval", + deserialize_with = "deserialize_duration", + serialize_with = "as_std_duration" + )] + pub discovery_retry_interval: Duration, pub oob_dhcp_relay_address: Ipv4Addr, pub admin_dhcp_relay_address: Ipv4Addr, /// Relay address used when a host DHCPs directly through a plain NIC rather than a managed DPU. @@ -163,6 +171,12 @@ pub struct WiwynnGb200RackConfig { serialize_with = "as_std_duration" )] pub scout_run_interval: Duration, + #[serde( + default = "default_discovery_retry_interval", + deserialize_with = "deserialize_duration", + serialize_with = "as_std_duration" + )] + pub discovery_retry_interval: Duration, pub oob_dhcp_relay_address: Ipv4Addr, pub admin_dhcp_relay_address: Ipv4Addr, #[serde(default)] @@ -214,6 +228,7 @@ impl WiwynnGb200RackConfig { dpu_reboot_delay: self.dpu_reboot_delay, host_reboot_delay: self.host_reboot_delay, scout_run_interval: self.scout_run_interval, + discovery_retry_interval: self.discovery_retry_interval, oob_dhcp_relay_address: self.oob_dhcp_relay_address, admin_dhcp_relay_address: self.admin_dhcp_relay_address, host_inband_dhcp_relay_address: self.host_inband_dhcp_relay_address, @@ -239,6 +254,12 @@ pub struct LenovoGb300RackConfig { serialize_with = "as_std_duration" )] pub scout_run_interval: Duration, + #[serde( + default = "default_discovery_retry_interval", + deserialize_with = "deserialize_duration", + serialize_with = "as_std_duration" + )] + pub discovery_retry_interval: Duration, pub oob_dhcp_relay_address: Ipv4Addr, pub admin_dhcp_relay_address: Ipv4Addr, #[serde(default)] @@ -290,6 +311,7 @@ impl LenovoGb300RackConfig { dpu_reboot_delay: self.dpu_reboot_delay, host_reboot_delay: self.host_reboot_delay, scout_run_interval: self.scout_run_interval, + discovery_retry_interval: self.discovery_retry_interval, oob_dhcp_relay_address: self.oob_dhcp_relay_address, admin_dhcp_relay_address: self.admin_dhcp_relay_address, host_inband_dhcp_relay_address: self.host_inband_dhcp_relay_address, @@ -859,6 +881,10 @@ fn default_run_interval_working() -> Duration { Duration::from_secs(5) } +fn default_discovery_retry_interval() -> Duration { + Duration::from_secs(60) +} + fn default_run_interval_idle() -> Duration { Duration::from_secs(30) } @@ -1006,6 +1032,7 @@ scout_run_interval = "5s" dpu_reboot_delay: machine.dpu_reboot_delay, host_reboot_delay: machine.host_reboot_delay, scout_run_interval: machine.scout_run_interval, + discovery_retry_interval: machine.discovery_retry_interval, oob_dhcp_relay_address: machine.oob_dhcp_relay_address, admin_dhcp_relay_address: machine.admin_dhcp_relay_address, host_inband_dhcp_relay_address: machine.host_inband_dhcp_relay_address, @@ -1024,6 +1051,7 @@ scout_run_interval = "5s" dpu_reboot_delay: machine.dpu_reboot_delay, host_reboot_delay: machine.host_reboot_delay, scout_run_interval: machine.scout_run_interval, + discovery_retry_interval: machine.discovery_retry_interval, oob_dhcp_relay_address: machine.oob_dhcp_relay_address, admin_dhcp_relay_address: machine.admin_dhcp_relay_address, host_inband_dhcp_relay_address: machine.host_inband_dhcp_relay_address, @@ -1074,6 +1102,10 @@ scout_run_interval = "5s" #[test] fn test_serialize_config() { let cfg = rack_config(); + assert_eq!( + cfg.machines["config"].discovery_retry_interval, + Duration::from_secs(60) + ); cfg.validate().expect("Could not validate config"); let serialized = toml::to_string(&cfg).expect("Could not serialize config"); let round_tripped = toml::from_str::(&serialized) diff --git a/crates/machine-a-tron/src/machine_state_machine.rs b/crates/machine-a-tron/src/machine_state_machine.rs index 9c326e33b4..f1d9fdc0b9 100644 --- a/crates/machine-a-tron/src/machine_state_machine.rs +++ b/crates/machine-a-tron/src/machine_state_machine.rs @@ -597,7 +597,7 @@ impl MachineStateMachine { self.actions.pop_front(); self.fsm_event(Event::InitialDiscoveryCompleted) } - Err(_) => return Some(self.config.run_interval_working), + Err(_) => return Some(self.config.discovery_retry_interval), } } FsmAction::AgentControlRequest(os_image) => { diff --git a/dev/deployment/tilt/values.yaml b/dev/deployment/tilt/values.yaml index a996af4a94..8e984c0fe2 100644 --- a/dev/deployment/tilt/values.yaml +++ b/dev/deployment/tilt/values.yaml @@ -177,6 +177,8 @@ nico-machine-a-tron: enabled: true machineATron: nicoApiUrl: https://nico-api.nico-system.svc.cluster.local:1079 + machineDefaults: + discoveryRetryInterval: "5s" persistence: enabled: true certificate: diff --git a/helm/charts/nico-machine-a-tron/templates/configmap.yaml b/helm/charts/nico-machine-a-tron/templates/configmap.yaml index f77e924144..8cac3700fe 100644 --- a/helm/charts/nico-machine-a-tron/templates/configmap.yaml +++ b/helm/charts/nico-machine-a-tron/templates/configmap.yaml @@ -173,6 +173,7 @@ data: ids = {{ $section.ids | toJson }} dpu_reboot_delay = {{ $section.dpu_reboot_delay | default $defaults.dpuRebootDelay | int }} host_reboot_delay = {{ $section.host_reboot_delay | default $defaults.hostRebootDelay | int }} + discovery_retry_interval = {{ $section.discovery_retry_interval | default $defaults.discoveryRetryInterval | default "60s" | quote }} oob_dhcp_relay_address = {{ $section.oob_dhcp_relay_address | quote }} admin_dhcp_relay_address = {{ $section.admin_dhcp_relay_address | quote }} {{- end }} @@ -191,6 +192,7 @@ data: dpu_reboot_delay = {{ $section.dpuRebootDelay | default $defaults.dpuRebootDelay | default 1 }} host_reboot_delay = {{ $section.hostRebootDelay | default $defaults.hostRebootDelay | default 1 }} scout_run_interval = {{ $section.scoutRunInterval | default $defaults.scoutRunInterval | default "60s" | quote }} + discovery_retry_interval = {{ $section.discoveryRetryInterval | default $defaults.discoveryRetryInterval | default "60s" | quote }} run_interval_working = {{ $section.runIntervalWorking | default $defaults.runIntervalWorking | default "1s" | quote }} run_interval_idle = {{ $section.runIntervalIdle | default $defaults.runIntervalIdle | default "10s" | quote }} network_status_run_interval = {{ $section.networkStatusRunInterval | default $defaults.networkStatusRunInterval | default "20s" | quote }} diff --git a/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml index 79fb308eba..7bb61bd9a4 100644 --- a/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml +++ b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml @@ -18,6 +18,9 @@ tests: - matchRegex: path: data["mat.toml"] pattern: 'log_format = "logfmt"' + - matchRegex: + path: data["mat.toml"] + pattern: 'discovery_retry_interval = "60s"' - it: should allow compact log output set: @@ -135,6 +138,7 @@ tests: machineDefaults: runIntervalIdle: "30s" scoutRunInterval: "120s" + discoveryRetryInterval: "7s" pods: mat-0: machines: @@ -148,6 +152,9 @@ tests: - matchRegex: path: data["mat.toml"] pattern: 'scout_run_interval = "120s"' + - matchRegex: + path: data["mat.toml"] + pattern: 'discovery_retry_interval = "7s"' - it: should not include enable_ipmi_simulation by default asserts: diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index f70d96c82c..57f387b597 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -182,6 +182,8 @@ machineDefaults: dpuRebootDelay: 1 hostRebootDelay: 1 scoutRunInterval: "60s" + ## Delay after a failed DiscoverMachine call; the default matches the production DPU agent. + discoveryRetryInterval: "60s" runIntervalWorking: "1s" runIntervalIdle: "10s" networkStatusRunInterval: "20s"