fix(machine-controller): gate Assigned/Ready on provisioning-completion on evidence for custom iPXE boots

crates/api-core/src/cfg/file.rs

@@ -4708,6 +4708,9 @@ mod tests {
             max_bios_config_retries: 3,
             polling_bios_setup_stuck_threshold: Duration::minutes(15),
             boot_interface_observation_interval: Duration::hours(2),
+            provisioning_quiet_window: Duration::minutes(15),
+            max_provisioning_serves: 4,
+            provisioning_deadline: Duration::minutes(60),
         };
 
         let config_str = serde_json::to_string(&input).unwrap();
@@ -4756,6 +4759,9 @@ mod tests {
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),
                 boot_interface_observation_interval: Duration::hours(2),
+                provisioning_quiet_window: Duration::minutes(15),
+                max_provisioning_serves: 4,
+                provisioning_deadline: Duration::minutes(60),
             }
         );
     }
@@ -4780,6 +4786,9 @@ mod tests {
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),
                 boot_interface_observation_interval: Duration::minutes(10),
+                provisioning_quiet_window: Duration::minutes(15),
+                max_provisioning_serves: 4,
+                provisioning_deadline: Duration::minutes(60),
             }
         );
     }
@@ -5323,6 +5332,9 @@ mod tests {
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),
                 boot_interface_observation_interval: Duration::hours(2),
+                provisioning_quiet_window: Duration::minutes(15),
+                max_provisioning_serves: 4,
+                provisioning_deadline: Duration::minutes(60),
             }
         );
         assert_eq!(
@@ -5711,6 +5723,9 @@ mod tests {
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),
                 boot_interface_observation_interval: Duration::minutes(10),
+                provisioning_quiet_window: Duration::minutes(15),
+                max_provisioning_serves: 4,
+                provisioning_deadline: Duration::minutes(60),
             }
         );
         assert_eq!(
@@ -6100,6 +6115,9 @@ mod tests {
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),
                 boot_interface_observation_interval: Duration::hours(2),
+                provisioning_quiet_window: Duration::minutes(15),
+                max_provisioning_serves: 4,
+                provisioning_deadline: Duration::minutes(60),
             }
         );
         assert_eq!(

crates/api-core/src/handlers/client_resolution.rs

@@ -252,10 +252,16 @@ pub(super) async fn resolve_cloud_init_instructions(
             && let Some(managed_host_state) =
                 db::machine::lookup_managed_host_state(&mut *conn, instance.machine_id).await?
         {
+            // The provisioning wait is included because it is exactly when the
+            // tenant's operating system is installing and running cloud-init:
+            // that cloud-init needs the tenant's user-data, and the phone-home
+            // block NICo injects into it is what ends the wait.
             let is_assigned_and_ready = matches!(
                 managed_host_state,
                 ManagedHostState::Assigned {
-                    instance_state: InstanceState::Ready | InstanceState::WaitingForRebootToReady,
+                    instance_state: InstanceState::Ready
+                        | InstanceState::WaitingForRebootToReady
+                        | InstanceState::WaitingForProvisioningComplete { .. },
                 }
             );
 

crates/api-core/src/handlers/instance.rs

@@ -47,8 +47,8 @@ use model::instance::config::tenant_config::TenantConfig;
 use model::instance::snapshot::InstanceSnapshot;
 use model::machine::machine_search_config::MachineSearchConfig;
 use model::machine::{
-    HostHealthConfig, InstanceState, LoadSnapshotOptions, ManagedHostState,
-    ManagedHostStateSnapshot,
+    FailureCause, FailureDetails, HostHealthConfig, InstanceState, LoadSnapshotOptions,
+    ManagedHostState, ManagedHostStateSnapshot,
 };
 use model::metadata::Metadata;
 use model::network_segment::{NetworkSegmentSearchConfig, NetworkSegmentType};
@@ -975,10 +975,23 @@ pub(crate) async fn invoke_power(
     // For custom PXE or always-PXE instances in Ready state, we use the state machine to
     // verify boot order before rebooting. For regular reboots, we clear the use_custom_pxe_on_boot
     // flag so the iPXE handler returns "exit" (boot from disk).
+    //
+    // `WaitingForProvisioningComplete` and a failed provisioning boot are
+    // included so a tenant can restart provisioning without waiting for the
+    // in-flight attempt to time out: both hand back to the Assigned/Ready
+    // handler when they see the flag, which then runs the full flow again.
     let use_state_machine_for_reboot = matches!(
         snapshot.managed_state,
         ManagedHostState::Assigned {
-            instance_state: InstanceState::Ready,
+            instance_state: InstanceState::Ready
+                | InstanceState::WaitingForProvisioningComplete { .. }
+                | InstanceState::Failed {
+                    details: FailureDetails {
+                        cause: FailureCause::ProvisioningFailed { .. },
+                        ..
+                    },
+                    ..
+                },
         }
     ) && (run_provisioning_instructions_on_every_boot
         || request.boot_with_custom_ipxe);