diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index 43eeb9df60..bc2695ee4e 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -4708,6 +4708,9 @@ mod tests { max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), boot_interface_observation_interval: Duration::hours(2), + provisioning_quiet_window: Duration::minutes(15), + max_provisioning_serves: 4, + provisioning_deadline: Duration::minutes(60), }; let config_str = serde_json::to_string(&input).unwrap(); @@ -4756,6 +4759,9 @@ mod tests { max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), boot_interface_observation_interval: Duration::hours(2), + provisioning_quiet_window: Duration::minutes(15), + max_provisioning_serves: 4, + provisioning_deadline: Duration::minutes(60), } ); } @@ -4780,6 +4786,9 @@ mod tests { max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), boot_interface_observation_interval: Duration::minutes(10), + provisioning_quiet_window: Duration::minutes(15), + max_provisioning_serves: 4, + provisioning_deadline: Duration::minutes(60), } ); } @@ -5323,6 +5332,9 @@ mod tests { max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), boot_interface_observation_interval: Duration::hours(2), + provisioning_quiet_window: Duration::minutes(15), + max_provisioning_serves: 4, + provisioning_deadline: Duration::minutes(60), } ); assert_eq!( @@ -5711,6 +5723,9 @@ mod tests { max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), boot_interface_observation_interval: Duration::minutes(10), + provisioning_quiet_window: Duration::minutes(15), + max_provisioning_serves: 4, + provisioning_deadline: Duration::minutes(60), } ); assert_eq!( @@ -6100,6 +6115,9 @@ mod tests { max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), boot_interface_observation_interval: Duration::hours(2), + provisioning_quiet_window: Duration::minutes(15), + max_provisioning_serves: 4, + provisioning_deadline: Duration::minutes(60), } ); assert_eq!( diff --git a/crates/api-core/src/handlers/client_resolution.rs b/crates/api-core/src/handlers/client_resolution.rs index b88d4136a1..d2bd614282 100644 --- a/crates/api-core/src/handlers/client_resolution.rs +++ b/crates/api-core/src/handlers/client_resolution.rs @@ -252,10 +252,16 @@ pub(super) async fn resolve_cloud_init_instructions( && let Some(managed_host_state) = db::machine::lookup_managed_host_state(&mut *conn, instance.machine_id).await? { + // The provisioning wait is included because it is exactly when the + // tenant's operating system is installing and running cloud-init: + // that cloud-init needs the tenant's user-data, and the phone-home + // block NICo injects into it is what ends the wait. let is_assigned_and_ready = matches!( managed_host_state, ManagedHostState::Assigned { - instance_state: InstanceState::Ready | InstanceState::WaitingForRebootToReady, + instance_state: InstanceState::Ready + | InstanceState::WaitingForRebootToReady + | InstanceState::WaitingForProvisioningComplete { .. }, } ); diff --git a/crates/api-core/src/handlers/instance.rs b/crates/api-core/src/handlers/instance.rs index de1cec8b97..83072cab8a 100644 --- a/crates/api-core/src/handlers/instance.rs +++ b/crates/api-core/src/handlers/instance.rs @@ -47,8 +47,8 @@ use model::instance::config::tenant_config::TenantConfig; use model::instance::snapshot::InstanceSnapshot; use model::machine::machine_search_config::MachineSearchConfig; use model::machine::{ - HostHealthConfig, InstanceState, LoadSnapshotOptions, ManagedHostState, - ManagedHostStateSnapshot, + FailureCause, FailureDetails, HostHealthConfig, InstanceState, LoadSnapshotOptions, + ManagedHostState, ManagedHostStateSnapshot, }; use model::metadata::Metadata; use model::network_segment::{NetworkSegmentSearchConfig, NetworkSegmentType}; @@ -975,10 +975,23 @@ pub(crate) async fn invoke_power( // For custom PXE or always-PXE instances in Ready state, we use the state machine to // verify boot order before rebooting. For regular reboots, we clear the use_custom_pxe_on_boot // flag so the iPXE handler returns "exit" (boot from disk). + // + // `WaitingForProvisioningComplete` and a failed provisioning boot are + // included so a tenant can restart provisioning without waiting for the + // in-flight attempt to time out: both hand back to the Assigned/Ready + // handler when they see the flag, which then runs the full flow again. let use_state_machine_for_reboot = matches!( snapshot.managed_state, ManagedHostState::Assigned { - instance_state: InstanceState::Ready, + instance_state: InstanceState::Ready + | InstanceState::WaitingForProvisioningComplete { .. } + | InstanceState::Failed { + details: FailureDetails { + cause: FailureCause::ProvisioningFailed { .. }, + .. + }, + .. + }, } ) && (run_provisioning_instructions_on_every_boot || request.boot_with_custom_ipxe); diff --git a/crates/api-core/src/ipxe.rs b/crates/api-core/src/ipxe.rs index 2b8062d429..dceaf1841b 100644 --- a/crates/api-core/src/ipxe.rs +++ b/crates/api-core/src/ipxe.rs @@ -24,6 +24,7 @@ use carbide_ipxe_renderer::{ use carbide_uuid::machine::{MachineId, MachineInterfaceId, MachineType}; use db::{self}; use mac_address::MacAddress; +use model::instance::snapshot::InstanceSnapshot; use model::machine::machine_search_config::MachineSearchConfig; use model::machine::{ DpuInitState, FailureCause, FailureDetails, HostReprovisionState, InstanceState, @@ -73,6 +74,52 @@ impl TryFrom for PxeInstructionRequest { } } +/// The parts of a PXE request that [`PxeInstructions::render_provisioning_script`] +/// needs beyond the instance itself. +struct RenderProvisioningScript<'a> { + machine_id: MachineId, + interface_id: MachineInterfaceId, + state: &'a ManagedHostState, + console: &'a str, + qcow_imager_url: &'a str, +} + +/// iPXE instructions for a state NICo cannot boot from. +fn error_instructions( + machine_id: MachineId, + interface_id: MachineInterfaceId, + state: &ManagedHostState, +) -> String { + format!( + r#" +echo Machine ID: {machine_id} +echo Interface ID: {interface_id} +echo Current state: {state} +echo Could not continue boot due to invalid state || +sleep 5 || +exit || +"# + ) +} + +/// iPXE instructions that hand control back to the local disk. +fn exit_instructions( + machine_id: MachineId, + interface_id: MachineInterfaceId, + state: &ManagedHostState, +) -> String { + format!( + r#" +echo Machine ID: {machine_id} +echo Interface ID: {interface_id} +echo Current state: {state} +echo This state assumes an OS is provisioned and will exit into the OS in 5 seconds. To re-run iPXE instructions and OS installation, trigger a reboot request with flag rebootWithCustomIpxe/boot_with_custom_ipxe set. || +sleep 5 || +exit || +"# + ) +} + /// Converts an operating_systems row (type ipxe_os_definition) to IpxeScript for the renderer. fn operating_system_row_to_ipxe_script( row: &db::operating_system::OperatingSystem, @@ -272,42 +319,111 @@ impl PxeInstructions { .map_err(|e| CarbideError::internal(format!("failed to render iPXE script: {}", e))) } + async fn require_instance( + txn: &mut PgConnection, + machine_id: &MachineId, + ) -> Result { + db::instance::find_by_machine_id(txn, machine_id) + .await? + .ok_or(CarbideError::NotFoundError { + kind: "machine", + id: machine_id.to_string(), + }) + } + + /// Renders the script that installs the tenant's operating system. + /// + /// Shared by every state that serves a provisioning boot, so that the + /// tenant sees the same script whether the boot is being tracked by + /// [`InstanceState::WaitingForProvisioningComplete`] or by the older + /// consume-on-serve path in `Assigned{Ready}`. + async fn render_provisioning_script( + txn: &mut PgConnection, + request: RenderProvisioningScript<'_>, + instance: InstanceSnapshot, + ) -> Result { + let RenderProvisioningScript { + machine_id, + interface_id, + state, + console, + qcow_imager_url, + } = request; + + Ok(match instance.config.os.variant { + model::os::OperatingSystemVariant::Ipxe(ipxe) => { + let mut tenant_ipxe = ipxe.ipxe_script; + let vendor_serial_console = format!(" console={console}"); + if !tenant_ipxe.contains(&vendor_serial_console) { + let idx = tenant_ipxe.find(" console="); + if let Some(x) = idx { + // insert correct serial console into custom ipxe before any other console=tty* specified + tenant_ipxe.insert_str(x, &vendor_serial_console); + } else { + // this is a strange ipxe script with no console=tty defined, leave it as is + } + } + tenant_ipxe + } + model::os::OperatingSystemVariant::OperatingSystemId(os_id) => { + let row = db::operating_system::get(txn, os_id).await?; + if row.type_ == model::operating_system_definition::OS_TYPE_TEMPLATED_IPXE { + let ipxeos = operating_system_row_to_ipxe_script(&row)?; + Self::render_ipxe_script(&ipxeos, "${base-url}", console)? + } else { + row.ipxe_script.unwrap_or_default() + } + } + model::os::OperatingSystemVariant::OsImage(id) => { + let os_image = db::os_image::get(txn, id).await?; + if os_image.attributes.create_volume { + // this is a block storage os image + // boot will be via the block storage snapshot volume + // no ipxe script for os imaging + exit_instructions(machine_id, interface_id, state) + } else { + let mut qcow_imaging_ipxe = format!( + "{} console={},115200 image_url={} image_sha={}", + qcow_imager_url, + console, + os_image.attributes.source_url, + os_image.attributes.digest + ); + if let Some(x) = os_image.attributes.auth_token { + qcow_imaging_ipxe += format!(" image_auth_token={x}").as_str(); + } + if let Some(x) = os_image.attributes.auth_type { + qcow_imaging_ipxe += format!(" image_auth_type={x}").as_str(); + } + if let Some(x) = os_image.attributes.rootfs_id { + qcow_imaging_ipxe += format!(" rootfs_uuid={x}").as_str(); + } + if let Some(x) = os_image.attributes.rootfs_label { + qcow_imaging_ipxe += format!(" rootfs_label={x}").as_str(); + } + if let Some(x) = os_image.attributes.boot_disk { + qcow_imaging_ipxe += format!(" image_disk={x}").as_str(); + } + if let Some(x) = os_image.attributes.bootfs_id { + qcow_imaging_ipxe += format!(" bootfs_uuid={x}").as_str(); + } + if let Some(x) = os_image.attributes.efifs_id { + qcow_imaging_ipxe += format!(" efifs_uuid={x}").as_str(); + } + if instance.config.os.user_data.is_some() { + qcow_imaging_ipxe += " ds=nocloud-net;s=${cloudinit-url}"; + } + qcow_imaging_ipxe += "\r\nboot"; + qcow_imaging_ipxe + } + } + }) + } + pub(crate) async fn get_pxe_instructions( txn: &mut PgConnection, target: PxeInstructionsInput, ) -> Result { - let error_instructions = |machine_id: MachineId, - interface_id: MachineInterfaceId, - state: &ManagedHostState| - -> String { - format!( - r#" -echo Machine ID: {machine_id} -echo Interface ID: {interface_id} -echo Current state: {state} -echo Could not continue boot due to invalid state || -sleep 5 || -exit || -"# - ) - }; - - let exit_instructions = |machine_id: MachineId, - interface_id: MachineInterfaceId, - state: &ManagedHostState| - -> String { - format!( - r#" -echo Machine ID: {machine_id} -echo Interface ID: {interface_id} -echo Current state: {state} -echo This state assumes an OS is provisioned and will exit into the OS in 5 seconds. To re-run iPXE instructions and OS installation, trigger a reboot request with flag rebootWithCustomIpxe/boot_with_custom_ipxe set. || -sleep 5 || -exit || -"# - ) - }; - static UNKNOWN_HOST_INSTRUCTIONS: &str = r#" echo this is an unknown host interface, not PXE booting || sleep 5 || @@ -509,13 +625,46 @@ exit || machine.id.machine_type(), ), ManagedHostState::Assigned { instance_state } => match instance_state { + // A provisioning boot is in flight and nothing yet proves the + // tenant's OS installed, so serve the script again for every + // request that arrives. Unlike the `Ready` arm below, the + // one-shot `use_custom_pxe_on_boot` request is deliberately left + // armed: consuming it on the first serve is what turned a failed + // install into an endless network-boot loop, because every later + // boot then got "exit into the OS" against an empty disk. The + // machine controller clears it once provisioning is confirmed, + // and fails the instance once the serve budget runs out. + InstanceState::WaitingForProvisioningComplete { .. } => { + let instance = Self::require_instance(txn, &machine_id).await?; + + // Recording the serve is what lets the machine controller + // bound retries: a host that keeps coming back is a host + // whose install keeps failing, and one that stops coming + // back has booted something. + let serve_count = + db::instance::record_custom_pxe_serve(&machine_id, txn).await?; + tracing::info!( + %machine_id, + instance_id = %instance.id, + serve_count, + "Serving tenant iPXE script for a provisioning boot" + ); + + Self::render_provisioning_script( + txn, + RenderProvisioningScript { + machine_id, + interface_id: target.interface_id, + state: machine.current_state(), + console, + qcow_imager_url, + }, + instance, + ) + .await? + } InstanceState::Ready => { - let instance = db::instance::find_by_machine_id(txn, &machine_id) - .await? - .ok_or(CarbideError::NotFoundError { - kind: "machine", - id: machine_id.to_string(), - })?; + let instance = Self::require_instance(txn, &machine_id).await?; if instance .config @@ -526,91 +675,46 @@ exit || // For non-always-PXE instances, clear the use_custom_pxe_on_boot flag // now that we're serving the script. Always-PXE instances don't use // this flag (they rely on run_provisioning_instructions_on_every_boot). + // + // New provisioning boots wait in + // `WaitingForProvisioningComplete` and are served by the + // arm above instead. This consume-on-serve path remains + // only for instances that were already mid-flight when + // the site upgraded, and can be dropped a release later. if instance.use_custom_pxe_on_boot { db::instance::use_custom_ipxe_on_next_boot(&machine_id, false, txn) .await?; } - match instance.config.os.variant { - model::os::OperatingSystemVariant::Ipxe(ipxe) => { - let mut tenant_ipxe = ipxe.ipxe_script; - let vendor_serial_console = format!(" console={console}"); - if !tenant_ipxe.contains(&vendor_serial_console) { - let idx = tenant_ipxe.find(" console="); - if let Some(x) = idx { - // insert correct serial console into custom ipxe before any other console=tty* specified - tenant_ipxe.insert_str(x, &vendor_serial_console); - } else { - // this is a strange ipxe script with no console=tty defined, leave it as is - } - } - tenant_ipxe - } - model::os::OperatingSystemVariant::OperatingSystemId(os_id) => { - let row = db::operating_system::get(txn, os_id).await?; - if row.type_ - == model::operating_system_definition::OS_TYPE_TEMPLATED_IPXE - { - let ipxeos = operating_system_row_to_ipxe_script(&row)?; - Self::render_ipxe_script(&ipxeos, "${base-url}", console)? - } else { - row.ipxe_script.unwrap_or_default() - } - } - model::os::OperatingSystemVariant::OsImage(id) => { - let os_image = db::os_image::get(txn, id).await?; - if os_image.attributes.create_volume { - // this is a block storage os image - // boot will be via the block storage snapshot volume - // no ipxe script for os imaging - exit_instructions( - machine_id, - target.interface_id, - machine.current_state(), - ) - } else { - let mut qcow_imaging_ipxe = format!( - "{} console={},115200 image_url={} image_sha={}", - qcow_imager_url, - console, - os_image.attributes.source_url, - os_image.attributes.digest - ); - if let Some(x) = os_image.attributes.auth_token { - qcow_imaging_ipxe += - format!(" image_auth_token={x}").as_str(); - } - if let Some(x) = os_image.attributes.auth_type { - qcow_imaging_ipxe += - format!(" image_auth_type={x}").as_str(); - } - if let Some(x) = os_image.attributes.rootfs_id { - qcow_imaging_ipxe += format!(" rootfs_uuid={x}").as_str(); - } - if let Some(x) = os_image.attributes.rootfs_label { - qcow_imaging_ipxe += format!(" rootfs_label={x}").as_str(); - } - if let Some(x) = os_image.attributes.boot_disk { - qcow_imaging_ipxe += format!(" image_disk={x}").as_str(); - } - if let Some(x) = os_image.attributes.bootfs_id { - qcow_imaging_ipxe += format!(" bootfs_uuid={x}").as_str(); - } - if let Some(x) = os_image.attributes.efifs_id { - qcow_imaging_ipxe += format!(" efifs_uuid={x}").as_str(); - } - if instance.config.os.user_data.is_some() { - qcow_imaging_ipxe += " ds=nocloud-net;s=${cloudinit-url}"; - } - qcow_imaging_ipxe += "\r\nboot"; - qcow_imaging_ipxe - } - } - } + Self::render_provisioning_script( + txn, + RenderProvisioningScript { + machine_id, + interface_id: target.interface_id, + state: machine.current_state(), + console, + qcow_imager_url, + }, + instance, + ) + .await? } else { exit_instructions(machine_id, target.interface_id, machine.current_state()) } } + // A provisioning boot that ran out of retries is terminal until a + // tenant asks for another custom-iPXE boot. Serving the tenant's + // script again would resume the loop the failure exists to stop, + // and serving exit instructions would send a host with an empty + // disk into a disk it cannot boot, so say so instead. + InstanceState::Failed { + details: + FailureDetails { + cause: FailureCause::ProvisioningFailed { .. }, + .. + }, + .. + } => error_instructions(machine_id, target.interface_id, machine.current_state()), InstanceState::BootingWithDiscoveryImage { .. } | InstanceState::HostReprovision { .. } => { PxeInstructions::get_pxe_instruction_for_arch( diff --git a/crates/api-core/src/tests/common/api_fixtures/instance.rs b/crates/api-core/src/tests/common/api_fixtures/instance.rs index fa4da6fdfd..3358b9a73f 100644 --- a/crates/api-core/src/tests/common/api_fixtures/instance.rs +++ b/crates/api-core/src/tests/common/api_fixtures/instance.rs @@ -46,6 +46,7 @@ pub(in crate::tests) struct TestInstanceBuilder<'a, 'b> { tenant: rpc::TenantConfig, metadata: Option, mh: &'b TestManagedHost, + stop_at_provisioning_wait: bool, } impl<'a, 'b> TestInstanceBuilder<'a, 'b> { @@ -66,9 +67,19 @@ impl<'a, 'b> TestInstanceBuilder<'a, 'b> { tenant: default_tenant_config(), metadata: None, mh, + stop_at_provisioning_wait: false, } } + /// Stops advancing at [`InstanceState::WaitingForProvisioningComplete`], + /// where the host has been rebooted for its provisioning boot but nothing has + /// confirmed that the tenant's operating system installed. Use this to + /// exercise what a host sees while it is being provisioned. + pub(in crate::tests) fn stop_at_provisioning_wait(mut self) -> Self { + self.stop_at_provisioning_wait = true; + self + } + pub(in crate::tests) fn config(mut self, config: rpc::InstanceConfig) -> Self { self.config = config; self @@ -125,6 +136,12 @@ impl<'a, 'b> TestInstanceBuilder<'a, 'b> { if self.config.tenant.is_none() { self.config.tenant = Some(self.tenant); } + // An instance whose operating system reports provisioning completion by + // phoning home cannot reach Ready without that contact, so stop where + // the state machine does. Always-PXE instances never wait at all. + let awaits_phone_home = self.config.os.as_ref().is_some_and(|os| { + os.phone_home_enabled && !os.run_provisioning_instructions_on_every_boot + }); let instance_id = self .env .api @@ -142,7 +159,11 @@ impl<'a, 'b> TestInstanceBuilder<'a, 'b> { .id .expect("Missing instance ID"); - advance_created_instance_into_ready_state(self.env, self.mh).await; + if self.stop_at_provisioning_wait || awaits_phone_home { + advance_created_instance_into_provisioning_wait(self.env, self.mh).await; + } else { + advance_created_instance_into_ready_state(self.env, self.mh).await; + } let tinstance = TestInstance { id: instance_id, env: self.env, @@ -444,17 +465,58 @@ pub(in crate::tests) async fn advance_created_instance_into_ready_state( }) .await; - assert_eq!( - mh.host().parsed_history(Some(2)).await, + // Instance creation arms a provisioning boot, so the reboot is followed by a + // wait for evidence that the tenant OS installed. The test site config uses + // a zero quiet window, so a host that never network boots -- which is every + // host in these fixtures -- clears the wait on the next iteration. + // Always-PXE instances never arm the one-shot request and so skip the wait. + let awaits_provisioning = !instance_of(env, mh) + .await + .config + .os + .run_provisioning_instructions_on_every_boot; + let expected: Vec<&str> = if awaits_provisioning { vec![ + "Assigned/WaitingForRebootToReady", + "Assigned/WaitingForProvisioningComplete", + "Assigned/Ready", + ] + } else { + vec!["Assigned/WaitingForRebootToReady", "Assigned/Ready"] + }; + let actual: Vec = mh + .host() + .parsed_history(Some(expected.len())) + .await + .iter() + .map(ManagedHostState::to_string) + .collect(); + assert_eq!(actual, expected); +} + +pub(in crate::tests) async fn advance_created_instance_into_provisioning_wait( + env: &TestEnv, + mh: &TestManagedHost, +) { + advance_created_instance_into_state(env, mh, |machine| { + matches!( + machine.state.value, ManagedHostState::Assigned { - instance_state: model::machine::InstanceState::WaitingForRebootToReady, - }, - ManagedHostState::Assigned { - instance_state: model::machine::InstanceState::Ready, + instance_state: model::machine::InstanceState::WaitingForProvisioningComplete { .. }, } - ] - ); + ) + }) + .await; +} + +async fn instance_of(env: &TestEnv, mh: &TestManagedHost) -> InstanceSnapshot { + let mut txn = env.pool.begin().await.unwrap(); + let instance = db::instance::find_by_machine_id(txn.as_mut(), &mh.host().id) + .await + .unwrap() + .expect("machine has an instance"); + txn.rollback().await.unwrap(); + instance } pub(in crate::tests) async fn delete_instance( diff --git a/crates/api-core/src/tests/instance.rs b/crates/api-core/src/tests/instance.rs index b739da01c3..cdea960567 100644 --- a/crates/api-core/src/tests/instance.rs +++ b/crates/api-core/src/tests/instance.rs @@ -2158,9 +2158,27 @@ async fn test_instance_phone_home(_: PgPoolOptions, options: PgConnectOptions) { .await .unwrap(); + // The contact is the evidence that the tenant OS installed, so the state + // machine completes the provisioning wait on its next iteration. + env.run_machine_state_controller_iteration_until_state_matches( + &mh.host().id, + 2, + ManagedHostState::Assigned { + instance_state: model::machine::InstanceState::Ready, + }, + ) + .await; + let instance = tinstance.rpc_instance().await; assert_eq!(instance.status().tenant(), rpc::TenantState::Ready); + + // Provisioning completing releases the one-shot custom-iPXE request, so the + // machine no longer re-serves the tenant's script. + let mut txn = env.pool.begin().await.unwrap(); + let db_instance = tinstance.db_instance(&mut txn).await; + txn.rollback().await.unwrap(); + assert!(!db_instance.use_custom_pxe_on_boot); } #[crate::sqlx_test] diff --git a/crates/api-core/src/tests/instance_ipxe_behaviors.rs b/crates/api-core/src/tests/instance_ipxe_behaviors.rs index a3a73491e4..4f684a6ea8 100644 --- a/crates/api-core/src/tests/instance_ipxe_behaviors.rs +++ b/crates/api-core/src/tests/instance_ipxe_behaviors.rs @@ -26,8 +26,14 @@ use crate::tests::common::api_fixtures::instance::{ TestInstance, default_os_config, default_tenant_config, single_interface_network_config, }; +/// The tenant's script is served for as long as the host keeps asking for it, +/// and only stops being served once provisioning is confirmed. +/// +/// Consuming the one-shot request on the first serve was NVBug 6601291: a script +/// that failed to fetch its kernel spent the request, so every later boot got +/// exit instructions and the host network booted against an empty disk forever. #[crate::sqlx_test] -async fn test_instance_uses_custom_ipxe_only_once(pool: sqlx::PgPool) { +async fn custom_ipxe_is_reserved_until_provisioning_completes(pool: sqlx::PgPool) { let env = create_test_env(pool).await; let segment_id = env.create_vpc_and_tenant_segment().await; let mh = create_managed_host(&env).await; @@ -37,7 +43,7 @@ async fn test_instance_uses_custom_ipxe_only_once(pool: sqlx::PgPool) { txn.rollback().await.unwrap(); let host_arch = rpc::forge::MachineArchitecture::X86; - let tinstance = create_instance(&env, &mh, false, segment_id).await; + let tinstance = create_instance_awaiting_provisioning(&env, &mh, segment_id).await; assert!( !tinstance .rpc_instance() @@ -47,83 +53,107 @@ async fn test_instance_uses_custom_ipxe_only_once(pool: sqlx::PgPool) { .run_provisioning_instructions_on_every_boot ); - // First boot should return custom iPXE instructions - let pxe = host_interface.get_pxe_instructions(host_arch).await; - assert_eq!(pxe.pxe_script, "SomeRandomiPxe"); + // The install fails and the host comes back for instructions. Both boots get + // the tenant's script, and both are recorded. + for expected_serves in 1..=2 { + let pxe = host_interface.get_pxe_instructions(host_arch).await; + assert_eq!(pxe.pxe_script, "SomeRandomiPxe"); + assert_eq!(serve_count(&env, &tinstance).await, expected_serves); + } - // Second boot should return "exit" - let pxe = host_interface.get_pxe_instructions(host_arch).await; - assert!( - pxe.pxe_script.contains("Current state: Assigned/Ready"), - "Actual script: {}", - pxe.pxe_script - ); - assert!(pxe.pxe_script.contains( - "This state assumes an OS is provisioned and will exit into the OS in 5 seconds." - )); + // The host stops coming back, which is what a successful install looks like + // from outside: the instance becomes Ready and the request is released. + advance_to_instance_ready(&env, &mh).await; + let mut txn = env.pool.begin().await.unwrap(); + let instance = tinstance.db_instance(&mut txn).await; + txn.rollback().await.unwrap(); + assert!(!instance.use_custom_pxe_on_boot); + assert_eq!(instance.custom_pxe_serve_count, 0); + assert_eq!(instance.custom_pxe_last_served_at, None); + + assert_exits_into_os(&host_interface.get_pxe_instructions(host_arch).await.pxe_script); // A regular reboot attempt should still lead to returning "exit" invoke_instance_power(&env, tinstance.id, false).await; - let pxe = host_interface.get_pxe_instructions(host_arch).await; - assert!( - pxe.pxe_script.contains("Current state: Assigned/Ready"), - "Actual script: {}", - pxe.pxe_script - ); - assert!(pxe.pxe_script.contains( - "This state assumes an OS is provisioned and will exit into the OS in 5 seconds." - )); + assert_exits_into_os(&host_interface.get_pxe_instructions(host_arch).await.pxe_script); // A reboot with flag `boot_with_custom_ipxe` should provide the custom iPXE // The reboot is handled by the state machine, which makes sure the boot order is configured properly. invoke_instance_power(&env, tinstance.id, true).await; - env.run_machine_state_controller_iteration_until_state_condition(&mh.id, 5, |machine| { + advance_to_awaiting_provisioning(&env, &mh).await; + let pxe = host_interface.get_pxe_instructions(host_arch).await; + assert_eq!(pxe.pxe_script, "SomeRandomiPxe"); + advance_to_instance_ready(&env, &mh).await; + + // The next reboot should again lead to returning "exit" + invoke_instance_power(&env, tinstance.id, false).await; + assert_exits_into_os(&host_interface.get_pxe_instructions(host_arch).await.pxe_script); +} + +/// A host that never installs anything is failed rather than left looping, and a +/// tenant can restart provisioning from that failure. +#[crate::sqlx_test] +async fn repeated_pxe_boots_fail_provisioning_and_can_be_retried(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + let segment_id = env.create_vpc_and_tenant_segment().await; + let mh = create_managed_host(&env).await; + + let mut txn = env.pool.begin().await.unwrap(); + let host_interface = mh.host().first_interface(&mut txn).await; + txn.rollback().await.unwrap(); + let host_arch = rpc::forge::MachineArchitecture::X86; + + let tinstance = create_instance_awaiting_provisioning(&env, &mh, segment_id).await; + + // Every attempt fails, so the host asks for one more than its budget allows. + let budget = env + .config + .machine_state_controller + .max_provisioning_serves; + for _ in 0..=budget { + let pxe = host_interface.get_pxe_instructions(host_arch).await; + assert_eq!(pxe.pxe_script, "SomeRandomiPxe"); + } + assert_eq!(serve_count(&env, &tinstance).await, budget + 1); + + env.run_machine_state_controller_iteration_until_state_condition(&mh.id, 2, |machine| { matches!( machine.current_state(), model::machine::ManagedHostState::Assigned { - instance_state: model::machine::InstanceState::HostPlatformConfiguration { - platform_config_state: - model::machine::HostPlatformConfigurationState::CheckHostConfig + instance_state: model::machine::InstanceState::Failed { + details: model::machine::FailureDetails { + cause: model::machine::FailureCause::ProvisioningFailed { .. }, + .. + }, + .. } } ) }) .await; - mh.network_configured(&env).await; - env.run_machine_state_controller_iteration_until_state_condition(&mh.id, 5, |machine| { - matches!( - machine.current_state(), - model::machine::ManagedHostState::Assigned { - instance_state: model::machine::InstanceState::WaitingForDpusToUp - } - ) - }) - .await; - mh.network_configured(&env).await; - env.run_machine_state_controller_iteration_until_state_condition(&mh.id, 5, |machine| { - matches!( - machine.current_state(), - model::machine::ManagedHostState::Assigned { - instance_state: model::machine::InstanceState::Ready - } - ) - }) - .await; - let pxe = host_interface.get_pxe_instructions(host_arch).await; - assert_eq!(pxe.pxe_script, "SomeRandomiPxe"); - env.run_machine_state_controller_iteration().await; - // The next reboot should again lead to returning "exit" - invoke_instance_power(&env, tinstance.id, false).await; + // Launch Layer sees a terminal failure instead of an endless PENDING. + assert_eq!( + env.one_instance(tinstance.id).await.status().tenant(), + rpc::TenantState::Failed + ); + + // Re-serving the script here would resume the loop the failure exists to + // stop, and exit instructions would send the host into an unbootable disk. let pxe = host_interface.get_pxe_instructions(host_arch).await; assert!( - pxe.pxe_script.contains("Current state: Assigned/Ready"), + pxe.pxe_script + .contains("Could not continue boot due to invalid state"), "Actual script: {}", pxe.pxe_script ); - assert!(pxe.pxe_script.contains( - "This state assumes an OS is provisioned and will exit into the OS in 5 seconds." - )); + + // The tenant fixes the script's artifacts and asks for another attempt. + invoke_instance_power(&env, tinstance.id, true).await; + advance_to_awaiting_provisioning(&env, &mh).await; + assert_eq!(serve_count(&env, &tinstance).await, 0); + let pxe = host_interface.get_pxe_instructions(host_arch).await; + assert_eq!(pxe.pxe_script, "SomeRandomiPxe"); } #[crate::sqlx_test] @@ -182,6 +212,78 @@ async fn test_instance_always_boot_with_custom_ipxe(pool: sqlx::PgPool) { assert_eq!(pxe.pxe_script, "SomeRandomiPxe"); } +fn assert_exits_into_os(pxe_script: &str) { + assert!( + pxe_script.contains("Current state: Assigned/Ready"), + "Actual script: {pxe_script}" + ); + assert!( + pxe_script.contains( + "This state assumes an OS is provisioned and will exit into the OS in 5 seconds." + ), + "Actual script: {pxe_script}" + ); +} + +async fn serve_count(env: &TestEnv, tinstance: &TestInstance<'_, '_>) -> u32 { + let mut txn = env.pool.begin().await.unwrap(); + let serve_count = tinstance.db_instance(&mut txn).await.custom_pxe_serve_count; + txn.rollback().await.unwrap(); + serve_count +} + +/// Drives a host that has just been asked for a custom-iPXE reboot through boot +/// configuration to the provisioning wait, where its script is served. +async fn advance_to_awaiting_provisioning(env: &TestEnv, mh: &TestManagedHost) { + env.run_machine_state_controller_iteration_until_state_condition(&mh.id, 5, |machine| { + matches!( + machine.current_state(), + model::machine::ManagedHostState::Assigned { + instance_state: model::machine::InstanceState::HostPlatformConfiguration { + platform_config_state: + model::machine::HostPlatformConfigurationState::CheckHostConfig + } + } + ) + }) + .await; + mh.network_configured(env).await; + env.run_machine_state_controller_iteration_until_state_condition(&mh.id, 5, |machine| { + matches!( + machine.current_state(), + model::machine::ManagedHostState::Assigned { + instance_state: model::machine::InstanceState::WaitingForDpusToUp + } + ) + }) + .await; + mh.network_configured(env).await; + env.run_machine_state_controller_iteration_until_state_condition(&mh.id, 5, |machine| { + matches!( + machine.current_state(), + model::machine::ManagedHostState::Assigned { + instance_state: model::machine::InstanceState::WaitingForProvisioningComplete { .. } + } + ) + }) + .await; +} + +/// Completes the provisioning wait. The test site config uses a zero quiet +/// window, so a host that has stopped asking for iPXE instructions is treated as +/// provisioned on the next iteration. +async fn advance_to_instance_ready(env: &TestEnv, mh: &TestManagedHost) { + env.run_machine_state_controller_iteration_until_state_condition(&mh.id, 2, |machine| { + matches!( + machine.current_state(), + model::machine::ManagedHostState::Assigned { + instance_state: model::machine::InstanceState::Ready + } + ) + }) + .await; +} + async fn invoke_instance_power( env: &TestEnv, instance_id: InstanceId, @@ -204,10 +306,37 @@ pub(in crate::tests) async fn create_instance<'a, 'b>( run_provisioning_instructions_on_every_boot: bool, segment_id: NetworkSegmentId, ) -> TestInstance<'a, 'b> { + mh.instance_builer(env) + .config(instance_config( + run_provisioning_instructions_on_every_boot, + segment_id, + )) + .build() + .await +} + +/// Creates an instance and stops at the provisioning wait, where the host has +/// been rebooted and the tenant's iPXE script is served for every PXE request. +async fn create_instance_awaiting_provisioning<'a, 'b>( + env: &'a TestEnv, + mh: &'b TestManagedHost, + segment_id: NetworkSegmentId, +) -> TestInstance<'a, 'b> { + mh.instance_builer(env) + .config(instance_config(false, segment_id)) + .stop_at_provisioning_wait() + .build() + .await +} + +fn instance_config( + run_provisioning_instructions_on_every_boot: bool, + segment_id: NetworkSegmentId, +) -> rpc::InstanceConfig { let mut os: rpc::forge::InstanceOperatingSystemConfig = default_os_config(); os.run_provisioning_instructions_on_every_boot = run_provisioning_instructions_on_every_boot; - let config = rpc::InstanceConfig { + rpc::InstanceConfig { tenant: Some(default_tenant_config()), os: Some(os), network: Some(single_interface_network_config(segment_id)), @@ -217,6 +346,5 @@ pub(in crate::tests) async fn create_instance<'a, 'b>( nvlink: None, spxconfig: None, power_profile: None, - }; - mh.instance_builer(env).config(config).build().await + } } diff --git a/crates/api-db/migrations/20260817154233_custom_pxe_serve_tracking.sql b/crates/api-db/migrations/20260817154233_custom_pxe_serve_tracking.sql new file mode 100644 index 0000000000..257daa53f1 --- /dev/null +++ b/crates/api-db/migrations/20260817154233_custom_pxe_serve_tracking.sql @@ -0,0 +1,11 @@ +-- Track how often the tenant's iPXE script has been served for the provisioning +-- boot currently being awaited, so the machine controller can tell a successful +-- install from a host that keeps returning to network boot. +-- +-- Both columns are reset when a new provisioning boot is armed, so they describe +-- the attempt in flight rather than the instance's lifetime. The CHECK keeps the +-- count decodable as an unsigned value in the instance snapshot. +ALTER TABLE instances +ADD COLUMN custom_pxe_serve_count INTEGER NOT NULL DEFAULT 0 + CHECK (custom_pxe_serve_count >= 0), +ADD COLUMN custom_pxe_last_served_at TIMESTAMPTZ; diff --git a/crates/api-db/src/instance.rs b/crates/api-db/src/instance.rs index 498127b6a3..ed567852c9 100644 --- a/crates/api-db/src/instance.rs +++ b/crates/api-db/src/instance.rs @@ -450,6 +450,47 @@ pub async fn set_custom_pxe_reboot_requested( Ok(()) } +/// Records that the tenant's iPXE script was served for the provisioning boot +/// currently being awaited. +/// +/// The increment is computed in SQL so concurrent PXE requests from the same +/// host cannot lose a serve to a stale read-modify-write. Returns the resulting +/// count. +pub async fn record_custom_pxe_serve( + machine_id: &MachineId, + txn: &mut PgConnection, +) -> Result { + let query = "UPDATE instances \ + SET custom_pxe_serve_count = custom_pxe_serve_count + 1, \ + custom_pxe_last_served_at = now() \ + WHERE machine_id=$1 RETURNING custom_pxe_serve_count"; + let (serve_count,): (i32,) = sqlx::query_as(query) + .bind(machine_id) + .fetch_one(txn) + .await + .map_err(|e| DatabaseError::query(query, e))?; + + Ok(serve_count) +} + +/// Clears the serve bookkeeping so it describes only the provisioning boot about +/// to be armed, and not any earlier attempt. +pub async fn clear_custom_pxe_serve_tracking( + machine_id: &MachineId, + txn: &mut PgConnection, +) -> Result<(), DatabaseError> { + let query = "UPDATE instances \ + SET custom_pxe_serve_count = 0, custom_pxe_last_served_at = NULL \ + WHERE machine_id=$1 RETURNING machine_id"; + let _: (MachineId,) = sqlx::query_as(query) + .bind(machine_id) + .fetch_one(txn) + .await + .map_err(|e| DatabaseError::query(query, e))?; + + Ok(()) +} + /// Updates the desired network configuration for an instance pub async fn update_network_config( txn: &mut PgConnection, diff --git a/crates/api-model/src/instance/snapshot.rs b/crates/api-model/src/instance/snapshot.rs index 9fd35d814d..26d90ac276 100644 --- a/crates/api-model/src/instance/snapshot.rs +++ b/crates/api-model/src/instance/snapshot.rs @@ -95,6 +95,20 @@ pub struct InstanceSnapshot { /// The WaitingForRebootToReady handler clears this flag. pub custom_pxe_reboot_requested: bool, + /// How many times the tenant's iPXE script has been served for the + /// provisioning boot currently being tracked by + /// [`InstanceState::WaitingForProvisioningComplete`]. Reset when a new + /// provisioning boot is armed, incremented by the iPXE handler on each + /// serve, and read by the machine controller to bound retries. + /// + /// [`InstanceState::WaitingForProvisioningComplete`]: crate::machine::InstanceState::WaitingForProvisioningComplete + pub custom_pxe_serve_count: u32, + + /// When the tenant's iPXE script was last served. `None` means it has not + /// been served since the provisioning boot was armed, which for a host that + /// never network boots is indistinguishable from "this host boots from disk". + pub custom_pxe_last_served_at: Option>, + /// The timestamp when deletion for this instance was requested pub deleted: Option>, @@ -132,6 +146,11 @@ pub struct InstanceSnapshotPgJson { use_custom_pxe_on_boot: bool, #[serde(default)] custom_pxe_reboot_requested: bool, + /// Non-negative by database constraint, so a `u32` cannot fail to decode. + #[serde(default)] + custom_pxe_serve_count: u32, + #[serde(default)] + custom_pxe_last_served_at: Option>, tenant_org: Option, keyset_ids: Vec, hostname: Option, @@ -243,6 +262,8 @@ pub fn from_pg_json_and_os( }, use_custom_pxe_on_boot: value.use_custom_pxe_on_boot, custom_pxe_reboot_requested: value.custom_pxe_reboot_requested, + custom_pxe_serve_count: value.custom_pxe_serve_count, + custom_pxe_last_served_at: value.custom_pxe_last_served_at, deleted: value.deleted, update_network_config_request: value.update_network_config_request, }) @@ -359,6 +380,8 @@ impl TryFrom for InstanceSnapshot { }, use_custom_pxe_on_boot: value.use_custom_pxe_on_boot, custom_pxe_reboot_requested: value.custom_pxe_reboot_requested, + custom_pxe_serve_count: value.custom_pxe_serve_count, + custom_pxe_last_served_at: value.custom_pxe_last_served_at, deleted: value.deleted, update_network_config_request: value.update_network_config_request, // Unused as of today @@ -405,6 +428,8 @@ mod tests { phone_home_last_contact: None, use_custom_pxe_on_boot: false, custom_pxe_reboot_requested: false, + custom_pxe_serve_count: 0, + custom_pxe_last_served_at: None, tenant_org: Some("TenantA".to_string()), keyset_ids: vec![], hostname: None, diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 3a2d758b72..a28fa6df33 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -1881,6 +1881,14 @@ pub enum FailureCause { SpdmAttestationFailed { err: String }, BiosSetupFailed { err: String }, + + // ProvisioningFailed is returned when a provisioning boot never produced + // evidence that the tenant's operating system installed: either the host + // kept coming back for iPXE instructions until `serve_count` exhausted the + // configured budget, or the overall provisioning deadline elapsed. Without + // it, a machine whose install fails network-boots against an empty disk + // forever while reporting itself provisioned. + ProvisioningFailed { serve_count: u32, err: String }, } #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] @@ -2286,6 +2294,25 @@ pub enum InstanceState { WaitingForDpaToBeReady, WaitingForExtensionServicesConfig, WaitingForRebootToReady, + /// A provisioning boot has been armed and the host rebooted, but nothing yet + /// proves the tenant's operating system actually installed. NICo keeps + /// re-serving the tenant's iPXE script for every PXE boot that arrives in + /// this state instead of consuming the one-shot request on the first serve, + /// so a failed install retries rather than falling through to + /// "exit into the OS" against an empty disk. + /// + /// Serve bookkeeping (`custom_pxe_serve_count` and + /// `custom_pxe_last_served_at` on the `instances` row) is deliberately not + /// duplicated here: the API records each serve while this state is + /// persisted, so a copy in the state would be stale the moment a host PXE + /// boots. + WaitingForProvisioningComplete { + /// When the provisioning boot was armed. Bounds the quiet window for a + /// host that never asks for iPXE instructions at all. + started_at: DateTime, + /// When to give up and fail the instance without success evidence. + deadline: DateTime, + }, Ready, HostPlatformConfiguration { platform_config_state: HostPlatformConfigurationState, @@ -2494,7 +2521,16 @@ impl Display for MachineState { impl Display for InstanceState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Debug::fmt(self, f) + match self { + // The label reaches operators through `ManagedHostState`'s `Display`, + // the iPXE echo lines, and state history. Rendering the timestamps + // this variant carries would make the same logical state read as a + // different label on every attempt, so keep the label constant. + InstanceState::WaitingForProvisioningComplete { .. } => { + write!(f, "WaitingForProvisioningComplete") + } + _ => std::fmt::Debug::fmt(self, f), + } } } @@ -2540,6 +2576,7 @@ impl FailureCause { FailureCause::DpfProvisioning { .. } => "dpf_provisioning", FailureCause::SpdmAttestationFailed { .. } => "spdm_attestation_failed", FailureCause::BiosSetupFailed { .. } => "bios_setup_failed", + FailureCause::ProvisioningFailed { .. } => "provisioning_failed", } } } @@ -2569,6 +2606,7 @@ impl Display for FailureCause { write!(f, "SpdmAttestationFailed") } FailureCause::BiosSetupFailed { .. } => write!(f, "BiosSetupFailed"), + FailureCause::ProvisioningFailed { .. } => write!(f, "ProvisioningFailed"), } } } @@ -2990,6 +3028,10 @@ pub fn state_sla( InstanceState::HostPlatformConfiguration { .. } => { StateSla::with_sla(slas::ASSIGNED_HOST_PLATFORM_CONFIGURATION, time_in_state) } + InstanceState::WaitingForProvisioningComplete { .. } => StateSla::with_sla( + slas::ASSIGNED_WAITING_FOR_PROVISIONING_COMPLETE, + time_in_state, + ), _ => StateSla::with_sla(slas::ASSIGNED, time_in_state), }, ManagedHostState::WaitingForCleanup { .. } => { diff --git a/crates/api-model/src/machine/slas.rs b/crates/api-model/src/machine/slas.rs index 243492c287..8ca4fe3dae 100644 --- a/crates/api-model/src/machine/slas.rs +++ b/crates/api-model/src/machine/slas.rs @@ -59,6 +59,14 @@ pub const ASSIGNED: Duration = Duration::from_secs(30 * 60); // ASSIGNED state, HostPlatformConfiguration substate pub const ASSIGNED_HOST_PLATFORM_CONFIGURATION: Duration = Duration::from_secs(90 * 60); + +// ASSIGNED state, WaitingForProvisioningComplete substate. A tenant OS install +// plus its confirmation legitimately takes longer than the generic ASSIGNED +// budget, and the state already escalates itself to +// Failed/ProvisioningFailed once `provisioning_deadline` elapses (60 minutes by +// default). Keep this above that ceiling so the terminal failure, not an SLA +// breach on a healthy install, is what operators react to. +pub const ASSIGNED_WAITING_FOR_PROVISIONING_COMPLETE: Duration = Duration::from_secs(90 * 60); pub const VALIDATION: Duration = Duration::from_secs(30 * 60); pub const MAINTENANCE: Duration = Duration::from_secs(5 * 60); diff --git a/crates/machine-controller/src/config/controller.rs b/crates/machine-controller/src/config/controller.rs index e6bb7206ef..2098a1f674 100644 --- a/crates/machine-controller/src/config/controller.rs +++ b/crates/machine-controller/src/config/controller.rs @@ -116,6 +116,30 @@ pub struct MachineStateControllerConfig { serialize_with = "as_duration" )] pub boot_interface_observation_interval: Duration, + /// How long an instance awaiting provisioning completion must go without a + /// new iPXE serve before NICo concludes the host booted its own disk. Only + /// consulted when the instance is not enrolled in phone home, which reports + /// completion directly. + #[serde( + default = "MachineStateControllerConfig::provisioning_quiet_window_default", + deserialize_with = "deserialize_duration_chrono", + serialize_with = "as_duration" + )] + pub provisioning_quiet_window: Duration, + /// How many times the tenant's iPXE script may be served for one + /// provisioning boot before the instance is failed. A host that keeps + /// coming back for instructions is not installing its operating system. + #[serde(default = "MachineStateControllerConfig::max_provisioning_serves_default")] + pub max_provisioning_serves: u32, + /// How long an instance may await provisioning completion before it is + /// failed, regardless of serve count. Bounds hosts that neither complete + /// nor return to network boot. + #[serde( + default = "MachineStateControllerConfig::provisioning_deadline_default", + deserialize_with = "deserialize_duration_chrono", + serialize_with = "as_duration" + )] + pub provisioning_deadline: Duration, } impl MachineStateControllerConfig { @@ -137,6 +161,13 @@ impl MachineStateControllerConfig { // Keep periodic Redfish reads out of unrelated controller tests. // Focused tests explicitly age the observation they exercise. boot_interface_observation_interval: Duration::weeks(52), + // No simulated host network boots in unrelated controller tests, so + // a zero quiet window lets provisioning completion resolve on the + // next iteration. Focused tests set their own window. + provisioning_quiet_window: Duration::zero(), + max_provisioning_serves: + MachineStateControllerConfig::max_provisioning_serves_default(), + provisioning_deadline: Duration::weeks(52), } } @@ -180,6 +211,28 @@ impl MachineStateControllerConfig { pub fn boot_interface_observation_interval_default() -> Duration { Duration::minutes(10) } + + /// Default quiet window before a silent host is assumed to have booted its + /// own disk. A host stuck in the empty-disk loop returns to network boot + /// within a minute or two of a failed install, so this is generous while + /// still bounding how long a healthy instance waits to report Ready. + pub fn provisioning_quiet_window_default() -> Duration { + Duration::minutes(15) + } + + /// Default serve budget for one provisioning boot. The first serve is the + /// install itself; the rest absorb transient artifact-fetch failures before + /// the instance is declared failed. + pub fn max_provisioning_serves_default() -> u32 { + 4 + } + + /// Default ceiling on awaiting provisioning completion. Long enough for a + /// large image to install and phone home, short enough that a stuck + /// instance becomes visibly failed within the hour. + pub fn provisioning_deadline_default() -> Duration { + Duration::minutes(60) + } } impl Default for MachineStateControllerConfig { @@ -201,6 +254,11 @@ impl Default for MachineStateControllerConfig { MachineStateControllerConfig::polling_bios_setup_stuck_threshold_default(), boot_interface_observation_interval: MachineStateControllerConfig::boot_interface_observation_interval_default(), + provisioning_quiet_window: + MachineStateControllerConfig::provisioning_quiet_window_default(), + max_provisioning_serves: + MachineStateControllerConfig::max_provisioning_serves_default(), + provisioning_deadline: MachineStateControllerConfig::provisioning_deadline_default(), } } } diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index d0f8a5cac4..59c0f1f954 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -132,6 +132,7 @@ mod host_uefi_rotation; mod machine_validation; mod maintenance; mod power; +mod provisioning_completion; mod rotation; mod sku; #[cfg(test)] @@ -148,6 +149,9 @@ use host_boot_config::{ initial_set_boot_order_info, inspect_host_boot_config, run_host_boot_config_stage, should_skip_boot_order_remediation, }; +use provisioning_completion::{ + ProvisioningBounds, ProvisioningFailure, ProvisioningOutcome, ProvisioningSignals, +}; use state_controller::db_write_batch::DbWriteBatch; use crate::config::{BomValidationConfig, PowerManagerOptions}; @@ -7874,7 +7878,7 @@ impl StateHandler for InstanceStateHandler { } InstanceState::WaitingForRebootToReady => { // If custom_pxe_reboot_requested is set, this reboot was triggered by - // the tenant requested a boot with custom iPXE. Clear the request flag. + // a tenant-requested boot with custom iPXE. Clear the request flag. // The use_custom_pxe_on_boot flag was already set by the API handler. if instance.custom_pxe_reboot_requested { ctx.pending_db_writes @@ -7884,17 +7888,78 @@ impl StateHandler for InstanceStateHandler { }); } + // A provisioning boot is one whose whole purpose is to run the + // tenant's iPXE script: instance creation and + // `rebootWithCustomIpxe` both arm `use_custom_pxe_on_boot`. + // Always-PXE instances re-serve their script on every boot and + // never depend on that one-shot flag, and a deleted instance is + // on its way out through Ready. Everything else is a plain + // reboot of an already-installed OS and stays on the existing + // straight-to-Ready path. + let awaits_provisioning = (instance.use_custom_pxe_on_boot + || instance.custom_pxe_reboot_requested) + && !instance + .config + .os + .run_provisioning_instructions_on_every_boot + && instance.deleted.is_none(); + + if awaits_provisioning { + // Serves recorded for an earlier attempt would make this + // one look like it had already spent its budget. Reset + // before the host can come back and be served, not after. + ctx.pending_db_writes + .push(MachineWriteOp::ClearCustomPxeServeTracking { + machine_id: mh_snapshot.host_snapshot.id, + }); + } + // Reboot host handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart) .await?; - // Instance is ready. - // We can not determine if machine is rebooted successfully or not. Just leave - // it like this and declare Instance Ready. - let next_state = ManagedHostState::Assigned { - instance_state: InstanceState::Ready, - }; - Ok(StateHandlerOutcome::transition(next_state)) + if !awaits_provisioning { + // Nothing is being installed, so there is nothing to + // confirm. We cannot tell whether the machine rebooted + // successfully, so declare the instance Ready. + return Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::Ready, + }, + )); + } + + // The reboot is only the start of provisioning: the host still + // has to fetch the script, install, and boot its own disk. + // Reporting Ready here is what let a failed install fall + // through to "exit into the OS" against an empty disk. + let started_at = Utc::now(); + let deadline = started_at + + ctx + .services + .site_config + .machine_state_controller + .provisioning_deadline; + Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::WaitingForProvisioningComplete { + started_at, + deadline, + }, + }, + )) + } + InstanceState::WaitingForProvisioningComplete { + started_at, + deadline, + } => { + handle_waiting_for_provisioning_complete( + ctx, + mh_snapshot, + instance, + *started_at, + *deadline, + ) } InstanceState::Ready => { // Machine is up after reboot. Hurray. Instance is up. @@ -8524,6 +8589,29 @@ impl StateHandler for InstanceStateHandler { }; handle_bios_setup_failed_recovery(ctx, mh_snapshot, None, recovered).await } + // A provisioning failure is recoverable without operator + // surgery, unlike the causes below: the tenant either fixes + // whatever the iPXE script could not deliver and asks for + // another custom-iPXE boot, or gives the machine back. Hand + // to `Ready`, which owns both flows, rather than requiring + // an admin force-delete to leave this state. + FailureCause::ProvisioningFailed { .. } + if instance.custom_pxe_reboot_requested + || instance.deleted.is_some() => + { + tracing::info!( + instance_id = %instance.id, + machine_id = %host_machine_id, + reboot_requested = instance.custom_pxe_reboot_requested, + release_requested = instance.deleted.is_some(), + "Leaving provisioning failure for a newer tenant request" + ); + Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::Ready, + }, + )) + } _ => { // Only way to proceed for other causes is to // 1. Force-delete the machine. @@ -12448,6 +12536,109 @@ async fn handle_instance_host_boot_config_stage( } } +/// Acts on the verdict [`provisioning_completion::evaluate`] reaches for an +/// instance waiting for evidence that its provisioning boot installed an OS. +/// +/// Success releases the one-shot custom-iPXE request that the iPXE handler +/// deliberately leaves armed while this state is current, and resets the serve +/// bookkeeping for whatever provisioning boot comes next. Both failure bounds +/// end in [`FailureCause::ProvisioningFailed`], so a tenant sees a terminal +/// failure instead of an instance that claims to be provisioned while its disk +/// is empty. +fn handle_waiting_for_provisioning_complete( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + mh_snapshot: &ManagedHostStateSnapshot, + instance: &InstanceSnapshot, + started_at: DateTime, + deadline: DateTime, +) -> Result, StateHandlerError> { + let machine_id = mh_snapshot.host_snapshot.id; + let controller_config = &ctx.services.site_config.machine_state_controller; + let bounds = ProvisioningBounds { + quiet_window: controller_config.provisioning_quiet_window, + max_serves: controller_config.max_provisioning_serves, + }; + let serve_count = instance.custom_pxe_serve_count; + let now = Utc::now(); + let signals = ProvisioningSignals::from_instance(instance, started_at, deadline); + + let ready = ManagedHostState::Assigned { + instance_state: InstanceState::Ready, + }; + + match provisioning_completion::evaluate(signals, bounds, now) { + // The `Assigned{Ready}` handler owns both the reboot and the deletion + // flow, so let it start the replacement attempt rather than judging the + // one being replaced. This is also how a retry after a failure works. + ProvisioningOutcome::Superseded => { + tracing::info!( + instance_id = %instance.id, + %machine_id, + serve_count, + "Provisioning wait superseded by a newer tenant request" + ); + Ok(StateHandlerOutcome::transition(ready)) + } + ProvisioningOutcome::Complete(evidence) => { + tracing::info!( + instance_id = %instance.id, + %machine_id, + serve_count, + ?evidence, + "Provisioning boot completed; instance is ready" + ); + ctx.pending_db_writes + .push(MachineWriteOp::UseCustomIpxeOnNextBoot { + machine_id, + boot_with_custom_ipxe: false, + }); + ctx.pending_db_writes + .push(MachineWriteOp::ClearCustomPxeServeTracking { machine_id }); + Ok(StateHandlerOutcome::transition(ready)) + } + // Each PXE request re-serves the script on its own, so waiting is the + // retry; there is nothing for the controller to drive here. + ProvisioningOutcome::KeepWaiting => Ok(StateHandlerOutcome::wait(format!( + "Waiting for evidence that provisioning completed (serves: {serve_count}, deadline: {deadline})." + ))), + ProvisioningOutcome::Failed(failure) => { + let err = match failure { + ProvisioningFailure::ServeBudgetExhausted => format!( + "the tenant's iPXE script was served {serve_count} times without evidence \ + the operating system installed (budget: {})", + bounds.max_serves + ), + ProvisioningFailure::DeadlineElapsed => format!( + "no evidence the operating system installed within {} of the provisioning \ + boot (serves: {serve_count})", + deadline - started_at + ), + }; + tracing::warn!( + instance_id = %instance.id, + %machine_id, + serve_count, + %err, + "Provisioning boot failed" + ); + Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::Failed { + details: FailureDetails { + cause: FailureCause::ProvisioningFailed { serve_count, err }, + failed_at: now, + source: FailureSource::StateMachineArea( + StateMachineArea::AssignedInstance, + ), + }, + machine_id, + }, + }, + )) + } + } +} + async fn handle_instance_host_platform_config( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, mh_snapshot: &mut ManagedHostStateSnapshot, diff --git a/crates/machine-controller/src/handler/provisioning_completion.rs b/crates/machine-controller/src/handler/provisioning_completion.rs new file mode 100644 index 0000000000..70dfa9ca2b --- /dev/null +++ b/crates/machine-controller/src/handler/provisioning_completion.rs @@ -0,0 +1,274 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Decides when a provisioning boot has finished. +//! +//! Serving a tenant's iPXE script proves only that the host asked for it. NICo +//! never sees the install itself, so [`evaluate`] infers the answer from the +//! signals it does have and bounds how long it will keep inferring. + +use chrono::{DateTime, Duration, Utc}; +use model::instance::snapshot::InstanceSnapshot; + +/// Why an instance is considered provisioned. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum CompletionEvidence { + /// The booted operating system contacted NICo's metadata service. The + /// contact was cleared when the boot was armed, so this one is fresh. + PhoneHome, + /// The host stopped asking for iPXE instructions, which is what booting an + /// installed operating system looks like from outside the host. + HostWentQuiet, +} + +/// Why an instance's provisioning boot is being given up on. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum ProvisioningFailure { + /// The host asked for more attempts than its budget allows, which can only + /// happen if every attempt before the last one failed. + ServeBudgetExhausted, + /// Neither success nor further attempts arrived before the deadline. + DeadlineElapsed, +} + +/// What to do with an instance awaiting evidence that provisioning finished. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum ProvisioningOutcome { + /// A newer tenant request replaces this attempt; hand back to the + /// `Assigned{Ready}` handler, which owns both reboot and deletion. + Superseded, + Complete(CompletionEvidence), + /// Nothing to do. Each PXE request re-serves the script on its own, so + /// waiting is how a failed install is retried. + KeepWaiting, + Failed(ProvisioningFailure), +} + +/// Signals about one provisioning boot, read from the instance row. +#[derive(Copy, Clone, Debug)] +pub(super) struct ProvisioningSignals { + /// A tenant asked for another custom-iPXE boot. + pub(super) reboot_requested: bool, + /// A tenant asked for the instance to be released. + pub(super) deletion_requested: bool, + /// The instance's operating system reports completion by phoning home. + pub(super) phone_home_enabled: bool, + /// A phone-home contact is recorded for this boot. + pub(super) phone_home_contacted: bool, + /// How many times the tenant's script has been served for this boot. + pub(super) serve_count: u32, + /// When it was last served, or `None` if the host never asked. + pub(super) last_served_at: Option>, + /// When the boot was armed. + pub(super) started_at: DateTime, + /// When to give up without success evidence. + pub(super) deadline: DateTime, +} + +impl ProvisioningSignals { + pub(super) fn from_instance( + instance: &InstanceSnapshot, + started_at: DateTime, + deadline: DateTime, + ) -> Self { + Self { + reboot_requested: instance.custom_pxe_reboot_requested, + deletion_requested: instance.deleted.is_some(), + phone_home_enabled: instance.config.os.phone_home_enabled, + phone_home_contacted: instance.observations.phone_home_last_contact.is_some(), + serve_count: instance.custom_pxe_serve_count, + last_served_at: instance.custom_pxe_last_served_at, + started_at, + deadline, + } + } +} + +/// Site limits on how long completion may be inferred for. +#[derive(Copy, Clone, Debug)] +pub(super) struct ProvisioningBounds { + /// How long without a serve implies the host booted its own disk. Only + /// consulted for instances that do not phone home. + pub(super) quiet_window: Duration, + /// How many serves one provisioning boot may take. + pub(super) max_serves: u32, +} + +/// Classifies a provisioning boot from the signals recorded for it. +/// +/// Success is checked before failure so an install that completed on its last +/// permitted attempt is not failed for having used the whole budget. +pub(super) fn evaluate( + signals: ProvisioningSignals, + bounds: ProvisioningBounds, + now: DateTime, +) -> ProvisioningOutcome { + if signals.reboot_requested || signals.deletion_requested { + return ProvisioningOutcome::Superseded; + } + + if signals.phone_home_enabled { + if signals.phone_home_contacted { + return ProvisioningOutcome::Complete(CompletionEvidence::PhoneHome); + } + } else { + // A host that never asked for instructions has been quiet since the + // boot was armed. Silence cannot be told apart from a host that booted + // a disk it already had, which is why phone home is the better signal + // and this one is bounded by the deadline below. + let quiet_since = signals.last_served_at.unwrap_or(signals.started_at); + if now - quiet_since >= bounds.quiet_window { + return ProvisioningOutcome::Complete(CompletionEvidence::HostWentQuiet); + } + } + + if signals.serve_count > bounds.max_serves { + return ProvisioningOutcome::Failed(ProvisioningFailure::ServeBudgetExhausted); + } + if now >= signals.deadline { + return ProvisioningOutcome::Failed(ProvisioningFailure::DeadlineElapsed); + } + + ProvisioningOutcome::KeepWaiting +} + +#[cfg(test)] +mod tests { + use carbide_test_support::value_scenarios; + + use super::*; + + fn timestamp() -> DateTime { + DateTime::from_timestamp(1_722_000_000, 0).expect("fixture timestamp") + } + + /// Every combination of the signals that decide one provisioning boot. + /// + /// Ages are relative to `now`: `served_ago`/`started_ago` count backwards + /// from it, and `deadline_in` forwards, so a non-positive `deadline_in` + /// means the deadline has passed. + #[test] + fn provisioning_outcome_follows_the_recorded_signals() { + struct Input { + reboot_requested: bool, + deletion_requested: bool, + phone_home_enabled: bool, + phone_home_contacted: bool, + serve_count: u32, + served_ago: Option, + started_ago: Duration, + deadline_in: Duration, + } + + /// One serve, recent enough that the quiet window has not elapsed, with + /// budget and deadline to spare: on its own this waits. + fn in_flight() -> Input { + Input { + reboot_requested: false, + deletion_requested: false, + phone_home_enabled: false, + phone_home_contacted: false, + serve_count: 1, + served_ago: Some(Duration::minutes(1)), + started_ago: Duration::minutes(2), + deadline_in: Duration::minutes(30), + } + } + + let now = timestamp(); + let bounds = ProvisioningBounds { + quiet_window: Duration::minutes(15), + max_serves: 4, + }; + + value_scenarios!( + run = |input: Input| { + let signals = ProvisioningSignals { + reboot_requested: input.reboot_requested, + deletion_requested: input.deletion_requested, + phone_home_enabled: input.phone_home_enabled, + phone_home_contacted: input.phone_home_contacted, + serve_count: input.serve_count, + last_served_at: input.served_ago.map(|age| now - age), + started_at: now - input.started_ago, + deadline: now + input.deadline_in, + }; + evaluate(signals, bounds, now) + }; + "a newer tenant request wins over every other signal" { + Input { reboot_requested: true, ..in_flight() } + => ProvisioningOutcome::Superseded, + Input { deletion_requested: true, ..in_flight() } + => ProvisioningOutcome::Superseded, + // Even from a state that would otherwise fail, so an operator + // can retry instead of waiting for the attempt to time out. + Input { reboot_requested: true, serve_count: 99, deadline_in: -Duration::minutes(1), ..in_flight() } + => ProvisioningOutcome::Superseded, + } + "a phone-home contact completes the boot" { + Input { phone_home_enabled: true, phone_home_contacted: true, ..in_flight() } + => ProvisioningOutcome::Complete(CompletionEvidence::PhoneHome), + // The last permitted attempt is the one that installed. + Input { phone_home_enabled: true, phone_home_contacted: true, serve_count: 99, ..in_flight() } + => ProvisioningOutcome::Complete(CompletionEvidence::PhoneHome), + } + "an instance enrolled in phone home waits for that contact alone" { + // Quiet for far longer than the window, which would complete a + // non-enrolled instance, proves nothing here: the OS is + // supposed to say so itself. + Input { phone_home_enabled: true, served_ago: Some(Duration::hours(10)), ..in_flight() } + => ProvisioningOutcome::KeepWaiting, + } + "silence completes a boot that has no phone home to wait for" { + Input { served_ago: Some(bounds.quiet_window), ..in_flight() } + => ProvisioningOutcome::Complete(CompletionEvidence::HostWentQuiet), + // A host that never asked for instructions is measured from + // when the boot was armed. + Input { served_ago: None, serve_count: 0, started_ago: bounds.quiet_window, ..in_flight() } + => ProvisioningOutcome::Complete(CompletionEvidence::HostWentQuiet), + // A contact recorded by an instance that does not report + // completion that way is not evidence of anything. + Input { phone_home_contacted: true, ..in_flight() } + => ProvisioningOutcome::KeepWaiting, + Input { served_ago: Some(bounds.quiet_window - Duration::seconds(1)), ..in_flight() } + => ProvisioningOutcome::KeepWaiting, + Input { served_ago: None, serve_count: 0, started_ago: bounds.quiet_window - Duration::seconds(1), ..in_flight() } + => ProvisioningOutcome::KeepWaiting, + } + "a host that keeps asking for the script has run out of attempts" { + Input { serve_count: bounds.max_serves + 1, ..in_flight() } + => ProvisioningOutcome::Failed(ProvisioningFailure::ServeBudgetExhausted), + // Spending the budget is not itself a failure; the attempt it + // paid for still gets its window. + Input { serve_count: bounds.max_serves, ..in_flight() } + => ProvisioningOutcome::KeepWaiting, + } + "the deadline ends a boot that produced neither success nor attempts" { + Input { deadline_in: Duration::zero(), phone_home_enabled: true, ..in_flight() } + => ProvisioningOutcome::Failed(ProvisioningFailure::DeadlineElapsed), + Input { deadline_in: -Duration::hours(1), phone_home_enabled: true, ..in_flight() } + => ProvisioningOutcome::Failed(ProvisioningFailure::DeadlineElapsed), + // An exhausted budget is reported ahead of the deadline: it + // says why the boot failed, where the deadline only says when. + Input { serve_count: bounds.max_serves + 1, deadline_in: -Duration::hours(1), phone_home_enabled: true, ..in_flight() } + => ProvisioningOutcome::Failed(ProvisioningFailure::ServeBudgetExhausted), + Input { deadline_in: Duration::seconds(1), phone_home_enabled: true, ..in_flight() } + => ProvisioningOutcome::KeepWaiting, + } + ); + } +} diff --git a/crates/machine-controller/src/io.rs b/crates/machine-controller/src/io.rs index ef2527269a..dfa94add88 100644 --- a/crates/machine-controller/src/io.rs +++ b/crates/machine-controller/src/io.rs @@ -223,6 +223,9 @@ impl StateControllerIO for MachineStateControllerIO { "waitingforextensionservicesconfig" } InstanceState::WaitingForRebootToReady => "waitingforreboottoready", + InstanceState::WaitingForProvisioningComplete { .. } => { + "waitingforprovisioningcomplete" + } InstanceState::Ready => "ready", InstanceState::BootingWithDiscoveryImage { .. } => "bootingwithdiscoveryimage", InstanceState::SwitchToAdminNetwork => "switchtoadminnetwork", diff --git a/crates/machine-controller/src/write_ops.rs b/crates/machine-controller/src/write_ops.rs index 015a2ebf01..d30b599d4c 100644 --- a/crates/machine-controller/src/write_ops.rs +++ b/crates/machine-controller/src/write_ops.rs @@ -107,6 +107,9 @@ pub enum MachineWriteOp { machine_id: MachineId, boot_with_custom_ipxe: bool, }, + ClearCustomPxeServeTracking { + machine_id: MachineId, + }, } #[async_trait] @@ -213,6 +216,9 @@ impl WriteOp for MachineWriteOp { db::instance::use_custom_ipxe_on_next_boot(&machine_id, boot_with_custom_ipxe, txn) .await?; } + ClearCustomPxeServeTracking { machine_id } => { + db::instance::clear_custom_pxe_serve_tracking(&machine_id, txn).await?; + } }; Ok(()) } diff --git a/crates/rpc/src/model/instance/status/tenant.rs b/crates/rpc/src/model/instance/status/tenant.rs index 6f1a712e70..c240426a3a 100644 --- a/crates/rpc/src/model/instance/status/tenant.rs +++ b/crates/rpc/src/model/instance/status/tenant.rs @@ -74,6 +74,12 @@ pub fn instance_status_tenant_state( TenantState::Provisioning } } + // The tenant's operating system is still installing, and nothing yet + // proves it will succeed. Unlike the states above, this one is not + // projected as ready under operator-managed networking: that setting + // only says NICo has no data-plane readiness signal to wait for, and + // says nothing about whether an OS reached the disk. + InstanceState::WaitingForProvisioningComplete { .. } => TenantState::Provisioning, InstanceState::NetworkConfigUpdate { .. } => { if operator_managed_networking { tenant_ready_state() diff --git a/docs/architecture/state_machines/managedhost.md b/docs/architecture/state_machines/managedhost.md index 3486a74ff1..a992116eb3 100644 --- a/docs/architecture/state_machines/managedhost.md +++ b/docs/architecture/state_machines/managedhost.md @@ -443,6 +443,7 @@ stateDiagram-v2 state "WaitingForNetworkConfig" as A_WaitingForNetworkConfig state "WaitingForStorageConfig" as A_WaitingForStorageConfig state "WaitingForRebootToReady" as A_WaitingForRebootToReady + state "WaitingForProvisioningComplete" as A_WaitingForProvisioningComplete state "Assigned/Ready" as A_Ready state "WaitingForDpusToUp" as A_WaitingForDpusToUp state "BootingWithDiscoveryImage" as A_BootingWithDiscoveryImage @@ -490,7 +491,13 @@ stateDiagram-v2 A_WaitingForNetworkConfig --> A_WaitingForStorageConfig : No DPU OR Host network synced on DPU A_WaitingForStorageConfig --> A_WaitingForRebootToReady : Attach storage volumes - A_WaitingForRebootToReady --> A_Ready : Reboot machine + A_WaitingForRebootToReady --> A_Ready : Reboot machine (plain reboot, or always-PXE instance) + A_WaitingForRebootToReady --> A_WaitingForProvisioningComplete : Reboot machine (provisioning boot) + + A_WaitingForProvisioningComplete --> A_WaitingForProvisioningComplete : Waiting for install evidence (each PXE request re-serves the tenant script) + A_WaitingForProvisioningComplete --> A_Ready : Phone home received, OR quiet for provisioning_quiet_window + A_WaitingForProvisioningComplete --> A_Failed : Serves > max_provisioning_serves, OR provisioning_deadline elapsed (ProvisioningFailed) + A_WaitingForProvisioningComplete --> A_Ready : Custom iPXE reboot requested OR instance deleted A_Ready --> A_NCU_WaitingForNetworkSegmentToBeReady : Update network request A_Ready --> A_HPC_PowerCycle : (Instance deleted OR Host/DPU reporvisioning requested) AND need config bootorder @@ -553,8 +560,52 @@ stateDiagram-v2 AnyState --> A_Failed : Any failure condition A_Failed --> A_Failed : Wait (stuck, manual action needed) A_Failed --> A_HPC_SBO_SetBootOrder : BiosSetupFailed AND is_bios_setup ok + A_Failed --> A_Ready : ProvisioningFailed AND (custom iPXE reboot requested OR instance deleted) ``` +### Waiting for provisioning to complete + +A provisioning boot is one whose purpose is to run the tenant's iPXE script: a +new instance, or a reboot requested with `rebootWithCustomIpxe` +(`boot_with_custom_ipxe`). `WaitingForRebootToReady` reboots the host and then +enters `WaitingForProvisioningComplete` for those boots, and `Assigned/Ready` +for everything else. Instances whose OS sets +`run_provisioning_instructions_on_every_boot` never enter the state: their +script is re-served on every boot regardless, so they do not depend on the +one-shot `use_custom_pxe_on_boot` request. + +While the state is current, `/api/v0/pxe/boot` serves the tenant's script for +every request and leaves `use_custom_pxe_on_boot` armed, recording each serve in +`instances.custom_pxe_serve_count` and `instances.custom_pxe_last_served_at`. +This is what makes a failed install retry: consuming the request on the first +serve left every later boot with "exit into the OS" against an empty disk, so a +host whose install failed network booted forever. + +NICo cannot observe an install directly, so completion is inferred: + +| Instance | Evidence of success | +| --- | --- | +| OS has `phone_home_enabled` | `phone_home_last_contact` is set. The reboot API clears it when the boot is armed, so any value was recorded by an OS that came up on this attempt. | +| Everything else | No iPXE serve for `machine_state_controller.provisioning_quiet_window` (default 15 minutes), measured from the last serve, or from state entry if the host never asked for instructions. A host that stopped network booting booted something else. | + +On success the instance moves to `Assigned/Ready`, `use_custom_pxe_on_boot` is +released, and the serve bookkeeping is reset. Neither signal ever arrives for a +genuinely broken install, so two bounds end the wait in +`Assigned/Failed{ProvisioningFailed}`: a serve count above +`machine_state_controller.max_provisioning_serves` (default 4), and +`machine_state_controller.provisioning_deadline` (default 60 minutes) elapsing. +Tenant status reports `PROVISIONING` throughout the wait and `FAILED` after, +rather than a `READY` instance with nothing installed. + +`/api/v0/pxe/boot` answers a host in `Failed{ProvisioningFailed}` with error +instructions: re-serving the script would resume the loop the failure exists to +stop, and exit instructions would send a host into a disk it cannot boot. Unlike +the other failure causes, this one is not terminal for the machine: a +`rebootWithCustomIpxe` request restarts provisioning from `Assigned/Ready` -- +from either the failed state or a wait still in flight -- and releasing the +instance runs the ordinary deletion flow, so neither needs an admin +force-delete. + ## Host Reprovision State Details (HostReprovisionState) diff --git a/docs/configuration/tenant_management.md b/docs/configuration/tenant_management.md index 1f2dcbb6d6..2c7d860469 100644 --- a/docs/configuration/tenant_management.md +++ b/docs/configuration/tenant_management.md @@ -613,11 +613,33 @@ cloud-init runs the `phone_home` module in its final stage, after the rest of yo **Notes and caveats.** -- Phone-home only gates *status reporting*. It does not change how the host is provisioned or booted -- the reboot into the provisioned OS happens identically whether or not phone-home is enabled. -- If phone-home is enabled but the booted OS never runs cloud-init, or the guest cannot reach the metadata endpoint, the callback never arrives and the instance stays in its provisioning state, never reporting ready. +- The reboot into the provisioned OS happens identically whether or not phone-home is enabled. What differs is how NICo decides a provisioning boot finished: with phone-home, the callback is the signal; without it, NICo waits for the host to stop asking for iPXE instructions (see [Provisioning retries and failure](#provisioning-retries-and-failure)). +- If phone-home is enabled but the booted OS never runs cloud-init, or the guest cannot reach the metadata endpoint, the callback never arrives. The instance stays in its provisioning state until the platform operator's provisioning deadline elapses (60 minutes by default), and then reports `Error`. - The metadata endpoint (by default, `169.254.169.254:7777`) is not a tenant-facing API, and is reachable only from the provisioned host over its link-local metadata link. - Rebooting the instance with a one-time custom iPXE override (`instance update --reboot-with-custom-ipxe=true`) re-arms the gate when phone-home is enabled: NICo clears the recorded contact, so the OS must phone home again before the instance is reported ready. +### Provisioning retries and failure + +A provisioning boot -- initial provisioning, or a reboot with +`--reboot-with-custom-ipxe=true` -- runs your iPXE script and then has to install +an operating system. NICo cannot see inside that install, so it treats the host +returning for iPXE instructions as evidence that the attempt failed, and serves +your script again. A script that cannot fetch its kernel (an expired artifact +token, a moved URL) therefore retries instead of leaving the machine to network +boot against an empty disk forever. + +NICo concludes the install succeeded when the OS phones home, or, for instances +without phone-home, when the host has gone quiet for the site's quiet window (15 +minutes by default). Only then does the instance report `Ready`. + +If neither happens -- after a handful of attempts, or once the site's +provisioning deadline (60 minutes by default) elapses -- the instance reports +`Error` rather than a `Ready` machine with nothing installed. Fix what the script +could not deliver and request another +`instance update --reboot-with-custom-ipxe=true` to start over; the same request +also restarts an attempt that is still in flight. The exact retry budget, quiet +window, and deadline are set by your platform operator. + ### Batch Instance Creation For creating multiple identical instances at once, use `instance batch-create`. Unlike `create`, batch-create takes a single shared spec plus a count -- it provisions N instances with auto-generated names from the same instance type, tenant, and VPC. `nicocli instance batch-create --help` shows: