From c6a08d2f406d1940dbbcf01751a1e7b7af741ee0 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sun, 4 Oct 2026 16:26:14 +0300 Subject: [PATCH] fix: recover model instances after host reboot --- docs/how-to/serve-a-model-on-spark.md | 7 + src/spark/agent.rs | 290 +++++++++++++++++++++++++- src/spark/client.rs | 11 +- src/spark/executor.rs | 98 +++++++-- src/spark/launch.rs | 154 ++++++++++++-- src/spark/mod.rs | 1 - 6 files changed, 527 insertions(+), 34 deletions(-) diff --git a/docs/how-to/serve-a-model-on-spark.md b/docs/how-to/serve-a-model-on-spark.md index b324def..5dbfc32 100644 --- a/docs/how-to/serve-a-model-on-spark.md +++ b/docs/how-to/serve-a-model-on-spark.md @@ -98,6 +98,13 @@ healthy, and confirm the OpenAI-compatible gateway path answers. so its existing tool permissions are left alone. See [network controls and limitations](../reference/spark.md#internet-access-for-development). + After a host reboot, launch waits for an existing running-intent instance + to recover instead of starting another copy of the model. `--dry-run --json` + reports `action: "wait"` while recovery is in progress. The wait is bounded + by the maximum engine startup deadline plus 60 seconds. Check `ps --json` + if recovery fails or is suppressed; explicitly select an instance with + `--model ` when several instances of the model are recovering. + 7. Stop when you are done. An already-absent instance is an idempotent success: ```bash diff --git a/src/spark/agent.rs b/src/spark/agent.rs index 3bb9f20..2ea55fd 100644 --- a/src/spark/agent.rs +++ b/src/spark/agent.rs @@ -2722,8 +2722,9 @@ async fn reconcile_once( if action == ReconcileAction::KeepSuppressed { continue; } - if let Some(observed) = observed { - reconcile_running_engine(database, executor, routes, &instance, observed).await?; + if let Some(observed) = observed.as_ref().filter(|engine| engine.running) { + reconcile_running_engine(database, executor, routes, &instance, observed.clone()) + .await?; complete_recovered_operation( database, &operations, @@ -2742,6 +2743,19 @@ async fn reconcile_once( if !persistent_restart_allowed(database, executor).await? { continue; } + if observed.is_some() { + // The scan already verified the complete catalog and container + // identity. Remove only this exited generation, under the lease, + // before recreating it with its original engine and artifacts. + executor + .stop_instance(StopInstanceInput { + instance_id: instance.id.clone(), + generation: instance.generation, + grace_seconds: 0, + }) + .await + .map_err(|_| ())?; + } let failed = database .record_restart_failure(&instance.id, instance.generation, unix_millis() / 1_000) .await @@ -7461,6 +7475,42 @@ mod tests { } } + struct ExitedReconcileExecutor { + recovery: CountingReconcileExecutor, + stops: Arc, + swap_in_pages_delta: Option, + } + + impl sparkplane_ipc::Handler for ExitedReconcileExecutor { + async fn handle(&self, request: sparkplane_ipc::Request) -> sparkplane_ipc::Response { + let action = &request.params["action"]; + if action.get("reconcile_scan").is_some() { + return MatchedReconcileExecutor { + mutations: Arc::clone(&self.stops), + } + .handle(request) + .await; + } + if action.get("stop_instance").is_some() { + self.stops.fetch_add(1, Ordering::SeqCst); + return sparkplane_ipc::Response::Ok { + schema_version: sparkplane_ipc::SCHEMA_VERSION, + request_id: request.request_id, + result: serde_json::json!({"action":"stop_instance"}), + blob: None, + }; + } + let mut response = self.recovery.handle(request).await; + if let sparkplane_ipc::Response::Ok { result, .. } = &mut response + && result["action"] == "inspect_resources" + { + result["snapshot"]["swap_in_pages_delta"] = + serde_json::json!(self.swap_in_pages_delta); + } + response + } + } + struct MatchedReconcileExecutor { mutations: Arc, } @@ -7602,13 +7652,13 @@ mod tests { serde_json::json!({"action":"reconcile_scan","scan":{ "matched":[{ "instance_id":identity["instance_id"],"generation":identity["generation"], - "container_id":"container-g7","network_id":"network","address":"172.30.0.2","port":8000, + "container_id":"container-g7","network_id":"network","address":"","port":8000, "running":false,"restart_policy":"no","health_method":"GET","health_path":"/health", "allowed_routes":[["GET","/health"],["GET","/v1/models"],["POST","/v1/completions"]], "gateway_profile":crate::spark::gateway::GatewayProfile::text(), "served_model":"Ornith-1.5-9B","semantic_prompt":"Generate one completion token.", - "semantic_max_tokens":1,"startup_deadline_seconds":900,"init_pid":1, - "pid_start_time_ticks":1,"cgroup_path":"/system.slice/docker-container-g7.scope" + "semantic_max_tokens":1,"startup_deadline_seconds":900,"init_pid":0, + "pid_start_time_ticks":0,"cgroup_path":"" }],"missing":[],"quarantined":[]}}) } else { if action.get("disable_restart_policy").is_some() @@ -7863,6 +7913,154 @@ mod tests { database.shutdown().unwrap(); } + #[tokio::test] + async fn launch_waits_for_exact_reboot_generation_over_pinned_https() { + use sha2::{Digest, Sha256}; + use std::os::unix::fs::PermissionsExt; + let (state, database, root) = durable_state().await; + let model = ornith_model(); + database.promote_model(model.clone(), false).await.unwrap(); + let selected = database + .begin_serve(creating_instance(&model)) + .await + .unwrap() + .instance; + state + .routes + .mark_warming(&selected.name, selected.generation); + let routes = state.routes.clone(); + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let certificate = cert.pem(); + let tls = super::tls13_config( + certificate.as_bytes().to_vec(), + signing_key.serialize_pem().into_bytes(), + ) + .await + .unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let handle = axum_server::Handle::new(); + let server_handle = handle.clone(); + let server = tokio::spawn(async move { + axum_server::from_tcp_rustls(listener, tls) + .unwrap() + .handle(server_handle) + .serve(router(state).into_make_service_with_connect_info::()) + .await + .unwrap(); + }); + let config = root.join("launch-client"); + std::fs::create_dir_all(config.join("spark")).unwrap(); + std::fs::create_dir_all(config.join("credentials/spark")).unwrap(); + std::fs::write(config.join("spark/fixture.ca.pem"), &certificate).unwrap(); + let pin = format!("sha256:{:x}", Sha256::digest(certificate.as_bytes())); + std::fs::write(config.join("spark.toml"), format!( + "[hosts.fixture]\nurl='https://localhost:{}'\nca_cert_sha256='{pin}'\ncredential='spark/fixture'\nrequest_timeout_seconds=2\n", + address.port())).unwrap(); + std::fs::write(config.join("credentials/spark/fixture"), TOKEN).unwrap(); + std::fs::set_permissions( + config.join("credentials/spark/fixture"), + std::fs::Permissions::from_mode(0o600), + ) + .unwrap(); + let client_config = config.clone(); + let instance = selected.clone(); + let waiter = tokio::task::spawn_blocking(move || { + let client = + crate::spark::client::SparkClient::load(&client_config, "fixture").unwrap(); + crate::spark::launch::wait_for_ready_instance( + &client, + &instance, + Duration::from_secs(5), + ) + }); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!waiter.is_finished()); + database + .set_instance_observed( + &selected.id, + selected.generation, + crate::spark::wire::InstanceObservedState::Healthy, + Some("/openai/ornith/v1".into()), + None, + None, + ) + .await + .unwrap(); + let route = crate::spark::upstream::ObservedRoute::new( + &selected.id, + selected.generation, + "127.0.0.1".parse().unwrap(), + 9, + [("GET", "/health")], + ) + .unwrap(); + routes.publish( + &selected.name, + selected.model.clone(), + "served-model".into(), + route, + ); + let ready = waiter.await.unwrap().unwrap(); + assert_eq!( + (ready.id, ready.generation), + (selected.id.clone(), selected.generation) + ); + assert!(ready.healthy && ready.endpoint.is_some()); + assert_eq!(database.list_instances().await.unwrap().len(), 1); + + // A different generation must never satisfy an in-flight launch wait. + database.begin_stop(&selected.id).await.unwrap(); + let replacement = database + .begin_serve(creating_instance(&model)) + .await + .unwrap() + .instance; + let changed_config = config.clone(); + let changed = tokio::task::spawn_blocking(move || { + let client = + crate::spark::client::SparkClient::load(&changed_config, "fixture").unwrap(); + crate::spark::launch::wait_for_ready_instance( + &client, + &selected, + Duration::from_secs(5), + ) + }) + .await + .unwrap() + .unwrap_err(); + assert!(changed.message.contains("changed generation")); + let timeout_config = config.clone(); + let waiting = replacement.clone(); + let expired = tokio::task::spawn_blocking(move || { + let client = + crate::spark::client::SparkClient::load(&timeout_config, "fixture").unwrap(); + crate::spark::launch::wait_for_ready_instance(&client, &waiting, Duration::ZERO) + }) + .await + .unwrap() + .unwrap_err(); + assert!(expired.message.contains("still recovering")); + database.begin_stop(&replacement.id).await.unwrap(); + let stopped = tokio::task::spawn_blocking(move || { + let client = crate::spark::client::SparkClient::load(&config, "fixture").unwrap(); + crate::spark::launch::wait_for_ready_instance( + &client, + &replacement, + Duration::from_secs(5), + ) + }) + .await + .unwrap() + .unwrap_err(); + assert!(stopped.message.contains("cannot recover")); + handle.graceful_shutdown(None); + server.await.unwrap(); + database.shutdown().unwrap(); + } + #[tokio::test] async fn deferred_reconciliation_keeps_admission_closed_without_consuming_restart_budget() { for (matched, swap_in_pages_delta, guard_heartbeat) in [ @@ -8020,6 +8218,88 @@ mod tests { server.abort(); } + #[tokio::test] + async fn reboot_exited_generation_is_recreated_through_exact_typed_ipc() { + for swap_in_pages_delta in [None, Some(1), Some(0)] { + let (state, database, root) = durable_state().await; + let socket = root.join("executor.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let prepares = Arc::new(AtomicUsize::new(0)); + let stops = Arc::new(AtomicUsize::new(0)); + let prepared = Arc::new(std::sync::Mutex::new(Vec::new())); + let server = tokio::spawn( + sparkplane_ipc::Server::new(ExitedReconcileExecutor { + recovery: CountingReconcileExecutor { + scans: Arc::new(AtomicUsize::new(0)), + prepares: Arc::clone(&prepares), + prepared: Arc::clone(&prepared), + }, + stops: Arc::clone(&stops), + swap_in_pages_delta, + }) + .serve(listener), + ); + let model = ornith_model(); + database.promote_model(model.clone(), false).await.unwrap(); + database + .accept_operation( + "bootstrap", + "instance.serve", + "01K00000000000000000000000", + &"a".repeat(64), + Some("ornith".into()), + ) + .await + .unwrap(); + let instance = database + .begin_serve(creating_instance(&model)) + .await + .unwrap() + .instance; + let executor = crate::spark::executor::ExecutorClient::new(socket); + super::reconcile_once(&database, &executor, Some(&state.routes), &state.admission) + .await + .unwrap(); + if swap_in_pages_delta != Some(0) { + assert_eq!(stops.load(Ordering::SeqCst), 0); + assert_eq!(prepares.load(Ordering::SeqCst), 0); + assert_eq!( + database + .instance(&instance.id) + .await + .unwrap() + .restart_failures, + 0 + ); + database.shutdown().unwrap(); + server.abort(); + continue; + } + assert_eq!(stops.load(Ordering::SeqCst), 1); + assert_eq!(prepares.load(Ordering::SeqCst), 1); + let prepared = prepared.lock().unwrap().clone(); + assert_eq!(prepared[0]["instance_id"], instance.id); + assert_eq!(prepared[0]["generation"], instance.generation); + assert_eq!( + prepared[0]["engine_fingerprint"], + instance.engine_fingerprint + ); + assert_eq!( + prepared[0]["artifact_fingerprint"], + instance.artifact_fingerprint + ); + let current = database.instance(&instance.id).await.unwrap(); + assert_eq!(current.context_window, instance.context_window); + assert!(!current.healthy); + assert!(!matches!( + state.routes.lookup(&instance.name), + crate::spark::gateway::RouteLookup::Healthy(_) + )); + database.shutdown().unwrap(); + server.abort(); + } + } + #[tokio::test] async fn matched_event_cannot_complete_an_active_semantic_probe() { let (state, database, root) = durable_state().await; diff --git a/src/spark/client.rs b/src/spark/client.rs index cd6a188..cdc9274 100644 --- a/src/spark/client.rs +++ b/src/spark/client.rs @@ -663,10 +663,19 @@ impl SparkClient { &self, id: &str, last_event_id: u64, + ) -> Result { + self.follow_operation_with_timeout(id, last_event_id, self.request_timeout) + } + + pub(crate) fn follow_operation_with_timeout( + &self, + id: &str, + last_event_id: u64, + timeout: Duration, ) -> Result { validate_id(id)?; let deadline = Instant::now() - .checked_add(self.request_timeout) + .checked_add(timeout) .ok_or_else(|| usage("Spark request timeout is too large"))?; let route = format!("api/sparkplane/v1/operations/{id}/events"); let url = self diff --git a/src/spark/executor.rs b/src/spark/executor.rs index 3889e3b..10da9b6 100644 --- a/src/spark/executor.rs +++ b/src/spark/executor.rs @@ -2333,11 +2333,6 @@ fn observed_from_inspect( return Err(()); } let endpoint = networks.values().next().ok_or(())?; - let address = endpoint - .ip_address - .clone() - .filter(|value| !value.is_empty()) - .ok_or(())?; let network_id = endpoint .network_id .clone() @@ -2347,7 +2342,18 @@ fn observed_from_inspect( .state .as_ref() .and_then(|state| state.running) - .unwrap_or(false); + .ok_or(())?; + // Docker retains a stopped container's network membership, but drops its + // endpoint and process. Keep that exact generation observable for recovery. + let address = if running { + endpoint + .ip_address + .clone() + .filter(|value| !value.is_empty()) + .ok_or(())? + } else { + String::new() + }; let restart_policy = inspect .host_config .as_ref() @@ -2355,14 +2361,19 @@ fn observed_from_inspect( .and_then(|policy| policy.name) .map(|name| name.to_string()) .unwrap_or_else(|| "no".into()); - let init_pid = inspect - .state - .as_ref() - .and_then(|state| state.pid) - .and_then(|pid| u32::try_from(pid).ok()) - .filter(|pid| *pid > 0) - .ok_or(())?; - let (pid_start_time_ticks, cgroup_path) = process_identity(init_pid)?; + let (init_pid, pid_start_time_ticks, cgroup_path) = if running { + let init_pid = inspect + .state + .as_ref() + .and_then(|state| state.pid) + .and_then(|pid| u32::try_from(pid).ok()) + .filter(|pid| *pid > 0) + .ok_or(())?; + let (start_time, cgroup) = process_identity(init_pid)?; + (init_pid, start_time, cgroup) + } else { + (0, 0, String::new()) + }; if inspect .network_settings .as_ref() @@ -5031,6 +5042,65 @@ mod tests { assert_eq!(inspect_managed_running(&inspect, &input), Ok(false)); } + #[test] + fn reboot_exited_container_without_endpoint_or_process_is_observable() { + let mut inspect = bollard::models::ContainerInspectResponse { + id: Some("a".repeat(64)), + config: Some(bollard::models::ContainerConfig { + labels: Some(std::collections::HashMap::from([ + ( + "io.sparkplane.instance".into(), + format!("i_{}", "1".repeat(32)), + ), + ("io.sparkplane.generation".into(), "12".into()), + ])), + ..Default::default() + }), + state: Some(bollard::models::ContainerState { + running: Some(false), + pid: Some(0), + ..Default::default() + }), + network_settings: Some(bollard::models::NetworkSettings { + networks: Some(std::collections::HashMap::from([( + "sparkplane-internal".into(), + EndpointSettings { + network_id: Some("network-id".into()), + ip_address: Some(String::new()), + ..Default::default() + }, + )])), + ..Default::default() + }), + ..Default::default() + }; + let observed = observed_from_inspect(inspect.clone(), 8000).unwrap(); + assert!(!observed.running); + assert_eq!(observed.generation, 12); + assert!(observed.address.is_empty()); + assert_eq!((observed.init_pid, observed.pid_start_time_ticks), (0, 0)); + assert!(observed.cgroup_path.is_empty()); + + inspect.state.as_mut().unwrap().running = None; + assert!(observed_from_inspect(inspect.clone(), 8000).is_err()); + + // A live engine still needs its endpoint and exact process identity. + inspect.state.as_mut().unwrap().running = Some(true); + assert!(observed_from_inspect(inspect.clone(), 8000).is_err()); + inspect + .network_settings + .as_mut() + .unwrap() + .networks + .as_mut() + .unwrap() + .values_mut() + .next() + .unwrap() + .ip_address = Some("172.30.0.2".into()); + assert!(observed_from_inspect(inspect, 8000).is_err()); + } + fn managed_event(action: &str) -> EventMessage { EventMessage { typ: Some(EventMessageTypeEnum::CONTAINER), diff --git a/src/spark/launch.rs b/src/spark/launch.rs index 9c860d8..44a06f8 100644 --- a/src/spark/launch.rs +++ b/src/spark/launch.rs @@ -6,6 +6,8 @@ use std::{ io::{self, IsTerminal, Read, Write}, path::{Path, PathBuf}, process::{Command, ExitStatus, Stdio}, + thread, + time::{Duration, Instant}, }; use serde::{Deserialize, Serialize}; @@ -139,13 +141,13 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client let instances = client.instances()?; let (model, selected_name) = resolve_launch_model(&models.models, &instances.instances, &model_reference)?; + let saved_name = if selected_name.is_some() { + None + } else { + saved.as_ref().map(|value| value.instance.as_str()) + }; let owned_name = selected_name.unwrap_or_else(|| launch_instance_name(&model)); - let mut instance = resolve_instance( - &instances.instances, - &model, - saved.as_ref().map(|value| value.instance.as_str()), - &owned_name, - )?; + let mut instance = resolve_instance(&instances.instances, &model, saved_name, &owned_name)?; let reused_instance = instance.is_some(); if args.dry_run { @@ -165,6 +167,11 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client )); } } + let action = match instance.as_ref() { + Some(value) if !value.healthy => "wait", + Some(_) => "reuse", + None => "serve", + }; return render_plan( &LaunchPlan { schema: LAUNCH_PLAN_SCHEMA, @@ -179,7 +186,7 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client .unwrap_or(owned_name), endpoint: instance.and_then(|value| value.endpoint), reused_instance, - action: if reused_instance { "reuse" } else { "serve" }, + action, token_id: None, extra_argument_count: args.extra_args.len(), allow_network: args.allow_network, @@ -200,7 +207,7 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client dry_run: false, }, )?; - client.follow_operation(&operation.id, 0)?; + client.follow_operation_with_timeout(&operation.id, 0, launch_readiness_timeout())?; instance = Some( client .instances()? @@ -226,6 +233,12 @@ pub fn run(host: &str, config_dir: &Path, args: LaunchArgs) -> Result<(), Client "Spark launch instance resolution ended without an instance", ) })?; + let instance = if instance.healthy && instance.endpoint.is_some() { + instance + } else { + eprintln!("Waiting for Spark instance {} to recover…", instance.name); + wait_for_ready_instance(&client, &instance, launch_readiness_timeout())? + }; let (token_id, token) = ensure_inference_token(&client, host, config_dir, &mut state)?; configure_integration(args.integration, config_dir, host, &instance, &model)?; @@ -569,21 +582,41 @@ fn resolve_instance( saved_name: Option<&str>, owned_name: &str, ) -> Result, ClientError> { - let healthy = instances + let available = instances .iter() - .filter(|value| value.model_id == model.id && value.healthy && value.endpoint.is_some()) + .filter(|value| { + value.model_id == model.id + && value.desired == super::wire::InstanceDesiredState::Running + && !value.restart_suppressed + && value.quarantine.is_none() + }) .cloned() .collect::>(); if let Some(saved) = saved_name - && let Some(instance) = healthy.iter().find(|value| value.name == saved) + && let Some(instance) = available.iter().find(|value| value.name == saved) { return Ok(Some(instance.clone())); } - if let Some(instance) = healthy.iter().find(|value| value.name == owned_name) { + if let Some(instance) = available.iter().find(|value| value.name == owned_name) { return Ok(Some(instance.clone())); } + let healthy = available + .iter() + .filter(|value| value.healthy && value.endpoint.is_some()) + .cloned() + .collect::>(); match healthy.as_slice() { - [] => Ok(None), + [] => match available.as_slice() { + [] => Ok(None), + [instance] => Ok(Some(instance.clone())), + many => Err(usage(format!( + "multiple recovering Spark instances serve the requested model: {}; select an instance with --model ", + many.iter() + .map(|value| value.name.as_str()) + .collect::>() + .join(", ") + ))), + }, [instance] => Ok(Some(instance.clone())), many => Err(usage(format!( "multiple healthy Spark instances serve the requested model: {}", @@ -595,6 +628,63 @@ fn resolve_instance( } } +fn launch_readiness_timeout() -> Duration { + Duration::from_secs(super::MAX_ENGINE_STARTUP_DEADLINE_SECONDS + 60) +} + +pub(crate) fn wait_for_ready_instance( + client: &SparkClient, + selected: &InstanceDocument, + timeout: Duration, +) -> Result { + let deadline = Instant::now() + timeout; + loop { + let current = client + .instances()? + .instances + .into_iter() + .find(|value| value.id == selected.id) + .ok_or_else(|| { + failure( + EXIT_REJECTED, + "selected Spark instance disappeared during recovery", + ) + })?; + if current.generation != selected.generation || current.model_id != selected.model_id { + return Err(failure( + EXIT_REJECTED, + "selected Spark instance changed generation during recovery; retry launch", + )); + } + if current.desired != super::wire::InstanceDesiredState::Running + || current.restart_suppressed + || current.quarantine.is_some() + { + return Err(failure( + EXIT_REJECTED, + format!( + "Spark instance {} cannot recover; inspect its state with `sparkplane ps --json`", + selected.name + ), + )); + } + if current.healthy && current.endpoint.is_some() { + return Ok(current); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(failure( + EXIT_REJECTED, + format!( + "Spark instance {} is still recovering; inspect its state with `sparkplane ps --json`", + selected.name + ), + )); + } + thread::sleep(remaining.min(Duration::from_secs(1))); + } +} + fn launch_instance_name(model: &ModelDocument) -> String { let mut slug = model .aliases @@ -1751,6 +1841,44 @@ mod tests { assert_eq!(actual.name, "saved"); } + #[test] + fn reboot_recovering_instance_is_selected_instead_of_a_second_model_copy() { + let model = model("m_one", "qwen3.8:flash-next"); + let mut recovering = instance("qwen38-vllm", &model.id); + recovering.healthy = false; + recovering.endpoint = None; + recovering.observed = InstanceObservedState::Degraded; + let instances = [recovering.clone()]; + let actual = resolve_instance(&instances, &model, None, &launch_instance_name(&model)) + .unwrap() + .unwrap(); + assert_eq!( + (actual.name, actual.generation), + (recovering.name, recovering.generation) + ); + } + + #[test] + fn stopped_or_suppressed_instances_are_not_waited_on_by_launch() { + let model = model("m_one", "qwen3.8:flash-next"); + let mut unavailable = instance("qwen38-vllm", &model.id); + unavailable.healthy = false; + unavailable.endpoint = None; + unavailable.desired = InstanceDesiredState::Stopped; + assert!( + resolve_instance(std::slice::from_ref(&unavailable), &model, None, "owned") + .unwrap() + .is_none() + ); + unavailable.desired = InstanceDesiredState::Running; + unavailable.restart_suppressed = true; + assert!( + resolve_instance(&[unavailable], &model, None, "owned") + .unwrap() + .is_none() + ); + } + #[test] fn launch_name_is_stable_and_safe() { let name = launch_instance_name(&model("m_one", "Ornith-1.5:9B")); diff --git a/src/spark/mod.rs b/src/spark/mod.rs index adf3084..8eff81b 100644 --- a/src/spark/mod.rs +++ b/src/spark/mod.rs @@ -47,5 +47,4 @@ pub const EXIT_USAGE: i32 = 2; pub const EXIT_REJECTED: i32 = 3; pub const EXIT_UNREACHABLE: i32 = 4; pub const EXIT_OPERATION_FAILED: i32 = 5; -#[cfg(feature = "appliance")] pub const MAX_ENGINE_STARTUP_DEADLINE_SECONDS: u64 = 1_800;