diff --git a/.config/mise/tasks/ci.toml b/.config/mise/tasks/ci.toml index a29693fa..d1cca0f0 100644 --- a/.config/mise/tasks/ci.toml +++ b/.config/mise/tasks/ci.toml @@ -161,6 +161,10 @@ description = "CI's `mutants` job's command: the wrapper over cargo-mutants. The # `--in-place` mutates this checkout, which the runner throws away, so rust-cache can restore # dependencies into its target. Run from a worktree you keep, pass it without `--in-place`, or # run `mise run mutants`. +# The CLI's parity test spawns microvms-core's `parity-tunnel-peer` example from the test +# binary's target directory, and a mutant's `cargo test -p microvms-cli` never builds another +# package's example, so the fixture is built first into the target `--in-place` tests use. +depends = ["parity:fixtures"] tools = { "cargo:cargo-mutants" = "{{vars.cargo_mutants}}" } run = "./tools/check-mutants.py" diff --git a/bindings/microvms-js/index.d.ts b/bindings/microvms-js/index.d.ts index 5b6a9a4a..0cd9ae18 100644 --- a/bindings/microvms-js/index.d.ts +++ b/bindings/microvms-js/index.d.ts @@ -192,8 +192,8 @@ export declare class BuildHookTimeout { * MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image * administration (list, delete, versions and their status, builds). * - * Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, - * STATE-12). Use it when a process has only an identifier. + * Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7). Use + * it when a process has only an identifier. */ export declare class ControlPlane { /** @@ -207,7 +207,10 @@ export declare class ControlPlane { list(options?: ListMicrovmsOptions | undefined | null): Promise> /** `SuspendMicrovm`. Resolves once accepted; `waitForState` for SUSPENDED. */ suspend(microvmId: string): Promise - /** `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. */ + /** + * `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. A VM that ended + * before the resume, whatever ended it, rejects with `ERR_WINDOW_CLOSED` (STATE-12). + */ resume(microvmId: string): Promise /** `TerminateMicrovm`. Resolves once accepted; `waitForState` for TERMINATED. */ terminate(microvmId: string): Promise @@ -996,11 +999,21 @@ export declare class Sandbox { * # What the core refuses, before any wire call * * A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, and - * even a call the service accepted would hand back a different machine. A resume from - * anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended window - * (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates a - * suspended VM once that window passes, so there is nothing left to resume, and calling - * would cost the full poll timeout to learn something worse. + * even a call the service accepted would hand back a different machine. And a resume + * from anything but SUSPENDED (STATE-7). + * + * # What the service decides (STATE-12) + * + * How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy + * suspended once `suspendedSec` passes, and a VM this sandbox suspended stayed resumable + * 1283 s into a 60 s window (measured 2026-10-12, us-east-1). `ERR_WINDOW_CLOSED` is the + * service's answer once a resume reaches it: the service refused the resume, and the VM + * is TERMINATING or TERMINATED, whatever ended it. The sandbox is TERMINATED after it. + * + * A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as + * RUNNING, so `resume` rejects with `ERR_INVALID_ARG` before any call (STATE-7). That + * suspend reaches a caller only through `Sandbox.adopt`, which reads the state from the + * service, or `ControlPlane.resume`, which holds no lifecycle to refuse from. * * Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory * token survived the freeze, and re-delivering it would hit the daemon's one-shot diff --git a/bindings/microvms-js/src/control.rs b/bindings/microvms-js/src/control.rs index 66a7dd9c..ead8c404 100644 --- a/bindings/microvms-js/src/control.rs +++ b/bindings/microvms-js/src/control.rs @@ -279,8 +279,8 @@ pub struct DeleteImageOptions { /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image /// administration (list, delete, versions and their status, builds). /// -/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, -/// STATE-12). Use it when a process has only an identifier. +/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7). Use +/// it when a process has only an identifier. #[napi] pub struct ControlPlane { inner: Arc, @@ -344,7 +344,8 @@ impl ControlPlane { self.inner.suspend(µvm_id).await.map_err(js_async) } - /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. + /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. A VM that ended + /// before the resume, whatever ended it, rejects with `ERR_WINDOW_CLOSED` (STATE-12). #[napi] pub async fn resume(&self, microvm_id: String) -> Result<(), AsyncError> { self.inner.resume(µvm_id).await.map_err(js_async) diff --git a/bindings/microvms-js/src/sandbox.rs b/bindings/microvms-js/src/sandbox.rs index 55d1e84b..517085e0 100644 --- a/bindings/microvms-js/src/sandbox.rs +++ b/bindings/microvms-js/src/sandbox.rs @@ -1144,11 +1144,21 @@ impl Sandbox { /// # What the core refuses, before any wire call /// /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, and - /// even a call the service accepted would hand back a different machine. A resume from - /// anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended window - /// (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates a - /// suspended VM once that window passes, so there is nothing left to resume, and calling - /// would cost the full poll timeout to learn something worse. + /// even a call the service accepted would hand back a different machine. And a resume + /// from anything but SUSPENDED (STATE-7). + /// + /// # What the service decides (STATE-12) + /// + /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy + /// suspended once `suspendedSec` passes, and a VM this sandbox suspended stayed resumable + /// 1283 s into a 60 s window (measured 2026-10-12, us-east-1). `ERR_WINDOW_CLOSED` is the + /// service's answer once a resume reaches it: the service refused the resume, and the VM + /// is TERMINATING or TERMINATED, whatever ended it. The sandbox is TERMINATED after it. + /// + /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as + /// RUNNING, so `resume` rejects with `ERR_INVALID_ARG` before any call (STATE-7). That + /// suspend reaches a caller only through `Sandbox.adopt`, which reads the state from the + /// service, or `ControlPlane.resume`, which holds no lifecycle to refuse from. /// /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory /// token survived the freeze, and re-delivering it would hit the daemon's one-shot diff --git a/bindings/microvms-py/microvms.pyi b/bindings/microvms-py/microvms.pyi index 29645041..a575905e 100644 --- a/bindings/microvms-py/microvms.pyi +++ b/bindings/microvms-py/microvms.pyi @@ -497,9 +497,9 @@ class ControlPlane: MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image administration (list, delete, versions and their status, builds). - Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, - STATE-12): a suspend of a SUSPENDED VM is the service's to refuse. Use it when a - process has only an identifier, such as a durable workflow step in a fresh process. + Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7): a + suspend of a SUSPENDED VM is the service's to refuse. Use it when a process has only an + identifier, such as a durable workflow step in a fresh process. """ def __new__(cls, /, region: Region) -> ControlPlane: """ @@ -582,7 +582,8 @@ class ControlPlane: """ def resume(self, /, microvm_id: str) -> None: """ - `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. + `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. A VM that ended + before the resume, whatever ended it, raises `WindowClosedError` (STATE-12). """ async def resume_async(self, /, microvm_id: str) -> None: """ @@ -2650,11 +2651,22 @@ class Sandbox: # What the core refuses, before any wire call A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, - and even a call the service accepted would hand back a different machine. A resume - from anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended - window (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates - a suspended VM once that window passes, so there is nothing left to resume, and - calling would cost the full poll timeout to learn something worse. + and even a call the service accepted would hand back a different machine. And a + resume from anything but SUSPENDED (STATE-7). + + # What the service decides (STATE-12) + + How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy + suspended once `suspended_sec` passes, and a VM this sandbox suspended stayed + resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1). + `WindowClosedError` is the service's answer once a resume reaches it: the service + refused the resume, and the VM is TERMINATING or TERMINATED, whatever ended it. The + sandbox is TERMINATED after it. + + A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as + RUNNING, so `resume` raises `InvalidArgError` before any call (STATE-7). That suspend + reaches a caller only through `Sandbox.adopt`, which reads the state from the service, + or `ControlPlane.resume`, which holds no lifecycle to refuse from. Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory token survived the freeze, and re-delivering it would hit the daemon's one-shot @@ -3849,6 +3861,8 @@ class UnexpectedError(MicrovmError): class WindowClosedError(MicrovmError): """ - ERR_WINDOW_CLOSED — the launch-time suspended window passed, so there is nothing to - resume. A longer window has to be set at launch on the next VM. + ERR_WINDOW_CLOSED — the MicroVM ended before the resume (the suspended window, the + maximum duration, or a terminate), and it can't be resumed; launch a new one. The + suspended window ends a VM the idle policy suspended; an explicit suspend stayed + resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1). """ diff --git a/bindings/microvms-py/src/control.rs b/bindings/microvms-py/src/control.rs index 5fbe7832..ce0752e7 100644 --- a/bindings/microvms-py/src/control.rs +++ b/bindings/microvms-py/src/control.rs @@ -444,9 +444,9 @@ impl PyImageBuild { /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image /// administration (list, delete, versions and their status, builds). /// -/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, -/// STATE-12): a suspend of a SUSPENDED VM is the service's to refuse. Use it when a -/// process has only an identifier, such as a durable workflow step in a fresh process. +/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7): a +/// suspend of a SUSPENDED VM is the service's to refuse. Use it when a process has only an +/// identifier, such as a durable workflow step in a fresh process. #[pyclass(frozen, name = "ControlPlane", module = "microvms")] pub struct PyControlPlane { inner: Arc, @@ -703,7 +703,8 @@ impl PyControlPlane { Ok(runtime::spawn_shielded(self.suspend_op(microvm_id)).await?) } - /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. + /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. A VM that ended + /// before the resume, whatever ended it, raises `WindowClosedError` (STATE-12). fn resume(&self, py: Python<'_>, microvm_id: String) -> PyCoreResult<()> { Ok(runtime::block_on(py, self.resume_op(microvm_id))?) } diff --git a/bindings/microvms-py/src/errors.rs b/bindings/microvms-py/src/errors.rs index a6647853..be8962a1 100644 --- a/bindings/microvms-py/src/errors.rs +++ b/bindings/microvms-py/src/errors.rs @@ -87,8 +87,10 @@ create_exception!( microvms, WindowClosedError, MicrovmError, - "ERR_WINDOW_CLOSED — the launch-time suspended window passed, so there is nothing to \ - resume. A longer window has to be set at launch on the next VM." + "ERR_WINDOW_CLOSED — the MicroVM ended before the resume (the suspended window, the \ + maximum duration, or a terminate), and it can't be resumed; launch a new one. The \ + suspended window ends a VM the idle policy suspended; an explicit suspend stayed \ + resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1)." ); create_exception!( microvms, diff --git a/bindings/microvms-py/src/sandbox.rs b/bindings/microvms-py/src/sandbox.rs index f7d6371e..449afbd6 100644 --- a/bindings/microvms-py/src/sandbox.rs +++ b/bindings/microvms-py/src/sandbox.rs @@ -1904,11 +1904,22 @@ impl PySandbox { /// # What the core refuses, before any wire call /// /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, - /// and even a call the service accepted would hand back a different machine. A resume - /// from anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended - /// window (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates - /// a suspended VM once that window passes, so there is nothing left to resume, and - /// calling would cost the full poll timeout to learn something worse. + /// and even a call the service accepted would hand back a different machine. And a + /// resume from anything but SUSPENDED (STATE-7). + /// + /// # What the service decides (STATE-12) + /// + /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy + /// suspended once `suspended_sec` passes, and a VM this sandbox suspended stayed + /// resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1). + /// `WindowClosedError` is the service's answer once a resume reaches it: the service + /// refused the resume, and the VM is TERMINATING or TERMINATED, whatever ended it. The + /// sandbox is TERMINATED after it. + /// + /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as + /// RUNNING, so `resume` raises `InvalidArgError` before any call (STATE-7). That suspend + /// reaches a caller only through `Sandbox.adopt`, which reads the state from the service, + /// or `ControlPlane.resume`, which holds no lifecycle to refuse from. /// /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory /// token survived the freeze, and re-delivering it would hit the daemon's one-shot diff --git a/changelog.d/597.fixed.md b/changelog.d/597.fixed.md new file mode 100644 index 00000000..a08ad17c --- /dev/null +++ b/changelog.d/597.fixed.md @@ -0,0 +1,14 @@ +- **A resume past the suspended window reaches the service (#597).** `Sandbox::resume` refused + with `ERR_WINDOW_CLOSED` once `suspendedDurationSeconds` had passed since the sandbox's own + `suspend()`, before any call. The service didn't hold an explicit suspend to that window: + measured 2026-10-12 in us-east-1, two VMs suspended with `SuspendMicrovm`, under the idle + case's own policy, resumed about 910 s into a 60 s window, and a third at 1283 s, so the + client refused resumes the service honors. AWS doesn't promise that behavior, and the client + no longer depends on either answer. The window ends a VM the idle + policy suspended, and the service then refuses the resume with a `ValidationException`. + `ERR_WINDOW_CLOSED` now reports that refusal, for a VM `GetMicrovm` reports TERMINATING or + TERMINATED, with its state and `stateReason`, and the sandbox is TERMINATED after it. The + mapping is core's `ControlPlane::resume`, so `microvm resume`, which exited `ERR_PLATFORM` + (9) there, now exits `ERR_WINDOW_CLOSED` (8), and both bindings' `ControlPlane.resume` raise + the same code. A VM a terminate or its maximum duration ended gets the same refusal, so the + error names no cause, and it carries the service's refusal as its source. diff --git a/crates/microvms-app/src/clock.rs b/crates/microvms-app/src/clock.rs index 56853039..75afb7c8 100644 --- a/crates/microvms-app/src/clock.rs +++ b/crates/microvms-app/src/clock.rs @@ -7,11 +7,11 @@ //! sources: the control plane's read `std::time::Instant` and the session's read tokio's. //! Under a deterministic simulator those disagree. tokio's clock is virtual there and `std`'s //! isn't, so a control-plane wait whose sleeps advanced virtual time measured its deadline -//! on the host's real time and never reached it. `Sandbox` already times the suspended window -//! on the plane's clock rather than its own, so a window isn't measured on one clock while -//! the polls run on another. One trait, with tokio's clock in production, keeps that property -//! for everything that reads time. The trait is re-exported at `control::Clock` and at -//! `session::Clock`, the paths each half had. +//! on the host's real time and never reached it. `Sandbox` already waits on the plane's clock +//! rather than one of its own, so a deadline isn't measured on one clock while the polls sleep +//! on another. One trait, with tokio's clock in production, keeps that property for everything +//! that reads time. The trait is re-exported at `control::Clock` and at `session::Clock`, the +//! paths each half had. //! //! # Why a trait rather than `tokio::time::pause` //! @@ -24,11 +24,11 @@ //! # Monotonic and wall time are separate readings //! //! [`Clock::elapsed`] is monotonic, the same reasoning the Python client's injectable -//! `time.monotonic` carried: the suspended window is a *duration*, and a wall clock that steps -//! backward would reopen a closed one. [`Clock::unix_now`] is the wall reading, for the values -//! that are dates rather than durations: a name record's timestamp, an exec id, the cost -//! engine's today. Deriving one from the other would make a stepped wall clock move a -//! deadline. +//! `time.monotonic` carried: a wait's deadline and a proxy token's age are *durations*, and a +//! wall clock that steps backward would hold a passed deadline open. [`Clock::unix_now`] is the +//! wall reading, for the values that are dates rather than durations: a name record's +//! timestamp, an exec id, the cost engine's today. Deriving one from the other would make a +//! stepped wall clock move a deadline. use std::fmt; use std::future::Future; diff --git a/crates/microvms-app/src/control/microvm.rs b/crates/microvms-app/src/control/microvm.rs index 8e86ecf1..96119564 100644 --- a/crates/microvms-app/src/control/microvm.rs +++ b/crates/microvms-app/src/control/microvm.rs @@ -799,11 +799,56 @@ impl ControlPlane { } /// `ResumeMicrovm`. Thaws the VM; no token re-delivery and no re-bootstrap. + /// + /// # A VM that ended before the resume is `WindowClosed` (STATE-12) + /// + /// Nothing is checked against the suspended window before the call, because the service + /// decides whether a suspended VM can still resume. It terminates a VM the idle policy + /// suspended once `suspendedDurationSeconds` passes, and it doesn't hold an explicit + /// `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, an explicitly + /// suspended VM resumed 1283 s into a 60 s window (#597). A resume of a VM that has + /// terminated is refused with a `ValidationException` ("The MicroVM has been + /// terminated and its state cannot be changed."), the same status as any refused value, + /// and it was the same refusal for a VM the service ended and one `TerminateMicrovm` + /// ended (measured 2026-10-12, us-east-1). So a refusal reads the VM's state rather than + /// the message, and a VM that's TERMINATING or TERMINATED is [`ErrorKind::WindowClosed`] + /// with its state and `stateReason`, carrying the refusal as its source. The error names + /// no cause, because the client doesn't know which of the suspended window, the maximum + /// duration, or a terminate ended the VM; the `stateReason` is the service's word on it. + /// Every other refusal comes back as the service gave it. pub async fn resume(&self, id: &str) -> Result<(), Error> { super::require_valid_identifier("microvmIdentifier", id)?; let call = Call::post_empty("ResumeMicrovm", paths::resume(id)); - send_with_retry(self.transport(), call).await?; - Ok(()) + match send_with_retry(self.transport(), call).await { + Ok(_) => Ok(()), + Err(refused) if refused.kind() == ErrorKind::Platform => { + Err(self.ended_before_resume(id, refused).await) + } + Err(error) => Err(error), + } + } + + /// The error a refused resume reports: [`ErrorKind::WindowClosed`] with `refused` as its + /// source when `GetMicrovm` reports the VM dead, and `refused` otherwise, including when + /// that read fails. + async fn ended_before_resume(&self, id: &str, refused: Error) -> Error { + let Ok(vm) = self.get_microvm(id).await else { + return refused; + }; + if !crate::constants::DEAD_STATES.contains(&vm.state.as_str()) { + return refused; + } + Error::new( + ErrorKind::WindowClosed, + format!( + "microvm {id} can't be resumed: the service refused ResumeMicrovm, and \ + GetMicrovm reports the VM {} ({}). A VM that has ended doesn't come back; \ + launch a new one.", + vm.state, + vm.reason(), + ), + ) + .with_source(refused) } /// `TerminateMicrovm`. @@ -1745,6 +1790,156 @@ mod tests { ); } + /// `ResumeMicrovm`'s refusal of a VM the service terminated, as it answered on 2026-10-12 + /// in us-east-1 (#597): a 400 whose error type is `ValidationException`. + fn terminated_refusal() -> Answer { + Answer { + status: 400, + body: r#"{"message": "The MicroVM mvm-abc123 has been terminated and its state cannot be changed."}"# + .to_string(), + error_type: Some("ValidationException".to_string()), + } + } + + /// **STATE-12 at the control plane.** A resume the service refuses for a VM `GetMicrovm` + /// reports in either dead state is `WindowClosed`, naming the state and its `stateReason`. + /// The CLI's `resume`, both bindings' `ControlPlane.resume` and `Sandbox::resume` call + /// this, so every resume that reaches the service reports an ended VM the same way. A + /// `Sandbox` whose VM the idle policy suspended still holds it as RUNNING, and its resume + /// is refused from RUNNING before any call: that suspend reaches a caller only through + /// `Sandbox::adopt`, which reads the state from the service, or `ControlPlane::resume`, + /// which holds no lifecycle to refuse from. + #[tokio::test] + async fn a_resume_refused_for_a_vm_that_terminated_while_suspended_is_window_closed() { + for state in crate::constants::DEAD_STATES { + let (plane, fake, _) = planted(); + fake.answer("ResumeMicrovm", terminated_refusal()).answer( + "GetMicrovm", + Answer::ok(fake::microvm_response( + state, + Some("MicroVM exceeded maximum lifetime."), + )), + ); + + let error = plane + .resume("mvm-abc123") + .await + .expect_err("the VM is gone"); + + assert_eq!(error.kind(), ErrorKind::WindowClosed, "{state}: {error}"); + let message = error.to_string(); + assert!(message.contains(state), "{state}: {message}"); + assert!( + message.contains("MicroVM exceeded maximum lifetime."), + "{state}: the stateReason: {message}" + ); + assert_eq!( + fake.operations(), + ["ResumeMicrovm", "GetMicrovm"], + "{state}: one refused resume, then one read" + ); + } + } + + /// **STATE-12 for a VM a terminate ended.** On 2026-10-12 in us-east-1, `GetMicrovm` on a + /// VM `microvm terminate` ended answered TERMINATED with `stateReason` `Success.` about + /// 4 h later, and `ResumeMicrovm` refused it with the same `ValidationException` as a VM + /// the service ended. The error states what the client knows, the state, the + /// `stateReason` and the refusal, and names no suspended window it can't know about. The + /// refusal is the error's source, and the `stateReason`'s own period isn't doubled. + #[tokio::test] + async fn a_resume_refused_for_a_vm_a_terminate_ended_names_no_cause_and_keeps_the_refusal() { + let (plane, fake, _) = planted(); + fake.answer("ResumeMicrovm", terminated_refusal()).answer( + "GetMicrovm", + Answer::ok(fake::microvm_response("TERMINATED", Some("Success."))), + ); + + let error = plane + .resume("mvm-abc123") + .await + .expect_err("the VM is gone"); + + assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}"); + let message = error.to_string(); + assert_eq!( + message, + "microvm mvm-abc123 can't be resumed: the service refused ResumeMicrovm, and \ + GetMicrovm reports the VM TERMINATED (Success.). A VM that has ended doesn't come \ + back; launch a new one." + ); + assert!(!message.contains(".."), "{message}"); + assert!( + !message.contains("suspended"), + "no cause it can't know: {message}" + ); + + let source = std::error::Error::source(&error).expect("the refusal is the source"); + let refused = source + .downcast_ref::() + .expect("the source is the service's refusal"); + assert_eq!(refused.kind(), ErrorKind::Platform); + assert!( + refused + .to_string() + .contains("has been terminated and its state cannot be changed"), + "the service's own text: {refused}" + ); + } + + /// A refused resume of a VM `GetMicrovm` reports alive, or whose state that read can't + /// get, comes back as the service's refusal: only a VM reported dead is `WindowClosed`. + #[tokio::test] + async fn a_resume_refused_for_a_live_or_unreadable_vm_keeps_the_services_answer() { + let (plane, fake, _) = planted(); + fake.answer( + "ResumeMicrovm", + Answer::failure(409, "The MicroVM is changing state."), + ) + .answer( + "GetMicrovm", + Answer::ok(fake::microvm_response("SUSPENDED", None)), + ); + let error = plane + .resume("mvm-abc123") + .await + .expect_err("the service refused"); + assert_eq!(error.kind(), ErrorKind::Platform, "{error}"); + assert!(error.to_string().contains("ConflictException"), "{error}"); + + let (plane, fake, _) = planted(); + fake.answer("ResumeMicrovm", terminated_refusal()) + .answer("GetMicrovm", Answer::failure(404, "not found")); + let error = plane + .resume("mvm-abc123") + .await + .expect_err("the service refused"); + assert_eq!(error.kind(), ErrorKind::Platform, "{error}"); + let message = error.to_string(); + assert!( + message.contains("ResumeMicrovm failed") && !message.contains("GetMicrovm"), + "the resume's refusal rather than the read's: {message}" + ); + } + + /// A resume denied for its credentials is not the service's word on the VM, so no state + /// is read and the denial comes back as it was. + #[tokio::test] + async fn a_resume_denied_for_its_credentials_reads_no_state() { + let (plane, fake, _) = planted(); + fake.answer( + "ResumeMicrovm", + Answer::failure(403, "User is not authorized to perform ResumeMicrovm"), + ) + .answer( + "GetMicrovm", + Answer::ok(fake::microvm_response("TERMINATED", None)), + ); + let error = plane.resume("mvm-abc123").await.expect_err("denied"); + assert_eq!(error.kind(), ErrorKind::Credentials, "{error}"); + assert_eq!(fake.call_count("GetMicrovm"), 0); + } + /// **TRAP-7.** The minted token is read out of the `authToken` **map**, and /// [`ProxyToken::headers`] emits both headers. #[tokio::test] diff --git a/crates/microvms-app/src/control/ops.rs b/crates/microvms-app/src/control/ops.rs index ae1c3c11..36833c62 100644 --- a/crates/microvms-app/src/control/ops.rs +++ b/crates/microvms-app/src/control/ops.rs @@ -367,11 +367,9 @@ pub struct RunMicrovmWire { /// "autoResumeEnabled": false}` (measured 2026-08-15, us-east-1, one read-only /// `GetMicrovm`). /// -/// The claim mattered because STATE-12 was resting on it: if only the client could name the -/// suspended window, the client's own record is the only authority on it. The readback -/// means there is a second one, and [`crate::control::Microvm::idle_policy`] carries it — -/// so a suspended window that disagrees with what was asked for is now observable rather -/// than assumed. +/// The readback means the client's own record isn't the only authority on the suspended +/// window: [`crate::control::Microvm::idle_policy`] carries the service's, so a window that +/// disagrees with what was asked for is observable rather than assumed. #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct IdlePolicy { diff --git a/crates/microvms-app/src/sandbox.rs b/crates/microvms-app/src/sandbox.rs index 182d8523..544535f9 100644 --- a/crates/microvms-app/src/sandbox.rs +++ b/crates/microvms-app/src/sandbox.rs @@ -3,8 +3,8 @@ //! model describes made true of the code. //! //! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the -//! bindings use: build, run, suspend, resume, terminate, the launch-time suspended -//! window (STATE-12), and teardown in the order that does not leak. +//! bindings use: build, run, suspend, resume, terminate, a resume of a VM that ended +//! before it (STATE-12), and teardown in the order that does not leak. //! //! # The lifecycle is a field, and every transition is guarded //! @@ -32,16 +32,20 @@ //! control-plane calls rather than answered by AWS. The test asserts the call count, which //! is the observable that distinguishes the two. //! -//! # The suspended window is checked before the call (STATE-12) +//! # The service decides whether a suspended VM resumes (STATE-12) //! -//! The launch-time `idlePolicy` *terminates* a suspended VM once `suspendedDurationSeconds` -//! passes, which means "resume later" silently stops working. A resume past the window is -//! refused locally, before `ResumeMicrovm`, because the alternative is calling and reading -//! the failure: the service answers about a terminated id, which is not the same statement -//! as "the window you set at launch closed", and getting there costs the full poll timeout -//! first. The window comes from this sandbox's own `RunMicrovm` request, falling back to the -//! `idlePolicy` that `GetMicrovm` reports (measured 2026-08-15, `docs/PLATFORM.md`). Only -//! the suspend's start time is the client's alone. +//! The launch-time `idlePolicy` *terminates* a VM the idle policy suspended once +//! `suspendedDurationSeconds` passes, and the service doesn't hold an explicit +//! `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, a VM this client +//! suspended resumed 1283 s into a 60 s window (#597). So a resume is never refused here for +//! the time it has spent suspended. It goes to the service, and the service's refusal for a +//! VM that ended, whatever ended it, comes back from [`ControlPlane::resume`] as +//! [`ErrorKind::WindowClosed`], which this sandbox records as a termination (STATE-11). +//! +//! A suspend the idle policy makes isn't seen here: the sandbox still holds the VM as +//! RUNNING, and its resume is refused before any call (STATE-7). That suspend reaches a +//! caller only through [`Sandbox::adopt`], which reads the state from the service, or +//! [`ControlPlane::resume`], which holds no lifecycle to refuse from. //! //! # Teardown never raises, and the log group is last //! @@ -266,7 +270,8 @@ pub struct RunRequest { pub shell: bool, /// `idlePolicy.maxIdleDurationSeconds`. pub max_idle_sec: u32, - /// `idlePolicy.suspendedDurationSeconds` — the window STATE-12 refuses past. + /// `idlePolicy.suspendedDurationSeconds`: how long a VM the idle policy suspended lasts + /// before the service terminates it. An explicit suspend isn't held to it (STATE-12). pub suspended_sec: u32, /// `idlePolicy.autoResumeEnabled`. pub auto_resume: bool, @@ -404,7 +409,8 @@ impl RunRequest { self } - /// Sets the suspended window this sandbox will refuse a resume past (STATE-12). + /// Sets `idlePolicy.suspendedDurationSeconds`, how long a VM the idle policy suspended + /// lasts before the service terminates it. See [`RunRequest::suspended_sec`]. #[must_use] pub fn with_suspended_sec(mut self, seconds: u32) -> Self { self.suspended_sec = seconds; @@ -618,7 +624,8 @@ impl TokenMinter for ControlPlaneMinter { /// One MicroVM's whole life. /// -/// See the module docs for the state machine, the window, and why teardown is explicit. +/// See the module docs for the state machine, the resume of an ended VM, and why teardown is +/// explicit. pub struct Sandbox { control: Arc, image: Option, @@ -644,8 +651,8 @@ pub struct Sandbox { was_terminated: bool, bootstrap_count: u32, - /// The window from *our own* `RunMicrovm` request. `GetMicrovm` also reports it (in - /// `idlePolicy`), which is the fallback when this sandbox has no request of its own. + /// The suspended window from *our own* `RunMicrovm` request, or `None` for a sandbox that + /// launched nothing. `GetMicrovm` reports the service's in `idlePolicy`. suspended_window: Option, /// Whether the accepted launch carried a caller client token, and so may have adopted /// a VM an earlier attempt launched (#195). @@ -659,8 +666,6 @@ pub struct Sandbox { /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence /// is checked against it. idle_window: Option, - /// The clock reading when the suspend call was accepted, or `None` when not suspended. - suspended_at: Option, /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down /// one. torn_down: bool, @@ -749,10 +754,10 @@ impl Sandbox { /// `Sandbox::from_name`, which resolve credentials for a region, are in /// `microvms_core::prelude`. /// - /// The plane's own clock is what times the suspended window, rather than a second one + /// The plane's own clock is what every wait here sleeps on, rather than a second one /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil - /// tests record: a window measured on one clock while the poll loop sleeps on another - /// calls a closed window open. + /// tests record: a deadline measured on one clock while the poll loop sleeps on another + /// reads the wrong elapsed time. pub fn with_control_plane(control: ControlPlane) -> Self { Self { control: Arc::new(control), @@ -772,7 +777,6 @@ impl Sandbox { launch_posture: None, pending_agent_token: None, idle_window: None, - suspended_at: None, torn_down: false, adopted: false, detached: false, @@ -1351,12 +1355,6 @@ impl Sandbox { /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once /// when the service reports RUNNING, the same as a launch this sandbox made itself. /// - /// # The suspended window (STATE-12) - /// - /// The window comes from the `idlePolicy` `GetMicrovm` reports. A VM adopted while - /// already SUSPENDED has no suspend time this client observed, so the local check has - /// nothing to measure and the service answers the resume instead. - /// /// `endpoint` must agree with the service's when the service reports one: a mismatch /// means the identifiers came from two different records. `agent_token` is the bearer /// credential the VM was launched with; it never appears in `Debug` or an error. @@ -1605,10 +1603,6 @@ impl Sandbox { // neither suspend nor resume accepts, bricking the handle over one bad request. self.control.suspend(&id).await?; self.set_lifecycle(Lifecycle::Suspending); - // Stamped after the call and before the wait, not after the wait: the idlePolicy's - // window starts when the platform begins suspending, so timing it from SUSPENDED - // would under-count the transition and call a closed window open. - self.suspended_at = Some(self.control.clock().elapsed()); let settled = self .control @@ -1655,12 +1649,16 @@ impl Sandbox { /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be /// refused — a 409 that reads like a broken VM. /// - /// # The window is checked first (STATE-12) + /// # The service decides, and a VM it ended is `WindowClosed` (STATE-12) /// - /// Before any wire call, because the answer is already known and calling costs the poll - /// timeout to learn something worse. The falsification is a service reporting - /// TERMINATED: without this check the resume burns the full deadline and then reports a - /// state the client could have named at once. + /// No resume is refused here for the time the VM has spent suspended. The service ends a + /// VM the idle policy suspended once `suspendedDurationSeconds` passes, and it kept a VM + /// this client suspended resumable 1283 s into a 60 s window (measured 2026-10-12, + /// us-east-1, #597). A resume the service refuses because the VM ended, whatever ended + /// it, is [`ErrorKind::WindowClosed`] ([`ControlPlane::resume`]), and this records the VM + /// as terminated, so a second resume is refused here with no call (STATE-11). A VM the + /// idle policy suspended is still RUNNING here, so its resume is refused before any call + /// (STATE-7); see the module docs. /// /// # The proxy token is dropped (STATE-8) /// @@ -1673,7 +1671,7 @@ impl Sandbox { let id = self.require_microvm("resume")?; // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal - // comes before the window check and before any call. + // comes before any call. if self.was_terminated || self.lifecycle == Lifecycle::Terminated { return Err(Error::invalid_arg(format!( "microvm {id} was terminated, and a terminated VM never returns to RUNNING \ @@ -1688,15 +1686,23 @@ impl Sandbox { ))); } - // STATE-12, first and locally. - self.require_open_suspended_window(&id)?; - - self.control.resume(&id).await?; + // STATE-12: a `WindowClosed` refusal is a VM that ended before this resume. It's + // recorded as TERMINATED whichever of TERMINATING or TERMINATED `GetMicrovm` read, + // because the service has said the VM's state can't change, and as a termination, so + // the next resume is refused before any call (STATE-11). Any other refusal leaves the + // sandbox SUSPENDED, and a retry goes to the service again. + if let Err(refused) = self.control.resume(&id).await { + if refused.kind() == ErrorKind::WindowClosed { + self.was_terminated = true; + self.set_lifecycle(Lifecycle::Terminated); + } + return Err(refused); + } // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the - // state this call was made from, so failing on it would fail every resume. A VM the - // idlePolicy terminated during suspension never reaches RUNNING, and waiting only - // for RUNNING there burns the full timeout and then reports a timeout message - // hiding a cause the service had already stated in `stateReason`. + // state this call was made from, so failing on it would fail every resume. A VM that + // terminates once the resume is accepted never reaches RUNNING, and waiting only for + // RUNNING there burns the full timeout and then reports a timeout message hiding a + // cause the service had already stated in `stateReason`. let running = self .control .wait_for_state( @@ -1716,10 +1722,6 @@ impl Sandbox { if let Some(session) = self.session.as_mut() { session.rebind(endpoint); } - // Cleared on success so the next cycle's window is measured from the next suspend. - // Leaving it set would accumulate every suspension's elapsed time into one total and - // reject a resume whose own window is wide open. - self.suspended_at = None; self.session.as_mut().ok_or_else(|| { Error::new( @@ -1730,41 +1732,6 @@ impl Sandbox { }) } - /// Rejects a resume the launch-time `idlePolicy` has already made impossible (STATE-12). - /// - /// The message names the elapsed time, the window, and the `idlePolicy` finding, because - /// "cannot resume" alone sends a reader looking for the flag that reopens it. - fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> { - let reported = self - .microvm - .as_ref() - .and_then(|vm| vm.idle_policy.as_ref()) - .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds))); - let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at) - else { - // No window or no suspend time means this sandbox cannot know how long the VM - // has been suspended, and guessing would refuse a resume the service would honour. - return Ok(()); - }; - let elapsed = self.control.clock().elapsed().saturating_sub(since); - if elapsed <= window { - return Ok(()); - } - Err(Error::new( - ErrorKind::WindowClosed, - format!( - "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \ - window set at launch — the idlePolicy terminates a suspended VM once that window \ - passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \ - Refused before ResumeMicrovm, because calling would cost the full poll timeout \ - to learn the same thing less clearly. A longer window has to be set at launch \ - on the next VM; there is no call that extends this one.", - elapsed.as_secs(), - window.as_secs(), - ), - )) - } - // ── terminate (STATE-9, STATE-10) ──────────────────────────────────────── /// Tears down, best-effort, never erroring. @@ -1907,11 +1874,14 @@ impl Sandbox { let vm = self.microvm.as_ref()?; self.lifecycle.is_live().then(|| { format!( - "warning: the Sandbox for microvm {} was dropped in {} without terminate() or \ - detach(). Nothing was torn down — Drop cannot await, so a teardown here would \ - deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \ - ceiling: terminate it with `microvm terminate {}`.", - vm.id, self.lifecycle, vm.id, + "warning: the Sandbox for microvm {id} was dropped in {state} without \ + terminate() or detach(). Nothing was torn down — Drop cannot await, so a \ + teardown here would deadlock inside a runtime. {state} is the state this \ + sandbox last saw, and the platform may have suspended or ended the VM since. A \ + VM that's still live bills until it ends, at its maximumDurationInSeconds \ + ceiling at the latest: terminate it with `microvm terminate {id}`.", + id = vm.id, + state = self.lifecycle, ) }) } @@ -2476,7 +2446,9 @@ mod tests { } /// **A dropped live sandbox warns through the plane's adapters**, not a stream of its - /// own, so the driving adapter decides where the warning goes (ARCH-7). + /// own, so the driving adapter decides where the warning goes (ARCH-7). The warning names + /// the state as the one the sandbox last saw, since the platform can suspend or end the VM + /// without telling it, and says only that a VM still live bills. /// /// **Falsification**: write the warning straight to stderr in `Drop` and the recorded /// list comes back empty (and the crate's `clippy.toml` refuses `std::io::stderr`). @@ -2504,6 +2476,11 @@ mod tests { let warnings = adapters.warnings(); assert_eq!(warnings.len(), 1, "{warnings:?}"); assert!(warnings[0].contains(&id), "{warnings:?}"); + assert!( + warnings[0].contains("RUNNING is the state this sandbox last saw") + && warnings[0].contains("A VM that's still live bills"), + "{warnings:?}" + ); } /// **Detach hands off, and nothing warns or moves afterwards.** A launched sandbox @@ -2606,9 +2583,9 @@ mod tests { /// answer, so a queue left at `[RUNNING]` after the launch and then appended to becomes /// `[RUNNING, SUSPENDED]` — the suspend wait pops the stale RUNNING, sleeps a poll /// interval, and the fake's `sleep` **advances the clock**. That five seconds is - /// invisible until it lands in the middle of the STATE-12 window arithmetic, which is - /// exactly where it first showed up. Queueing both up front means every wait matches on - /// its first poll and no test clock moves except when a test moves it. + /// invisible until a test reads the clock, which then counts the fake's poll interval as + /// time the VM spent suspended. Queueing both up front means every wait matches on its + /// first poll and no test clock moves except when a test moves it. async fn suspended_with_window( suspended_sec: u32, ) -> (Sandbox, Arc, Arc) { @@ -2644,7 +2621,7 @@ mod tests { assert_eq!( clock.now(), Duration::ZERO, - "no wait may have slept, or the window arithmetic below is measuring the fake's \ + "no wait may have slept, or a test that reads the clock is measuring the fake's \ poll interval as well as the elapsed suspension" ); (sandbox, recorder, clock) @@ -2916,14 +2893,11 @@ mod tests { assert_eq!(sandbox.bootstrap_count(), 1, "STATE-3: never above one"); } - /// **STATE-12 for an adopted VM.** With no window from a request of its own, the - /// sandbox refuses a late resume using the `idlePolicy` window `GetMicrovm` reported. - /// - /// **Falsification** — 2026-09-24. Drop the reported-window fallback in - /// `require_open_suspended_window` and this resume reaches `ResumeMicrovm`; restored - /// after. + /// **STATE-12 for an adopted VM.** An adopted VM this sandbox suspended resumes past the + /// 600 s `idlePolicy` window `GetMicrovm` reported: the resume goes to the service, which + /// doesn't hold an explicit suspend to that window (#597). #[tokio::test] - async fn an_adopted_vm_refuses_a_resume_past_the_reported_window() { + async fn an_adopted_vm_resumes_past_the_reported_window_through_the_service() { let (plane, recorder, clock) = adopt_plane(); recorder .answer( @@ -2934,7 +2908,12 @@ mod tests { "GetMicrovm", Answer::ok(fake::microvm_response("SUSPENDED", None)), ) - .answer("SuspendMicrovm", Answer::ok(fake::empty_response())); + .answer( + "GetMicrovm", + Answer::ok(fake::microvm_response("RUNNING", None)), + ) + .answer("SuspendMicrovm", Answer::ok(fake::empty_response())) + .answer("ResumeMicrovm", Answer::ok(fake::empty_response())); let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN) .await .expect("adopts"); @@ -2942,12 +2921,12 @@ mod tests { sandbox.suspend().await.expect("suspends"); clock.advance(Duration::from_secs(601)); - let error = sandbox + sandbox .resume() .await - .expect_err("past the 600 s window the service reported"); - assert_eq!(error.kind(), ErrorKind::WindowClosed); - assert_eq!(recorder.call_count("ResumeMicrovm"), 0); + .expect("past the 600 s window the service reported, the service resumes it"); + assert_eq!(sandbox.lifecycle(), Lifecycle::Running); + assert_eq!(recorder.call_count("ResumeMicrovm"), 1); } /// A VM adopted while already SUSPENDED has no locally observed suspend time, so the @@ -3825,21 +3804,6 @@ mod tests { ); } - /// **STATE-12, the guard proof.** A resume past the launch-time suspended window is - /// refused locally, with **zero** resume calls, and the message names the window. - /// - /// Four assertions, and the packet asks for all four because each rules out a different - /// wrong implementation. `ErrorKind::WindowClosed` rather than `Timeout` separates this - /// from the client that waits. Zero `ResumeMicrovm` calls is what "before any wire call" - /// means. The elapsed reading rules out an implementation that polled to the deadline - /// first — the fake's clock advances on every `sleep`, so a poll loop here would show up - /// as time passed. And the message naming both numbers plus `idlePolicy` is what sends a - /// reader to the finding rather than looking for the flag that reopens the window. - /// - /// **Falsification** — delete the `require_open_suspended_window` call from `resume` and - /// the fake answers TERMINATED, so this test fails with an `ErrorKind::LaunchDied` after - /// a `ResumeMicrovm` call, naming neither the window nor the seconds elapsed. Verified; - /// see the packet's guard proofs. /// A keepalive reads this receiver instead of the lock, so every transition must reach /// it: a suspend it missed would be undone by the keepalive's next poll auto-resuming /// the VM. @@ -3876,9 +3840,24 @@ mod tests { assert_eq!(*watch.borrow(), sandbox.lifecycle()); } + /// `ResumeMicrovm`'s refusal of a VM the service terminated, as it answered on 2026-10-12 + /// in us-east-1 (#597): a 400 whose error type is `ValidationException`. + fn terminated_refusal() -> Answer { + Answer { + status: 400, + body: r#"{"message": "The MicroVM mvm-abc123 has been terminated and its state cannot be changed."}"# + .to_string(), + error_type: Some("ValidationException".to_string()), + } + } + + /// **STATE-12, the regression (#597).** A resume 1283 s into a 60 s window, on a sandbox + /// that suspended the VM itself, reaches `ResumeMicrovm` and comes back RUNNING. Those are + /// the numbers measured on 2026-10-12 in us-east-1: the service doesn't hold an explicit + /// suspend to `suspendedDurationSeconds`, so a client that refused here refused a resume + /// the service honors. #[tokio::test] - async fn a_resume_past_the_suspended_window_is_refused_before_the_wire() { - // A short window, so the arithmetic is legible: sixty seconds asked for at launch. + async fn a_resume_past_the_window_of_an_explicit_suspend_reaches_the_service_and_resumes() { let (mut sandbox, recorder, clock) = suspended_with_window(60).await; assert_eq!(sandbox.suspended_window(), Some(Duration::from_secs(60))); assert_eq!( @@ -3887,68 +3866,128 @@ mod tests { "the launch's own maxIdleDurationSeconds, which a keepalive checks its cadence against" ); - // The window closes. This is the whole of what the clock injection buys. - clock.advance(Duration::from_secs(61)); - - // What a service says about a VM the idlePolicy already terminated — which is what a - // client without the local check would spend its poll timeout discovering. + clock.advance(Duration::from_secs(1283)); recorder .answer("ResumeMicrovm", Answer::ok(fake::empty_response())) + .answer( + "GetMicrovm", + Answer::ok(fake::microvm_response("RUNNING", None)), + ); + sandbox + .resume() + .await + .expect("the service resumes an explicit suspend past its window"); + + assert_eq!(sandbox.lifecycle(), Lifecycle::Running); + assert!(!sandbox.was_terminated()); + assert_eq!( + recorder.call_count("ResumeMicrovm"), + 1, + "the resume has to reach the service" + ); + } + + /// **STATE-12, a VM the service ended.** The service refuses the resume with the + /// `ValidationException` it gave on 2026-10-12 for a VM the idle policy terminated, and + /// `GetMicrovm` reports TERMINATED. That's `ERR_WINDOW_CLOSED` naming the state and the + /// `stateReason`, and the sandbox records the termination: a second resume is refused + /// with no call (STATE-11), and dropping it warns about no live VM. + #[tokio::test] + async fn a_resume_the_service_refuses_for_an_ended_vm_is_window_closed_and_recorded() { + let (mut sandbox, recorder, _clock) = planted(); + // Every `GetMicrovm` answer queued up front, in the order the launch wait, the suspend + // wait and the refused resume's read take them; `suspended_with_window` says why. + recorder + .answer( + "RunMicrovm", + Answer::ok(fake::microvm_response("PENDING", None)), + ) + .answer( + "GetMicrovm", + Answer::ok(fake::microvm_response("RUNNING", None)), + ) + .answer( + "GetMicrovm", + Answer::ok(fake::microvm_response("SUSPENDED", None)), + ) .answer( "GetMicrovm", Answer::ok(fake::microvm_response( "TERMINATED", - Some("suspended window elapsed"), + Some("MicroVM exceeded maximum lifetime."), )), - ); - let elapsed_before = clock.now(); - let resumes_before = recorder.call_count("ResumeMicrovm"); + ) + .answer( + "CreateMicrovmAuthToken", + Answer::ok(fake::auth_token_response("proxy-token")), + ) + .answer("SuspendMicrovm", Answer::ok(fake::empty_response())) + .answer("ResumeMicrovm", terminated_refusal()); + sandbox + .run( + RunRequest::new() + .with_image("arn:image") + .with_suspended_sec(60), + ) + .await + .expect("launches"); + sandbox.suspend().await.expect("suspends"); let error = sandbox .resume() .await - .expect_err("the window the launch set has closed"); - - assert_eq!(error.kind(), ErrorKind::WindowClosed); + .expect_err("the service ended the VM"); + assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}"); assert_eq!(error.code(), "ERR_WINDOW_CLOSED"); let message = error.to_string(); - assert!(message.contains("61s"), "the elapsed time: {message}"); + assert!(message.contains("TERMINATED"), "the state: {message}"); assert!( - message.contains("60s suspendedDurationSeconds"), - "the window has to be named: {message}" - ); - assert!(message.contains("idlePolicy"), "the finding: {message}"); - assert!( - message.contains("Refused before ResumeMicrovm"), - "why the refusal is local: {message}" + message.contains("MicroVM exceeded maximum lifetime."), + "the stateReason: {message}" ); + assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated); + assert!(sandbox.was_terminated()); + assert_eq!(recorder.call_count("ResumeMicrovm"), 1); + + let again = sandbox.resume().await.expect_err("STATE-11"); + assert!(again.to_string().contains("STATE-11"), "{again}"); assert_eq!( recorder.call_count("ResumeMicrovm"), - resumes_before, - "the refusal must come before ResumeMicrovm" + 1, + "the second resume makes no call" ); assert_eq!( - clock.now(), - elapsed_before, - "the refusal must be immediate rather than polled to the deadline" + sandbox.drop_warning(), + None, + "a VM the service ended bills nothing, so dropping its sandbox isn't a leak" ); } - /// **STATE-12's fallback.** With no window from its own request, the sandbox uses the - /// `suspendedDurationSeconds` that `GetMicrovm` reported (600 in the fake). + /// A resume the service refuses while `GetMicrovm` still reports SUSPENDED is the + /// service's answer, and the sandbox stays SUSPENDED: only a VM the service reports dead + /// is recorded as terminated (STATE-12), so a retry reaches the service and resumes. #[tokio::test] - async fn the_window_falls_back_to_the_idle_policy_the_service_reported() { - let (mut sandbox, recorder, clock) = suspended_with_window(60).await; - sandbox.suspended_window = None; - recorder.answer("ResumeMicrovm", Answer::ok(fake::empty_response())); - clock.advance(Duration::from_secs(601)); - let error = sandbox - .resume() - .await - .expect_err("past the reported window"); - assert_eq!(error.kind(), ErrorKind::WindowClosed); - assert!(error.to_string().contains("600s"), "{error}"); - assert_eq!(recorder.call_count("ResumeMicrovm"), 0); + async fn a_resume_refused_for_a_live_vm_leaves_the_sandbox_suspended_and_resumable() { + let (mut sandbox, recorder, _clock) = suspended().await; + recorder + .answer( + "ResumeMicrovm", + Answer::failure(409, "The MicroVM is changing state."), + ) + .answer("ResumeMicrovm", Answer::ok(fake::empty_response())); + + let error = sandbox.resume().await.expect_err("the service refused"); + assert_eq!(error.kind(), ErrorKind::Platform, "{error}"); + assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended); + assert!(!sandbox.was_terminated()); + + recorder.answer( + "GetMicrovm", + Answer::ok(fake::microvm_response("RUNNING", None)), + ); + sandbox.resume().await.expect("the retry resumes"); + assert_eq!(sandbox.lifecycle(), Lifecycle::Running); + assert_eq!(recorder.call_count("ResumeMicrovm"), 2); } /// **#195 at the sandbox.** A client-token launch that adopts an idle-suspended VM is @@ -4037,121 +4076,6 @@ mod tests { assert_eq!(sandbox.bootstrap_count(), 1, "counted once (STATE-3)"); } - /// A resume **inside** the window goes through, so the guard is a comparison rather than - /// a blanket refusal of every resume. - /// - /// The boundary is inclusive: elapsed exactly equal to the window is still open, matching - /// the Python's `elapsed <= window`. - #[tokio::test] - async fn a_resume_inside_the_window_is_allowed_and_the_boundary_is_inclusive() { - let (mut sandbox, recorder, clock) = suspended_with_window(60).await; - - // Exactly at the window. - clock.advance(Duration::from_secs(60)); - recorder - .answer("ResumeMicrovm", Answer::ok(fake::empty_response())) - .answer( - "GetMicrovm", - Answer::ok(fake::microvm_response("RUNNING", None)), - ); - sandbox - .resume() - .await - .expect("60s == the 60s window is open"); - assert_eq!(sandbox.lifecycle(), Lifecycle::Running); - assert_eq!(recorder.call_count("ResumeMicrovm"), 1); - } - - /// A successful resume clears the stamp, so the next cycle's window is measured from the - /// next suspend rather than from the first one. - /// - /// The failure this rules out is subtle and would only show on a second cycle: - /// accumulating every suspension's elapsed time into one total refuses a resume whose own - /// window is wide open. The clock is advanced past the window *between* the two cycles, - /// which is what makes the accumulating implementation fail here. - #[tokio::test] - async fn a_successful_resume_clears_the_stamp_so_a_second_cycle_measures_its_own_window() { - let (mut sandbox, recorder, clock) = planted(); - // Every `GetMicrovm` answer for both cycles, in the order the waits consume them: - // the launch's RUNNING, then SUSPENDED / RUNNING twice. Queued up front for the - // reason `suspended_with_window` documents — a stale answer left at the head makes a - // wait poll twice, and the fake's `sleep` moves the very clock this test is measuring. - recorder - .answer( - "RunMicrovm", - Answer::ok(fake::microvm_response("PENDING", None)), - ) - .answer( - "CreateMicrovmAuthToken", - Answer::ok(fake::auth_token_response("proxy-token")), - ) - .answer("SuspendMicrovm", Answer::ok(fake::empty_response())) - .answer("ResumeMicrovm", Answer::ok(fake::empty_response())) - .answer( - "GetMicrovm", - Answer::ok(fake::microvm_response("RUNNING", None)), - ) - .answer( - "GetMicrovm", - Answer::ok(fake::microvm_response("SUSPENDED", None)), - ) - .answer( - "GetMicrovm", - Answer::ok(fake::microvm_response("RUNNING", None)), - ) - .answer( - "GetMicrovm", - Answer::ok(fake::microvm_response("SUSPENDED", None)), - ) - .answer( - "GetMicrovm", - Answer::ok(fake::microvm_response("RUNNING", None)), - ); - sandbox - .run( - RunRequest::new() - .with_image("arn:image") - .with_suspended_sec(60), - ) - .await - .expect("launches"); - - for cycle in 0..2 { - sandbox - .suspend() - .await - .unwrap_or_else(|error| panic!("cycle {cycle} suspend: {error}")); - assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended, "cycle {cycle}"); - - // Forty-five seconds each time — inside the sixty-second window on its own, and - // ninety in total. An implementation that never cleared the stamp measures the - // total and refuses the second resume, which is the whole point of the loop. - clock.advance(Duration::from_secs(45)); - - sandbox.resume().await.unwrap_or_else(|error| { - panic!("cycle {cycle} resume, whose own window is wide open: {error}") - }); - assert_eq!(sandbox.lifecycle(), Lifecycle::Running, "cycle {cycle}"); - assert_eq!( - sandbox.bootstrap_count(), - 1, - "cycle {cycle}: no re-bootstrap" - ); - } - - assert_eq!( - clock.now(), - Duration::from_secs(90), - "ninety seconds of accumulated suspension against a sixty-second window: this is \ - the total an implementation that never cleared the stamp would have compared" - ); - assert_eq!( - recorder.call_count("ResumeMicrovm"), - 2, - "both cycles resumed" - ); - } - /// **STATE-11, the resume-after-terminate case the packet names.** A terminated VM never /// returns to RUNNING, and the recorder sees **zero** resume wire calls. /// diff --git a/crates/microvms-cli/README.md b/crates/microvms-cli/README.md index f5ee4f8e..b83de5c3 100644 --- a/crates/microvms-cli/README.md +++ b/crates/microvms-cli/README.md @@ -50,7 +50,9 @@ and `.venv`; other project files are included. `agent-up` leaves the VM running. Terminate it when finished; its reusable image remains. By default the VM has a one-hour lifetime ceiling, suspends after ten minutes of inbound inactivity, and terminates after ten more -minutes suspended. See the [agent guide](https://laithalsaadoon.github.io/microvms-agentd/learn/operations/run-coding-agents-on-bedrock/) +minutes in that idle suspension. A VM you suspend with `microvm suspend` +isn't held to that window: one stayed resumable 1283 s into a 60 s window +(measured 2026-10-12, us-east-1). See the [agent guide](https://laithalsaadoon.github.io/microvms-agentd/learn/operations/run-coding-agents-on-bedrock/) for both agents together, longer sessions, and credential refresh. ## Run commands or connect an SDK diff --git a/crates/microvms-cli/src/cli.rs b/crates/microvms-cli/src/cli.rs index f3a820e7..25b94f0d 100644 --- a/crates/microvms-cli/src/cli.rs +++ b/crates/microvms-cli/src/cli.rs @@ -301,8 +301,10 @@ pub enum Command { /// Thaw a suspended MicroVM and report its endpoint. /// - /// The launch-time `suspendedDurationSeconds` window terminates a suspended VM once it - /// passes, so "resume later" silently stops working and the VM is gone rather than slow. + /// The launch-time `suspendedDurationSeconds` window terminates a VM the idle policy + /// suspended once it passes. VMs suspended with `microvm suspend` resumed 900 s and more + /// into a 60 s window (2026-10-12), which AWS doesn't promise, so the service decides. A + /// resume of a VM that ended, whatever ended it, exits `ERR_WINDOW_CLOSED`; launch a new one. Resume(ResumeArgs), /// Tear down a MicroVM, and optionally its image and build log group. @@ -1054,7 +1056,8 @@ pub struct RunArgs { #[arg(long, default_value_t = 600)] pub max_idle_sec: u32, - /// Terminate the VM after this long suspended. A resume past it cannot work. + /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit + /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. #[arg(long, default_value_t = 600)] pub suspended_sec: u32, @@ -2503,7 +2506,8 @@ pub struct AgentUpArgs { #[arg(long, default_value_t = 600)] pub max_idle_sec: u32, - /// Terminate the VM after this long suspended. A resume past it cannot work. + /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit + /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. #[arg(long, default_value_t = 600)] pub suspended_sec: u32, diff --git a/crates/microvms-cli/src/commands/lifecycle.rs b/crates/microvms-cli/src/commands/lifecycle.rs index ef6a544b..c4c0b8b2 100644 --- a/crates/microvms-cli/src/commands/lifecycle.rs +++ b/crates/microvms-cli/src/commands/lifecycle.rs @@ -2084,10 +2084,12 @@ pub async fn suspend( /// Thaws a suspended MicroVM and reports its endpoint. /// /// The suspended window is **not** checked here, and that is correct rather than a gap: the -/// window is in `GetMicrovm`'s `idlePolicy`, but when the suspend began is known only to the -/// process that issued it. Guessing would either reject an open window or accept a closed -/// one, both worse than letting the service answer — which is what `fail_on: DEAD_STATES` -/// does, failing fast with `stateReason` instead of burning the poll timeout. +/// service ends a VM the idle policy suspended once `suspendedDurationSeconds` passes and keeps +/// an explicitly suspended one resumable past it (measured 2026-10-12, us-east-1), so only the +/// service can answer. Core's `ControlPlane::resume` reports its refusal of a VM that ended +/// before the resume as `ERR_WINDOW_CLOSED`. A VM that dies after the resume is +/// accepted fails the wait at once through `fail_on: DEAD_STATES`, with `stateReason`, instead +/// of burning the poll timeout. pub async fn resume( ctx: &mut Ctx<'_, O, E>, args: &ResumeArgs, diff --git a/crates/microvms-cli/src/config.rs b/crates/microvms-cli/src/config.rs index 6020ef20..9693850b 100644 --- a/crates/microvms-cli/src/config.rs +++ b/crates/microvms-cli/src/config.rs @@ -83,7 +83,8 @@ pub struct ProjectConfig { pub auto_resume: Option, /// `run --max-idle-sec`: suspend after this much inbound-traffic idleness. pub max_idle_sec: Option, - /// `run --suspended-sec`: terminate after this long suspended. + /// `run --suspended-sec`: terminate a VM the idle policy suspended once it's been suspended + /// this long. pub suspended_sec: Option, /// `run --max-duration-sec`: hard ceiling on the VM's life. pub max_duration_sec: Option, diff --git a/crates/microvms-cli/src/envelope.rs b/crates/microvms-cli/src/envelope.rs index 7f487a89..d6153f90 100644 --- a/crates/microvms-cli/src/envelope.rs +++ b/crates/microvms-cli/src/envelope.rs @@ -463,7 +463,7 @@ mod tests { fn a_platform_trap_failure_names_its_finding_in_the_envelope() { let envelope = error(&classify(&Error::new( ErrorKind::WindowClosed, - "past the 300s window", + "microvm mvm-1 can't be resumed", ))); assert_eq!(envelope["code"], "ERR_WINDOW_CLOSED"); assert_eq!(envelope["exitCode"], 8); diff --git a/crates/microvms-cli/src/exit.rs b/crates/microvms-cli/src/exit.rs index 887f3cb6..84342b94 100644 --- a/crates/microvms-cli/src/exit.rs +++ b/crates/microvms-cli/src/exit.rs @@ -254,7 +254,8 @@ pub const EXIT_TABLE: [ExitRow; 17] = [ ExitRow { exit: Exit::WindowClosed, code: Some("ERR_WINDOW_CLOSED"), - meaning: "the launch-time suspended window passed, so there is nothing to resume", + meaning: "the MicroVM ended before the resume (the suspended window, the maximum \ + duration, or a terminate), and it can't be resumed; launch a new one", finding: "`idlePolicy`", }, ExitRow { @@ -442,9 +443,7 @@ pub fn classify(error: &Error) -> CliError { (Exit::LaunchDied, _) => { &["`microvm logs ` names the build log group the hook wrote to"] } - (Exit::WindowClosed, _) => { - &["a longer window is set at launch with --suspended-sec; no call extends this one"] - } + (Exit::WindowClosed, _) => &["the VM ended and no call brings it back: launch a new one"], _ => &[], }; classified @@ -698,6 +697,39 @@ mod tests { ); } + /// A launch that died and a VM that ended before its resume each carry the remedy that + /// says what to run next, which is all that separates them from an exit with no hint. The + /// closed window's remedy names no flag: the error can't tell which of the suspended + /// window, the maximum duration, or a terminate ended the VM. + #[test] + fn a_died_launch_and_a_closed_window_each_carry_their_remedy() { + let died = classify(&Error::new( + ErrorKind::LaunchDied, + "the VM died during startup", + )); + assert_eq!(died.exit, Exit::LaunchDied); + assert!( + died.suggestions.iter().any(|s| s.contains("microvm logs")), + "{died:?}" + ); + let closed = classify(&Error::new( + ErrorKind::WindowClosed, + "microvm mvm-1 can't be resumed", + )); + assert_eq!(closed.exit, Exit::WindowClosed); + assert!( + closed + .suggestions + .iter() + .any(|s| s.contains("launch a new one")), + "{closed:?}" + ); + assert!( + closed.suggestions.iter().all(|s| !s.contains("--")), + "no flag for a cause the error doesn't know: {closed:?}" + ); + } + /// A parse failure is an argument error carrying clap's own message. #[test] fn a_parse_failure_is_an_argument_error_carrying_claps_message() { diff --git a/crates/microvms-cli/src/guards/exit_codes.rs b/crates/microvms-cli/src/guards/exit_codes.rs index 24016aa5..5a66ac92 100644 --- a/crates/microvms-cli/src/guards/exit_codes.rs +++ b/crates/microvms-cli/src/guards/exit_codes.rs @@ -44,10 +44,10 @@ async fn each_induced_failure_class_earns_its_own_code_and_finding() { "`runHookPayload` arrives wrapped, not as the body", ), ( - "expired suspended window", + "a VM that ended before its resume", Error::new( ErrorKind::WindowClosed, - "suspended 301s, past the 300s suspendedDurationSeconds window", + "microvm mvm-1 can't be resumed: the service refused ResumeMicrovm", ), Exit::WindowClosed, "`idlePolicy`", diff --git a/crates/microvms-cli/src/guards/history.rs b/crates/microvms-cli/src/guards/history.rs index 2df3d60d..86a0019a 100644 --- a/crates/microvms-cli/src/guards/history.rs +++ b/crates/microvms-cli/src/guards/history.rs @@ -588,6 +588,50 @@ async fn a_resume_polls_the_thawed_daemon_and_lands_its_hook_observations() { assert_eq!(read[0]["event"], "resumed"); } +/// **`resume` of a VM the service ended exits `ERR_WINDOW_CLOSED` and records no resume.** +/// (#597, STATE-12) The service refuses the resume with the `ValidationException` it gave on +/// 2026-10-12 for a VM the idle policy terminated, and `GetMicrovm` reports TERMINATED. The +/// CLI calls core's `ControlPlane::resume`, so it exits 8 with the code both bindings raise, +/// rather than `ERR_PLATFORM`. +#[tokio::test] +async fn a_resume_of_a_vm_the_service_ended_exits_window_closed_and_records_no_resume() { + let dir = TempDir::new("history-resume-ended"); + let transport = Arc::new(ScriptedTransport::new()); + transport + .answer( + "ResumeMicrovm", + 400, + r#"{"message": "The MicroVM mvm-abc123 has been terminated and its state cannot be changed."}"#, + ) + .answer("GetMicrovm", 200, µvm_body("TERMINATED")); + let seam = ScriptedSeam { + transport: Arc::clone(&transport), + clock: Arc::new(YieldingClock::default()), + }; + let command = Command::Resume(ResumeArgs { + microvm_id: "mvm-abc123".into(), + timeout: Duration::from_secs(30), + state_dir: Some(dir.0.clone()), + region: region_flags(), + }); + + let (result, _) = dispatch_with(&seam, &command, full_infra()).await; + let failure = result.expect_err("the VM is gone"); + + assert_eq!(failure.exit, Exit::WindowClosed, "{}", failure.message); + assert_eq!(failure.code(), "ERR_WINDOW_CLOSED"); + assert!( + failure.message.contains("TERMINATED"), + "the state GetMicrovm reported: {}", + failure.message + ); + let read = crate::history::read_events(&dir.0, "mvm-abc123"); + assert!( + read.iter().all(|event| event["event"] != "resumed"), + "a refused resume records no `resumed` event: {read:?}" + ); +} + /// **`microvm health` carries each hook handler's outcome and the identity steps.** /// (#198, #205) A hook without a handler keeps its two-key shape; one with a handler /// gains a camelCase `handler`; `identitySteps` names the failed step. diff --git a/crates/microvms-domain/src/error.rs b/crates/microvms-domain/src/error.rs index 15ed493e..bbbd0ed5 100644 --- a/crates/microvms-domain/src/error.rs +++ b/crates/microvms-domain/src/error.rs @@ -130,7 +130,9 @@ pub enum ErrorKind { BuildWedged, /// The MicroVM reached a terminal state before RUNNING; read `stateReason`. LaunchDied, - /// The launch-time suspended window passed, so there is nothing to resume. + /// The MicroVM ended before the resume (the suspended window, the maximum duration, or a + /// terminate), and it can't be resumed; launch a new one. The service refused the resume, + /// and `GetMicrovm` reports the VM TERMINATING or TERMINATED. WindowClosed, /// A control-plane failure with no more specific class. Platform, diff --git a/crates/model-conformance/tests/client_lifecycle.rs b/crates/model-conformance/tests/client_lifecycle.rs index f7344e96..352fc8a7 100644 --- a/crates/model-conformance/tests/client_lifecycle.rs +++ b/crates/model-conformance/tests/client_lifecycle.rs @@ -29,8 +29,12 @@ //! answers SUSPENDED and the two steps are one call. Otherwise it answers SUSPENDING until the //! wait times out, which leaves the sandbox SUSPENDING, where the model is. //! - `ResumeRequested` is `resume`, the same way: RUNNING when `ResumeComplete` is next, and -//! SUSPENDED until the wait times out otherwise. A closed window first moves the test clock -//! past the launch's suspended window, which is the only way the sandbox can know it closed. +//! SUSPENDED until the wait times out otherwise. A resume the service refuses is `resume` +//! with `ResumeMicrovm` answering the `ValidationException` the service gave for a VM it had +//! terminated (2026-10-12, #597), which the sandbox has to report as `ERR_WINDOW_CLOSED` and +//! record as TERMINATED. +//! - `SuspendedVmEnded` calls nothing: from then on the fake's `GetMicrovm` answers TERMINATED, +//! the platform's side of a VM the idle policy ended while it was suspended. //! - `TerminateRequested` is `terminate`, waiting for TERMINATED when `TerminateComplete` is the //! next step and returning once the call is accepted otherwise. //! - A platform report the model ignores changes nothing, so nothing is called. @@ -45,10 +49,9 @@ //! `&mut self` across its wait, so no second call can land between a request and its //! completion, and a completion the sandbox stopped waiting for is never observed. Those //! interleavings are the platform side of the model (a resume that completes after a -//! terminate), not our code. The same goes for a closed window that opens again: the model draws -//! the window per request, and the test clock only runs forward. Such paths are counted and -//! skipped. The planted configs (`guards_skipped`, `double_bootstrap_planted`) describe clients -//! other than this one, so they aren't replayed either. +//! terminate), not our code. Such paths are counted and skipped. The planted configs +//! (`guards_skipped`, `double_bootstrap_planted`) describe clients other than this one, so they +//! aren't replayed either. use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::sync::Arc; @@ -65,10 +68,6 @@ use microvms_app::{Error, ErrorKind, Region}; use serde_json::Value; use stateright::{Checker, Expectation, Model, PathRecorder}; -/// The suspended window every replayed launch asks for. Wide enough that the waits a path can -/// time out inside one suspension (three resumes at five minutes each) leave it open. -const WINDOW: Duration = Duration::from_secs(3_600); - /// How long `wait_until_running` may take. The fake answers RUNNING on the first poll. const READY: Duration = Duration::from_secs(60); @@ -85,7 +84,9 @@ enum Call { Suspend { completes: bool }, SuspendRefused, Resume { completes: bool }, - ResumeRefused { closes: bool }, + ResumeRefused, + ResumeEnded, + Ended, Terminate { completes: bool }, Nothing, } @@ -102,8 +103,9 @@ impl Call { Call::SuspendRefused => "suspend refused", Call::Resume { completes: true } => "resume completed", Call::Resume { completes: false } => "resume unanswered", - Call::ResumeRefused { closes: true } => "resume refused window closed", - Call::ResumeRefused { closes: false } => "resume refused", + Call::ResumeRefused => "resume refused", + Call::ResumeEnded => "resume refused by the service", + Call::Ended => "platform ends the suspended VM", Call::Terminate { completes: true } => "terminate completed", Call::Terminate { completes: false } => "terminate", Call::Nothing => "platform report ignored", @@ -113,7 +115,7 @@ impl Call { fn refused(self) -> bool { matches!( self, - Call::HookRefused | Call::SuspendRefused | Call::ResumeRefused { .. } + Call::HookRefused | Call::SuspendRefused | Call::ResumeRefused ) } } @@ -121,7 +123,7 @@ impl Call { /// Every call the replay has to make. A walk that shrank to a corner of the model would pass /// every step it took; this is what it would miss. A planner that came to skip paths is held /// apart: every path the recorder kept has to plan, and [`SKIPPED`] pins the rest. -const COVERED: [&str; 14] = [ +const COVERED: [&str; 15] = [ "launch", "hook", "hook refused", @@ -132,7 +134,8 @@ const COVERED: [&str; 14] = [ "resume completed", "resume unanswered", "resume refused", - "resume refused window closed", + "resume refused by the service", + "platform ends the suspended VM", "terminate", "terminate completed", "platform report ignored", @@ -142,10 +145,7 @@ const COVERED: [&str; 14] = [ /// gives. Every recorded path plans, so these are all paths that end in a step no recorded path /// takes. A change to the model or the planner that moves these counts is a change to what the /// replay drives, so it edits this table in the same diff. -const SKIPPED: [(&str, usize); 2] = [ - ("a closed window opens again", 11), - ("a completion arrives after another step", 122), -]; +const SKIPPED: [(&str, usize); 1] = [("a completion arrives after another step", 121)]; /// One sandbox call, and the model's state after the steps it stands for. struct Step<'a> { @@ -161,8 +161,6 @@ fn verdict(state: &State) -> Verdict { /// The path as sandbox calls, or why it can't be one. fn plan(path: &Path) -> Result>, &'static str> { let mut steps = Vec::new(); - // Whether the clock has passed the window of the suspension the sandbox is in. - let mut window_closed = false; let mut at = 0; while at + 1 < path.len() { let (before, Some(action)) = &path[at] else { @@ -182,23 +180,17 @@ fn plan(path: &Path) -> Result>, &'static str> { (Action::HookSucceeded, Verdict::Ignored) => (Call::HookRefused, 1), (Action::ExecRequested, Verdict::Issued) => (Call::Exec, 1), (Action::SuspendRequested, Verdict::Issued) => { - window_closed = false; let completes = completed_by(Action::SuspendComplete); (Call::Suspend { completes }, 1 + usize::from(completes)) } (Action::SuspendRequested, Verdict::RefusedLocally) => (Call::SuspendRefused, 1), - (Action::ResumeRequested { .. }, Verdict::Issued) if window_closed => { - return Err("a closed window opens again"); - } - (Action::ResumeRequested { .. }, Verdict::Issued) => { + (Action::ResumeRequested, Verdict::Issued) => { let completes = completed_by(Action::ResumeComplete); (Call::Resume { completes }, 1 + usize::from(completes)) } - (Action::ResumeRequested { window_open }, Verdict::RefusedLocally) => { - let closes = !window_open && before.vm_state == VmState::Suspended; - window_closed |= closes; - (Call::ResumeRefused { closes }, 1) - } + (Action::ResumeRequested, Verdict::RefusedLocally) => (Call::ResumeRefused, 1), + (Action::ResumeRequested, Verdict::RefusedByService) => (Call::ResumeEnded, 1), + (Action::SuspendedVmEnded, Verdict::Issued) => (Call::Ended, 1), (Action::ResumeComplete, Verdict::Ignored) => (Call::Nothing, 1), (Action::TerminateRequested, Verdict::Issued) => { let completes = completed_by(Action::TerminateComplete); @@ -308,6 +300,17 @@ fn lifecycle(state: VmState) -> Lifecycle { } } +/// `ResumeMicrovm`'s refusal of a VM the service terminated, as it answered on 2026-10-12 in +/// us-east-1 (#597): a 400 whose error type is `ValidationException`. +fn terminated_refusal() -> Answer { + Answer { + status: 400, + body: r#"{"message": "The MicroVM mvm-abc123 has been terminated and its state cannot be changed."}"# + .to_string(), + error_type: Some("ValidationException".to_string()), + } +} + /// A fresh sandbox for one path, over a fake that answers every call a path makes. struct Replay { sandbox: Sandbox, @@ -319,12 +322,34 @@ struct Replay { } impl Replay { - fn new() -> Self { + /// A replay of `steps`, whose `ResumeMicrovm` answers are queued here, in the order the + /// path's resumes take them. + /// + /// Up front, because the fake serves a queued answer only once every answer before it has + /// been read, and repeats the last: a refusal queued after an acceptance the path already + /// read would be read after one more acceptance. A path's one refused resume is its last + /// resume call, since the sandbox refuses every resume after it with no call (STATE-11). + fn new(steps: &[Step<'_>]) -> Self { let plane = Arc::new(FakeControlPlane::new()); + let mut resumes: Vec = steps + .iter() + .filter_map(|step| match step.call { + Call::Resume { .. } => Some(Answer::ok(empty_response())), + Call::ResumeEnded => Some(terminated_refusal()), + _ => None, + }) + .collect(); + // A path with no resume still answers one, so a sandbox that resumes anyway is caught + // by the call count rather than by the fake's panic. + if resumes.is_empty() { + resumes.push(Answer::ok(empty_response())); + } + for answer in resumes { + plane.answer("ResumeMicrovm", answer); + } plane .answer("RunMicrovm", Answer::ok(microvm_response("PENDING", None))) .answer("SuspendMicrovm", Answer::ok(empty_response())) - .answer("ResumeMicrovm", Answer::ok(empty_response())) .answer("TerminateMicrovm", Answer::ok(empty_response())) .answer( "CreateMicrovmAuthToken", @@ -374,9 +399,7 @@ impl Replay { let calls = self.plane.calls().len(); match step.call { Call::Launch => { - let mut request = RunRequest::new() - .with_image("arn:image") - .with_suspended_sec(u32::try_from(WINDOW.as_secs()).expect("fits")); + let mut request = RunRequest::new().with_image("arn:image"); request.wait = false; self.sandbox .run(request) @@ -416,14 +439,38 @@ impl Replay { self.reports(if completes { "RUNNING" } else { "SUSPENDED" }); waited(self.sandbox.resume().await.map(|_| ()), completes, "resume")?; } - Call::ResumeRefused { closes, .. } => { - if closes { - self.clock.advance(WINDOW + Duration::from_secs(1)); + Call::ResumeRefused => refusal(self.sandbox.resume().await.map(|_| ()), None)?, + Call::ResumeEnded => match self.sandbox.resume().await { + Err(error) if error.kind() == ErrorKind::WindowClosed => {} + Err(error) => { + return Err(format!( + "the service refused the resume of a VM the platform ended, and the \ + sandbox reported {:?} rather than WindowClosed: {error}", + error.kind() + )); } - refusal( - self.sandbox.resume().await.map(|_| ()), - closes.then_some(ErrorKind::WindowClosed), - )?; + Ok(_) => return Err("the sandbox resumed a VM the platform ended".into()), + }, + Call::Ended => { + // The fake serves the answer the last wait ended on once more before a newer + // one, so the platform's new state is read once here, by a plane of the + // replay's own, and the sandbox's next read is TERMINATED. + self.plane.answer( + "GetMicrovm", + Answer::ok(microvm_response( + "TERMINATED", + Some("MicroVM exceeded maximum lifetime."), + )), + ); + let platform = control_plane( + Arc::clone(&self.plane) as Arc, + Region::UsEast1, + self.clock.clone(), + ); + platform + .get_microvm("mvm-abc123") + .await + .map_err(|error| format!("the platform's own read failed: {error}"))?; } Call::Terminate { completes } => { let opts = if completes { @@ -564,7 +611,7 @@ fn waited(result: Result<(), Error>, completes: bool, what: &str) -> Result<(), } async fn replay(steps: &[Step<'_>]) -> Result<(), String> { - let mut sandbox = Replay::new(); + let mut sandbox = Replay::new(steps); for (index, step) in steps.iter().enumerate() { let outcome = match sandbox.apply(step).await { Ok(()) => sandbox.observe(step.after), @@ -591,9 +638,10 @@ async fn replay(steps: &[Step<'_>]) -> Result<(), String> { /// after every call: the transitions STATE-1, STATE-2, STATE-4, STATE-6, STATE-7, STATE-9 and /// STATE-10 (the lifecycle and the three flags), STATE-3 (the bootstrap count never passes one), /// STATE-5 (no suspend call leaves a state other than RUNNING), STATE-7's payload count, STATE-8 -/// (a completed resume leaves no proxy token cached), and STATE-11 and STATE-12 (the resume -/// calls a terminated VM or a closed window never gets). The guarded model offers a -/// launch only while none was made, so no path calls `run` twice: for STATE-3 the replay holds +/// (a completed resume leaves no proxy token cached), STATE-11 (the resume calls a terminated +/// VM never gets), and STATE-12 (a resume from SUSPENDED reaches `ResumeMicrovm`, and the +/// service's refusal of a VM the platform ended is `WindowClosed` and leaves the sandbox +/// TERMINATED). The guarded model offers a launch only while none was made, so no path calls `run` twice: for STATE-3 the replay holds /// the count against an extra bootstrap from a resume or a hook, and the refusal of a second /// `run` is `a_second_run_on_one_sandbox_is_refused_before_any_call` in the app's sandbox.rs. /// @@ -758,7 +806,7 @@ fn claims(items: &[syn::Item], known: &BTreeSet, claimed: &mut BTreeSet< /// The client model's safety properties that carry a requirement key, each exactly as named in /// `crates/model/src/client.rs`. A key that moves onto another property changes this set, though the /// key still starts some property's name. -const KEYED: [&str; 15] = [ +const KEYED: [&str; 16] = [ "STATE-1 an accepted launch leaves the VM PENDING with its image recorded", "STATE-2 a successful hook leaves the VM RUNNING with its token installed", "STATE-4 an accepted suspend leaves the VM SUSPENDING", @@ -766,11 +814,12 @@ const KEYED: [&str; 15] = [ "STATE-7 a completed resume leaves the VM RUNNING", "STATE-9 an accepted terminate leaves the VM TERMINATING and recorded terminated", "STATE-10 a reported termination leaves the VM TERMINATED", + "STATE-12 a resume refused for an ended VM leaves it TERMINATED and recorded terminated", "STATE-3 bootstrap happens at most once", "STATE-5 no suspend call outside RUNNING", "STATE-11 a terminated VM never reaches RUNNING", "STATE-11 no resume call after a terminate", - "STATE-12 no resume call with the window closed", + "STATE-12 a resume from SUSPENDED reaches the service", "STATE-7 a resume re-delivers no run-hook payload", "STATE-7 the installed token is never replaced", "STATE-8 a resume completion drops the proxy token", diff --git a/crates/model/src/client.rs b/crates/model/src/client.rs index eb683a59..dc21b48c 100644 --- a/crates/model/src/client.rs +++ b/crates/model/src/client.rs @@ -41,15 +41,19 @@ //! only worth reading next to `a resume is attempted after a terminate`, which proves the //! checker got there. //! -//! # The window is a boolean, not a clock +//! # The end of a suspended VM is the platform's action, not a clock //! -//! [`Action::ResumeRequested`] carries `window_open`, which is the model's whole -//! representation of `suspendedDurationSeconds`. Time is not in the state space, and it -//! should not be: what the model has to settle is whether the client *checks* before -//! calling, and both answers to that check are reachable in one step. The arithmetic — a -//! monotonic clock, a stamp taken before the wait, the inclusive boundary — is what the -//! Rust test suite drives with an injectable clock. Putting a counter in here instead would -//! multiply the state space by the window's width and prove the same one bit. +//! [`Action::SuspendedVmEnded`] is the platform ending a VM the client suspended, whatever +//! ends it: its maximum duration, say, or a terminate from another client. It sets +//! [`State::platform_ended`], which the client can't read: `WindowClosed` comes only from the +//! service's answer once the client's next resume reaches it (STATE-12). A suspend the idle +//! policy makes isn't an action here, because the client doesn't see it: a `Sandbox` learns +//! of one only through `Sandbox::adopt` or `ControlPlane::resume`, and until then holds the +//! VM as RUNNING. Time is not in the state space, and it should not be. The client doesn't +//! decide on the window, because the service doesn't hold an explicit suspend to it +//! (measured 2026-10-12, us-east-1, #597), so the model settles two things that need no +//! clock: a resume from SUSPENDED always reaches the service, and the service's refusal +//! leaves the client TERMINATED. use stateright::{Model, Property}; @@ -84,8 +88,7 @@ pub struct Wire { pub launches: u8, /// `SuspendMicrovm` calls. Must only ever increment from RUNNING (STATE-5). pub suspends: u8, - /// `ResumeMicrovm` calls. Must never increment once `was_terminated` (STATE-11), nor - /// with the suspended window closed (STATE-12). + /// `ResumeMicrovm` calls. Must never increment once `was_terminated` (STATE-11). pub resumes: u8, /// `TerminateMicrovm` calls. pub terminates: u8, @@ -100,8 +103,11 @@ pub struct Wire { pub enum Verdict { /// The client issued the call. Issued, - /// The client refused locally, before any call. Every STATE-5/11/12 refusal is this. + /// The client refused locally, before any call. Every STATE-5/7/11 refusal is this. RefusedLocally, + /// The client issued the call and the service refused it: a resume of a VM the platform + /// ended while it was suspended (STATE-12). + RefusedByService, /// The action did not apply — the platform reporting something the client is not /// waiting for. Ignored, @@ -130,10 +136,10 @@ pub enum Action { /// The platform reporting suspension complete (STATE-6). SuspendComplete, /// The caller asking for a resume (STATE-7, STATE-11, STATE-12). - /// - /// `window_open` is the launch-time `suspendedDurationSeconds` window, as the client - /// knows it. See the module docs on why a boolean rather than a clock. - ResumeRequested { window_open: bool }, + ResumeRequested, + /// The platform ending a SUSPENDED VM, which the client doesn't see until its next resume + /// is refused (STATE-12). See the module docs on why an action rather than a clock. + SuspendedVmEnded, /// The platform reporting the resume complete, which is where the cached proxy token is /// dropped (STATE-8). ResumeComplete, @@ -184,6 +190,11 @@ pub struct State { /// waiting for one *and* still SUSPENDED. The terminate wins, which is what the real /// client does: `terminate` clears the session and the lifecycle before anything else. pub resume_in_flight: bool, + /// Whether the platform ended the VM while it was SUSPENDED ([`Action::SuspendedVmEnded`]). + /// + /// The platform's state, not the client's: no guard reads it, and the client learns it + /// only when the service refuses its next resume (STATE-12). + pub platform_ended: bool, // ── audit counters, in the daemon model's idiom ─────────────────────────── // @@ -196,8 +207,6 @@ pub struct State { pub suspends_outside_running: u8, /// Incremented when a resume call was issued on a terminated VM (STATE-11). pub resumes_after_terminate: u8, - /// Incremented when a resume call was issued with the window closed (STATE-12). - pub resumes_window_closed: u8, /// Incremented if the installed token is ever replaced (STATE-3, STATE-7). pub token_replacements: u8, @@ -212,17 +221,16 @@ pub struct State { pub struct Config { /// How many suspend/resume cycles a caller may drive. /// - /// Two, not one: the defect a single cycle cannot see is a client that accumulates - /// every suspension's elapsed time into one total and refuses the second resume even - /// though its own window is wide open. That is a real bug the Rust suite has a test - /// for, and one cycle is blind to it. + /// Two, not one: the defect a single cycle cannot see is state the first cycle leaves + /// behind, such as a proxy token the first resume should have dropped, and one cycle is + /// blind to it. pub max_cycles: u8, /// Whether the client is allowed to skip its local guards. /// /// The falsification switch, and the reason the model is worth running: with this true - /// the client issues suspends from non-RUNNING states, resumes after a terminate, and - /// resumes with a closed window, and stateright hands back the path. That is how the - /// properties are shown to be capable of failing rather than merely green. + /// the client issues suspends from non-RUNNING states and resumes after a terminate, and + /// stateright hands back the path. That is how the properties are shown to be capable of + /// failing rather than merely green. pub skip_local_guards: bool, /// Whether a second bootstrap may be attempted — the planted double-bootstrap the /// packet's guard proofs name. @@ -296,9 +304,9 @@ impl Model for ClientLifecycle { mints: 0, installed_token: None, resume_in_flight: false, + platform_ended: false, suspends_outside_running: 0, resumes_after_terminate: 0, - resumes_window_closed: 0, token_replacements: 0, last: None, cycles: 0, @@ -350,9 +358,7 @@ impl Model for ClientLifecycle { actions.push(Action::TerminateRequested); } if state.wire.resumes <= self.cfg.max_cycles { - for window_open in [true, false] { - actions.push(Action::ResumeRequested { window_open }); - } + actions.push(Action::ResumeRequested); } } @@ -360,6 +366,16 @@ impl Model for ClientLifecycle { if state.vm_state == VmState::Suspending { actions.push(Action::SuspendComplete); } + // The platform ending a SUSPENDED VM, once. Not while a resume is in flight: the + // service accepted that resume, so the VM is no longer waiting out a suspension, and + // the sandbox's wait leaves no gap for the end to land in. + if state.vm_state == VmState::Suspended + && !state.was_terminated + && !state.resume_in_flight + && !state.platform_ended + { + actions.push(Action::SuspendedVmEnded); + } if state.wire.resumes > 0 && state.vm_state != VmState::Running { actions.push(Action::ResumeComplete); } @@ -459,31 +475,42 @@ impl Model for ClientLifecycle { } // STATE-7, STATE-11, STATE-12. - Action::ResumeRequested { window_open } => { + Action::ResumeRequested => { let terminated = last.was_terminated || last.vm_state == VmState::Terminated; let wrong_state = last.vm_state != VmState::Suspended; - if self.guarded() && (terminated || wrong_state || !window_open) { - // All three refusals are local and cost no call. The window is checked - // *first* in the real client, but the model does not order the three - // because the observable — zero resume calls — is the same. + if self.guarded() && (terminated || wrong_state) { + // Both refusals are local and cost no call. Nothing else is: how long the + // VM has spent suspended is the service's to judge (STATE-12). Verdict::RefusedLocally } else { next.wire.resumes += 1; - next.resume_in_flight = true; // Recorded at the decision, for the same reason the suspend is: the // resulting state cannot say whether the call should have been made. if terminated { next.resumes_after_terminate += 1; } - if !window_open { - next.resumes_window_closed += 1; - } // STATE-7's other half: no payload is added here. A resume that // re-delivered would show up as `payloads > launches`. - Verdict::Issued + if last.platform_ended { + // STATE-12. The service's refusal is the first the client hears of the + // end, and it records the VM the way a completed terminate does: gone, + // and never offered a resume again (STATE-11). + (next.vm_state, next.was_terminated) = (VmState::Terminated, true); + Verdict::RefusedByService + } else { + next.resume_in_flight = true; + Verdict::Issued + } } } + // STATE-12's platform half. The client's state doesn't move: nothing it holds + // says the VM is gone until a call is answered. + Action::SuspendedVmEnded => { + next.platform_ended = true; + Verdict::Issued + } + // STATE-8. The resume completing is where the cached token is dropped, and the // next request mints — which is the only externally visible difference. Action::ResumeComplete => { @@ -621,6 +648,15 @@ impl Model for ClientLifecycle { _ => true, }, ), + Property::::always( + "STATE-12 a resume refused for an ended VM leaves it TERMINATED and recorded terminated", + |_, state| match state.last { + Some((Action::ResumeRequested, Verdict::RefusedByService)) => { + state.vm_state == VmState::Terminated && state.was_terminated + } + _ => true, + }, + ), // ── the three Z3 already proved, restated over interleavings ───── // // Stated unconditionally rather than consulting the config, for the reason the @@ -665,8 +701,19 @@ impl Model for ClientLifecycle { state.resumes_after_terminate == 0 }), Property::::always( - "STATE-12 no resume call with the window closed", - |_, state| state.resumes_window_closed == 0, + "STATE-12 a resume from SUSPENDED reaches the service", + |_, state| { + // #597: the client refused a resume the service honors, because it judged + // the suspended window itself. A local refusal leaves the state as it + // was, so the state after one is the state it was refused in, and a VM + // still SUSPENDED and not terminated there was refused on nothing but + // the client's own judgment. + !matches!( + state.last, + Some((Action::ResumeRequested, Verdict::RefusedLocally)) + ) || state.was_terminated + || state.vm_state != VmState::Suspended + }, ), Property::::always( "STATE-7 a resume re-delivers no run-hook payload", @@ -721,8 +768,7 @@ impl Model for ClientLifecycle { (state.wire.suspends == 0 || launched) && (state.wire.resumes == 0 || state.wire.suspends > 0 - || state.resumes_after_terminate > 0 - || state.resumes_window_closed > 0) + || state.resumes_after_terminate > 0) && state.mints <= state.wire.resumes }), // ── coverage, so none of the above can pass by measuring nothing ── @@ -744,21 +790,17 @@ impl Model for ClientLifecycle { // checker really did try it, and really was refused. matches!( state.last, - Some((Action::ResumeRequested { .. }, Verdict::RefusedLocally)) + Some((Action::ResumeRequested, Verdict::RefusedLocally)) ) && state.was_terminated }), - Property::::sometimes( - "a resume is attempted with the window closed", - |_, state| { - matches!( - state.last, - Some(( - Action::ResumeRequested { window_open: false }, - Verdict::RefusedLocally - )) - ) - }, - ), + Property::::sometimes("a resume reaches a VM the platform ended", |_, state| { + // The witness that makes both STATE-12 properties non-vacuous: the checker + // reached a VM the platform ended, and its resume went to the service. + matches!( + state.last, + Some((Action::ResumeRequested, Verdict::RefusedByService)) + ) + }), Property::::sometimes("a suspend is attempted outside RUNNING", |_, state| { matches!( state.last, @@ -822,29 +864,59 @@ mod tests { "the counterexample must terminate before it resumes, got {steps:?}" ); assert!( - steps - .iter() - .any(|a| matches!(a, Action::ResumeRequested { .. })), + steps.iter().any(|a| matches!(a, Action::ResumeRequested)), "and it must then resume, got {steps:?}" ); } - /// **STATE-12's guard proof, in the model.** The unguarded client issues a resume with - /// the window closed, and the counterexample names the closed window. + /// **STATE-12, in the model.** A VM the platform ended while SUSPENDED still looks + /// SUSPENDED to the client, so its resume reaches the service, which refuses it; the + /// refusal leaves the client TERMINATED, and the next resume is refused with no call + /// (STATE-11). Driven through the transition function, since the sequence reads more + /// clearly than a search for it. #[test] - fn skipping_the_local_guards_issues_a_resume_with_the_window_closed() { - let checker = ClientLifecycle::new(Config::guards_skipped()) - .checker() - .spawn_bfs() - .join(); - let path = checker.assert_any_discovery("STATE-12 no resume call with the window closed"); - let steps = path.into_actions(); - assert!( - steps - .iter() - .any(|a| matches!(a, Action::ResumeRequested { window_open: false })), - "the counterexample must resume on a closed window, got {steps:?}" + fn a_resume_of_a_vm_the_platform_ended_reaches_the_service_and_records_the_end() { + let model = ClientLifecycle::new(Config::guards_held()); + let start = model.init_states().pop().expect("one init state"); + let mut state = start; + for action in [ + Action::LaunchAccepted, + Action::HookSucceeded, + Action::SuspendRequested, + Action::SuspendComplete, + Action::SuspendedVmEnded, + ] { + state = model + .next_state(&state, action) + .unwrap_or_else(|| panic!("{action:?} applies")); + } + assert_eq!( + state.vm_state, + VmState::Suspended, + "the client doesn't see the end" ); + assert!(state.platform_ended && !state.was_terminated); + + let refused = model + .next_state(&state, Action::ResumeRequested) + .expect("the resume is a step"); + assert_eq!( + refused.wire.resumes, 1, + "the resume reaches the service rather than being refused here" + ); + assert_eq!( + refused.last.expect("a verdict").1, + Verdict::RefusedByService + ); + assert_eq!(refused.vm_state, VmState::Terminated); + assert!(refused.was_terminated); + assert!(!refused.resume_in_flight, "nothing is waited for"); + + let again = model + .next_state(&refused, Action::ResumeRequested) + .expect("a refusal still records a verdict"); + assert_eq!(again.wire.resumes, 1, "STATE-11: no second call"); + assert_eq!(again.last.expect("a verdict").1, Verdict::RefusedLocally); } /// **STATE-5's guard proof, in the model.** Without the RUNNING check the client issues @@ -920,7 +992,7 @@ mod tests { ); } - /// A resume from SUSPENDED with an open window reaches RUNNING, re-delivers nothing, and + /// A resume from SUSPENDED reaches RUNNING, re-delivers nothing, and /// drops the proxy token — asserted against the transition function directly, since a /// three-step sequence is clearer read than searched for. #[test] @@ -955,8 +1027,8 @@ mod tests { assert_eq!(suspended.vm_state, VmState::Suspended); let resuming = model - .next_state(&suspended, Action::ResumeRequested { window_open: true }) - .expect("an open window resumes"); + .next_state(&suspended, Action::ResumeRequested) + .expect("a resume from SUSPENDED is issued"); assert_eq!(resuming.wire.resumes, 1); let resumed = model .next_state(&resuming, Action::ResumeComplete) @@ -1037,24 +1109,22 @@ mod tests { .expect("terminated"); assert!(terminated.was_terminated); - for window_open in [true, false] { - let refused = model - .next_state(&terminated, Action::ResumeRequested { window_open }) - .expect("a refusal still records a verdict"); - assert_eq!( - refused.wire.resumes, 0, - "window_open={window_open}: no resume may reach the wire once terminated" - ); - assert_ne!( - refused.vm_state, - VmState::Running, - "window_open={window_open}: a terminated VM must not return to RUNNING" - ); - } + let refused = model + .next_state(&terminated, Action::ResumeRequested) + .expect("a refusal still records a verdict"); + assert_eq!( + refused.wire.resumes, 0, + "no resume may reach the wire once terminated" + ); + assert_ne!( + refused.vm_state, + VmState::Running, + "a terminated VM must not return to RUNNING" + ); } /// The state space really is explored rather than cut short by the cycle bound: two - /// cycles are reachable, which is what the second-cycle window defect needs. + /// cycles are reachable, which is what a defect the first cycle leaves behind needs. #[test] fn two_suspend_resume_cycles_are_reachable() { ClientLifecycle::new(Config::guards_held()) diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md index 99878d75..83845fd9 100644 --- a/docs/TRACEABILITY.md +++ b/docs/TRACEABILITY.md @@ -105,7 +105,7 @@ traced requirement is missing a layer. Requirements are defined in | STATE-9 | 1 | waived | waived | 2 | 1 | waived | | STATE-10 | 1 | waived | waived | 2 | 1 | 1 | | STATE-11 | 1 | waived | waived | 2 | 3 | waived | -| STATE-12 | 1 | waived | waived | 2 | 6 | waived | +| STATE-12 | 1 | waived | waived | 4 | 6 | waived | | TRAP-1 | waived | waived | waived | 9 | 19 | waived | | TRAP-2 | waived | waived | waived | 1 | 5 | waived | | TRAP-3 | waived | waived | waived | 1 | 6 | 1 | @@ -1200,14 +1200,14 @@ The microvms-core shall not return a terminated VM to RUNNING. ## STATE-12 -If a resume is attempted after the launch-time suspended window has elapsed, then the microvms-core shall reject the resume with an error naming the elapsed suspended window. +If a resume is requested for a suspended MicroVM that the service terminated while it was suspended, then the microvms-core shall issue the resume to the service and report its refusal as a window-closed error naming the state and stateReason that GetMicrovm reports. - **model:** `crates/model/src/client.rs` - **gherkin:** waived: the model walks every interleaving of caller requests and platform reports up to two cycles, and model-conformance's replay drives each path against `Sandbox`; a scenario would restate one of those paths - **fuzz:** waived: the input is a sequence of requests and reports, which the model enumerates exhaustively and the replay drives; there is no byte stream to fuzz -- **test:** `crates/microvms-app/src/sandbox.rs`, `crates/model-conformance/tests/client_lifecycle.rs` -- **impl:** `bindings/microvms-js/src/control.rs`, `bindings/microvms-js/src/sandbox.rs`, `bindings/microvms-py/src/control.rs`, `bindings/microvms-py/src/sandbox.rs`, `crates/microvms-app/src/control/ops.rs`, `crates/microvms-app/src/sandbox.rs` -- **live:** waived: the window is launch-time `suspendedDurationSeconds`, and the refusal comes before any call once the client's clock passes it; the model, its replay and the app's tests drive that clock, which a live run can only wait out +- **test:** `crates/microvms-app/src/control/microvm.rs`, `crates/microvms-app/src/sandbox.rs`, `crates/microvms-cli/src/guards/history.rs`, `crates/model-conformance/tests/client_lifecycle.rs` +- **impl:** `bindings/microvms-js/src/control.rs`, `bindings/microvms-js/src/sandbox.rs`, `bindings/microvms-py/src/control.rs`, `bindings/microvms-py/src/sandbox.rs`, `crates/microvms-app/src/control/microvm.rs`, `crates/microvms-app/src/sandbox.rs` +- **live:** waived: the service's two answers are the 2026-10-12 us-east-1 measurement in #597 (an idle-suspended VM refused once its window passed, an explicit suspend resumed 1283 s into a 60 s window), which the app's tests, the CLI's and the replay script verbatim; a live check holds a billed VM through an idle suspend and its whole window, which the suite doesn't run yet ## TRAP-1 diff --git a/docs/behavior/processes.md b/docs/behavior/processes.md index a1f2aa8b..7860a03e 100644 --- a/docs/behavior/processes.md +++ b/docs/behavior/processes.md @@ -44,14 +44,14 @@ Entry point: `crates/microvms-app/src/sandbox.rs:1108` ## Suspend and resume -Entry point: `crates/microvms-app/src/sandbox.rs:1587` - -1. `suspend` requires a running, attached VM before sending the suspension request. `crates/microvms-app/src/sandbox.rs:1588` -2. Once the request succeeds, it records `Suspending` and the suspension time, then waits for `SUSPENDED` or `TERMINATED`. `crates/microvms-app/src/sandbox.rs:1606` -3. The returned state updates the local lifecycle, including the terminal marker when the VM terminated during suspension. `crates/microvms-app/src/sandbox.rs:1627` -4. `resume` rejects a terminated VM or a lifecycle other than suspended, then checks the suspension window when both a window and an observed suspension time are available. `crates/microvms-app/src/sandbox.rs:1671` `crates/microvms-app/src/sandbox.rs:1737` -5. It sends the resume request and waits for `RUNNING`, treating dead states as failures. `crates/microvms-app/src/sandbox.rs:1694` -6. The running response updates the VM record, rebinds the session to its endpoint, and clears the previous suspension time. `crates/microvms-app/src/sandbox.rs:1710` +Entry point: `crates/microvms-app/src/sandbox.rs:1585` + +1. `suspend` requires a running, attached VM before sending the suspension request. `crates/microvms-app/src/sandbox.rs:1586` +2. Once the request succeeds, it records `Suspending`, then waits for `SUSPENDED` or `TERMINATED`. `crates/microvms-app/src/sandbox.rs:1605` +3. The returned state updates the local lifecycle, including the terminal marker when the VM terminated during suspension. `crates/microvms-app/src/sandbox.rs:1621` +4. `resume` rejects a terminated VM or a lifecycle other than suspended. It doesn't check the time spent suspended; the service decides. `crates/microvms-app/src/sandbox.rs:1675` `crates/microvms-app/src/sandbox.rs:1682` +5. It sends the resume request and waits for `RUNNING`, treating dead states as failures. `crates/microvms-app/src/sandbox.rs:1694` `crates/microvms-app/src/sandbox.rs:1708` +6. The running response updates the VM record and rebinds the session to its endpoint. A resume the service refuses for a VM that ended records the sandbox `Terminated`. `crates/microvms-app/src/sandbox.rs:1716` `crates/microvms-app/src/sandbox.rs:1697` ### Related diff --git a/docs/insights/contract-map.md b/docs/insights/contract-map.md index af478e82..d76bf27f 100644 --- a/docs/insights/contract-map.md +++ b/docs/insights/contract-map.md @@ -76,7 +76,9 @@ pub enum ErrorKind { BuildWedged, /// The MicroVM reached a terminal state before RUNNING; read `stateReason`. LaunchDied, - /// The launch-time suspended window passed, so there is nothing to resume. + /// The MicroVM ended before the resume (the suspended window, the maximum duration, or a + /// terminate), and it can't be resumed; launch a new one. The service refused the resume, + /// and `GetMicrovm` reports the VM TERMINATING or TERMINATED. WindowClosed, /// A control-plane failure with no more specific class. Platform, diff --git a/docs/insights/debugging-guide.md b/docs/insights/debugging-guide.md index 57279e83..c5307b22 100644 --- a/docs/insights/debugging-guide.md +++ b/docs/insights/debugging-guide.md @@ -15,7 +15,7 @@ This distinction separates a failed workload from a failed platform operation. | `ERR_BUILD_WEDGED` while an image remains in `CREATING` | The stall probe found the client-token replay signature | Inspect the first image version's build states. The diagnosis requires a nonempty list with every build still `PENDING`; an unreadable or empty list does not establish it | `crates/microvms-app/src/control/image.rs:464-500` | | An image reports a failed state | The failed version or its builds may carry the cause | Read the reported `stateReason` details and the build log group; the client gathers version and build reasons | `crates/microvms-app/src/control/image.rs:600-620`, `crates/microvms-app/src/control/image.rs:637-667` | | `ERR_LAUNCH_DIED` while waiting for a lifecycle state | The service reported a state in the wait's failure set | Read the state and `stateReason` included in the error before treating it as a connection problem | `crates/microvms-app/src/control/microvm.rs:675-681`, `crates/microvms-app/src/control/microvm.rs:698-714` | -| `ERR_WINDOW_CLOSED` on resume | The locally known suspended duration exceeds the launch-time window | Read the elapsed duration and window in the error; a longer window applies to a later launch | `crates/microvms-app/src/sandbox.rs:1737-1765` | +| `ERR_WINDOW_CLOSED` on resume | The service refused the resume, and `GetMicrovm` reports the VM gone: it ended first, and nothing brings it back | Read the state and `stateReason` in the error before you launch a new VM. A VM ended with `microvm terminate` gets the same refusal as one the platform ended, so the error can't say which ended it. The idle policy's `suspendedDurationSeconds` ends a VM it suspended, and `maximumDurationInSeconds` ends any VM; an explicit suspend could still resume past the window (measured 2026-10-12, us-east-1) | `crates/microvms-app/src/control/microvm.rs:819-852`, `crates/microvms-app/src/sandbox.rs:1694-1700` | | Control requests return 503 before bootstrap | The daemon has no installed run token yet | Read `bootstrapped` from health and distinguish `NotBootstrapped` from a permanent refusal | `crates/agentd/src/auth.rs:68-80`, `crates/protocol/src/health.rs:20`, `crates/microvms-domain/src/error.rs:384-388` | | A write returns 507 | The destination filesystem is below its configured reserve | Read the available and reserved byte values in the response, then inspect `health.disk`; missing disk data means unmeasurable | `crates/agentd/src/fs.rs:97-123`, `crates/protocol/src/health.rs:21-34` | | `ERR_EXEC_FAILED` | The workload completed with a nonzero exit status | Inspect the workload's stdout, stderr, exit code, and signal before investigating infrastructure | `crates/microvms-cli/src/exit.rs:97-102`, `crates/protocol/src/exec.rs:56-68` | diff --git a/docs/manifest.json b/docs/manifest.json index 4e8778ef..4f5db148 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -339,7 +339,7 @@ { "choices": null, "default": "600", - "help": "Terminate the VM after this long suspended. A resume past it cannot work", + "help": "Terminate a VM the idle policy suspended once it's been suspended this long. An explicit suspend outlasted it when measured (2026-10-12), which AWS doesn't promise", "name": "suspended-sec", "positional": false, "required": false, @@ -988,7 +988,7 @@ { "choices": null, "default": "600", - "help": "Terminate the VM after this long suspended. A resume past it cannot work", + "help": "Terminate a VM the idle policy suspended once it's been suspended this long. An explicit suspend outlasted it when measured (2026-10-12), which AWS doesn't promise", "name": "suspended-sec", "positional": false, "required": false, @@ -4623,7 +4623,7 @@ "code": "ERR_WINDOW_CLOSED", "exit": 8, "finding": "`idlePolicy`", - "meaning": "the launch-time suspended window passed, so there is nothing to resume" + "meaning": "the MicroVM ended before the resume (the suspended window, the maximum duration, or a terminate), and it can't be resumed; launch a new one" }, { "code": "ERR_PLATFORM", diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 5a5cbc13..e4bf5484 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -56,7 +56,7 @@ Flags: - `--no-vm-logs`: Turn the VM's own logging off. `crates/microvms-cli/src/cli.rs:770`. - `--timeout`: How long to wait for the exec, in seconds. `crates/microvms-cli/src/cli.rs:1050`. - `--max-idle-sec`: Suspend the VM after this much inbound-traffic idleness. `crates/microvms-cli/src/cli.rs:1054`. -- `--suspended-sec`: Terminate the VM after this long suspended. A resume past it cannot work. `crates/microvms-cli/src/cli.rs:1058`. +- `--suspended-sec`: Terminate a VM the idle policy suspended once it's been suspended this long. An explicit suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. `crates/microvms-cli/src/cli.rs:1059`. - `--auto-resume`: Let the platform resume a suspended VM on an incoming request. `crates/microvms-cli/src/cli.rs:1062`. - `--max-duration-sec`: Hard ceiling on the VM's life. Refused above 28800 (eight hours) before any call. `crates/microvms-cli/src/cli.rs:1066`. - `--port`: The daemon's port inside the guest. `crates/microvms-cli/src/cli.rs:1070`. @@ -167,7 +167,7 @@ Flags: - `--memory-mib`: MiB of memory the workload needs, instead of --memory: any figure, which selects the smallest class whose baseline covers it (1500 selects 2048). Combines with --cpus. `crates/microvms-cli/src/cli.rs:511`. - `--token-ttl-hours`: How long the Bedrock bearer token lives, in hours. Default and ceiling 12. `crates/microvms-cli/src/cli.rs:2499`. - `--max-idle-sec`: Suspend the VM after this much inbound-traffic idleness. `crates/microvms-cli/src/cli.rs:2503`. -- `--suspended-sec`: Terminate the VM after this long suspended. A resume past it cannot work. `crates/microvms-cli/src/cli.rs:2507`. +- `--suspended-sec`: Terminate a VM the idle policy suspended once it's been suspended this long. An explicit suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. `crates/microvms-cli/src/cli.rs:2509`. - `--auto-resume`: Let the platform resume a suspended VM on an incoming request. `crates/microvms-cli/src/cli.rs:2511`. - `--max-duration-sec`: Hard ceiling on the VM's life. Refused above 28800 (eight hours) before any call. `crates/microvms-cli/src/cli.rs:2515`. - `--port`: The daemon's port inside the guest. `crates/microvms-cli/src/cli.rs:2519`. diff --git a/site/authored/learn/operations/configure-the-project-file.md b/site/authored/learn/operations/configure-the-project-file.md index 469ac8b7..70963e84 100644 --- a/site/authored/learn/operations/configure-the-project-file.md +++ b/site/authored/learn/operations/configure-the-project-file.md @@ -52,7 +52,7 @@ Most field names match their flags; the repeatable connector flag uses the plura | `shell` | `run --shell` | Launch shell-capable, so `microvm shell` can attach later. | | `auto-resume` | `run --auto-resume` | Let the platform resume a suspended VM on an incoming request. | | `max-idle-sec` | `run --max-idle-sec` | Suspend after this much inbound-traffic idleness. | -| `suspended-sec` | `run --suspended-sec` | The platform terminates a VM the idle policy suspended after this long suspended. | +| `suspended-sec` | `run --suspended-sec` | Terminate a VM the idle policy suspended once it's been suspended this long. An explicit `microvm suspend` isn't held to it. | | `max-duration-sec` | `run --max-duration-sec` | Hard ceiling on the VM's life. Refused outside 1 through 28800, eight hours being the platform's ceiling. | | `[env]` | `run --launch-env` | The launch environment, as a table. Merged per key, with a `--launch-env` pair winning on a shared key. A key containing `=` or an empty key is refused. | | `artifacts` | none | Globs for `run `: which files to bring back from the VM's synced working directory. Deliberately no flag spelling. Each glob must compile. | diff --git a/site/authored/learn/operations/recover-a-leaked-vm.md b/site/authored/learn/operations/recover-a-leaked-vm.md index b76c9d58..7dc43f35 100644 --- a/site/authored/learn/operations/recover-a-leaked-vm.md +++ b/site/authored/learn/operations/recover-a-leaked-vm.md @@ -68,7 +68,7 @@ before using `--delete`. It is not an inventory of every resource in the account `run` tears down by default, and an interruption after launch is `ERR_INTERRUPTED` (exit 11): teardown ran, and any leak is named in the envelope's `data.leaked`. A process killed before it could report leaves the ledger entry that `ls` reads. A run under `--keep` is a leak you asked for, and its name is released when the terminate is accepted, so a `--vm-name` refused with `ERR_NAME_TAKEN` (exit 14) means a record still holds it. A VM that ended some other way, at its maximum duration or through another client, leaves its record behind: `microvm names` lists the registry without the records' secrets, and `microvm names --delete NAME` clears one. -Some things that look like leaks are not. A VM the idle policy suspended is terminated by the platform once it has been suspended for its `--suspended-sec` window, and a resume fails because there is nothing to resume (the SDKs report `ERR_WINDOW_CLOSED` when the same `Sandbox` launched the VM). And a build log group under `/aws/lambda-microvms/` outliving `terraform destroy` is expected; it is why `verify-clean` exists. +Some things that look like leaks are not. A VM the idle policy suspended and then kept past its `--suspended-sec` window was terminated by the platform, and a resume fails because there is nothing to resume: `microvm resume` and the SDKs' `ControlPlane.resume` report `ERR_WINDOW_CLOSED` (exit 8) with the state and `stateReason` the service reported. A `Sandbox` that launched the VM doesn't see the idle suspend and refuses its own resume before any call (#603). VMs you suspended with `microvm suspend` outlasted that window, resuming about 910 s into a 60 s window (measured 2026-10-12, us-east-1), which AWS doesn't promise. And a build log group under `/aws/lambda-microvms/` outliving `terraform destroy` is expected; it is why `verify-clean` exists. ## 6. A name registered on another machine diff --git a/site/authored/learn/operations/run-coding-agents-on-bedrock.md b/site/authored/learn/operations/run-coding-agents-on-bedrock.md index cd239e54..db661a19 100644 --- a/site/authored/learn/operations/run-coding-agents-on-bedrock.md +++ b/site/authored/learn/operations/run-coding-agents-on-bedrock.md @@ -128,7 +128,7 @@ runner must own health checks, artifact collection, cancellation and cleanup. | --- | --- | --- | | `--memory` | `1024` | 1 GiB billing baseline, 4 GiB guest ceiling | | `--max-idle-sec` | `600` | Suspend after this many seconds without inbound traffic | -| `--suspended-sec` | `600` | Terminate an idle-suspended VM after this many seconds suspended | +| `--suspended-sec` | `600` | Terminate a VM the idle policy suspended after this many seconds suspended | | `--max-duration-sec` | `3600` | Maximum VM lifetime; upper limit is 28800 seconds (8 hours) | | `--auto-resume` | off | Let an inbound request wake a suspended VM | | `--token-ttl-hours` | `12` | Maximum Bedrock token lifetime; signing credentials may expire sooner | diff --git a/site/authored/learn/tutorial/long-lived-vm.md b/site/authored/learn/tutorial/long-lived-vm.md index df2f8fa9..851759a0 100644 --- a/site/authored/learn/tutorial/long-lived-vm.md +++ b/site/authored/learn/tutorial/long-lived-vm.md @@ -76,7 +76,7 @@ Suspend freezes the VM. Memory, the filesystem, the agent token, and the endpoin adds a snapshot write and read. Compare those costs with remaining running before choosing a suspension policy. -The platform also suspends on its own. `--max-idle-sec` (default 600) suspends the VM after that much inbound-traffic idleness, `--suspended-sec` (default 600) is the suspended window: a VM the idle policy suspended is terminated once it has been suspended that long, and `--auto-resume` lets the platform resume a suspended VM on an incoming request. Once the platform has terminated the VM, `microvm resume` fails with the service's error or on its dead-state wait, and no call extends the window once the VM is launched. A VM suspended by `microvm suspend` was still `SUSPENDED` long past the same window and resumed; [Suspend and resume](/internals/suspend-resume/) has the measurements. The SDKs refuse such a resume up front with `ERR_WINDOW_CLOSED` when the same `Sandbox` launched the VM. `--max-duration-sec` (default 3600) is the hard ceiling on the VM's life and is refused above 28800 (eight hours) before any call. +The platform also suspends on its own. `--max-idle-sec` (default 600) suspends the VM after that much inbound-traffic idleness, `--suspended-sec` (default 600) is the suspended window: a VM the idle policy suspended is terminated once it has been suspended that long, and `--auto-resume` lets the platform resume a suspended VM on an incoming request. Once the platform has terminated the VM, `microvm resume` and the SDKs' `ControlPlane.resume` report `ERR_WINDOW_CLOSED` (exit 8) with the state the service reported, a `Sandbox` that launched the VM refuses its resume before any call because it doesn't see the idle suspend (#603), and no call extends the window once the VM is launched. A VM suspended by `microvm suspend` was still `SUSPENDED` long past the same window and resumed, which AWS doesn't promise; [Suspend and resume](/internals/suspend-resume/) has the measurements. Every resume goes to the service, so a VM the service still holds resumes however long it was suspended. `--max-duration-sec` (default 3600) is the hard ceiling on the VM's life and is refused above 28800 (eight hours) before any call. Idleness is measured by inbound traffic through the endpoint proxy, which terminates outside the VM. A keepalive has to come from outside too: `microvm health --name dev` polls `/v1/health`, which is unauthenticated and resets the idle timer, and reports `busy` and `execs` so the poll can stop once the VM is drained. A guest process cannot keep its own VM alive, and a multi-hour exec with no outside traffic is frozen at the idle window with its process intact. diff --git a/verify/guards/faults/agentd-model.toml b/verify/guards/faults/agentd-model.toml index ad01984b..fed9d9b9 100644 --- a/verify/guards/faults/agentd-model.toml +++ b/verify/guards/faults/agentd-model.toml @@ -125,3 +125,16 @@ transform = { file = "crates/model/src/client.rs", replace = "next.vm_state = Vm id = "model-client-state-10-termination-stays-terminating" message = "\"STATE-10 a reported termination leaves the VM TERMINATED\" counterexample" transform = { file = "crates/model/src/client.rs", replace = "next.vm_state = VmState::Terminated;", with = "next.vm_state = VmState::Terminating;" } + +# A resume the service refuses for a VM the platform ended leaves the client SUSPENDED (#597). +[[family.fault]] +id = "model-client-state-12-refused-resume-unrecorded" +message = "\"STATE-12 a resume refused for an ended VM leaves it TERMINATED and recorded terminated\" counterexample" +transform = { file = "crates/model/src/client.rs", replace = "(next.vm_state, next.was_terminated) = (VmState::Terminated, true);", with = "next.was_terminated = true;" } + +# A client that refuses a resume from SUSPENDED on its own reading of the VM's end, the shape +# #597 removed: the client refused a resume past the window that the service honors. +[[family.fault]] +id = "model-client-state-12-resume-refused-locally" +message = "\"STATE-12 a resume from SUSPENDED reaches the service\" counterexample" +transform = { file = "crates/model/src/client.rs", replace = "if self.guarded() && (terminated || wrong_state) {", with = "if self.guarded() && (terminated || wrong_state || last.platform_ended) {" } diff --git a/verify/guards/faults/f2p-a-resume-of-a-vm-the-service-ended-exits-window-closed.patch b/verify/guards/faults/f2p-a-resume-of-a-vm-the-service-ended-exits-window-closed.patch new file mode 100644 index 00000000..303b48bf --- /dev/null +++ b/verify/guards/faults/f2p-a-resume-of-a-vm-the-service-ended-exits-window-closed.patch @@ -0,0 +1,641 @@ +diff --git a/bindings/microvms-js/src/control.rs b/bindings/microvms-js/src/control.rs +index ead8c40..66a7dd9 100644 +--- a/bindings/microvms-js/src/control.rs ++++ b/bindings/microvms-js/src/control.rs +@@ -279,8 +279,8 @@ pub struct DeleteImageOptions { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7). Use +-/// it when a process has only an identifier. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12). Use it when a process has only an identifier. + #[napi] + pub struct ControlPlane { + inner: Arc, +@@ -344,8 +344,7 @@ impl ControlPlane { + self.inner.suspend(µvm_id).await.map_err(js_async) + } + +- /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, rejects with `ERR_WINDOW_CLOSED` (STATE-12). ++ /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. + #[napi] + pub async fn resume(&self, microvm_id: String) -> Result<(), AsyncError> { + self.inner.resume(µvm_id).await.map_err(js_async) +diff --git a/bindings/microvms-js/src/sandbox.rs b/bindings/microvms-js/src/sandbox.rs +index 517085e..55d1e84 100644 +--- a/bindings/microvms-js/src/sandbox.rs ++++ b/bindings/microvms-js/src/sandbox.rs +@@ -1144,21 +1144,11 @@ impl Sandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, and +- /// even a call the service accepted would hand back a different machine. And a resume +- /// from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspendedSec` passes, and a VM this sandbox suspended stayed resumable +- /// 1283 s into a 60 s window (measured 2026-10-12, us-east-1). `ERR_WINDOW_CLOSED` is the +- /// service's answer once a resume reaches it: the service refused the resume, and the VM +- /// is TERMINATING or TERMINATED, whatever ended it. The sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` rejects with `ERR_INVALID_ARG` before any call (STATE-7). That +- /// suspend reaches a caller only through `Sandbox.adopt`, which reads the state from the +- /// service, or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// even a call the service accepted would hand back a different machine. A resume from ++ /// anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended window ++ /// (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates a ++ /// suspended VM once that window passes, so there is nothing left to resume, and calling ++ /// would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/bindings/microvms-py/src/control.rs b/bindings/microvms-py/src/control.rs +index ce0752e..5fbe783 100644 +--- a/bindings/microvms-py/src/control.rs ++++ b/bindings/microvms-py/src/control.rs +@@ -444,9 +444,9 @@ impl PyImageBuild { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7): a +-/// suspend of a SUSPENDED VM is the service's to refuse. Use it when a process has only an +-/// identifier, such as a durable workflow step in a fresh process. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12): a suspend of a SUSPENDED VM is the service's to refuse. Use it when a ++/// process has only an identifier, such as a durable workflow step in a fresh process. + #[pyclass(frozen, name = "ControlPlane", module = "microvms")] + pub struct PyControlPlane { + inner: Arc, +@@ -703,8 +703,7 @@ impl PyControlPlane { + Ok(runtime::spawn_shielded(self.suspend_op(microvm_id)).await?) + } + +- /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, raises `WindowClosedError` (STATE-12). ++ /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. + fn resume(&self, py: Python<'_>, microvm_id: String) -> PyCoreResult<()> { + Ok(runtime::block_on(py, self.resume_op(microvm_id))?) + } +diff --git a/bindings/microvms-py/src/errors.rs b/bindings/microvms-py/src/errors.rs +index be8962a..a664785 100644 +--- a/bindings/microvms-py/src/errors.rs ++++ b/bindings/microvms-py/src/errors.rs +@@ -87,10 +87,8 @@ create_exception!( + microvms, + WindowClosedError, + MicrovmError, +- "ERR_WINDOW_CLOSED — the MicroVM ended before the resume (the suspended window, the \ +- maximum duration, or a terminate), and it can't be resumed; launch a new one. The \ +- suspended window ends a VM the idle policy suspended; an explicit suspend stayed \ +- resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1)." ++ "ERR_WINDOW_CLOSED — the launch-time suspended window passed, so there is nothing to \ ++ resume. A longer window has to be set at launch on the next VM." + ); + create_exception!( + microvms, +diff --git a/bindings/microvms-py/src/sandbox.rs b/bindings/microvms-py/src/sandbox.rs +index 449afbd..f7d6371 100644 +--- a/bindings/microvms-py/src/sandbox.rs ++++ b/bindings/microvms-py/src/sandbox.rs +@@ -1904,22 +1904,11 @@ impl PySandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, +- /// and even a call the service accepted would hand back a different machine. And a +- /// resume from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspended_sec` passes, and a VM this sandbox suspended stayed +- /// resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1). +- /// `WindowClosedError` is the service's answer once a resume reaches it: the service +- /// refused the resume, and the VM is TERMINATING or TERMINATED, whatever ended it. The +- /// sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` raises `InvalidArgError` before any call (STATE-7). That suspend +- /// reaches a caller only through `Sandbox.adopt`, which reads the state from the service, +- /// or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// and even a call the service accepted would hand back a different machine. A resume ++ /// from anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended ++ /// window (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates ++ /// a suspended VM once that window passes, so there is nothing left to resume, and ++ /// calling would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/crates/microvms-app/src/clock.rs b/crates/microvms-app/src/clock.rs +index 75afb7c..5685303 100644 +--- a/crates/microvms-app/src/clock.rs ++++ b/crates/microvms-app/src/clock.rs +@@ -7,11 +7,11 @@ + //! sources: the control plane's read `std::time::Instant` and the session's read tokio's. + //! Under a deterministic simulator those disagree. tokio's clock is virtual there and `std`'s + //! isn't, so a control-plane wait whose sleeps advanced virtual time measured its deadline +-//! on the host's real time and never reached it. `Sandbox` already waits on the plane's clock +-//! rather than one of its own, so a deadline isn't measured on one clock while the polls sleep +-//! on another. One trait, with tokio's clock in production, keeps that property for everything +-//! that reads time. The trait is re-exported at `control::Clock` and at `session::Clock`, the +-//! paths each half had. ++//! on the host's real time and never reached it. `Sandbox` already times the suspended window ++//! on the plane's clock rather than its own, so a window isn't measured on one clock while ++//! the polls run on another. One trait, with tokio's clock in production, keeps that property ++//! for everything that reads time. The trait is re-exported at `control::Clock` and at ++//! `session::Clock`, the paths each half had. + //! + //! # Why a trait rather than `tokio::time::pause` + //! +@@ -24,11 +24,11 @@ + //! # Monotonic and wall time are separate readings + //! + //! [`Clock::elapsed`] is monotonic, the same reasoning the Python client's injectable +-//! `time.monotonic` carried: a wait's deadline and a proxy token's age are *durations*, and a +-//! wall clock that steps backward would hold a passed deadline open. [`Clock::unix_now`] is the +-//! wall reading, for the values that are dates rather than durations: a name record's +-//! timestamp, an exec id, the cost engine's today. Deriving one from the other would make a +-//! stepped wall clock move a deadline. ++//! `time.monotonic` carried: the suspended window is a *duration*, and a wall clock that steps ++//! backward would reopen a closed one. [`Clock::unix_now`] is the wall reading, for the values ++//! that are dates rather than durations: a name record's timestamp, an exec id, the cost ++//! engine's today. Deriving one from the other would make a stepped wall clock move a ++//! deadline. + + use std::fmt; + use std::future::Future; +diff --git a/crates/microvms-app/src/control/microvm.rs b/crates/microvms-app/src/control/microvm.rs +index 9611956..36597d0 100644 +--- a/crates/microvms-app/src/control/microvm.rs ++++ b/crates/microvms-app/src/control/microvm.rs +@@ -799,56 +799,11 @@ impl ControlPlane { + } + + /// `ResumeMicrovm`. Thaws the VM; no token re-delivery and no re-bootstrap. +- /// +- /// # A VM that ended before the resume is `WindowClosed` (STATE-12) +- /// +- /// Nothing is checked against the suspended window before the call, because the service +- /// decides whether a suspended VM can still resume. It terminates a VM the idle policy +- /// suspended once `suspendedDurationSeconds` passes, and it doesn't hold an explicit +- /// `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, an explicitly +- /// suspended VM resumed 1283 s into a 60 s window (#597). A resume of a VM that has +- /// terminated is refused with a `ValidationException` ("The MicroVM has been +- /// terminated and its state cannot be changed."), the same status as any refused value, +- /// and it was the same refusal for a VM the service ended and one `TerminateMicrovm` +- /// ended (measured 2026-10-12, us-east-1). So a refusal reads the VM's state rather than +- /// the message, and a VM that's TERMINATING or TERMINATED is [`ErrorKind::WindowClosed`] +- /// with its state and `stateReason`, carrying the refusal as its source. The error names +- /// no cause, because the client doesn't know which of the suspended window, the maximum +- /// duration, or a terminate ended the VM; the `stateReason` is the service's word on it. +- /// Every other refusal comes back as the service gave it. + pub async fn resume(&self, id: &str) -> Result<(), Error> { + super::require_valid_identifier("microvmIdentifier", id)?; + let call = Call::post_empty("ResumeMicrovm", paths::resume(id)); +- match send_with_retry(self.transport(), call).await { +- Ok(_) => Ok(()), +- Err(refused) if refused.kind() == ErrorKind::Platform => { +- Err(self.ended_before_resume(id, refused).await) +- } +- Err(error) => Err(error), +- } +- } +- +- /// The error a refused resume reports: [`ErrorKind::WindowClosed`] with `refused` as its +- /// source when `GetMicrovm` reports the VM dead, and `refused` otherwise, including when +- /// that read fails. +- async fn ended_before_resume(&self, id: &str, refused: Error) -> Error { +- let Ok(vm) = self.get_microvm(id).await else { +- return refused; +- }; +- if !crate::constants::DEAD_STATES.contains(&vm.state.as_str()) { +- return refused; +- } +- Error::new( +- ErrorKind::WindowClosed, +- format!( +- "microvm {id} can't be resumed: the service refused ResumeMicrovm, and \ +- GetMicrovm reports the VM {} ({}). A VM that has ended doesn't come back; \ +- launch a new one.", +- vm.state, +- vm.reason(), +- ), +- ) +- .with_source(refused) ++ send_with_retry(self.transport(), call).await?; ++ Ok(()) + } + + /// `TerminateMicrovm`. +diff --git a/crates/microvms-app/src/control/ops.rs b/crates/microvms-app/src/control/ops.rs +index 36833c6..ae1c3c1 100644 +--- a/crates/microvms-app/src/control/ops.rs ++++ b/crates/microvms-app/src/control/ops.rs +@@ -367,9 +367,11 @@ pub struct RunMicrovmWire { + /// "autoResumeEnabled": false}` (measured 2026-08-15, us-east-1, one read-only + /// `GetMicrovm`). + /// +-/// The readback means the client's own record isn't the only authority on the suspended +-/// window: [`crate::control::Microvm::idle_policy`] carries the service's, so a window that +-/// disagrees with what was asked for is observable rather than assumed. ++/// The claim mattered because STATE-12 was resting on it: if only the client could name the ++/// suspended window, the client's own record is the only authority on it. The readback ++/// means there is a second one, and [`crate::control::Microvm::idle_policy`] carries it — ++/// so a suspended window that disagrees with what was asked for is now observable rather ++/// than assumed. + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct IdlePolicy { +diff --git a/crates/microvms-app/src/sandbox.rs b/crates/microvms-app/src/sandbox.rs +index 544535f..dc316a4 100644 +--- a/crates/microvms-app/src/sandbox.rs ++++ b/crates/microvms-app/src/sandbox.rs +@@ -3,8 +3,8 @@ + //! model describes made true of the code. + //! + //! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the +-//! bindings use: build, run, suspend, resume, terminate, a resume of a VM that ended +-//! before it (STATE-12), and teardown in the order that does not leak. ++//! bindings use: build, run, suspend, resume, terminate, the launch-time suspended ++//! window (STATE-12), and teardown in the order that does not leak. + //! + //! # The lifecycle is a field, and every transition is guarded + //! +@@ -32,20 +32,16 @@ + //! control-plane calls rather than answered by AWS. The test asserts the call count, which + //! is the observable that distinguishes the two. + //! +-//! # The service decides whether a suspended VM resumes (STATE-12) ++//! # The suspended window is checked before the call (STATE-12) + //! +-//! The launch-time `idlePolicy` *terminates* a VM the idle policy suspended once +-//! `suspendedDurationSeconds` passes, and the service doesn't hold an explicit +-//! `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, a VM this client +-//! suspended resumed 1283 s into a 60 s window (#597). So a resume is never refused here for +-//! the time it has spent suspended. It goes to the service, and the service's refusal for a +-//! VM that ended, whatever ended it, comes back from [`ControlPlane::resume`] as +-//! [`ErrorKind::WindowClosed`], which this sandbox records as a termination (STATE-11). +-//! +-//! A suspend the idle policy makes isn't seen here: the sandbox still holds the VM as +-//! RUNNING, and its resume is refused before any call (STATE-7). That suspend reaches a +-//! caller only through [`Sandbox::adopt`], which reads the state from the service, or +-//! [`ControlPlane::resume`], which holds no lifecycle to refuse from. ++//! The launch-time `idlePolicy` *terminates* a suspended VM once `suspendedDurationSeconds` ++//! passes, which means "resume later" silently stops working. A resume past the window is ++//! refused locally, before `ResumeMicrovm`, because the alternative is calling and reading ++//! the failure: the service answers about a terminated id, which is not the same statement ++//! as "the window you set at launch closed", and getting there costs the full poll timeout ++//! first. The window comes from this sandbox's own `RunMicrovm` request, falling back to the ++//! `idlePolicy` that `GetMicrovm` reports (measured 2026-08-15, `docs/PLATFORM.md`). Only ++//! the suspend's start time is the client's alone. + //! + //! # Teardown never raises, and the log group is last + //! +@@ -270,8 +266,7 @@ pub struct RunRequest { + pub shell: bool, + /// `idlePolicy.maxIdleDurationSeconds`. + pub max_idle_sec: u32, +- /// `idlePolicy.suspendedDurationSeconds`: how long a VM the idle policy suspended lasts +- /// before the service terminates it. An explicit suspend isn't held to it (STATE-12). ++ /// `idlePolicy.suspendedDurationSeconds` — the window STATE-12 refuses past. + pub suspended_sec: u32, + /// `idlePolicy.autoResumeEnabled`. + pub auto_resume: bool, +@@ -409,8 +404,7 @@ impl RunRequest { + self + } + +- /// Sets `idlePolicy.suspendedDurationSeconds`, how long a VM the idle policy suspended +- /// lasts before the service terminates it. See [`RunRequest::suspended_sec`]. ++ /// Sets the suspended window this sandbox will refuse a resume past (STATE-12). + #[must_use] + pub fn with_suspended_sec(mut self, seconds: u32) -> Self { + self.suspended_sec = seconds; +@@ -624,8 +618,7 @@ impl TokenMinter for ControlPlaneMinter { + + /// One MicroVM's whole life. + /// +-/// See the module docs for the state machine, the resume of an ended VM, and why teardown is +-/// explicit. ++/// See the module docs for the state machine, the window, and why teardown is explicit. + pub struct Sandbox { + control: Arc, + image: Option, +@@ -651,8 +644,8 @@ pub struct Sandbox { + was_terminated: bool, + bootstrap_count: u32, + +- /// The suspended window from *our own* `RunMicrovm` request, or `None` for a sandbox that +- /// launched nothing. `GetMicrovm` reports the service's in `idlePolicy`. ++ /// The window from *our own* `RunMicrovm` request. `GetMicrovm` also reports it (in ++ /// `idlePolicy`), which is the fallback when this sandbox has no request of its own. + suspended_window: Option, + /// Whether the accepted launch carried a caller client token, and so may have adopted + /// a VM an earlier attempt launched (#195). +@@ -666,6 +659,8 @@ pub struct Sandbox { + /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence + /// is checked against it. + idle_window: Option, ++ /// The clock reading when the suspend call was accepted, or `None` when not suspended. ++ suspended_at: Option, + /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down + /// one. + torn_down: bool, +@@ -754,10 +749,10 @@ impl Sandbox { + /// `Sandbox::from_name`, which resolve credentials for a region, are in + /// `microvms_core::prelude`. + /// +- /// The plane's own clock is what every wait here sleeps on, rather than a second one ++ /// The plane's own clock is what times the suspended window, rather than a second one + /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil +- /// tests record: a deadline measured on one clock while the poll loop sleeps on another +- /// reads the wrong elapsed time. ++ /// tests record: a window measured on one clock while the poll loop sleeps on another ++ /// calls a closed window open. + pub fn with_control_plane(control: ControlPlane) -> Self { + Self { + control: Arc::new(control), +@@ -777,6 +772,7 @@ impl Sandbox { + launch_posture: None, + pending_agent_token: None, + idle_window: None, ++ suspended_at: None, + torn_down: false, + adopted: false, + detached: false, +@@ -1355,6 +1351,12 @@ impl Sandbox { + /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once + /// when the service reports RUNNING, the same as a launch this sandbox made itself. + /// ++ /// # The suspended window (STATE-12) ++ /// ++ /// The window comes from the `idlePolicy` `GetMicrovm` reports. A VM adopted while ++ /// already SUSPENDED has no suspend time this client observed, so the local check has ++ /// nothing to measure and the service answers the resume instead. ++ /// + /// `endpoint` must agree with the service's when the service reports one: a mismatch + /// means the identifiers came from two different records. `agent_token` is the bearer + /// credential the VM was launched with; it never appears in `Debug` or an error. +@@ -1603,6 +1605,10 @@ impl Sandbox { + // neither suspend nor resume accepts, bricking the handle over one bad request. + self.control.suspend(&id).await?; + self.set_lifecycle(Lifecycle::Suspending); ++ // Stamped after the call and before the wait, not after the wait: the idlePolicy's ++ // window starts when the platform begins suspending, so timing it from SUSPENDED ++ // would under-count the transition and call a closed window open. ++ self.suspended_at = Some(self.control.clock().elapsed()); + + let settled = self + .control +@@ -1649,16 +1655,12 @@ impl Sandbox { + /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be + /// refused — a 409 that reads like a broken VM. + /// +- /// # The service decides, and a VM it ended is `WindowClosed` (STATE-12) ++ /// # The window is checked first (STATE-12) + /// +- /// No resume is refused here for the time the VM has spent suspended. The service ends a +- /// VM the idle policy suspended once `suspendedDurationSeconds` passes, and it kept a VM +- /// this client suspended resumable 1283 s into a 60 s window (measured 2026-10-12, +- /// us-east-1, #597). A resume the service refuses because the VM ended, whatever ended +- /// it, is [`ErrorKind::WindowClosed`] ([`ControlPlane::resume`]), and this records the VM +- /// as terminated, so a second resume is refused here with no call (STATE-11). A VM the +- /// idle policy suspended is still RUNNING here, so its resume is refused before any call +- /// (STATE-7); see the module docs. ++ /// Before any wire call, because the answer is already known and calling costs the poll ++ /// timeout to learn something worse. The falsification is a service reporting ++ /// TERMINATED: without this check the resume burns the full deadline and then reports a ++ /// state the client could have named at once. + /// + /// # The proxy token is dropped (STATE-8) + /// +@@ -1671,7 +1673,7 @@ impl Sandbox { + let id = self.require_microvm("resume")?; + + // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal +- // comes before any call. ++ // comes before the window check and before any call. + if self.was_terminated || self.lifecycle == Lifecycle::Terminated { + return Err(Error::invalid_arg(format!( + "microvm {id} was terminated, and a terminated VM never returns to RUNNING \ +@@ -1686,23 +1688,15 @@ impl Sandbox { + ))); + } + +- // STATE-12: a `WindowClosed` refusal is a VM that ended before this resume. It's +- // recorded as TERMINATED whichever of TERMINATING or TERMINATED `GetMicrovm` read, +- // because the service has said the VM's state can't change, and as a termination, so +- // the next resume is refused before any call (STATE-11). Any other refusal leaves the +- // sandbox SUSPENDED, and a retry goes to the service again. +- if let Err(refused) = self.control.resume(&id).await { +- if refused.kind() == ErrorKind::WindowClosed { +- self.was_terminated = true; +- self.set_lifecycle(Lifecycle::Terminated); +- } +- return Err(refused); +- } ++ // STATE-12, first and locally. ++ self.require_open_suspended_window(&id)?; ++ ++ self.control.resume(&id).await?; + // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the +- // state this call was made from, so failing on it would fail every resume. A VM that +- // terminates once the resume is accepted never reaches RUNNING, and waiting only for +- // RUNNING there burns the full timeout and then reports a timeout message hiding a +- // cause the service had already stated in `stateReason`. ++ // state this call was made from, so failing on it would fail every resume. A VM the ++ // idlePolicy terminated during suspension never reaches RUNNING, and waiting only ++ // for RUNNING there burns the full timeout and then reports a timeout message ++ // hiding a cause the service had already stated in `stateReason`. + let running = self + .control + .wait_for_state( +@@ -1722,6 +1716,10 @@ impl Sandbox { + if let Some(session) = self.session.as_mut() { + session.rebind(endpoint); + } ++ // Cleared on success so the next cycle's window is measured from the next suspend. ++ // Leaving it set would accumulate every suspension's elapsed time into one total and ++ // reject a resume whose own window is wide open. ++ self.suspended_at = None; + + self.session.as_mut().ok_or_else(|| { + Error::new( +@@ -1732,6 +1730,41 @@ impl Sandbox { + }) + } + ++ /// Rejects a resume the launch-time `idlePolicy` has already made impossible (STATE-12). ++ /// ++ /// The message names the elapsed time, the window, and the `idlePolicy` finding, because ++ /// "cannot resume" alone sends a reader looking for the flag that reopens it. ++ fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> { ++ let reported = self ++ .microvm ++ .as_ref() ++ .and_then(|vm| vm.idle_policy.as_ref()) ++ .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds))); ++ let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at) ++ else { ++ // No window or no suspend time means this sandbox cannot know how long the VM ++ // has been suspended, and guessing would refuse a resume the service would honour. ++ return Ok(()); ++ }; ++ let elapsed = self.control.clock().elapsed().saturating_sub(since); ++ if elapsed <= window { ++ return Ok(()); ++ } ++ Err(Error::new( ++ ErrorKind::WindowClosed, ++ format!( ++ "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \ ++ window set at launch — the idlePolicy terminates a suspended VM once that window \ ++ passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \ ++ Refused before ResumeMicrovm, because calling would cost the full poll timeout \ ++ to learn the same thing less clearly. A longer window has to be set at launch \ ++ on the next VM; there is no call that extends this one.", ++ elapsed.as_secs(), ++ window.as_secs(), ++ ), ++ )) ++ } ++ + // ── terminate (STATE-9, STATE-10) ──────────────────────────────────────── + + /// Tears down, best-effort, never erroring. +@@ -1874,14 +1907,11 @@ impl Sandbox { + let vm = self.microvm.as_ref()?; + self.lifecycle.is_live().then(|| { + format!( +- "warning: the Sandbox for microvm {id} was dropped in {state} without \ +- terminate() or detach(). Nothing was torn down — Drop cannot await, so a \ +- teardown here would deadlock inside a runtime. {state} is the state this \ +- sandbox last saw, and the platform may have suspended or ended the VM since. A \ +- VM that's still live bills until it ends, at its maximumDurationInSeconds \ +- ceiling at the latest: terminate it with `microvm terminate {id}`.", +- id = vm.id, +- state = self.lifecycle, ++ "warning: the Sandbox for microvm {} was dropped in {} without terminate() or \ ++ detach(). Nothing was torn down — Drop cannot await, so a teardown here would \ ++ deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \ ++ ceiling: terminate it with `microvm terminate {}`.", ++ vm.id, self.lifecycle, vm.id, + ) + }) + } +diff --git a/crates/microvms-cli/src/cli.rs b/crates/microvms-cli/src/cli.rs +index 25b94f0..f3a820e 100644 +--- a/crates/microvms-cli/src/cli.rs ++++ b/crates/microvms-cli/src/cli.rs +@@ -301,10 +301,8 @@ pub enum Command { + + /// Thaw a suspended MicroVM and report its endpoint. + /// +- /// The launch-time `suspendedDurationSeconds` window terminates a VM the idle policy +- /// suspended once it passes. VMs suspended with `microvm suspend` resumed 900 s and more +- /// into a 60 s window (2026-10-12), which AWS doesn't promise, so the service decides. A +- /// resume of a VM that ended, whatever ended it, exits `ERR_WINDOW_CLOSED`; launch a new one. ++ /// The launch-time `suspendedDurationSeconds` window terminates a suspended VM once it ++ /// passes, so "resume later" silently stops working and the VM is gone rather than slow. + Resume(ResumeArgs), + + /// Tear down a MicroVM, and optionally its image and build log group. +@@ -1056,8 +1054,7 @@ pub struct RunArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +@@ -2506,8 +2503,7 @@ pub struct AgentUpArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +diff --git a/crates/microvms-cli/src/commands/lifecycle.rs b/crates/microvms-cli/src/commands/lifecycle.rs +index c4c0b8b..ef6a544 100644 +--- a/crates/microvms-cli/src/commands/lifecycle.rs ++++ b/crates/microvms-cli/src/commands/lifecycle.rs +@@ -2084,12 +2084,10 @@ pub async fn suspend( + /// Thaws a suspended MicroVM and reports its endpoint. + /// + /// The suspended window is **not** checked here, and that is correct rather than a gap: the +-/// service ends a VM the idle policy suspended once `suspendedDurationSeconds` passes and keeps +-/// an explicitly suspended one resumable past it (measured 2026-10-12, us-east-1), so only the +-/// service can answer. Core's `ControlPlane::resume` reports its refusal of a VM that ended +-/// before the resume as `ERR_WINDOW_CLOSED`. A VM that dies after the resume is +-/// accepted fails the wait at once through `fail_on: DEAD_STATES`, with `stateReason`, instead +-/// of burning the poll timeout. ++/// window is in `GetMicrovm`'s `idlePolicy`, but when the suspend began is known only to the ++/// process that issued it. Guessing would either reject an open window or accept a closed ++/// one, both worse than letting the service answer — which is what `fail_on: DEAD_STATES` ++/// does, failing fast with `stateReason` instead of burning the poll timeout. + pub async fn resume( + ctx: &mut Ctx<'_, O, E>, + args: &ResumeArgs, +diff --git a/crates/microvms-cli/src/config.rs b/crates/microvms-cli/src/config.rs +index 9693850..6020ef2 100644 +--- a/crates/microvms-cli/src/config.rs ++++ b/crates/microvms-cli/src/config.rs +@@ -83,8 +83,7 @@ pub struct ProjectConfig { + pub auto_resume: Option, + /// `run --max-idle-sec`: suspend after this much inbound-traffic idleness. + pub max_idle_sec: Option, +- /// `run --suspended-sec`: terminate a VM the idle policy suspended once it's been suspended +- /// this long. ++ /// `run --suspended-sec`: terminate after this long suspended. + pub suspended_sec: Option, + /// `run --max-duration-sec`: hard ceiling on the VM's life. + pub max_duration_sec: Option, +diff --git a/crates/microvms-cli/src/exit.rs b/crates/microvms-cli/src/exit.rs +index 84342b9..4bba5a5 100644 +--- a/crates/microvms-cli/src/exit.rs ++++ b/crates/microvms-cli/src/exit.rs +@@ -254,8 +254,7 @@ pub const EXIT_TABLE: [ExitRow; 17] = [ + ExitRow { + exit: Exit::WindowClosed, + code: Some("ERR_WINDOW_CLOSED"), +- meaning: "the MicroVM ended before the resume (the suspended window, the maximum \ +- duration, or a terminate), and it can't be resumed; launch a new one", ++ meaning: "the launch-time suspended window passed, so there is nothing to resume", + finding: "`idlePolicy`", + }, + ExitRow { +@@ -443,7 +442,9 @@ pub fn classify(error: &Error) -> CliError { + (Exit::LaunchDied, _) => { + &["`microvm logs ` names the build log group the hook wrote to"] + } +- (Exit::WindowClosed, _) => &["the VM ended and no call brings it back: launch a new one"], ++ (Exit::WindowClosed, _) => { ++ &["a longer window is set at launch with --suspended-sec; no call extends this one"] ++ } + _ => &[], + }; + classified +diff --git a/crates/microvms-domain/src/error.rs b/crates/microvms-domain/src/error.rs +index bbbd0ed..15ed493 100644 +--- a/crates/microvms-domain/src/error.rs ++++ b/crates/microvms-domain/src/error.rs +@@ -130,9 +130,7 @@ pub enum ErrorKind { + BuildWedged, + /// The MicroVM reached a terminal state before RUNNING; read `stateReason`. + LaunchDied, +- /// The MicroVM ended before the resume (the suspended window, the maximum duration, or a +- /// terminate), and it can't be resumed; launch a new one. The service refused the resume, +- /// and `GetMicrovm` reports the VM TERMINATING or TERMINATED. ++ /// The launch-time suspended window passed, so there is nothing to resume. + WindowClosed, + /// A control-plane failure with no more specific class. + Platform, diff --git a/verify/guards/faults/f2p-a-resume-past-the-window-of-an-explicit-suspend-reaches.patch b/verify/guards/faults/f2p-a-resume-past-the-window-of-an-explicit-suspend-reaches.patch new file mode 100644 index 00000000..303b48bf --- /dev/null +++ b/verify/guards/faults/f2p-a-resume-past-the-window-of-an-explicit-suspend-reaches.patch @@ -0,0 +1,641 @@ +diff --git a/bindings/microvms-js/src/control.rs b/bindings/microvms-js/src/control.rs +index ead8c40..66a7dd9 100644 +--- a/bindings/microvms-js/src/control.rs ++++ b/bindings/microvms-js/src/control.rs +@@ -279,8 +279,8 @@ pub struct DeleteImageOptions { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7). Use +-/// it when a process has only an identifier. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12). Use it when a process has only an identifier. + #[napi] + pub struct ControlPlane { + inner: Arc, +@@ -344,8 +344,7 @@ impl ControlPlane { + self.inner.suspend(µvm_id).await.map_err(js_async) + } + +- /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, rejects with `ERR_WINDOW_CLOSED` (STATE-12). ++ /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. + #[napi] + pub async fn resume(&self, microvm_id: String) -> Result<(), AsyncError> { + self.inner.resume(µvm_id).await.map_err(js_async) +diff --git a/bindings/microvms-js/src/sandbox.rs b/bindings/microvms-js/src/sandbox.rs +index 517085e..55d1e84 100644 +--- a/bindings/microvms-js/src/sandbox.rs ++++ b/bindings/microvms-js/src/sandbox.rs +@@ -1144,21 +1144,11 @@ impl Sandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, and +- /// even a call the service accepted would hand back a different machine. And a resume +- /// from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspendedSec` passes, and a VM this sandbox suspended stayed resumable +- /// 1283 s into a 60 s window (measured 2026-10-12, us-east-1). `ERR_WINDOW_CLOSED` is the +- /// service's answer once a resume reaches it: the service refused the resume, and the VM +- /// is TERMINATING or TERMINATED, whatever ended it. The sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` rejects with `ERR_INVALID_ARG` before any call (STATE-7). That +- /// suspend reaches a caller only through `Sandbox.adopt`, which reads the state from the +- /// service, or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// even a call the service accepted would hand back a different machine. A resume from ++ /// anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended window ++ /// (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates a ++ /// suspended VM once that window passes, so there is nothing left to resume, and calling ++ /// would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/bindings/microvms-py/src/control.rs b/bindings/microvms-py/src/control.rs +index ce0752e..5fbe783 100644 +--- a/bindings/microvms-py/src/control.rs ++++ b/bindings/microvms-py/src/control.rs +@@ -444,9 +444,9 @@ impl PyImageBuild { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7): a +-/// suspend of a SUSPENDED VM is the service's to refuse. Use it when a process has only an +-/// identifier, such as a durable workflow step in a fresh process. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12): a suspend of a SUSPENDED VM is the service's to refuse. Use it when a ++/// process has only an identifier, such as a durable workflow step in a fresh process. + #[pyclass(frozen, name = "ControlPlane", module = "microvms")] + pub struct PyControlPlane { + inner: Arc, +@@ -703,8 +703,7 @@ impl PyControlPlane { + Ok(runtime::spawn_shielded(self.suspend_op(microvm_id)).await?) + } + +- /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, raises `WindowClosedError` (STATE-12). ++ /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. + fn resume(&self, py: Python<'_>, microvm_id: String) -> PyCoreResult<()> { + Ok(runtime::block_on(py, self.resume_op(microvm_id))?) + } +diff --git a/bindings/microvms-py/src/errors.rs b/bindings/microvms-py/src/errors.rs +index be8962a..a664785 100644 +--- a/bindings/microvms-py/src/errors.rs ++++ b/bindings/microvms-py/src/errors.rs +@@ -87,10 +87,8 @@ create_exception!( + microvms, + WindowClosedError, + MicrovmError, +- "ERR_WINDOW_CLOSED — the MicroVM ended before the resume (the suspended window, the \ +- maximum duration, or a terminate), and it can't be resumed; launch a new one. The \ +- suspended window ends a VM the idle policy suspended; an explicit suspend stayed \ +- resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1)." ++ "ERR_WINDOW_CLOSED — the launch-time suspended window passed, so there is nothing to \ ++ resume. A longer window has to be set at launch on the next VM." + ); + create_exception!( + microvms, +diff --git a/bindings/microvms-py/src/sandbox.rs b/bindings/microvms-py/src/sandbox.rs +index 449afbd..f7d6371 100644 +--- a/bindings/microvms-py/src/sandbox.rs ++++ b/bindings/microvms-py/src/sandbox.rs +@@ -1904,22 +1904,11 @@ impl PySandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, +- /// and even a call the service accepted would hand back a different machine. And a +- /// resume from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspended_sec` passes, and a VM this sandbox suspended stayed +- /// resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1). +- /// `WindowClosedError` is the service's answer once a resume reaches it: the service +- /// refused the resume, and the VM is TERMINATING or TERMINATED, whatever ended it. The +- /// sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` raises `InvalidArgError` before any call (STATE-7). That suspend +- /// reaches a caller only through `Sandbox.adopt`, which reads the state from the service, +- /// or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// and even a call the service accepted would hand back a different machine. A resume ++ /// from anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended ++ /// window (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates ++ /// a suspended VM once that window passes, so there is nothing left to resume, and ++ /// calling would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/crates/microvms-app/src/clock.rs b/crates/microvms-app/src/clock.rs +index 75afb7c..5685303 100644 +--- a/crates/microvms-app/src/clock.rs ++++ b/crates/microvms-app/src/clock.rs +@@ -7,11 +7,11 @@ + //! sources: the control plane's read `std::time::Instant` and the session's read tokio's. + //! Under a deterministic simulator those disagree. tokio's clock is virtual there and `std`'s + //! isn't, so a control-plane wait whose sleeps advanced virtual time measured its deadline +-//! on the host's real time and never reached it. `Sandbox` already waits on the plane's clock +-//! rather than one of its own, so a deadline isn't measured on one clock while the polls sleep +-//! on another. One trait, with tokio's clock in production, keeps that property for everything +-//! that reads time. The trait is re-exported at `control::Clock` and at `session::Clock`, the +-//! paths each half had. ++//! on the host's real time and never reached it. `Sandbox` already times the suspended window ++//! on the plane's clock rather than its own, so a window isn't measured on one clock while ++//! the polls run on another. One trait, with tokio's clock in production, keeps that property ++//! for everything that reads time. The trait is re-exported at `control::Clock` and at ++//! `session::Clock`, the paths each half had. + //! + //! # Why a trait rather than `tokio::time::pause` + //! +@@ -24,11 +24,11 @@ + //! # Monotonic and wall time are separate readings + //! + //! [`Clock::elapsed`] is monotonic, the same reasoning the Python client's injectable +-//! `time.monotonic` carried: a wait's deadline and a proxy token's age are *durations*, and a +-//! wall clock that steps backward would hold a passed deadline open. [`Clock::unix_now`] is the +-//! wall reading, for the values that are dates rather than durations: a name record's +-//! timestamp, an exec id, the cost engine's today. Deriving one from the other would make a +-//! stepped wall clock move a deadline. ++//! `time.monotonic` carried: the suspended window is a *duration*, and a wall clock that steps ++//! backward would reopen a closed one. [`Clock::unix_now`] is the wall reading, for the values ++//! that are dates rather than durations: a name record's timestamp, an exec id, the cost ++//! engine's today. Deriving one from the other would make a stepped wall clock move a ++//! deadline. + + use std::fmt; + use std::future::Future; +diff --git a/crates/microvms-app/src/control/microvm.rs b/crates/microvms-app/src/control/microvm.rs +index 9611956..36597d0 100644 +--- a/crates/microvms-app/src/control/microvm.rs ++++ b/crates/microvms-app/src/control/microvm.rs +@@ -799,56 +799,11 @@ impl ControlPlane { + } + + /// `ResumeMicrovm`. Thaws the VM; no token re-delivery and no re-bootstrap. +- /// +- /// # A VM that ended before the resume is `WindowClosed` (STATE-12) +- /// +- /// Nothing is checked against the suspended window before the call, because the service +- /// decides whether a suspended VM can still resume. It terminates a VM the idle policy +- /// suspended once `suspendedDurationSeconds` passes, and it doesn't hold an explicit +- /// `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, an explicitly +- /// suspended VM resumed 1283 s into a 60 s window (#597). A resume of a VM that has +- /// terminated is refused with a `ValidationException` ("The MicroVM has been +- /// terminated and its state cannot be changed."), the same status as any refused value, +- /// and it was the same refusal for a VM the service ended and one `TerminateMicrovm` +- /// ended (measured 2026-10-12, us-east-1). So a refusal reads the VM's state rather than +- /// the message, and a VM that's TERMINATING or TERMINATED is [`ErrorKind::WindowClosed`] +- /// with its state and `stateReason`, carrying the refusal as its source. The error names +- /// no cause, because the client doesn't know which of the suspended window, the maximum +- /// duration, or a terminate ended the VM; the `stateReason` is the service's word on it. +- /// Every other refusal comes back as the service gave it. + pub async fn resume(&self, id: &str) -> Result<(), Error> { + super::require_valid_identifier("microvmIdentifier", id)?; + let call = Call::post_empty("ResumeMicrovm", paths::resume(id)); +- match send_with_retry(self.transport(), call).await { +- Ok(_) => Ok(()), +- Err(refused) if refused.kind() == ErrorKind::Platform => { +- Err(self.ended_before_resume(id, refused).await) +- } +- Err(error) => Err(error), +- } +- } +- +- /// The error a refused resume reports: [`ErrorKind::WindowClosed`] with `refused` as its +- /// source when `GetMicrovm` reports the VM dead, and `refused` otherwise, including when +- /// that read fails. +- async fn ended_before_resume(&self, id: &str, refused: Error) -> Error { +- let Ok(vm) = self.get_microvm(id).await else { +- return refused; +- }; +- if !crate::constants::DEAD_STATES.contains(&vm.state.as_str()) { +- return refused; +- } +- Error::new( +- ErrorKind::WindowClosed, +- format!( +- "microvm {id} can't be resumed: the service refused ResumeMicrovm, and \ +- GetMicrovm reports the VM {} ({}). A VM that has ended doesn't come back; \ +- launch a new one.", +- vm.state, +- vm.reason(), +- ), +- ) +- .with_source(refused) ++ send_with_retry(self.transport(), call).await?; ++ Ok(()) + } + + /// `TerminateMicrovm`. +diff --git a/crates/microvms-app/src/control/ops.rs b/crates/microvms-app/src/control/ops.rs +index 36833c6..ae1c3c1 100644 +--- a/crates/microvms-app/src/control/ops.rs ++++ b/crates/microvms-app/src/control/ops.rs +@@ -367,9 +367,11 @@ pub struct RunMicrovmWire { + /// "autoResumeEnabled": false}` (measured 2026-08-15, us-east-1, one read-only + /// `GetMicrovm`). + /// +-/// The readback means the client's own record isn't the only authority on the suspended +-/// window: [`crate::control::Microvm::idle_policy`] carries the service's, so a window that +-/// disagrees with what was asked for is observable rather than assumed. ++/// The claim mattered because STATE-12 was resting on it: if only the client could name the ++/// suspended window, the client's own record is the only authority on it. The readback ++/// means there is a second one, and [`crate::control::Microvm::idle_policy`] carries it — ++/// so a suspended window that disagrees with what was asked for is now observable rather ++/// than assumed. + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct IdlePolicy { +diff --git a/crates/microvms-app/src/sandbox.rs b/crates/microvms-app/src/sandbox.rs +index 544535f..dc316a4 100644 +--- a/crates/microvms-app/src/sandbox.rs ++++ b/crates/microvms-app/src/sandbox.rs +@@ -3,8 +3,8 @@ + //! model describes made true of the code. + //! + //! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the +-//! bindings use: build, run, suspend, resume, terminate, a resume of a VM that ended +-//! before it (STATE-12), and teardown in the order that does not leak. ++//! bindings use: build, run, suspend, resume, terminate, the launch-time suspended ++//! window (STATE-12), and teardown in the order that does not leak. + //! + //! # The lifecycle is a field, and every transition is guarded + //! +@@ -32,20 +32,16 @@ + //! control-plane calls rather than answered by AWS. The test asserts the call count, which + //! is the observable that distinguishes the two. + //! +-//! # The service decides whether a suspended VM resumes (STATE-12) ++//! # The suspended window is checked before the call (STATE-12) + //! +-//! The launch-time `idlePolicy` *terminates* a VM the idle policy suspended once +-//! `suspendedDurationSeconds` passes, and the service doesn't hold an explicit +-//! `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, a VM this client +-//! suspended resumed 1283 s into a 60 s window (#597). So a resume is never refused here for +-//! the time it has spent suspended. It goes to the service, and the service's refusal for a +-//! VM that ended, whatever ended it, comes back from [`ControlPlane::resume`] as +-//! [`ErrorKind::WindowClosed`], which this sandbox records as a termination (STATE-11). +-//! +-//! A suspend the idle policy makes isn't seen here: the sandbox still holds the VM as +-//! RUNNING, and its resume is refused before any call (STATE-7). That suspend reaches a +-//! caller only through [`Sandbox::adopt`], which reads the state from the service, or +-//! [`ControlPlane::resume`], which holds no lifecycle to refuse from. ++//! The launch-time `idlePolicy` *terminates* a suspended VM once `suspendedDurationSeconds` ++//! passes, which means "resume later" silently stops working. A resume past the window is ++//! refused locally, before `ResumeMicrovm`, because the alternative is calling and reading ++//! the failure: the service answers about a terminated id, which is not the same statement ++//! as "the window you set at launch closed", and getting there costs the full poll timeout ++//! first. The window comes from this sandbox's own `RunMicrovm` request, falling back to the ++//! `idlePolicy` that `GetMicrovm` reports (measured 2026-08-15, `docs/PLATFORM.md`). Only ++//! the suspend's start time is the client's alone. + //! + //! # Teardown never raises, and the log group is last + //! +@@ -270,8 +266,7 @@ pub struct RunRequest { + pub shell: bool, + /// `idlePolicy.maxIdleDurationSeconds`. + pub max_idle_sec: u32, +- /// `idlePolicy.suspendedDurationSeconds`: how long a VM the idle policy suspended lasts +- /// before the service terminates it. An explicit suspend isn't held to it (STATE-12). ++ /// `idlePolicy.suspendedDurationSeconds` — the window STATE-12 refuses past. + pub suspended_sec: u32, + /// `idlePolicy.autoResumeEnabled`. + pub auto_resume: bool, +@@ -409,8 +404,7 @@ impl RunRequest { + self + } + +- /// Sets `idlePolicy.suspendedDurationSeconds`, how long a VM the idle policy suspended +- /// lasts before the service terminates it. See [`RunRequest::suspended_sec`]. ++ /// Sets the suspended window this sandbox will refuse a resume past (STATE-12). + #[must_use] + pub fn with_suspended_sec(mut self, seconds: u32) -> Self { + self.suspended_sec = seconds; +@@ -624,8 +618,7 @@ impl TokenMinter for ControlPlaneMinter { + + /// One MicroVM's whole life. + /// +-/// See the module docs for the state machine, the resume of an ended VM, and why teardown is +-/// explicit. ++/// See the module docs for the state machine, the window, and why teardown is explicit. + pub struct Sandbox { + control: Arc, + image: Option, +@@ -651,8 +644,8 @@ pub struct Sandbox { + was_terminated: bool, + bootstrap_count: u32, + +- /// The suspended window from *our own* `RunMicrovm` request, or `None` for a sandbox that +- /// launched nothing. `GetMicrovm` reports the service's in `idlePolicy`. ++ /// The window from *our own* `RunMicrovm` request. `GetMicrovm` also reports it (in ++ /// `idlePolicy`), which is the fallback when this sandbox has no request of its own. + suspended_window: Option, + /// Whether the accepted launch carried a caller client token, and so may have adopted + /// a VM an earlier attempt launched (#195). +@@ -666,6 +659,8 @@ pub struct Sandbox { + /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence + /// is checked against it. + idle_window: Option, ++ /// The clock reading when the suspend call was accepted, or `None` when not suspended. ++ suspended_at: Option, + /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down + /// one. + torn_down: bool, +@@ -754,10 +749,10 @@ impl Sandbox { + /// `Sandbox::from_name`, which resolve credentials for a region, are in + /// `microvms_core::prelude`. + /// +- /// The plane's own clock is what every wait here sleeps on, rather than a second one ++ /// The plane's own clock is what times the suspended window, rather than a second one + /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil +- /// tests record: a deadline measured on one clock while the poll loop sleeps on another +- /// reads the wrong elapsed time. ++ /// tests record: a window measured on one clock while the poll loop sleeps on another ++ /// calls a closed window open. + pub fn with_control_plane(control: ControlPlane) -> Self { + Self { + control: Arc::new(control), +@@ -777,6 +772,7 @@ impl Sandbox { + launch_posture: None, + pending_agent_token: None, + idle_window: None, ++ suspended_at: None, + torn_down: false, + adopted: false, + detached: false, +@@ -1355,6 +1351,12 @@ impl Sandbox { + /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once + /// when the service reports RUNNING, the same as a launch this sandbox made itself. + /// ++ /// # The suspended window (STATE-12) ++ /// ++ /// The window comes from the `idlePolicy` `GetMicrovm` reports. A VM adopted while ++ /// already SUSPENDED has no suspend time this client observed, so the local check has ++ /// nothing to measure and the service answers the resume instead. ++ /// + /// `endpoint` must agree with the service's when the service reports one: a mismatch + /// means the identifiers came from two different records. `agent_token` is the bearer + /// credential the VM was launched with; it never appears in `Debug` or an error. +@@ -1603,6 +1605,10 @@ impl Sandbox { + // neither suspend nor resume accepts, bricking the handle over one bad request. + self.control.suspend(&id).await?; + self.set_lifecycle(Lifecycle::Suspending); ++ // Stamped after the call and before the wait, not after the wait: the idlePolicy's ++ // window starts when the platform begins suspending, so timing it from SUSPENDED ++ // would under-count the transition and call a closed window open. ++ self.suspended_at = Some(self.control.clock().elapsed()); + + let settled = self + .control +@@ -1649,16 +1655,12 @@ impl Sandbox { + /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be + /// refused — a 409 that reads like a broken VM. + /// +- /// # The service decides, and a VM it ended is `WindowClosed` (STATE-12) ++ /// # The window is checked first (STATE-12) + /// +- /// No resume is refused here for the time the VM has spent suspended. The service ends a +- /// VM the idle policy suspended once `suspendedDurationSeconds` passes, and it kept a VM +- /// this client suspended resumable 1283 s into a 60 s window (measured 2026-10-12, +- /// us-east-1, #597). A resume the service refuses because the VM ended, whatever ended +- /// it, is [`ErrorKind::WindowClosed`] ([`ControlPlane::resume`]), and this records the VM +- /// as terminated, so a second resume is refused here with no call (STATE-11). A VM the +- /// idle policy suspended is still RUNNING here, so its resume is refused before any call +- /// (STATE-7); see the module docs. ++ /// Before any wire call, because the answer is already known and calling costs the poll ++ /// timeout to learn something worse. The falsification is a service reporting ++ /// TERMINATED: without this check the resume burns the full deadline and then reports a ++ /// state the client could have named at once. + /// + /// # The proxy token is dropped (STATE-8) + /// +@@ -1671,7 +1673,7 @@ impl Sandbox { + let id = self.require_microvm("resume")?; + + // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal +- // comes before any call. ++ // comes before the window check and before any call. + if self.was_terminated || self.lifecycle == Lifecycle::Terminated { + return Err(Error::invalid_arg(format!( + "microvm {id} was terminated, and a terminated VM never returns to RUNNING \ +@@ -1686,23 +1688,15 @@ impl Sandbox { + ))); + } + +- // STATE-12: a `WindowClosed` refusal is a VM that ended before this resume. It's +- // recorded as TERMINATED whichever of TERMINATING or TERMINATED `GetMicrovm` read, +- // because the service has said the VM's state can't change, and as a termination, so +- // the next resume is refused before any call (STATE-11). Any other refusal leaves the +- // sandbox SUSPENDED, and a retry goes to the service again. +- if let Err(refused) = self.control.resume(&id).await { +- if refused.kind() == ErrorKind::WindowClosed { +- self.was_terminated = true; +- self.set_lifecycle(Lifecycle::Terminated); +- } +- return Err(refused); +- } ++ // STATE-12, first and locally. ++ self.require_open_suspended_window(&id)?; ++ ++ self.control.resume(&id).await?; + // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the +- // state this call was made from, so failing on it would fail every resume. A VM that +- // terminates once the resume is accepted never reaches RUNNING, and waiting only for +- // RUNNING there burns the full timeout and then reports a timeout message hiding a +- // cause the service had already stated in `stateReason`. ++ // state this call was made from, so failing on it would fail every resume. A VM the ++ // idlePolicy terminated during suspension never reaches RUNNING, and waiting only ++ // for RUNNING there burns the full timeout and then reports a timeout message ++ // hiding a cause the service had already stated in `stateReason`. + let running = self + .control + .wait_for_state( +@@ -1722,6 +1716,10 @@ impl Sandbox { + if let Some(session) = self.session.as_mut() { + session.rebind(endpoint); + } ++ // Cleared on success so the next cycle's window is measured from the next suspend. ++ // Leaving it set would accumulate every suspension's elapsed time into one total and ++ // reject a resume whose own window is wide open. ++ self.suspended_at = None; + + self.session.as_mut().ok_or_else(|| { + Error::new( +@@ -1732,6 +1730,41 @@ impl Sandbox { + }) + } + ++ /// Rejects a resume the launch-time `idlePolicy` has already made impossible (STATE-12). ++ /// ++ /// The message names the elapsed time, the window, and the `idlePolicy` finding, because ++ /// "cannot resume" alone sends a reader looking for the flag that reopens it. ++ fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> { ++ let reported = self ++ .microvm ++ .as_ref() ++ .and_then(|vm| vm.idle_policy.as_ref()) ++ .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds))); ++ let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at) ++ else { ++ // No window or no suspend time means this sandbox cannot know how long the VM ++ // has been suspended, and guessing would refuse a resume the service would honour. ++ return Ok(()); ++ }; ++ let elapsed = self.control.clock().elapsed().saturating_sub(since); ++ if elapsed <= window { ++ return Ok(()); ++ } ++ Err(Error::new( ++ ErrorKind::WindowClosed, ++ format!( ++ "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \ ++ window set at launch — the idlePolicy terminates a suspended VM once that window \ ++ passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \ ++ Refused before ResumeMicrovm, because calling would cost the full poll timeout \ ++ to learn the same thing less clearly. A longer window has to be set at launch \ ++ on the next VM; there is no call that extends this one.", ++ elapsed.as_secs(), ++ window.as_secs(), ++ ), ++ )) ++ } ++ + // ── terminate (STATE-9, STATE-10) ──────────────────────────────────────── + + /// Tears down, best-effort, never erroring. +@@ -1874,14 +1907,11 @@ impl Sandbox { + let vm = self.microvm.as_ref()?; + self.lifecycle.is_live().then(|| { + format!( +- "warning: the Sandbox for microvm {id} was dropped in {state} without \ +- terminate() or detach(). Nothing was torn down — Drop cannot await, so a \ +- teardown here would deadlock inside a runtime. {state} is the state this \ +- sandbox last saw, and the platform may have suspended or ended the VM since. A \ +- VM that's still live bills until it ends, at its maximumDurationInSeconds \ +- ceiling at the latest: terminate it with `microvm terminate {id}`.", +- id = vm.id, +- state = self.lifecycle, ++ "warning: the Sandbox for microvm {} was dropped in {} without terminate() or \ ++ detach(). Nothing was torn down — Drop cannot await, so a teardown here would \ ++ deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \ ++ ceiling: terminate it with `microvm terminate {}`.", ++ vm.id, self.lifecycle, vm.id, + ) + }) + } +diff --git a/crates/microvms-cli/src/cli.rs b/crates/microvms-cli/src/cli.rs +index 25b94f0..f3a820e 100644 +--- a/crates/microvms-cli/src/cli.rs ++++ b/crates/microvms-cli/src/cli.rs +@@ -301,10 +301,8 @@ pub enum Command { + + /// Thaw a suspended MicroVM and report its endpoint. + /// +- /// The launch-time `suspendedDurationSeconds` window terminates a VM the idle policy +- /// suspended once it passes. VMs suspended with `microvm suspend` resumed 900 s and more +- /// into a 60 s window (2026-10-12), which AWS doesn't promise, so the service decides. A +- /// resume of a VM that ended, whatever ended it, exits `ERR_WINDOW_CLOSED`; launch a new one. ++ /// The launch-time `suspendedDurationSeconds` window terminates a suspended VM once it ++ /// passes, so "resume later" silently stops working and the VM is gone rather than slow. + Resume(ResumeArgs), + + /// Tear down a MicroVM, and optionally its image and build log group. +@@ -1056,8 +1054,7 @@ pub struct RunArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +@@ -2506,8 +2503,7 @@ pub struct AgentUpArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +diff --git a/crates/microvms-cli/src/commands/lifecycle.rs b/crates/microvms-cli/src/commands/lifecycle.rs +index c4c0b8b..ef6a544 100644 +--- a/crates/microvms-cli/src/commands/lifecycle.rs ++++ b/crates/microvms-cli/src/commands/lifecycle.rs +@@ -2084,12 +2084,10 @@ pub async fn suspend( + /// Thaws a suspended MicroVM and reports its endpoint. + /// + /// The suspended window is **not** checked here, and that is correct rather than a gap: the +-/// service ends a VM the idle policy suspended once `suspendedDurationSeconds` passes and keeps +-/// an explicitly suspended one resumable past it (measured 2026-10-12, us-east-1), so only the +-/// service can answer. Core's `ControlPlane::resume` reports its refusal of a VM that ended +-/// before the resume as `ERR_WINDOW_CLOSED`. A VM that dies after the resume is +-/// accepted fails the wait at once through `fail_on: DEAD_STATES`, with `stateReason`, instead +-/// of burning the poll timeout. ++/// window is in `GetMicrovm`'s `idlePolicy`, but when the suspend began is known only to the ++/// process that issued it. Guessing would either reject an open window or accept a closed ++/// one, both worse than letting the service answer — which is what `fail_on: DEAD_STATES` ++/// does, failing fast with `stateReason` instead of burning the poll timeout. + pub async fn resume( + ctx: &mut Ctx<'_, O, E>, + args: &ResumeArgs, +diff --git a/crates/microvms-cli/src/config.rs b/crates/microvms-cli/src/config.rs +index 9693850..6020ef2 100644 +--- a/crates/microvms-cli/src/config.rs ++++ b/crates/microvms-cli/src/config.rs +@@ -83,8 +83,7 @@ pub struct ProjectConfig { + pub auto_resume: Option, + /// `run --max-idle-sec`: suspend after this much inbound-traffic idleness. + pub max_idle_sec: Option, +- /// `run --suspended-sec`: terminate a VM the idle policy suspended once it's been suspended +- /// this long. ++ /// `run --suspended-sec`: terminate after this long suspended. + pub suspended_sec: Option, + /// `run --max-duration-sec`: hard ceiling on the VM's life. + pub max_duration_sec: Option, +diff --git a/crates/microvms-cli/src/exit.rs b/crates/microvms-cli/src/exit.rs +index 84342b9..4bba5a5 100644 +--- a/crates/microvms-cli/src/exit.rs ++++ b/crates/microvms-cli/src/exit.rs +@@ -254,8 +254,7 @@ pub const EXIT_TABLE: [ExitRow; 17] = [ + ExitRow { + exit: Exit::WindowClosed, + code: Some("ERR_WINDOW_CLOSED"), +- meaning: "the MicroVM ended before the resume (the suspended window, the maximum \ +- duration, or a terminate), and it can't be resumed; launch a new one", ++ meaning: "the launch-time suspended window passed, so there is nothing to resume", + finding: "`idlePolicy`", + }, + ExitRow { +@@ -443,7 +442,9 @@ pub fn classify(error: &Error) -> CliError { + (Exit::LaunchDied, _) => { + &["`microvm logs ` names the build log group the hook wrote to"] + } +- (Exit::WindowClosed, _) => &["the VM ended and no call brings it back: launch a new one"], ++ (Exit::WindowClosed, _) => { ++ &["a longer window is set at launch with --suspended-sec; no call extends this one"] ++ } + _ => &[], + }; + classified +diff --git a/crates/microvms-domain/src/error.rs b/crates/microvms-domain/src/error.rs +index bbbd0ed..15ed493 100644 +--- a/crates/microvms-domain/src/error.rs ++++ b/crates/microvms-domain/src/error.rs +@@ -130,9 +130,7 @@ pub enum ErrorKind { + BuildWedged, + /// The MicroVM reached a terminal state before RUNNING; read `stateReason`. + LaunchDied, +- /// The MicroVM ended before the resume (the suspended window, the maximum duration, or a +- /// terminate), and it can't be resumed; launch a new one. The service refused the resume, +- /// and `GetMicrovm` reports the VM TERMINATING or TERMINATED. ++ /// The launch-time suspended window passed, so there is nothing to resume. + WindowClosed, + /// A control-plane failure with no more specific class. + Platform, diff --git a/verify/guards/faults/f2p-a-resume-refused-for-a-vm-that-terminated-while.patch b/verify/guards/faults/f2p-a-resume-refused-for-a-vm-that-terminated-while.patch new file mode 100644 index 00000000..303b48bf --- /dev/null +++ b/verify/guards/faults/f2p-a-resume-refused-for-a-vm-that-terminated-while.patch @@ -0,0 +1,641 @@ +diff --git a/bindings/microvms-js/src/control.rs b/bindings/microvms-js/src/control.rs +index ead8c40..66a7dd9 100644 +--- a/bindings/microvms-js/src/control.rs ++++ b/bindings/microvms-js/src/control.rs +@@ -279,8 +279,8 @@ pub struct DeleteImageOptions { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7). Use +-/// it when a process has only an identifier. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12). Use it when a process has only an identifier. + #[napi] + pub struct ControlPlane { + inner: Arc, +@@ -344,8 +344,7 @@ impl ControlPlane { + self.inner.suspend(µvm_id).await.map_err(js_async) + } + +- /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, rejects with `ERR_WINDOW_CLOSED` (STATE-12). ++ /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. + #[napi] + pub async fn resume(&self, microvm_id: String) -> Result<(), AsyncError> { + self.inner.resume(µvm_id).await.map_err(js_async) +diff --git a/bindings/microvms-js/src/sandbox.rs b/bindings/microvms-js/src/sandbox.rs +index 517085e..55d1e84 100644 +--- a/bindings/microvms-js/src/sandbox.rs ++++ b/bindings/microvms-js/src/sandbox.rs +@@ -1144,21 +1144,11 @@ impl Sandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, and +- /// even a call the service accepted would hand back a different machine. And a resume +- /// from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspendedSec` passes, and a VM this sandbox suspended stayed resumable +- /// 1283 s into a 60 s window (measured 2026-10-12, us-east-1). `ERR_WINDOW_CLOSED` is the +- /// service's answer once a resume reaches it: the service refused the resume, and the VM +- /// is TERMINATING or TERMINATED, whatever ended it. The sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` rejects with `ERR_INVALID_ARG` before any call (STATE-7). That +- /// suspend reaches a caller only through `Sandbox.adopt`, which reads the state from the +- /// service, or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// even a call the service accepted would hand back a different machine. A resume from ++ /// anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended window ++ /// (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates a ++ /// suspended VM once that window passes, so there is nothing left to resume, and calling ++ /// would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/bindings/microvms-py/src/control.rs b/bindings/microvms-py/src/control.rs +index ce0752e..5fbe783 100644 +--- a/bindings/microvms-py/src/control.rs ++++ b/bindings/microvms-py/src/control.rs +@@ -444,9 +444,9 @@ impl PyImageBuild { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7): a +-/// suspend of a SUSPENDED VM is the service's to refuse. Use it when a process has only an +-/// identifier, such as a durable workflow step in a fresh process. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12): a suspend of a SUSPENDED VM is the service's to refuse. Use it when a ++/// process has only an identifier, such as a durable workflow step in a fresh process. + #[pyclass(frozen, name = "ControlPlane", module = "microvms")] + pub struct PyControlPlane { + inner: Arc, +@@ -703,8 +703,7 @@ impl PyControlPlane { + Ok(runtime::spawn_shielded(self.suspend_op(microvm_id)).await?) + } + +- /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, raises `WindowClosedError` (STATE-12). ++ /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. + fn resume(&self, py: Python<'_>, microvm_id: String) -> PyCoreResult<()> { + Ok(runtime::block_on(py, self.resume_op(microvm_id))?) + } +diff --git a/bindings/microvms-py/src/errors.rs b/bindings/microvms-py/src/errors.rs +index be8962a..a664785 100644 +--- a/bindings/microvms-py/src/errors.rs ++++ b/bindings/microvms-py/src/errors.rs +@@ -87,10 +87,8 @@ create_exception!( + microvms, + WindowClosedError, + MicrovmError, +- "ERR_WINDOW_CLOSED — the MicroVM ended before the resume (the suspended window, the \ +- maximum duration, or a terminate), and it can't be resumed; launch a new one. The \ +- suspended window ends a VM the idle policy suspended; an explicit suspend stayed \ +- resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1)." ++ "ERR_WINDOW_CLOSED — the launch-time suspended window passed, so there is nothing to \ ++ resume. A longer window has to be set at launch on the next VM." + ); + create_exception!( + microvms, +diff --git a/bindings/microvms-py/src/sandbox.rs b/bindings/microvms-py/src/sandbox.rs +index 449afbd..f7d6371 100644 +--- a/bindings/microvms-py/src/sandbox.rs ++++ b/bindings/microvms-py/src/sandbox.rs +@@ -1904,22 +1904,11 @@ impl PySandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, +- /// and even a call the service accepted would hand back a different machine. And a +- /// resume from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspended_sec` passes, and a VM this sandbox suspended stayed +- /// resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1). +- /// `WindowClosedError` is the service's answer once a resume reaches it: the service +- /// refused the resume, and the VM is TERMINATING or TERMINATED, whatever ended it. The +- /// sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` raises `InvalidArgError` before any call (STATE-7). That suspend +- /// reaches a caller only through `Sandbox.adopt`, which reads the state from the service, +- /// or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// and even a call the service accepted would hand back a different machine. A resume ++ /// from anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended ++ /// window (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates ++ /// a suspended VM once that window passes, so there is nothing left to resume, and ++ /// calling would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/crates/microvms-app/src/clock.rs b/crates/microvms-app/src/clock.rs +index 75afb7c..5685303 100644 +--- a/crates/microvms-app/src/clock.rs ++++ b/crates/microvms-app/src/clock.rs +@@ -7,11 +7,11 @@ + //! sources: the control plane's read `std::time::Instant` and the session's read tokio's. + //! Under a deterministic simulator those disagree. tokio's clock is virtual there and `std`'s + //! isn't, so a control-plane wait whose sleeps advanced virtual time measured its deadline +-//! on the host's real time and never reached it. `Sandbox` already waits on the plane's clock +-//! rather than one of its own, so a deadline isn't measured on one clock while the polls sleep +-//! on another. One trait, with tokio's clock in production, keeps that property for everything +-//! that reads time. The trait is re-exported at `control::Clock` and at `session::Clock`, the +-//! paths each half had. ++//! on the host's real time and never reached it. `Sandbox` already times the suspended window ++//! on the plane's clock rather than its own, so a window isn't measured on one clock while ++//! the polls run on another. One trait, with tokio's clock in production, keeps that property ++//! for everything that reads time. The trait is re-exported at `control::Clock` and at ++//! `session::Clock`, the paths each half had. + //! + //! # Why a trait rather than `tokio::time::pause` + //! +@@ -24,11 +24,11 @@ + //! # Monotonic and wall time are separate readings + //! + //! [`Clock::elapsed`] is monotonic, the same reasoning the Python client's injectable +-//! `time.monotonic` carried: a wait's deadline and a proxy token's age are *durations*, and a +-//! wall clock that steps backward would hold a passed deadline open. [`Clock::unix_now`] is the +-//! wall reading, for the values that are dates rather than durations: a name record's +-//! timestamp, an exec id, the cost engine's today. Deriving one from the other would make a +-//! stepped wall clock move a deadline. ++//! `time.monotonic` carried: the suspended window is a *duration*, and a wall clock that steps ++//! backward would reopen a closed one. [`Clock::unix_now`] is the wall reading, for the values ++//! that are dates rather than durations: a name record's timestamp, an exec id, the cost ++//! engine's today. Deriving one from the other would make a stepped wall clock move a ++//! deadline. + + use std::fmt; + use std::future::Future; +diff --git a/crates/microvms-app/src/control/microvm.rs b/crates/microvms-app/src/control/microvm.rs +index 9611956..36597d0 100644 +--- a/crates/microvms-app/src/control/microvm.rs ++++ b/crates/microvms-app/src/control/microvm.rs +@@ -799,56 +799,11 @@ impl ControlPlane { + } + + /// `ResumeMicrovm`. Thaws the VM; no token re-delivery and no re-bootstrap. +- /// +- /// # A VM that ended before the resume is `WindowClosed` (STATE-12) +- /// +- /// Nothing is checked against the suspended window before the call, because the service +- /// decides whether a suspended VM can still resume. It terminates a VM the idle policy +- /// suspended once `suspendedDurationSeconds` passes, and it doesn't hold an explicit +- /// `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, an explicitly +- /// suspended VM resumed 1283 s into a 60 s window (#597). A resume of a VM that has +- /// terminated is refused with a `ValidationException` ("The MicroVM has been +- /// terminated and its state cannot be changed."), the same status as any refused value, +- /// and it was the same refusal for a VM the service ended and one `TerminateMicrovm` +- /// ended (measured 2026-10-12, us-east-1). So a refusal reads the VM's state rather than +- /// the message, and a VM that's TERMINATING or TERMINATED is [`ErrorKind::WindowClosed`] +- /// with its state and `stateReason`, carrying the refusal as its source. The error names +- /// no cause, because the client doesn't know which of the suspended window, the maximum +- /// duration, or a terminate ended the VM; the `stateReason` is the service's word on it. +- /// Every other refusal comes back as the service gave it. + pub async fn resume(&self, id: &str) -> Result<(), Error> { + super::require_valid_identifier("microvmIdentifier", id)?; + let call = Call::post_empty("ResumeMicrovm", paths::resume(id)); +- match send_with_retry(self.transport(), call).await { +- Ok(_) => Ok(()), +- Err(refused) if refused.kind() == ErrorKind::Platform => { +- Err(self.ended_before_resume(id, refused).await) +- } +- Err(error) => Err(error), +- } +- } +- +- /// The error a refused resume reports: [`ErrorKind::WindowClosed`] with `refused` as its +- /// source when `GetMicrovm` reports the VM dead, and `refused` otherwise, including when +- /// that read fails. +- async fn ended_before_resume(&self, id: &str, refused: Error) -> Error { +- let Ok(vm) = self.get_microvm(id).await else { +- return refused; +- }; +- if !crate::constants::DEAD_STATES.contains(&vm.state.as_str()) { +- return refused; +- } +- Error::new( +- ErrorKind::WindowClosed, +- format!( +- "microvm {id} can't be resumed: the service refused ResumeMicrovm, and \ +- GetMicrovm reports the VM {} ({}). A VM that has ended doesn't come back; \ +- launch a new one.", +- vm.state, +- vm.reason(), +- ), +- ) +- .with_source(refused) ++ send_with_retry(self.transport(), call).await?; ++ Ok(()) + } + + /// `TerminateMicrovm`. +diff --git a/crates/microvms-app/src/control/ops.rs b/crates/microvms-app/src/control/ops.rs +index 36833c6..ae1c3c1 100644 +--- a/crates/microvms-app/src/control/ops.rs ++++ b/crates/microvms-app/src/control/ops.rs +@@ -367,9 +367,11 @@ pub struct RunMicrovmWire { + /// "autoResumeEnabled": false}` (measured 2026-08-15, us-east-1, one read-only + /// `GetMicrovm`). + /// +-/// The readback means the client's own record isn't the only authority on the suspended +-/// window: [`crate::control::Microvm::idle_policy`] carries the service's, so a window that +-/// disagrees with what was asked for is observable rather than assumed. ++/// The claim mattered because STATE-12 was resting on it: if only the client could name the ++/// suspended window, the client's own record is the only authority on it. The readback ++/// means there is a second one, and [`crate::control::Microvm::idle_policy`] carries it — ++/// so a suspended window that disagrees with what was asked for is now observable rather ++/// than assumed. + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct IdlePolicy { +diff --git a/crates/microvms-app/src/sandbox.rs b/crates/microvms-app/src/sandbox.rs +index 544535f..dc316a4 100644 +--- a/crates/microvms-app/src/sandbox.rs ++++ b/crates/microvms-app/src/sandbox.rs +@@ -3,8 +3,8 @@ + //! model describes made true of the code. + //! + //! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the +-//! bindings use: build, run, suspend, resume, terminate, a resume of a VM that ended +-//! before it (STATE-12), and teardown in the order that does not leak. ++//! bindings use: build, run, suspend, resume, terminate, the launch-time suspended ++//! window (STATE-12), and teardown in the order that does not leak. + //! + //! # The lifecycle is a field, and every transition is guarded + //! +@@ -32,20 +32,16 @@ + //! control-plane calls rather than answered by AWS. The test asserts the call count, which + //! is the observable that distinguishes the two. + //! +-//! # The service decides whether a suspended VM resumes (STATE-12) ++//! # The suspended window is checked before the call (STATE-12) + //! +-//! The launch-time `idlePolicy` *terminates* a VM the idle policy suspended once +-//! `suspendedDurationSeconds` passes, and the service doesn't hold an explicit +-//! `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, a VM this client +-//! suspended resumed 1283 s into a 60 s window (#597). So a resume is never refused here for +-//! the time it has spent suspended. It goes to the service, and the service's refusal for a +-//! VM that ended, whatever ended it, comes back from [`ControlPlane::resume`] as +-//! [`ErrorKind::WindowClosed`], which this sandbox records as a termination (STATE-11). +-//! +-//! A suspend the idle policy makes isn't seen here: the sandbox still holds the VM as +-//! RUNNING, and its resume is refused before any call (STATE-7). That suspend reaches a +-//! caller only through [`Sandbox::adopt`], which reads the state from the service, or +-//! [`ControlPlane::resume`], which holds no lifecycle to refuse from. ++//! The launch-time `idlePolicy` *terminates* a suspended VM once `suspendedDurationSeconds` ++//! passes, which means "resume later" silently stops working. A resume past the window is ++//! refused locally, before `ResumeMicrovm`, because the alternative is calling and reading ++//! the failure: the service answers about a terminated id, which is not the same statement ++//! as "the window you set at launch closed", and getting there costs the full poll timeout ++//! first. The window comes from this sandbox's own `RunMicrovm` request, falling back to the ++//! `idlePolicy` that `GetMicrovm` reports (measured 2026-08-15, `docs/PLATFORM.md`). Only ++//! the suspend's start time is the client's alone. + //! + //! # Teardown never raises, and the log group is last + //! +@@ -270,8 +266,7 @@ pub struct RunRequest { + pub shell: bool, + /// `idlePolicy.maxIdleDurationSeconds`. + pub max_idle_sec: u32, +- /// `idlePolicy.suspendedDurationSeconds`: how long a VM the idle policy suspended lasts +- /// before the service terminates it. An explicit suspend isn't held to it (STATE-12). ++ /// `idlePolicy.suspendedDurationSeconds` — the window STATE-12 refuses past. + pub suspended_sec: u32, + /// `idlePolicy.autoResumeEnabled`. + pub auto_resume: bool, +@@ -409,8 +404,7 @@ impl RunRequest { + self + } + +- /// Sets `idlePolicy.suspendedDurationSeconds`, how long a VM the idle policy suspended +- /// lasts before the service terminates it. See [`RunRequest::suspended_sec`]. ++ /// Sets the suspended window this sandbox will refuse a resume past (STATE-12). + #[must_use] + pub fn with_suspended_sec(mut self, seconds: u32) -> Self { + self.suspended_sec = seconds; +@@ -624,8 +618,7 @@ impl TokenMinter for ControlPlaneMinter { + + /// One MicroVM's whole life. + /// +-/// See the module docs for the state machine, the resume of an ended VM, and why teardown is +-/// explicit. ++/// See the module docs for the state machine, the window, and why teardown is explicit. + pub struct Sandbox { + control: Arc, + image: Option, +@@ -651,8 +644,8 @@ pub struct Sandbox { + was_terminated: bool, + bootstrap_count: u32, + +- /// The suspended window from *our own* `RunMicrovm` request, or `None` for a sandbox that +- /// launched nothing. `GetMicrovm` reports the service's in `idlePolicy`. ++ /// The window from *our own* `RunMicrovm` request. `GetMicrovm` also reports it (in ++ /// `idlePolicy`), which is the fallback when this sandbox has no request of its own. + suspended_window: Option, + /// Whether the accepted launch carried a caller client token, and so may have adopted + /// a VM an earlier attempt launched (#195). +@@ -666,6 +659,8 @@ pub struct Sandbox { + /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence + /// is checked against it. + idle_window: Option, ++ /// The clock reading when the suspend call was accepted, or `None` when not suspended. ++ suspended_at: Option, + /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down + /// one. + torn_down: bool, +@@ -754,10 +749,10 @@ impl Sandbox { + /// `Sandbox::from_name`, which resolve credentials for a region, are in + /// `microvms_core::prelude`. + /// +- /// The plane's own clock is what every wait here sleeps on, rather than a second one ++ /// The plane's own clock is what times the suspended window, rather than a second one + /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil +- /// tests record: a deadline measured on one clock while the poll loop sleeps on another +- /// reads the wrong elapsed time. ++ /// tests record: a window measured on one clock while the poll loop sleeps on another ++ /// calls a closed window open. + pub fn with_control_plane(control: ControlPlane) -> Self { + Self { + control: Arc::new(control), +@@ -777,6 +772,7 @@ impl Sandbox { + launch_posture: None, + pending_agent_token: None, + idle_window: None, ++ suspended_at: None, + torn_down: false, + adopted: false, + detached: false, +@@ -1355,6 +1351,12 @@ impl Sandbox { + /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once + /// when the service reports RUNNING, the same as a launch this sandbox made itself. + /// ++ /// # The suspended window (STATE-12) ++ /// ++ /// The window comes from the `idlePolicy` `GetMicrovm` reports. A VM adopted while ++ /// already SUSPENDED has no suspend time this client observed, so the local check has ++ /// nothing to measure and the service answers the resume instead. ++ /// + /// `endpoint` must agree with the service's when the service reports one: a mismatch + /// means the identifiers came from two different records. `agent_token` is the bearer + /// credential the VM was launched with; it never appears in `Debug` or an error. +@@ -1603,6 +1605,10 @@ impl Sandbox { + // neither suspend nor resume accepts, bricking the handle over one bad request. + self.control.suspend(&id).await?; + self.set_lifecycle(Lifecycle::Suspending); ++ // Stamped after the call and before the wait, not after the wait: the idlePolicy's ++ // window starts when the platform begins suspending, so timing it from SUSPENDED ++ // would under-count the transition and call a closed window open. ++ self.suspended_at = Some(self.control.clock().elapsed()); + + let settled = self + .control +@@ -1649,16 +1655,12 @@ impl Sandbox { + /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be + /// refused — a 409 that reads like a broken VM. + /// +- /// # The service decides, and a VM it ended is `WindowClosed` (STATE-12) ++ /// # The window is checked first (STATE-12) + /// +- /// No resume is refused here for the time the VM has spent suspended. The service ends a +- /// VM the idle policy suspended once `suspendedDurationSeconds` passes, and it kept a VM +- /// this client suspended resumable 1283 s into a 60 s window (measured 2026-10-12, +- /// us-east-1, #597). A resume the service refuses because the VM ended, whatever ended +- /// it, is [`ErrorKind::WindowClosed`] ([`ControlPlane::resume`]), and this records the VM +- /// as terminated, so a second resume is refused here with no call (STATE-11). A VM the +- /// idle policy suspended is still RUNNING here, so its resume is refused before any call +- /// (STATE-7); see the module docs. ++ /// Before any wire call, because the answer is already known and calling costs the poll ++ /// timeout to learn something worse. The falsification is a service reporting ++ /// TERMINATED: without this check the resume burns the full deadline and then reports a ++ /// state the client could have named at once. + /// + /// # The proxy token is dropped (STATE-8) + /// +@@ -1671,7 +1673,7 @@ impl Sandbox { + let id = self.require_microvm("resume")?; + + // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal +- // comes before any call. ++ // comes before the window check and before any call. + if self.was_terminated || self.lifecycle == Lifecycle::Terminated { + return Err(Error::invalid_arg(format!( + "microvm {id} was terminated, and a terminated VM never returns to RUNNING \ +@@ -1686,23 +1688,15 @@ impl Sandbox { + ))); + } + +- // STATE-12: a `WindowClosed` refusal is a VM that ended before this resume. It's +- // recorded as TERMINATED whichever of TERMINATING or TERMINATED `GetMicrovm` read, +- // because the service has said the VM's state can't change, and as a termination, so +- // the next resume is refused before any call (STATE-11). Any other refusal leaves the +- // sandbox SUSPENDED, and a retry goes to the service again. +- if let Err(refused) = self.control.resume(&id).await { +- if refused.kind() == ErrorKind::WindowClosed { +- self.was_terminated = true; +- self.set_lifecycle(Lifecycle::Terminated); +- } +- return Err(refused); +- } ++ // STATE-12, first and locally. ++ self.require_open_suspended_window(&id)?; ++ ++ self.control.resume(&id).await?; + // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the +- // state this call was made from, so failing on it would fail every resume. A VM that +- // terminates once the resume is accepted never reaches RUNNING, and waiting only for +- // RUNNING there burns the full timeout and then reports a timeout message hiding a +- // cause the service had already stated in `stateReason`. ++ // state this call was made from, so failing on it would fail every resume. A VM the ++ // idlePolicy terminated during suspension never reaches RUNNING, and waiting only ++ // for RUNNING there burns the full timeout and then reports a timeout message ++ // hiding a cause the service had already stated in `stateReason`. + let running = self + .control + .wait_for_state( +@@ -1722,6 +1716,10 @@ impl Sandbox { + if let Some(session) = self.session.as_mut() { + session.rebind(endpoint); + } ++ // Cleared on success so the next cycle's window is measured from the next suspend. ++ // Leaving it set would accumulate every suspension's elapsed time into one total and ++ // reject a resume whose own window is wide open. ++ self.suspended_at = None; + + self.session.as_mut().ok_or_else(|| { + Error::new( +@@ -1732,6 +1730,41 @@ impl Sandbox { + }) + } + ++ /// Rejects a resume the launch-time `idlePolicy` has already made impossible (STATE-12). ++ /// ++ /// The message names the elapsed time, the window, and the `idlePolicy` finding, because ++ /// "cannot resume" alone sends a reader looking for the flag that reopens it. ++ fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> { ++ let reported = self ++ .microvm ++ .as_ref() ++ .and_then(|vm| vm.idle_policy.as_ref()) ++ .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds))); ++ let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at) ++ else { ++ // No window or no suspend time means this sandbox cannot know how long the VM ++ // has been suspended, and guessing would refuse a resume the service would honour. ++ return Ok(()); ++ }; ++ let elapsed = self.control.clock().elapsed().saturating_sub(since); ++ if elapsed <= window { ++ return Ok(()); ++ } ++ Err(Error::new( ++ ErrorKind::WindowClosed, ++ format!( ++ "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \ ++ window set at launch — the idlePolicy terminates a suspended VM once that window \ ++ passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \ ++ Refused before ResumeMicrovm, because calling would cost the full poll timeout \ ++ to learn the same thing less clearly. A longer window has to be set at launch \ ++ on the next VM; there is no call that extends this one.", ++ elapsed.as_secs(), ++ window.as_secs(), ++ ), ++ )) ++ } ++ + // ── terminate (STATE-9, STATE-10) ──────────────────────────────────────── + + /// Tears down, best-effort, never erroring. +@@ -1874,14 +1907,11 @@ impl Sandbox { + let vm = self.microvm.as_ref()?; + self.lifecycle.is_live().then(|| { + format!( +- "warning: the Sandbox for microvm {id} was dropped in {state} without \ +- terminate() or detach(). Nothing was torn down — Drop cannot await, so a \ +- teardown here would deadlock inside a runtime. {state} is the state this \ +- sandbox last saw, and the platform may have suspended or ended the VM since. A \ +- VM that's still live bills until it ends, at its maximumDurationInSeconds \ +- ceiling at the latest: terminate it with `microvm terminate {id}`.", +- id = vm.id, +- state = self.lifecycle, ++ "warning: the Sandbox for microvm {} was dropped in {} without terminate() or \ ++ detach(). Nothing was torn down — Drop cannot await, so a teardown here would \ ++ deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \ ++ ceiling: terminate it with `microvm terminate {}`.", ++ vm.id, self.lifecycle, vm.id, + ) + }) + } +diff --git a/crates/microvms-cli/src/cli.rs b/crates/microvms-cli/src/cli.rs +index 25b94f0..f3a820e 100644 +--- a/crates/microvms-cli/src/cli.rs ++++ b/crates/microvms-cli/src/cli.rs +@@ -301,10 +301,8 @@ pub enum Command { + + /// Thaw a suspended MicroVM and report its endpoint. + /// +- /// The launch-time `suspendedDurationSeconds` window terminates a VM the idle policy +- /// suspended once it passes. VMs suspended with `microvm suspend` resumed 900 s and more +- /// into a 60 s window (2026-10-12), which AWS doesn't promise, so the service decides. A +- /// resume of a VM that ended, whatever ended it, exits `ERR_WINDOW_CLOSED`; launch a new one. ++ /// The launch-time `suspendedDurationSeconds` window terminates a suspended VM once it ++ /// passes, so "resume later" silently stops working and the VM is gone rather than slow. + Resume(ResumeArgs), + + /// Tear down a MicroVM, and optionally its image and build log group. +@@ -1056,8 +1054,7 @@ pub struct RunArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +@@ -2506,8 +2503,7 @@ pub struct AgentUpArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +diff --git a/crates/microvms-cli/src/commands/lifecycle.rs b/crates/microvms-cli/src/commands/lifecycle.rs +index c4c0b8b..ef6a544 100644 +--- a/crates/microvms-cli/src/commands/lifecycle.rs ++++ b/crates/microvms-cli/src/commands/lifecycle.rs +@@ -2084,12 +2084,10 @@ pub async fn suspend( + /// Thaws a suspended MicroVM and reports its endpoint. + /// + /// The suspended window is **not** checked here, and that is correct rather than a gap: the +-/// service ends a VM the idle policy suspended once `suspendedDurationSeconds` passes and keeps +-/// an explicitly suspended one resumable past it (measured 2026-10-12, us-east-1), so only the +-/// service can answer. Core's `ControlPlane::resume` reports its refusal of a VM that ended +-/// before the resume as `ERR_WINDOW_CLOSED`. A VM that dies after the resume is +-/// accepted fails the wait at once through `fail_on: DEAD_STATES`, with `stateReason`, instead +-/// of burning the poll timeout. ++/// window is in `GetMicrovm`'s `idlePolicy`, but when the suspend began is known only to the ++/// process that issued it. Guessing would either reject an open window or accept a closed ++/// one, both worse than letting the service answer — which is what `fail_on: DEAD_STATES` ++/// does, failing fast with `stateReason` instead of burning the poll timeout. + pub async fn resume( + ctx: &mut Ctx<'_, O, E>, + args: &ResumeArgs, +diff --git a/crates/microvms-cli/src/config.rs b/crates/microvms-cli/src/config.rs +index 9693850..6020ef2 100644 +--- a/crates/microvms-cli/src/config.rs ++++ b/crates/microvms-cli/src/config.rs +@@ -83,8 +83,7 @@ pub struct ProjectConfig { + pub auto_resume: Option, + /// `run --max-idle-sec`: suspend after this much inbound-traffic idleness. + pub max_idle_sec: Option, +- /// `run --suspended-sec`: terminate a VM the idle policy suspended once it's been suspended +- /// this long. ++ /// `run --suspended-sec`: terminate after this long suspended. + pub suspended_sec: Option, + /// `run --max-duration-sec`: hard ceiling on the VM's life. + pub max_duration_sec: Option, +diff --git a/crates/microvms-cli/src/exit.rs b/crates/microvms-cli/src/exit.rs +index 84342b9..4bba5a5 100644 +--- a/crates/microvms-cli/src/exit.rs ++++ b/crates/microvms-cli/src/exit.rs +@@ -254,8 +254,7 @@ pub const EXIT_TABLE: [ExitRow; 17] = [ + ExitRow { + exit: Exit::WindowClosed, + code: Some("ERR_WINDOW_CLOSED"), +- meaning: "the MicroVM ended before the resume (the suspended window, the maximum \ +- duration, or a terminate), and it can't be resumed; launch a new one", ++ meaning: "the launch-time suspended window passed, so there is nothing to resume", + finding: "`idlePolicy`", + }, + ExitRow { +@@ -443,7 +442,9 @@ pub fn classify(error: &Error) -> CliError { + (Exit::LaunchDied, _) => { + &["`microvm logs ` names the build log group the hook wrote to"] + } +- (Exit::WindowClosed, _) => &["the VM ended and no call brings it back: launch a new one"], ++ (Exit::WindowClosed, _) => { ++ &["a longer window is set at launch with --suspended-sec; no call extends this one"] ++ } + _ => &[], + }; + classified +diff --git a/crates/microvms-domain/src/error.rs b/crates/microvms-domain/src/error.rs +index bbbd0ed..15ed493 100644 +--- a/crates/microvms-domain/src/error.rs ++++ b/crates/microvms-domain/src/error.rs +@@ -130,9 +130,7 @@ pub enum ErrorKind { + BuildWedged, + /// The MicroVM reached a terminal state before RUNNING; read `stateReason`. + LaunchDied, +- /// The MicroVM ended before the resume (the suspended window, the maximum duration, or a +- /// terminate), and it can't be resumed; launch a new one. The service refused the resume, +- /// and `GetMicrovm` reports the VM TERMINATING or TERMINATED. ++ /// The launch-time suspended window passed, so there is nothing to resume. + WindowClosed, + /// A control-plane failure with no more specific class. + Platform, diff --git a/verify/guards/faults/f2p-a-resume-the-service-refuses-for-an-ended-vm-is-window.patch b/verify/guards/faults/f2p-a-resume-the-service-refuses-for-an-ended-vm-is-window.patch new file mode 100644 index 00000000..303b48bf --- /dev/null +++ b/verify/guards/faults/f2p-a-resume-the-service-refuses-for-an-ended-vm-is-window.patch @@ -0,0 +1,641 @@ +diff --git a/bindings/microvms-js/src/control.rs b/bindings/microvms-js/src/control.rs +index ead8c40..66a7dd9 100644 +--- a/bindings/microvms-js/src/control.rs ++++ b/bindings/microvms-js/src/control.rs +@@ -279,8 +279,8 @@ pub struct DeleteImageOptions { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7). Use +-/// it when a process has only an identifier. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12). Use it when a process has only an identifier. + #[napi] + pub struct ControlPlane { + inner: Arc, +@@ -344,8 +344,7 @@ impl ControlPlane { + self.inner.suspend(µvm_id).await.map_err(js_async) + } + +- /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, rejects with `ERR_WINDOW_CLOSED` (STATE-12). ++ /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. + #[napi] + pub async fn resume(&self, microvm_id: String) -> Result<(), AsyncError> { + self.inner.resume(µvm_id).await.map_err(js_async) +diff --git a/bindings/microvms-js/src/sandbox.rs b/bindings/microvms-js/src/sandbox.rs +index 517085e..55d1e84 100644 +--- a/bindings/microvms-js/src/sandbox.rs ++++ b/bindings/microvms-js/src/sandbox.rs +@@ -1144,21 +1144,11 @@ impl Sandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, and +- /// even a call the service accepted would hand back a different machine. And a resume +- /// from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspendedSec` passes, and a VM this sandbox suspended stayed resumable +- /// 1283 s into a 60 s window (measured 2026-10-12, us-east-1). `ERR_WINDOW_CLOSED` is the +- /// service's answer once a resume reaches it: the service refused the resume, and the VM +- /// is TERMINATING or TERMINATED, whatever ended it. The sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` rejects with `ERR_INVALID_ARG` before any call (STATE-7). That +- /// suspend reaches a caller only through `Sandbox.adopt`, which reads the state from the +- /// service, or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// even a call the service accepted would hand back a different machine. A resume from ++ /// anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended window ++ /// (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates a ++ /// suspended VM once that window passes, so there is nothing left to resume, and calling ++ /// would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/bindings/microvms-py/src/control.rs b/bindings/microvms-py/src/control.rs +index ce0752e..5fbe783 100644 +--- a/bindings/microvms-py/src/control.rs ++++ b/bindings/microvms-py/src/control.rs +@@ -444,9 +444,9 @@ impl PyImageBuild { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7): a +-/// suspend of a SUSPENDED VM is the service's to refuse. Use it when a process has only an +-/// identifier, such as a durable workflow step in a fresh process. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12): a suspend of a SUSPENDED VM is the service's to refuse. Use it when a ++/// process has only an identifier, such as a durable workflow step in a fresh process. + #[pyclass(frozen, name = "ControlPlane", module = "microvms")] + pub struct PyControlPlane { + inner: Arc, +@@ -703,8 +703,7 @@ impl PyControlPlane { + Ok(runtime::spawn_shielded(self.suspend_op(microvm_id)).await?) + } + +- /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, raises `WindowClosedError` (STATE-12). ++ /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. + fn resume(&self, py: Python<'_>, microvm_id: String) -> PyCoreResult<()> { + Ok(runtime::block_on(py, self.resume_op(microvm_id))?) + } +diff --git a/bindings/microvms-py/src/errors.rs b/bindings/microvms-py/src/errors.rs +index be8962a..a664785 100644 +--- a/bindings/microvms-py/src/errors.rs ++++ b/bindings/microvms-py/src/errors.rs +@@ -87,10 +87,8 @@ create_exception!( + microvms, + WindowClosedError, + MicrovmError, +- "ERR_WINDOW_CLOSED — the MicroVM ended before the resume (the suspended window, the \ +- maximum duration, or a terminate), and it can't be resumed; launch a new one. The \ +- suspended window ends a VM the idle policy suspended; an explicit suspend stayed \ +- resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1)." ++ "ERR_WINDOW_CLOSED — the launch-time suspended window passed, so there is nothing to \ ++ resume. A longer window has to be set at launch on the next VM." + ); + create_exception!( + microvms, +diff --git a/bindings/microvms-py/src/sandbox.rs b/bindings/microvms-py/src/sandbox.rs +index 449afbd..f7d6371 100644 +--- a/bindings/microvms-py/src/sandbox.rs ++++ b/bindings/microvms-py/src/sandbox.rs +@@ -1904,22 +1904,11 @@ impl PySandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, +- /// and even a call the service accepted would hand back a different machine. And a +- /// resume from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspended_sec` passes, and a VM this sandbox suspended stayed +- /// resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1). +- /// `WindowClosedError` is the service's answer once a resume reaches it: the service +- /// refused the resume, and the VM is TERMINATING or TERMINATED, whatever ended it. The +- /// sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` raises `InvalidArgError` before any call (STATE-7). That suspend +- /// reaches a caller only through `Sandbox.adopt`, which reads the state from the service, +- /// or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// and even a call the service accepted would hand back a different machine. A resume ++ /// from anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended ++ /// window (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates ++ /// a suspended VM once that window passes, so there is nothing left to resume, and ++ /// calling would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/crates/microvms-app/src/clock.rs b/crates/microvms-app/src/clock.rs +index 75afb7c..5685303 100644 +--- a/crates/microvms-app/src/clock.rs ++++ b/crates/microvms-app/src/clock.rs +@@ -7,11 +7,11 @@ + //! sources: the control plane's read `std::time::Instant` and the session's read tokio's. + //! Under a deterministic simulator those disagree. tokio's clock is virtual there and `std`'s + //! isn't, so a control-plane wait whose sleeps advanced virtual time measured its deadline +-//! on the host's real time and never reached it. `Sandbox` already waits on the plane's clock +-//! rather than one of its own, so a deadline isn't measured on one clock while the polls sleep +-//! on another. One trait, with tokio's clock in production, keeps that property for everything +-//! that reads time. The trait is re-exported at `control::Clock` and at `session::Clock`, the +-//! paths each half had. ++//! on the host's real time and never reached it. `Sandbox` already times the suspended window ++//! on the plane's clock rather than its own, so a window isn't measured on one clock while ++//! the polls run on another. One trait, with tokio's clock in production, keeps that property ++//! for everything that reads time. The trait is re-exported at `control::Clock` and at ++//! `session::Clock`, the paths each half had. + //! + //! # Why a trait rather than `tokio::time::pause` + //! +@@ -24,11 +24,11 @@ + //! # Monotonic and wall time are separate readings + //! + //! [`Clock::elapsed`] is monotonic, the same reasoning the Python client's injectable +-//! `time.monotonic` carried: a wait's deadline and a proxy token's age are *durations*, and a +-//! wall clock that steps backward would hold a passed deadline open. [`Clock::unix_now`] is the +-//! wall reading, for the values that are dates rather than durations: a name record's +-//! timestamp, an exec id, the cost engine's today. Deriving one from the other would make a +-//! stepped wall clock move a deadline. ++//! `time.monotonic` carried: the suspended window is a *duration*, and a wall clock that steps ++//! backward would reopen a closed one. [`Clock::unix_now`] is the wall reading, for the values ++//! that are dates rather than durations: a name record's timestamp, an exec id, the cost ++//! engine's today. Deriving one from the other would make a stepped wall clock move a ++//! deadline. + + use std::fmt; + use std::future::Future; +diff --git a/crates/microvms-app/src/control/microvm.rs b/crates/microvms-app/src/control/microvm.rs +index 9611956..36597d0 100644 +--- a/crates/microvms-app/src/control/microvm.rs ++++ b/crates/microvms-app/src/control/microvm.rs +@@ -799,56 +799,11 @@ impl ControlPlane { + } + + /// `ResumeMicrovm`. Thaws the VM; no token re-delivery and no re-bootstrap. +- /// +- /// # A VM that ended before the resume is `WindowClosed` (STATE-12) +- /// +- /// Nothing is checked against the suspended window before the call, because the service +- /// decides whether a suspended VM can still resume. It terminates a VM the idle policy +- /// suspended once `suspendedDurationSeconds` passes, and it doesn't hold an explicit +- /// `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, an explicitly +- /// suspended VM resumed 1283 s into a 60 s window (#597). A resume of a VM that has +- /// terminated is refused with a `ValidationException` ("The MicroVM has been +- /// terminated and its state cannot be changed."), the same status as any refused value, +- /// and it was the same refusal for a VM the service ended and one `TerminateMicrovm` +- /// ended (measured 2026-10-12, us-east-1). So a refusal reads the VM's state rather than +- /// the message, and a VM that's TERMINATING or TERMINATED is [`ErrorKind::WindowClosed`] +- /// with its state and `stateReason`, carrying the refusal as its source. The error names +- /// no cause, because the client doesn't know which of the suspended window, the maximum +- /// duration, or a terminate ended the VM; the `stateReason` is the service's word on it. +- /// Every other refusal comes back as the service gave it. + pub async fn resume(&self, id: &str) -> Result<(), Error> { + super::require_valid_identifier("microvmIdentifier", id)?; + let call = Call::post_empty("ResumeMicrovm", paths::resume(id)); +- match send_with_retry(self.transport(), call).await { +- Ok(_) => Ok(()), +- Err(refused) if refused.kind() == ErrorKind::Platform => { +- Err(self.ended_before_resume(id, refused).await) +- } +- Err(error) => Err(error), +- } +- } +- +- /// The error a refused resume reports: [`ErrorKind::WindowClosed`] with `refused` as its +- /// source when `GetMicrovm` reports the VM dead, and `refused` otherwise, including when +- /// that read fails. +- async fn ended_before_resume(&self, id: &str, refused: Error) -> Error { +- let Ok(vm) = self.get_microvm(id).await else { +- return refused; +- }; +- if !crate::constants::DEAD_STATES.contains(&vm.state.as_str()) { +- return refused; +- } +- Error::new( +- ErrorKind::WindowClosed, +- format!( +- "microvm {id} can't be resumed: the service refused ResumeMicrovm, and \ +- GetMicrovm reports the VM {} ({}). A VM that has ended doesn't come back; \ +- launch a new one.", +- vm.state, +- vm.reason(), +- ), +- ) +- .with_source(refused) ++ send_with_retry(self.transport(), call).await?; ++ Ok(()) + } + + /// `TerminateMicrovm`. +diff --git a/crates/microvms-app/src/control/ops.rs b/crates/microvms-app/src/control/ops.rs +index 36833c6..ae1c3c1 100644 +--- a/crates/microvms-app/src/control/ops.rs ++++ b/crates/microvms-app/src/control/ops.rs +@@ -367,9 +367,11 @@ pub struct RunMicrovmWire { + /// "autoResumeEnabled": false}` (measured 2026-08-15, us-east-1, one read-only + /// `GetMicrovm`). + /// +-/// The readback means the client's own record isn't the only authority on the suspended +-/// window: [`crate::control::Microvm::idle_policy`] carries the service's, so a window that +-/// disagrees with what was asked for is observable rather than assumed. ++/// The claim mattered because STATE-12 was resting on it: if only the client could name the ++/// suspended window, the client's own record is the only authority on it. The readback ++/// means there is a second one, and [`crate::control::Microvm::idle_policy`] carries it — ++/// so a suspended window that disagrees with what was asked for is now observable rather ++/// than assumed. + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct IdlePolicy { +diff --git a/crates/microvms-app/src/sandbox.rs b/crates/microvms-app/src/sandbox.rs +index 544535f..dc316a4 100644 +--- a/crates/microvms-app/src/sandbox.rs ++++ b/crates/microvms-app/src/sandbox.rs +@@ -3,8 +3,8 @@ + //! model describes made true of the code. + //! + //! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the +-//! bindings use: build, run, suspend, resume, terminate, a resume of a VM that ended +-//! before it (STATE-12), and teardown in the order that does not leak. ++//! bindings use: build, run, suspend, resume, terminate, the launch-time suspended ++//! window (STATE-12), and teardown in the order that does not leak. + //! + //! # The lifecycle is a field, and every transition is guarded + //! +@@ -32,20 +32,16 @@ + //! control-plane calls rather than answered by AWS. The test asserts the call count, which + //! is the observable that distinguishes the two. + //! +-//! # The service decides whether a suspended VM resumes (STATE-12) ++//! # The suspended window is checked before the call (STATE-12) + //! +-//! The launch-time `idlePolicy` *terminates* a VM the idle policy suspended once +-//! `suspendedDurationSeconds` passes, and the service doesn't hold an explicit +-//! `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, a VM this client +-//! suspended resumed 1283 s into a 60 s window (#597). So a resume is never refused here for +-//! the time it has spent suspended. It goes to the service, and the service's refusal for a +-//! VM that ended, whatever ended it, comes back from [`ControlPlane::resume`] as +-//! [`ErrorKind::WindowClosed`], which this sandbox records as a termination (STATE-11). +-//! +-//! A suspend the idle policy makes isn't seen here: the sandbox still holds the VM as +-//! RUNNING, and its resume is refused before any call (STATE-7). That suspend reaches a +-//! caller only through [`Sandbox::adopt`], which reads the state from the service, or +-//! [`ControlPlane::resume`], which holds no lifecycle to refuse from. ++//! The launch-time `idlePolicy` *terminates* a suspended VM once `suspendedDurationSeconds` ++//! passes, which means "resume later" silently stops working. A resume past the window is ++//! refused locally, before `ResumeMicrovm`, because the alternative is calling and reading ++//! the failure: the service answers about a terminated id, which is not the same statement ++//! as "the window you set at launch closed", and getting there costs the full poll timeout ++//! first. The window comes from this sandbox's own `RunMicrovm` request, falling back to the ++//! `idlePolicy` that `GetMicrovm` reports (measured 2026-08-15, `docs/PLATFORM.md`). Only ++//! the suspend's start time is the client's alone. + //! + //! # Teardown never raises, and the log group is last + //! +@@ -270,8 +266,7 @@ pub struct RunRequest { + pub shell: bool, + /// `idlePolicy.maxIdleDurationSeconds`. + pub max_idle_sec: u32, +- /// `idlePolicy.suspendedDurationSeconds`: how long a VM the idle policy suspended lasts +- /// before the service terminates it. An explicit suspend isn't held to it (STATE-12). ++ /// `idlePolicy.suspendedDurationSeconds` — the window STATE-12 refuses past. + pub suspended_sec: u32, + /// `idlePolicy.autoResumeEnabled`. + pub auto_resume: bool, +@@ -409,8 +404,7 @@ impl RunRequest { + self + } + +- /// Sets `idlePolicy.suspendedDurationSeconds`, how long a VM the idle policy suspended +- /// lasts before the service terminates it. See [`RunRequest::suspended_sec`]. ++ /// Sets the suspended window this sandbox will refuse a resume past (STATE-12). + #[must_use] + pub fn with_suspended_sec(mut self, seconds: u32) -> Self { + self.suspended_sec = seconds; +@@ -624,8 +618,7 @@ impl TokenMinter for ControlPlaneMinter { + + /// One MicroVM's whole life. + /// +-/// See the module docs for the state machine, the resume of an ended VM, and why teardown is +-/// explicit. ++/// See the module docs for the state machine, the window, and why teardown is explicit. + pub struct Sandbox { + control: Arc, + image: Option, +@@ -651,8 +644,8 @@ pub struct Sandbox { + was_terminated: bool, + bootstrap_count: u32, + +- /// The suspended window from *our own* `RunMicrovm` request, or `None` for a sandbox that +- /// launched nothing. `GetMicrovm` reports the service's in `idlePolicy`. ++ /// The window from *our own* `RunMicrovm` request. `GetMicrovm` also reports it (in ++ /// `idlePolicy`), which is the fallback when this sandbox has no request of its own. + suspended_window: Option, + /// Whether the accepted launch carried a caller client token, and so may have adopted + /// a VM an earlier attempt launched (#195). +@@ -666,6 +659,8 @@ pub struct Sandbox { + /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence + /// is checked against it. + idle_window: Option, ++ /// The clock reading when the suspend call was accepted, or `None` when not suspended. ++ suspended_at: Option, + /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down + /// one. + torn_down: bool, +@@ -754,10 +749,10 @@ impl Sandbox { + /// `Sandbox::from_name`, which resolve credentials for a region, are in + /// `microvms_core::prelude`. + /// +- /// The plane's own clock is what every wait here sleeps on, rather than a second one ++ /// The plane's own clock is what times the suspended window, rather than a second one + /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil +- /// tests record: a deadline measured on one clock while the poll loop sleeps on another +- /// reads the wrong elapsed time. ++ /// tests record: a window measured on one clock while the poll loop sleeps on another ++ /// calls a closed window open. + pub fn with_control_plane(control: ControlPlane) -> Self { + Self { + control: Arc::new(control), +@@ -777,6 +772,7 @@ impl Sandbox { + launch_posture: None, + pending_agent_token: None, + idle_window: None, ++ suspended_at: None, + torn_down: false, + adopted: false, + detached: false, +@@ -1355,6 +1351,12 @@ impl Sandbox { + /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once + /// when the service reports RUNNING, the same as a launch this sandbox made itself. + /// ++ /// # The suspended window (STATE-12) ++ /// ++ /// The window comes from the `idlePolicy` `GetMicrovm` reports. A VM adopted while ++ /// already SUSPENDED has no suspend time this client observed, so the local check has ++ /// nothing to measure and the service answers the resume instead. ++ /// + /// `endpoint` must agree with the service's when the service reports one: a mismatch + /// means the identifiers came from two different records. `agent_token` is the bearer + /// credential the VM was launched with; it never appears in `Debug` or an error. +@@ -1603,6 +1605,10 @@ impl Sandbox { + // neither suspend nor resume accepts, bricking the handle over one bad request. + self.control.suspend(&id).await?; + self.set_lifecycle(Lifecycle::Suspending); ++ // Stamped after the call and before the wait, not after the wait: the idlePolicy's ++ // window starts when the platform begins suspending, so timing it from SUSPENDED ++ // would under-count the transition and call a closed window open. ++ self.suspended_at = Some(self.control.clock().elapsed()); + + let settled = self + .control +@@ -1649,16 +1655,12 @@ impl Sandbox { + /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be + /// refused — a 409 that reads like a broken VM. + /// +- /// # The service decides, and a VM it ended is `WindowClosed` (STATE-12) ++ /// # The window is checked first (STATE-12) + /// +- /// No resume is refused here for the time the VM has spent suspended. The service ends a +- /// VM the idle policy suspended once `suspendedDurationSeconds` passes, and it kept a VM +- /// this client suspended resumable 1283 s into a 60 s window (measured 2026-10-12, +- /// us-east-1, #597). A resume the service refuses because the VM ended, whatever ended +- /// it, is [`ErrorKind::WindowClosed`] ([`ControlPlane::resume`]), and this records the VM +- /// as terminated, so a second resume is refused here with no call (STATE-11). A VM the +- /// idle policy suspended is still RUNNING here, so its resume is refused before any call +- /// (STATE-7); see the module docs. ++ /// Before any wire call, because the answer is already known and calling costs the poll ++ /// timeout to learn something worse. The falsification is a service reporting ++ /// TERMINATED: without this check the resume burns the full deadline and then reports a ++ /// state the client could have named at once. + /// + /// # The proxy token is dropped (STATE-8) + /// +@@ -1671,7 +1673,7 @@ impl Sandbox { + let id = self.require_microvm("resume")?; + + // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal +- // comes before any call. ++ // comes before the window check and before any call. + if self.was_terminated || self.lifecycle == Lifecycle::Terminated { + return Err(Error::invalid_arg(format!( + "microvm {id} was terminated, and a terminated VM never returns to RUNNING \ +@@ -1686,23 +1688,15 @@ impl Sandbox { + ))); + } + +- // STATE-12: a `WindowClosed` refusal is a VM that ended before this resume. It's +- // recorded as TERMINATED whichever of TERMINATING or TERMINATED `GetMicrovm` read, +- // because the service has said the VM's state can't change, and as a termination, so +- // the next resume is refused before any call (STATE-11). Any other refusal leaves the +- // sandbox SUSPENDED, and a retry goes to the service again. +- if let Err(refused) = self.control.resume(&id).await { +- if refused.kind() == ErrorKind::WindowClosed { +- self.was_terminated = true; +- self.set_lifecycle(Lifecycle::Terminated); +- } +- return Err(refused); +- } ++ // STATE-12, first and locally. ++ self.require_open_suspended_window(&id)?; ++ ++ self.control.resume(&id).await?; + // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the +- // state this call was made from, so failing on it would fail every resume. A VM that +- // terminates once the resume is accepted never reaches RUNNING, and waiting only for +- // RUNNING there burns the full timeout and then reports a timeout message hiding a +- // cause the service had already stated in `stateReason`. ++ // state this call was made from, so failing on it would fail every resume. A VM the ++ // idlePolicy terminated during suspension never reaches RUNNING, and waiting only ++ // for RUNNING there burns the full timeout and then reports a timeout message ++ // hiding a cause the service had already stated in `stateReason`. + let running = self + .control + .wait_for_state( +@@ -1722,6 +1716,10 @@ impl Sandbox { + if let Some(session) = self.session.as_mut() { + session.rebind(endpoint); + } ++ // Cleared on success so the next cycle's window is measured from the next suspend. ++ // Leaving it set would accumulate every suspension's elapsed time into one total and ++ // reject a resume whose own window is wide open. ++ self.suspended_at = None; + + self.session.as_mut().ok_or_else(|| { + Error::new( +@@ -1732,6 +1730,41 @@ impl Sandbox { + }) + } + ++ /// Rejects a resume the launch-time `idlePolicy` has already made impossible (STATE-12). ++ /// ++ /// The message names the elapsed time, the window, and the `idlePolicy` finding, because ++ /// "cannot resume" alone sends a reader looking for the flag that reopens it. ++ fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> { ++ let reported = self ++ .microvm ++ .as_ref() ++ .and_then(|vm| vm.idle_policy.as_ref()) ++ .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds))); ++ let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at) ++ else { ++ // No window or no suspend time means this sandbox cannot know how long the VM ++ // has been suspended, and guessing would refuse a resume the service would honour. ++ return Ok(()); ++ }; ++ let elapsed = self.control.clock().elapsed().saturating_sub(since); ++ if elapsed <= window { ++ return Ok(()); ++ } ++ Err(Error::new( ++ ErrorKind::WindowClosed, ++ format!( ++ "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \ ++ window set at launch — the idlePolicy terminates a suspended VM once that window \ ++ passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \ ++ Refused before ResumeMicrovm, because calling would cost the full poll timeout \ ++ to learn the same thing less clearly. A longer window has to be set at launch \ ++ on the next VM; there is no call that extends this one.", ++ elapsed.as_secs(), ++ window.as_secs(), ++ ), ++ )) ++ } ++ + // ── terminate (STATE-9, STATE-10) ──────────────────────────────────────── + + /// Tears down, best-effort, never erroring. +@@ -1874,14 +1907,11 @@ impl Sandbox { + let vm = self.microvm.as_ref()?; + self.lifecycle.is_live().then(|| { + format!( +- "warning: the Sandbox for microvm {id} was dropped in {state} without \ +- terminate() or detach(). Nothing was torn down — Drop cannot await, so a \ +- teardown here would deadlock inside a runtime. {state} is the state this \ +- sandbox last saw, and the platform may have suspended or ended the VM since. A \ +- VM that's still live bills until it ends, at its maximumDurationInSeconds \ +- ceiling at the latest: terminate it with `microvm terminate {id}`.", +- id = vm.id, +- state = self.lifecycle, ++ "warning: the Sandbox for microvm {} was dropped in {} without terminate() or \ ++ detach(). Nothing was torn down — Drop cannot await, so a teardown here would \ ++ deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \ ++ ceiling: terminate it with `microvm terminate {}`.", ++ vm.id, self.lifecycle, vm.id, + ) + }) + } +diff --git a/crates/microvms-cli/src/cli.rs b/crates/microvms-cli/src/cli.rs +index 25b94f0..f3a820e 100644 +--- a/crates/microvms-cli/src/cli.rs ++++ b/crates/microvms-cli/src/cli.rs +@@ -301,10 +301,8 @@ pub enum Command { + + /// Thaw a suspended MicroVM and report its endpoint. + /// +- /// The launch-time `suspendedDurationSeconds` window terminates a VM the idle policy +- /// suspended once it passes. VMs suspended with `microvm suspend` resumed 900 s and more +- /// into a 60 s window (2026-10-12), which AWS doesn't promise, so the service decides. A +- /// resume of a VM that ended, whatever ended it, exits `ERR_WINDOW_CLOSED`; launch a new one. ++ /// The launch-time `suspendedDurationSeconds` window terminates a suspended VM once it ++ /// passes, so "resume later" silently stops working and the VM is gone rather than slow. + Resume(ResumeArgs), + + /// Tear down a MicroVM, and optionally its image and build log group. +@@ -1056,8 +1054,7 @@ pub struct RunArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +@@ -2506,8 +2503,7 @@ pub struct AgentUpArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +diff --git a/crates/microvms-cli/src/commands/lifecycle.rs b/crates/microvms-cli/src/commands/lifecycle.rs +index c4c0b8b..ef6a544 100644 +--- a/crates/microvms-cli/src/commands/lifecycle.rs ++++ b/crates/microvms-cli/src/commands/lifecycle.rs +@@ -2084,12 +2084,10 @@ pub async fn suspend( + /// Thaws a suspended MicroVM and reports its endpoint. + /// + /// The suspended window is **not** checked here, and that is correct rather than a gap: the +-/// service ends a VM the idle policy suspended once `suspendedDurationSeconds` passes and keeps +-/// an explicitly suspended one resumable past it (measured 2026-10-12, us-east-1), so only the +-/// service can answer. Core's `ControlPlane::resume` reports its refusal of a VM that ended +-/// before the resume as `ERR_WINDOW_CLOSED`. A VM that dies after the resume is +-/// accepted fails the wait at once through `fail_on: DEAD_STATES`, with `stateReason`, instead +-/// of burning the poll timeout. ++/// window is in `GetMicrovm`'s `idlePolicy`, but when the suspend began is known only to the ++/// process that issued it. Guessing would either reject an open window or accept a closed ++/// one, both worse than letting the service answer — which is what `fail_on: DEAD_STATES` ++/// does, failing fast with `stateReason` instead of burning the poll timeout. + pub async fn resume( + ctx: &mut Ctx<'_, O, E>, + args: &ResumeArgs, +diff --git a/crates/microvms-cli/src/config.rs b/crates/microvms-cli/src/config.rs +index 9693850..6020ef2 100644 +--- a/crates/microvms-cli/src/config.rs ++++ b/crates/microvms-cli/src/config.rs +@@ -83,8 +83,7 @@ pub struct ProjectConfig { + pub auto_resume: Option, + /// `run --max-idle-sec`: suspend after this much inbound-traffic idleness. + pub max_idle_sec: Option, +- /// `run --suspended-sec`: terminate a VM the idle policy suspended once it's been suspended +- /// this long. ++ /// `run --suspended-sec`: terminate after this long suspended. + pub suspended_sec: Option, + /// `run --max-duration-sec`: hard ceiling on the VM's life. + pub max_duration_sec: Option, +diff --git a/crates/microvms-cli/src/exit.rs b/crates/microvms-cli/src/exit.rs +index 84342b9..4bba5a5 100644 +--- a/crates/microvms-cli/src/exit.rs ++++ b/crates/microvms-cli/src/exit.rs +@@ -254,8 +254,7 @@ pub const EXIT_TABLE: [ExitRow; 17] = [ + ExitRow { + exit: Exit::WindowClosed, + code: Some("ERR_WINDOW_CLOSED"), +- meaning: "the MicroVM ended before the resume (the suspended window, the maximum \ +- duration, or a terminate), and it can't be resumed; launch a new one", ++ meaning: "the launch-time suspended window passed, so there is nothing to resume", + finding: "`idlePolicy`", + }, + ExitRow { +@@ -443,7 +442,9 @@ pub fn classify(error: &Error) -> CliError { + (Exit::LaunchDied, _) => { + &["`microvm logs ` names the build log group the hook wrote to"] + } +- (Exit::WindowClosed, _) => &["the VM ended and no call brings it back: launch a new one"], ++ (Exit::WindowClosed, _) => { ++ &["a longer window is set at launch with --suspended-sec; no call extends this one"] ++ } + _ => &[], + }; + classified +diff --git a/crates/microvms-domain/src/error.rs b/crates/microvms-domain/src/error.rs +index bbbd0ed..15ed493 100644 +--- a/crates/microvms-domain/src/error.rs ++++ b/crates/microvms-domain/src/error.rs +@@ -130,9 +130,7 @@ pub enum ErrorKind { + BuildWedged, + /// The MicroVM reached a terminal state before RUNNING; read `stateReason`. + LaunchDied, +- /// The MicroVM ended before the resume (the suspended window, the maximum duration, or a +- /// terminate), and it can't be resumed; launch a new one. The service refused the resume, +- /// and `GetMicrovm` reports the VM TERMINATING or TERMINATED. ++ /// The launch-time suspended window passed, so there is nothing to resume. + WindowClosed, + /// A control-plane failure with no more specific class. + Platform, diff --git a/verify/guards/faults/f2p-an-adopted-vm-resumes-past-the-reported-window-through.patch b/verify/guards/faults/f2p-an-adopted-vm-resumes-past-the-reported-window-through.patch new file mode 100644 index 00000000..303b48bf --- /dev/null +++ b/verify/guards/faults/f2p-an-adopted-vm-resumes-past-the-reported-window-through.patch @@ -0,0 +1,641 @@ +diff --git a/bindings/microvms-js/src/control.rs b/bindings/microvms-js/src/control.rs +index ead8c40..66a7dd9 100644 +--- a/bindings/microvms-js/src/control.rs ++++ b/bindings/microvms-js/src/control.rs +@@ -279,8 +279,8 @@ pub struct DeleteImageOptions { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7). Use +-/// it when a process has only an identifier. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12). Use it when a process has only an identifier. + #[napi] + pub struct ControlPlane { + inner: Arc, +@@ -344,8 +344,7 @@ impl ControlPlane { + self.inner.suspend(µvm_id).await.map_err(js_async) + } + +- /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, rejects with `ERR_WINDOW_CLOSED` (STATE-12). ++ /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. + #[napi] + pub async fn resume(&self, microvm_id: String) -> Result<(), AsyncError> { + self.inner.resume(µvm_id).await.map_err(js_async) +diff --git a/bindings/microvms-js/src/sandbox.rs b/bindings/microvms-js/src/sandbox.rs +index 517085e..55d1e84 100644 +--- a/bindings/microvms-js/src/sandbox.rs ++++ b/bindings/microvms-js/src/sandbox.rs +@@ -1144,21 +1144,11 @@ impl Sandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, and +- /// even a call the service accepted would hand back a different machine. And a resume +- /// from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspendedSec` passes, and a VM this sandbox suspended stayed resumable +- /// 1283 s into a 60 s window (measured 2026-10-12, us-east-1). `ERR_WINDOW_CLOSED` is the +- /// service's answer once a resume reaches it: the service refused the resume, and the VM +- /// is TERMINATING or TERMINATED, whatever ended it. The sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` rejects with `ERR_INVALID_ARG` before any call (STATE-7). That +- /// suspend reaches a caller only through `Sandbox.adopt`, which reads the state from the +- /// service, or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// even a call the service accepted would hand back a different machine. A resume from ++ /// anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended window ++ /// (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates a ++ /// suspended VM once that window passes, so there is nothing left to resume, and calling ++ /// would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/bindings/microvms-py/src/control.rs b/bindings/microvms-py/src/control.rs +index ce0752e..5fbe783 100644 +--- a/bindings/microvms-py/src/control.rs ++++ b/bindings/microvms-py/src/control.rs +@@ -444,9 +444,9 @@ impl PyImageBuild { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7): a +-/// suspend of a SUSPENDED VM is the service's to refuse. Use it when a process has only an +-/// identifier, such as a durable workflow step in a fresh process. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12): a suspend of a SUSPENDED VM is the service's to refuse. Use it when a ++/// process has only an identifier, such as a durable workflow step in a fresh process. + #[pyclass(frozen, name = "ControlPlane", module = "microvms")] + pub struct PyControlPlane { + inner: Arc, +@@ -703,8 +703,7 @@ impl PyControlPlane { + Ok(runtime::spawn_shielded(self.suspend_op(microvm_id)).await?) + } + +- /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, raises `WindowClosedError` (STATE-12). ++ /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. + fn resume(&self, py: Python<'_>, microvm_id: String) -> PyCoreResult<()> { + Ok(runtime::block_on(py, self.resume_op(microvm_id))?) + } +diff --git a/bindings/microvms-py/src/errors.rs b/bindings/microvms-py/src/errors.rs +index be8962a..a664785 100644 +--- a/bindings/microvms-py/src/errors.rs ++++ b/bindings/microvms-py/src/errors.rs +@@ -87,10 +87,8 @@ create_exception!( + microvms, + WindowClosedError, + MicrovmError, +- "ERR_WINDOW_CLOSED — the MicroVM ended before the resume (the suspended window, the \ +- maximum duration, or a terminate), and it can't be resumed; launch a new one. The \ +- suspended window ends a VM the idle policy suspended; an explicit suspend stayed \ +- resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1)." ++ "ERR_WINDOW_CLOSED — the launch-time suspended window passed, so there is nothing to \ ++ resume. A longer window has to be set at launch on the next VM." + ); + create_exception!( + microvms, +diff --git a/bindings/microvms-py/src/sandbox.rs b/bindings/microvms-py/src/sandbox.rs +index 449afbd..f7d6371 100644 +--- a/bindings/microvms-py/src/sandbox.rs ++++ b/bindings/microvms-py/src/sandbox.rs +@@ -1904,22 +1904,11 @@ impl PySandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, +- /// and even a call the service accepted would hand back a different machine. And a +- /// resume from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspended_sec` passes, and a VM this sandbox suspended stayed +- /// resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1). +- /// `WindowClosedError` is the service's answer once a resume reaches it: the service +- /// refused the resume, and the VM is TERMINATING or TERMINATED, whatever ended it. The +- /// sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` raises `InvalidArgError` before any call (STATE-7). That suspend +- /// reaches a caller only through `Sandbox.adopt`, which reads the state from the service, +- /// or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// and even a call the service accepted would hand back a different machine. A resume ++ /// from anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended ++ /// window (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates ++ /// a suspended VM once that window passes, so there is nothing left to resume, and ++ /// calling would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/crates/microvms-app/src/clock.rs b/crates/microvms-app/src/clock.rs +index 75afb7c..5685303 100644 +--- a/crates/microvms-app/src/clock.rs ++++ b/crates/microvms-app/src/clock.rs +@@ -7,11 +7,11 @@ + //! sources: the control plane's read `std::time::Instant` and the session's read tokio's. + //! Under a deterministic simulator those disagree. tokio's clock is virtual there and `std`'s + //! isn't, so a control-plane wait whose sleeps advanced virtual time measured its deadline +-//! on the host's real time and never reached it. `Sandbox` already waits on the plane's clock +-//! rather than one of its own, so a deadline isn't measured on one clock while the polls sleep +-//! on another. One trait, with tokio's clock in production, keeps that property for everything +-//! that reads time. The trait is re-exported at `control::Clock` and at `session::Clock`, the +-//! paths each half had. ++//! on the host's real time and never reached it. `Sandbox` already times the suspended window ++//! on the plane's clock rather than its own, so a window isn't measured on one clock while ++//! the polls run on another. One trait, with tokio's clock in production, keeps that property ++//! for everything that reads time. The trait is re-exported at `control::Clock` and at ++//! `session::Clock`, the paths each half had. + //! + //! # Why a trait rather than `tokio::time::pause` + //! +@@ -24,11 +24,11 @@ + //! # Monotonic and wall time are separate readings + //! + //! [`Clock::elapsed`] is monotonic, the same reasoning the Python client's injectable +-//! `time.monotonic` carried: a wait's deadline and a proxy token's age are *durations*, and a +-//! wall clock that steps backward would hold a passed deadline open. [`Clock::unix_now`] is the +-//! wall reading, for the values that are dates rather than durations: a name record's +-//! timestamp, an exec id, the cost engine's today. Deriving one from the other would make a +-//! stepped wall clock move a deadline. ++//! `time.monotonic` carried: the suspended window is a *duration*, and a wall clock that steps ++//! backward would reopen a closed one. [`Clock::unix_now`] is the wall reading, for the values ++//! that are dates rather than durations: a name record's timestamp, an exec id, the cost ++//! engine's today. Deriving one from the other would make a stepped wall clock move a ++//! deadline. + + use std::fmt; + use std::future::Future; +diff --git a/crates/microvms-app/src/control/microvm.rs b/crates/microvms-app/src/control/microvm.rs +index 9611956..36597d0 100644 +--- a/crates/microvms-app/src/control/microvm.rs ++++ b/crates/microvms-app/src/control/microvm.rs +@@ -799,56 +799,11 @@ impl ControlPlane { + } + + /// `ResumeMicrovm`. Thaws the VM; no token re-delivery and no re-bootstrap. +- /// +- /// # A VM that ended before the resume is `WindowClosed` (STATE-12) +- /// +- /// Nothing is checked against the suspended window before the call, because the service +- /// decides whether a suspended VM can still resume. It terminates a VM the idle policy +- /// suspended once `suspendedDurationSeconds` passes, and it doesn't hold an explicit +- /// `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, an explicitly +- /// suspended VM resumed 1283 s into a 60 s window (#597). A resume of a VM that has +- /// terminated is refused with a `ValidationException` ("The MicroVM has been +- /// terminated and its state cannot be changed."), the same status as any refused value, +- /// and it was the same refusal for a VM the service ended and one `TerminateMicrovm` +- /// ended (measured 2026-10-12, us-east-1). So a refusal reads the VM's state rather than +- /// the message, and a VM that's TERMINATING or TERMINATED is [`ErrorKind::WindowClosed`] +- /// with its state and `stateReason`, carrying the refusal as its source. The error names +- /// no cause, because the client doesn't know which of the suspended window, the maximum +- /// duration, or a terminate ended the VM; the `stateReason` is the service's word on it. +- /// Every other refusal comes back as the service gave it. + pub async fn resume(&self, id: &str) -> Result<(), Error> { + super::require_valid_identifier("microvmIdentifier", id)?; + let call = Call::post_empty("ResumeMicrovm", paths::resume(id)); +- match send_with_retry(self.transport(), call).await { +- Ok(_) => Ok(()), +- Err(refused) if refused.kind() == ErrorKind::Platform => { +- Err(self.ended_before_resume(id, refused).await) +- } +- Err(error) => Err(error), +- } +- } +- +- /// The error a refused resume reports: [`ErrorKind::WindowClosed`] with `refused` as its +- /// source when `GetMicrovm` reports the VM dead, and `refused` otherwise, including when +- /// that read fails. +- async fn ended_before_resume(&self, id: &str, refused: Error) -> Error { +- let Ok(vm) = self.get_microvm(id).await else { +- return refused; +- }; +- if !crate::constants::DEAD_STATES.contains(&vm.state.as_str()) { +- return refused; +- } +- Error::new( +- ErrorKind::WindowClosed, +- format!( +- "microvm {id} can't be resumed: the service refused ResumeMicrovm, and \ +- GetMicrovm reports the VM {} ({}). A VM that has ended doesn't come back; \ +- launch a new one.", +- vm.state, +- vm.reason(), +- ), +- ) +- .with_source(refused) ++ send_with_retry(self.transport(), call).await?; ++ Ok(()) + } + + /// `TerminateMicrovm`. +diff --git a/crates/microvms-app/src/control/ops.rs b/crates/microvms-app/src/control/ops.rs +index 36833c6..ae1c3c1 100644 +--- a/crates/microvms-app/src/control/ops.rs ++++ b/crates/microvms-app/src/control/ops.rs +@@ -367,9 +367,11 @@ pub struct RunMicrovmWire { + /// "autoResumeEnabled": false}` (measured 2026-08-15, us-east-1, one read-only + /// `GetMicrovm`). + /// +-/// The readback means the client's own record isn't the only authority on the suspended +-/// window: [`crate::control::Microvm::idle_policy`] carries the service's, so a window that +-/// disagrees with what was asked for is observable rather than assumed. ++/// The claim mattered because STATE-12 was resting on it: if only the client could name the ++/// suspended window, the client's own record is the only authority on it. The readback ++/// means there is a second one, and [`crate::control::Microvm::idle_policy`] carries it — ++/// so a suspended window that disagrees with what was asked for is now observable rather ++/// than assumed. + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct IdlePolicy { +diff --git a/crates/microvms-app/src/sandbox.rs b/crates/microvms-app/src/sandbox.rs +index 544535f..dc316a4 100644 +--- a/crates/microvms-app/src/sandbox.rs ++++ b/crates/microvms-app/src/sandbox.rs +@@ -3,8 +3,8 @@ + //! model describes made true of the code. + //! + //! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the +-//! bindings use: build, run, suspend, resume, terminate, a resume of a VM that ended +-//! before it (STATE-12), and teardown in the order that does not leak. ++//! bindings use: build, run, suspend, resume, terminate, the launch-time suspended ++//! window (STATE-12), and teardown in the order that does not leak. + //! + //! # The lifecycle is a field, and every transition is guarded + //! +@@ -32,20 +32,16 @@ + //! control-plane calls rather than answered by AWS. The test asserts the call count, which + //! is the observable that distinguishes the two. + //! +-//! # The service decides whether a suspended VM resumes (STATE-12) ++//! # The suspended window is checked before the call (STATE-12) + //! +-//! The launch-time `idlePolicy` *terminates* a VM the idle policy suspended once +-//! `suspendedDurationSeconds` passes, and the service doesn't hold an explicit +-//! `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, a VM this client +-//! suspended resumed 1283 s into a 60 s window (#597). So a resume is never refused here for +-//! the time it has spent suspended. It goes to the service, and the service's refusal for a +-//! VM that ended, whatever ended it, comes back from [`ControlPlane::resume`] as +-//! [`ErrorKind::WindowClosed`], which this sandbox records as a termination (STATE-11). +-//! +-//! A suspend the idle policy makes isn't seen here: the sandbox still holds the VM as +-//! RUNNING, and its resume is refused before any call (STATE-7). That suspend reaches a +-//! caller only through [`Sandbox::adopt`], which reads the state from the service, or +-//! [`ControlPlane::resume`], which holds no lifecycle to refuse from. ++//! The launch-time `idlePolicy` *terminates* a suspended VM once `suspendedDurationSeconds` ++//! passes, which means "resume later" silently stops working. A resume past the window is ++//! refused locally, before `ResumeMicrovm`, because the alternative is calling and reading ++//! the failure: the service answers about a terminated id, which is not the same statement ++//! as "the window you set at launch closed", and getting there costs the full poll timeout ++//! first. The window comes from this sandbox's own `RunMicrovm` request, falling back to the ++//! `idlePolicy` that `GetMicrovm` reports (measured 2026-08-15, `docs/PLATFORM.md`). Only ++//! the suspend's start time is the client's alone. + //! + //! # Teardown never raises, and the log group is last + //! +@@ -270,8 +266,7 @@ pub struct RunRequest { + pub shell: bool, + /// `idlePolicy.maxIdleDurationSeconds`. + pub max_idle_sec: u32, +- /// `idlePolicy.suspendedDurationSeconds`: how long a VM the idle policy suspended lasts +- /// before the service terminates it. An explicit suspend isn't held to it (STATE-12). ++ /// `idlePolicy.suspendedDurationSeconds` — the window STATE-12 refuses past. + pub suspended_sec: u32, + /// `idlePolicy.autoResumeEnabled`. + pub auto_resume: bool, +@@ -409,8 +404,7 @@ impl RunRequest { + self + } + +- /// Sets `idlePolicy.suspendedDurationSeconds`, how long a VM the idle policy suspended +- /// lasts before the service terminates it. See [`RunRequest::suspended_sec`]. ++ /// Sets the suspended window this sandbox will refuse a resume past (STATE-12). + #[must_use] + pub fn with_suspended_sec(mut self, seconds: u32) -> Self { + self.suspended_sec = seconds; +@@ -624,8 +618,7 @@ impl TokenMinter for ControlPlaneMinter { + + /// One MicroVM's whole life. + /// +-/// See the module docs for the state machine, the resume of an ended VM, and why teardown is +-/// explicit. ++/// See the module docs for the state machine, the window, and why teardown is explicit. + pub struct Sandbox { + control: Arc, + image: Option, +@@ -651,8 +644,8 @@ pub struct Sandbox { + was_terminated: bool, + bootstrap_count: u32, + +- /// The suspended window from *our own* `RunMicrovm` request, or `None` for a sandbox that +- /// launched nothing. `GetMicrovm` reports the service's in `idlePolicy`. ++ /// The window from *our own* `RunMicrovm` request. `GetMicrovm` also reports it (in ++ /// `idlePolicy`), which is the fallback when this sandbox has no request of its own. + suspended_window: Option, + /// Whether the accepted launch carried a caller client token, and so may have adopted + /// a VM an earlier attempt launched (#195). +@@ -666,6 +659,8 @@ pub struct Sandbox { + /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence + /// is checked against it. + idle_window: Option, ++ /// The clock reading when the suspend call was accepted, or `None` when not suspended. ++ suspended_at: Option, + /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down + /// one. + torn_down: bool, +@@ -754,10 +749,10 @@ impl Sandbox { + /// `Sandbox::from_name`, which resolve credentials for a region, are in + /// `microvms_core::prelude`. + /// +- /// The plane's own clock is what every wait here sleeps on, rather than a second one ++ /// The plane's own clock is what times the suspended window, rather than a second one + /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil +- /// tests record: a deadline measured on one clock while the poll loop sleeps on another +- /// reads the wrong elapsed time. ++ /// tests record: a window measured on one clock while the poll loop sleeps on another ++ /// calls a closed window open. + pub fn with_control_plane(control: ControlPlane) -> Self { + Self { + control: Arc::new(control), +@@ -777,6 +772,7 @@ impl Sandbox { + launch_posture: None, + pending_agent_token: None, + idle_window: None, ++ suspended_at: None, + torn_down: false, + adopted: false, + detached: false, +@@ -1355,6 +1351,12 @@ impl Sandbox { + /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once + /// when the service reports RUNNING, the same as a launch this sandbox made itself. + /// ++ /// # The suspended window (STATE-12) ++ /// ++ /// The window comes from the `idlePolicy` `GetMicrovm` reports. A VM adopted while ++ /// already SUSPENDED has no suspend time this client observed, so the local check has ++ /// nothing to measure and the service answers the resume instead. ++ /// + /// `endpoint` must agree with the service's when the service reports one: a mismatch + /// means the identifiers came from two different records. `agent_token` is the bearer + /// credential the VM was launched with; it never appears in `Debug` or an error. +@@ -1603,6 +1605,10 @@ impl Sandbox { + // neither suspend nor resume accepts, bricking the handle over one bad request. + self.control.suspend(&id).await?; + self.set_lifecycle(Lifecycle::Suspending); ++ // Stamped after the call and before the wait, not after the wait: the idlePolicy's ++ // window starts when the platform begins suspending, so timing it from SUSPENDED ++ // would under-count the transition and call a closed window open. ++ self.suspended_at = Some(self.control.clock().elapsed()); + + let settled = self + .control +@@ -1649,16 +1655,12 @@ impl Sandbox { + /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be + /// refused — a 409 that reads like a broken VM. + /// +- /// # The service decides, and a VM it ended is `WindowClosed` (STATE-12) ++ /// # The window is checked first (STATE-12) + /// +- /// No resume is refused here for the time the VM has spent suspended. The service ends a +- /// VM the idle policy suspended once `suspendedDurationSeconds` passes, and it kept a VM +- /// this client suspended resumable 1283 s into a 60 s window (measured 2026-10-12, +- /// us-east-1, #597). A resume the service refuses because the VM ended, whatever ended +- /// it, is [`ErrorKind::WindowClosed`] ([`ControlPlane::resume`]), and this records the VM +- /// as terminated, so a second resume is refused here with no call (STATE-11). A VM the +- /// idle policy suspended is still RUNNING here, so its resume is refused before any call +- /// (STATE-7); see the module docs. ++ /// Before any wire call, because the answer is already known and calling costs the poll ++ /// timeout to learn something worse. The falsification is a service reporting ++ /// TERMINATED: without this check the resume burns the full deadline and then reports a ++ /// state the client could have named at once. + /// + /// # The proxy token is dropped (STATE-8) + /// +@@ -1671,7 +1673,7 @@ impl Sandbox { + let id = self.require_microvm("resume")?; + + // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal +- // comes before any call. ++ // comes before the window check and before any call. + if self.was_terminated || self.lifecycle == Lifecycle::Terminated { + return Err(Error::invalid_arg(format!( + "microvm {id} was terminated, and a terminated VM never returns to RUNNING \ +@@ -1686,23 +1688,15 @@ impl Sandbox { + ))); + } + +- // STATE-12: a `WindowClosed` refusal is a VM that ended before this resume. It's +- // recorded as TERMINATED whichever of TERMINATING or TERMINATED `GetMicrovm` read, +- // because the service has said the VM's state can't change, and as a termination, so +- // the next resume is refused before any call (STATE-11). Any other refusal leaves the +- // sandbox SUSPENDED, and a retry goes to the service again. +- if let Err(refused) = self.control.resume(&id).await { +- if refused.kind() == ErrorKind::WindowClosed { +- self.was_terminated = true; +- self.set_lifecycle(Lifecycle::Terminated); +- } +- return Err(refused); +- } ++ // STATE-12, first and locally. ++ self.require_open_suspended_window(&id)?; ++ ++ self.control.resume(&id).await?; + // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the +- // state this call was made from, so failing on it would fail every resume. A VM that +- // terminates once the resume is accepted never reaches RUNNING, and waiting only for +- // RUNNING there burns the full timeout and then reports a timeout message hiding a +- // cause the service had already stated in `stateReason`. ++ // state this call was made from, so failing on it would fail every resume. A VM the ++ // idlePolicy terminated during suspension never reaches RUNNING, and waiting only ++ // for RUNNING there burns the full timeout and then reports a timeout message ++ // hiding a cause the service had already stated in `stateReason`. + let running = self + .control + .wait_for_state( +@@ -1722,6 +1716,10 @@ impl Sandbox { + if let Some(session) = self.session.as_mut() { + session.rebind(endpoint); + } ++ // Cleared on success so the next cycle's window is measured from the next suspend. ++ // Leaving it set would accumulate every suspension's elapsed time into one total and ++ // reject a resume whose own window is wide open. ++ self.suspended_at = None; + + self.session.as_mut().ok_or_else(|| { + Error::new( +@@ -1732,6 +1730,41 @@ impl Sandbox { + }) + } + ++ /// Rejects a resume the launch-time `idlePolicy` has already made impossible (STATE-12). ++ /// ++ /// The message names the elapsed time, the window, and the `idlePolicy` finding, because ++ /// "cannot resume" alone sends a reader looking for the flag that reopens it. ++ fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> { ++ let reported = self ++ .microvm ++ .as_ref() ++ .and_then(|vm| vm.idle_policy.as_ref()) ++ .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds))); ++ let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at) ++ else { ++ // No window or no suspend time means this sandbox cannot know how long the VM ++ // has been suspended, and guessing would refuse a resume the service would honour. ++ return Ok(()); ++ }; ++ let elapsed = self.control.clock().elapsed().saturating_sub(since); ++ if elapsed <= window { ++ return Ok(()); ++ } ++ Err(Error::new( ++ ErrorKind::WindowClosed, ++ format!( ++ "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \ ++ window set at launch — the idlePolicy terminates a suspended VM once that window \ ++ passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \ ++ Refused before ResumeMicrovm, because calling would cost the full poll timeout \ ++ to learn the same thing less clearly. A longer window has to be set at launch \ ++ on the next VM; there is no call that extends this one.", ++ elapsed.as_secs(), ++ window.as_secs(), ++ ), ++ )) ++ } ++ + // ── terminate (STATE-9, STATE-10) ──────────────────────────────────────── + + /// Tears down, best-effort, never erroring. +@@ -1874,14 +1907,11 @@ impl Sandbox { + let vm = self.microvm.as_ref()?; + self.lifecycle.is_live().then(|| { + format!( +- "warning: the Sandbox for microvm {id} was dropped in {state} without \ +- terminate() or detach(). Nothing was torn down — Drop cannot await, so a \ +- teardown here would deadlock inside a runtime. {state} is the state this \ +- sandbox last saw, and the platform may have suspended or ended the VM since. A \ +- VM that's still live bills until it ends, at its maximumDurationInSeconds \ +- ceiling at the latest: terminate it with `microvm terminate {id}`.", +- id = vm.id, +- state = self.lifecycle, ++ "warning: the Sandbox for microvm {} was dropped in {} without terminate() or \ ++ detach(). Nothing was torn down — Drop cannot await, so a teardown here would \ ++ deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \ ++ ceiling: terminate it with `microvm terminate {}`.", ++ vm.id, self.lifecycle, vm.id, + ) + }) + } +diff --git a/crates/microvms-cli/src/cli.rs b/crates/microvms-cli/src/cli.rs +index 25b94f0..f3a820e 100644 +--- a/crates/microvms-cli/src/cli.rs ++++ b/crates/microvms-cli/src/cli.rs +@@ -301,10 +301,8 @@ pub enum Command { + + /// Thaw a suspended MicroVM and report its endpoint. + /// +- /// The launch-time `suspendedDurationSeconds` window terminates a VM the idle policy +- /// suspended once it passes. VMs suspended with `microvm suspend` resumed 900 s and more +- /// into a 60 s window (2026-10-12), which AWS doesn't promise, so the service decides. A +- /// resume of a VM that ended, whatever ended it, exits `ERR_WINDOW_CLOSED`; launch a new one. ++ /// The launch-time `suspendedDurationSeconds` window terminates a suspended VM once it ++ /// passes, so "resume later" silently stops working and the VM is gone rather than slow. + Resume(ResumeArgs), + + /// Tear down a MicroVM, and optionally its image and build log group. +@@ -1056,8 +1054,7 @@ pub struct RunArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +@@ -2506,8 +2503,7 @@ pub struct AgentUpArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +diff --git a/crates/microvms-cli/src/commands/lifecycle.rs b/crates/microvms-cli/src/commands/lifecycle.rs +index c4c0b8b..ef6a544 100644 +--- a/crates/microvms-cli/src/commands/lifecycle.rs ++++ b/crates/microvms-cli/src/commands/lifecycle.rs +@@ -2084,12 +2084,10 @@ pub async fn suspend( + /// Thaws a suspended MicroVM and reports its endpoint. + /// + /// The suspended window is **not** checked here, and that is correct rather than a gap: the +-/// service ends a VM the idle policy suspended once `suspendedDurationSeconds` passes and keeps +-/// an explicitly suspended one resumable past it (measured 2026-10-12, us-east-1), so only the +-/// service can answer. Core's `ControlPlane::resume` reports its refusal of a VM that ended +-/// before the resume as `ERR_WINDOW_CLOSED`. A VM that dies after the resume is +-/// accepted fails the wait at once through `fail_on: DEAD_STATES`, with `stateReason`, instead +-/// of burning the poll timeout. ++/// window is in `GetMicrovm`'s `idlePolicy`, but when the suspend began is known only to the ++/// process that issued it. Guessing would either reject an open window or accept a closed ++/// one, both worse than letting the service answer — which is what `fail_on: DEAD_STATES` ++/// does, failing fast with `stateReason` instead of burning the poll timeout. + pub async fn resume( + ctx: &mut Ctx<'_, O, E>, + args: &ResumeArgs, +diff --git a/crates/microvms-cli/src/config.rs b/crates/microvms-cli/src/config.rs +index 9693850..6020ef2 100644 +--- a/crates/microvms-cli/src/config.rs ++++ b/crates/microvms-cli/src/config.rs +@@ -83,8 +83,7 @@ pub struct ProjectConfig { + pub auto_resume: Option, + /// `run --max-idle-sec`: suspend after this much inbound-traffic idleness. + pub max_idle_sec: Option, +- /// `run --suspended-sec`: terminate a VM the idle policy suspended once it's been suspended +- /// this long. ++ /// `run --suspended-sec`: terminate after this long suspended. + pub suspended_sec: Option, + /// `run --max-duration-sec`: hard ceiling on the VM's life. + pub max_duration_sec: Option, +diff --git a/crates/microvms-cli/src/exit.rs b/crates/microvms-cli/src/exit.rs +index 84342b9..4bba5a5 100644 +--- a/crates/microvms-cli/src/exit.rs ++++ b/crates/microvms-cli/src/exit.rs +@@ -254,8 +254,7 @@ pub const EXIT_TABLE: [ExitRow; 17] = [ + ExitRow { + exit: Exit::WindowClosed, + code: Some("ERR_WINDOW_CLOSED"), +- meaning: "the MicroVM ended before the resume (the suspended window, the maximum \ +- duration, or a terminate), and it can't be resumed; launch a new one", ++ meaning: "the launch-time suspended window passed, so there is nothing to resume", + finding: "`idlePolicy`", + }, + ExitRow { +@@ -443,7 +442,9 @@ pub fn classify(error: &Error) -> CliError { + (Exit::LaunchDied, _) => { + &["`microvm logs ` names the build log group the hook wrote to"] + } +- (Exit::WindowClosed, _) => &["the VM ended and no call brings it back: launch a new one"], ++ (Exit::WindowClosed, _) => { ++ &["a longer window is set at launch with --suspended-sec; no call extends this one"] ++ } + _ => &[], + }; + classified +diff --git a/crates/microvms-domain/src/error.rs b/crates/microvms-domain/src/error.rs +index bbbd0ed..15ed493 100644 +--- a/crates/microvms-domain/src/error.rs ++++ b/crates/microvms-domain/src/error.rs +@@ -130,9 +130,7 @@ pub enum ErrorKind { + BuildWedged, + /// The MicroVM reached a terminal state before RUNNING; read `stateReason`. + LaunchDied, +- /// The MicroVM ended before the resume (the suspended window, the maximum duration, or a +- /// terminate), and it can't be resumed; launch a new one. The service refused the resume, +- /// and `GetMicrovm` reports the VM TERMINATING or TERMINATED. ++ /// The launch-time suspended window passed, so there is nothing to resume. + WindowClosed, + /// A control-plane failure with no more specific class. + Platform, diff --git a/verify/guards/faults/f2p-every-walked-path-of-the-client-model-replays-against.patch b/verify/guards/faults/f2p-every-walked-path-of-the-client-model-replays-against.patch new file mode 100644 index 00000000..303b48bf --- /dev/null +++ b/verify/guards/faults/f2p-every-walked-path-of-the-client-model-replays-against.patch @@ -0,0 +1,641 @@ +diff --git a/bindings/microvms-js/src/control.rs b/bindings/microvms-js/src/control.rs +index ead8c40..66a7dd9 100644 +--- a/bindings/microvms-js/src/control.rs ++++ b/bindings/microvms-js/src/control.rs +@@ -279,8 +279,8 @@ pub struct DeleteImageOptions { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7). Use +-/// it when a process has only an identifier. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12). Use it when a process has only an identifier. + #[napi] + pub struct ControlPlane { + inner: Arc, +@@ -344,8 +344,7 @@ impl ControlPlane { + self.inner.suspend(µvm_id).await.map_err(js_async) + } + +- /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, rejects with `ERR_WINDOW_CLOSED` (STATE-12). ++ /// `ResumeMicrovm`. Resolves once accepted; `waitForState` for RUNNING. + #[napi] + pub async fn resume(&self, microvm_id: String) -> Result<(), AsyncError> { + self.inner.resume(µvm_id).await.map_err(js_async) +diff --git a/bindings/microvms-js/src/sandbox.rs b/bindings/microvms-js/src/sandbox.rs +index 517085e..55d1e84 100644 +--- a/bindings/microvms-js/src/sandbox.rs ++++ b/bindings/microvms-js/src/sandbox.rs +@@ -1144,21 +1144,11 @@ impl Sandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, and +- /// even a call the service accepted would hand back a different machine. And a resume +- /// from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspendedSec` passes, and a VM this sandbox suspended stayed resumable +- /// 1283 s into a 60 s window (measured 2026-10-12, us-east-1). `ERR_WINDOW_CLOSED` is the +- /// service's answer once a resume reaches it: the service refused the resume, and the VM +- /// is TERMINATING or TERMINATED, whatever ended it. The sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` rejects with `ERR_INVALID_ARG` before any call (STATE-7). That +- /// suspend reaches a caller only through `Sandbox.adopt`, which reads the state from the +- /// service, or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// even a call the service accepted would hand back a different machine. A resume from ++ /// anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended window ++ /// (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates a ++ /// suspended VM once that window passes, so there is nothing left to resume, and calling ++ /// would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/bindings/microvms-py/src/control.rs b/bindings/microvms-py/src/control.rs +index ce0752e..5fbe783 100644 +--- a/bindings/microvms-py/src/control.rs ++++ b/bindings/microvms-py/src/control.rs +@@ -444,9 +444,9 @@ impl PyImageBuild { + /// MicroVM lifecycle by ID (get, list, suspend, resume, terminate, and wait) and image + /// administration (list, delete, versions and their status, builds). + /// +-/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7): a +-/// suspend of a SUSPENDED VM is the service's to refuse. Use it when a process has only an +-/// identifier, such as a durable workflow step in a fresh process. ++/// Holds no lifecycle state, so it checks nothing a `Sandbox` would (STATE-5, STATE-7, ++/// STATE-12): a suspend of a SUSPENDED VM is the service's to refuse. Use it when a ++/// process has only an identifier, such as a durable workflow step in a fresh process. + #[pyclass(frozen, name = "ControlPlane", module = "microvms")] + pub struct PyControlPlane { + inner: Arc, +@@ -703,8 +703,7 @@ impl PyControlPlane { + Ok(runtime::spawn_shielded(self.suspend_op(microvm_id)).await?) + } + +- /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. A VM that ended +- /// before the resume, whatever ended it, raises `WindowClosedError` (STATE-12). ++ /// `ResumeMicrovm`. Returns once accepted; `wait_for_state` for RUNNING. + fn resume(&self, py: Python<'_>, microvm_id: String) -> PyCoreResult<()> { + Ok(runtime::block_on(py, self.resume_op(microvm_id))?) + } +diff --git a/bindings/microvms-py/src/errors.rs b/bindings/microvms-py/src/errors.rs +index be8962a..a664785 100644 +--- a/bindings/microvms-py/src/errors.rs ++++ b/bindings/microvms-py/src/errors.rs +@@ -87,10 +87,8 @@ create_exception!( + microvms, + WindowClosedError, + MicrovmError, +- "ERR_WINDOW_CLOSED — the MicroVM ended before the resume (the suspended window, the \ +- maximum duration, or a terminate), and it can't be resumed; launch a new one. The \ +- suspended window ends a VM the idle policy suspended; an explicit suspend stayed \ +- resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1)." ++ "ERR_WINDOW_CLOSED — the launch-time suspended window passed, so there is nothing to \ ++ resume. A longer window has to be set at launch on the next VM." + ); + create_exception!( + microvms, +diff --git a/bindings/microvms-py/src/sandbox.rs b/bindings/microvms-py/src/sandbox.rs +index 449afbd..f7d6371 100644 +--- a/bindings/microvms-py/src/sandbox.rs ++++ b/bindings/microvms-py/src/sandbox.rs +@@ -1904,22 +1904,11 @@ impl PySandbox { + /// # What the core refuses, before any wire call + /// + /// A resume after `terminate` (STATE-11) — a terminated VM never returns to RUNNING, +- /// and even a call the service accepted would hand back a different machine. And a +- /// resume from anything but SUSPENDED (STATE-7). +- /// +- /// # What the service decides (STATE-12) +- /// +- /// How long the VM has been suspended. The `idlePolicy` terminates a VM the idle policy +- /// suspended once `suspended_sec` passes, and a VM this sandbox suspended stayed +- /// resumable 1283 s into a 60 s window (measured 2026-10-12, us-east-1). +- /// `WindowClosedError` is the service's answer once a resume reaches it: the service +- /// refused the resume, and the VM is TERMINATING or TERMINATED, whatever ended it. The +- /// sandbox is TERMINATED after it. +- /// +- /// A sandbox doesn't see a suspend the idle policy makes. It still holds the VM as +- /// RUNNING, so `resume` raises `InvalidArgError` before any call (STATE-7). That suspend +- /// reaches a caller only through `Sandbox.adopt`, which reads the state from the service, +- /// or `ControlPlane.resume`, which holds no lifecycle to refuse from. ++ /// and even a call the service accepted would hand back a different machine. A resume ++ /// from anything but SUSPENDED (STATE-7). And a resume past the launch-time suspended ++ /// window (STATE-12), which is the one worth knowing about: the `idlePolicy` terminates ++ /// a suspended VM once that window passes, so there is nothing left to resume, and ++ /// calling would cost the full poll timeout to learn something worse. + /// + /// Nothing is re-delivered: no run-hook payload, no token, no bootstrap. The in-memory + /// token survived the freeze, and re-delivering it would hit the daemon's one-shot +diff --git a/crates/microvms-app/src/clock.rs b/crates/microvms-app/src/clock.rs +index 75afb7c..5685303 100644 +--- a/crates/microvms-app/src/clock.rs ++++ b/crates/microvms-app/src/clock.rs +@@ -7,11 +7,11 @@ + //! sources: the control plane's read `std::time::Instant` and the session's read tokio's. + //! Under a deterministic simulator those disagree. tokio's clock is virtual there and `std`'s + //! isn't, so a control-plane wait whose sleeps advanced virtual time measured its deadline +-//! on the host's real time and never reached it. `Sandbox` already waits on the plane's clock +-//! rather than one of its own, so a deadline isn't measured on one clock while the polls sleep +-//! on another. One trait, with tokio's clock in production, keeps that property for everything +-//! that reads time. The trait is re-exported at `control::Clock` and at `session::Clock`, the +-//! paths each half had. ++//! on the host's real time and never reached it. `Sandbox` already times the suspended window ++//! on the plane's clock rather than its own, so a window isn't measured on one clock while ++//! the polls run on another. One trait, with tokio's clock in production, keeps that property ++//! for everything that reads time. The trait is re-exported at `control::Clock` and at ++//! `session::Clock`, the paths each half had. + //! + //! # Why a trait rather than `tokio::time::pause` + //! +@@ -24,11 +24,11 @@ + //! # Monotonic and wall time are separate readings + //! + //! [`Clock::elapsed`] is monotonic, the same reasoning the Python client's injectable +-//! `time.monotonic` carried: a wait's deadline and a proxy token's age are *durations*, and a +-//! wall clock that steps backward would hold a passed deadline open. [`Clock::unix_now`] is the +-//! wall reading, for the values that are dates rather than durations: a name record's +-//! timestamp, an exec id, the cost engine's today. Deriving one from the other would make a +-//! stepped wall clock move a deadline. ++//! `time.monotonic` carried: the suspended window is a *duration*, and a wall clock that steps ++//! backward would reopen a closed one. [`Clock::unix_now`] is the wall reading, for the values ++//! that are dates rather than durations: a name record's timestamp, an exec id, the cost ++//! engine's today. Deriving one from the other would make a stepped wall clock move a ++//! deadline. + + use std::fmt; + use std::future::Future; +diff --git a/crates/microvms-app/src/control/microvm.rs b/crates/microvms-app/src/control/microvm.rs +index 9611956..36597d0 100644 +--- a/crates/microvms-app/src/control/microvm.rs ++++ b/crates/microvms-app/src/control/microvm.rs +@@ -799,56 +799,11 @@ impl ControlPlane { + } + + /// `ResumeMicrovm`. Thaws the VM; no token re-delivery and no re-bootstrap. +- /// +- /// # A VM that ended before the resume is `WindowClosed` (STATE-12) +- /// +- /// Nothing is checked against the suspended window before the call, because the service +- /// decides whether a suspended VM can still resume. It terminates a VM the idle policy +- /// suspended once `suspendedDurationSeconds` passes, and it doesn't hold an explicit +- /// `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, an explicitly +- /// suspended VM resumed 1283 s into a 60 s window (#597). A resume of a VM that has +- /// terminated is refused with a `ValidationException` ("The MicroVM has been +- /// terminated and its state cannot be changed."), the same status as any refused value, +- /// and it was the same refusal for a VM the service ended and one `TerminateMicrovm` +- /// ended (measured 2026-10-12, us-east-1). So a refusal reads the VM's state rather than +- /// the message, and a VM that's TERMINATING or TERMINATED is [`ErrorKind::WindowClosed`] +- /// with its state and `stateReason`, carrying the refusal as its source. The error names +- /// no cause, because the client doesn't know which of the suspended window, the maximum +- /// duration, or a terminate ended the VM; the `stateReason` is the service's word on it. +- /// Every other refusal comes back as the service gave it. + pub async fn resume(&self, id: &str) -> Result<(), Error> { + super::require_valid_identifier("microvmIdentifier", id)?; + let call = Call::post_empty("ResumeMicrovm", paths::resume(id)); +- match send_with_retry(self.transport(), call).await { +- Ok(_) => Ok(()), +- Err(refused) if refused.kind() == ErrorKind::Platform => { +- Err(self.ended_before_resume(id, refused).await) +- } +- Err(error) => Err(error), +- } +- } +- +- /// The error a refused resume reports: [`ErrorKind::WindowClosed`] with `refused` as its +- /// source when `GetMicrovm` reports the VM dead, and `refused` otherwise, including when +- /// that read fails. +- async fn ended_before_resume(&self, id: &str, refused: Error) -> Error { +- let Ok(vm) = self.get_microvm(id).await else { +- return refused; +- }; +- if !crate::constants::DEAD_STATES.contains(&vm.state.as_str()) { +- return refused; +- } +- Error::new( +- ErrorKind::WindowClosed, +- format!( +- "microvm {id} can't be resumed: the service refused ResumeMicrovm, and \ +- GetMicrovm reports the VM {} ({}). A VM that has ended doesn't come back; \ +- launch a new one.", +- vm.state, +- vm.reason(), +- ), +- ) +- .with_source(refused) ++ send_with_retry(self.transport(), call).await?; ++ Ok(()) + } + + /// `TerminateMicrovm`. +diff --git a/crates/microvms-app/src/control/ops.rs b/crates/microvms-app/src/control/ops.rs +index 36833c6..ae1c3c1 100644 +--- a/crates/microvms-app/src/control/ops.rs ++++ b/crates/microvms-app/src/control/ops.rs +@@ -367,9 +367,11 @@ pub struct RunMicrovmWire { + /// "autoResumeEnabled": false}` (measured 2026-08-15, us-east-1, one read-only + /// `GetMicrovm`). + /// +-/// The readback means the client's own record isn't the only authority on the suspended +-/// window: [`crate::control::Microvm::idle_policy`] carries the service's, so a window that +-/// disagrees with what was asked for is observable rather than assumed. ++/// The claim mattered because STATE-12 was resting on it: if only the client could name the ++/// suspended window, the client's own record is the only authority on it. The readback ++/// means there is a second one, and [`crate::control::Microvm::idle_policy`] carries it — ++/// so a suspended window that disagrees with what was asked for is now observable rather ++/// than assumed. + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct IdlePolicy { +diff --git a/crates/microvms-app/src/sandbox.rs b/crates/microvms-app/src/sandbox.rs +index 544535f..dc316a4 100644 +--- a/crates/microvms-app/src/sandbox.rs ++++ b/crates/microvms-app/src/sandbox.rs +@@ -3,8 +3,8 @@ + //! model describes made true of the code. + //! + //! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the +-//! bindings use: build, run, suspend, resume, terminate, a resume of a VM that ended +-//! before it (STATE-12), and teardown in the order that does not leak. ++//! bindings use: build, run, suspend, resume, terminate, the launch-time suspended ++//! window (STATE-12), and teardown in the order that does not leak. + //! + //! # The lifecycle is a field, and every transition is guarded + //! +@@ -32,20 +32,16 @@ + //! control-plane calls rather than answered by AWS. The test asserts the call count, which + //! is the observable that distinguishes the two. + //! +-//! # The service decides whether a suspended VM resumes (STATE-12) ++//! # The suspended window is checked before the call (STATE-12) + //! +-//! The launch-time `idlePolicy` *terminates* a VM the idle policy suspended once +-//! `suspendedDurationSeconds` passes, and the service doesn't hold an explicit +-//! `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, a VM this client +-//! suspended resumed 1283 s into a 60 s window (#597). So a resume is never refused here for +-//! the time it has spent suspended. It goes to the service, and the service's refusal for a +-//! VM that ended, whatever ended it, comes back from [`ControlPlane::resume`] as +-//! [`ErrorKind::WindowClosed`], which this sandbox records as a termination (STATE-11). +-//! +-//! A suspend the idle policy makes isn't seen here: the sandbox still holds the VM as +-//! RUNNING, and its resume is refused before any call (STATE-7). That suspend reaches a +-//! caller only through [`Sandbox::adopt`], which reads the state from the service, or +-//! [`ControlPlane::resume`], which holds no lifecycle to refuse from. ++//! The launch-time `idlePolicy` *terminates* a suspended VM once `suspendedDurationSeconds` ++//! passes, which means "resume later" silently stops working. A resume past the window is ++//! refused locally, before `ResumeMicrovm`, because the alternative is calling and reading ++//! the failure: the service answers about a terminated id, which is not the same statement ++//! as "the window you set at launch closed", and getting there costs the full poll timeout ++//! first. The window comes from this sandbox's own `RunMicrovm` request, falling back to the ++//! `idlePolicy` that `GetMicrovm` reports (measured 2026-08-15, `docs/PLATFORM.md`). Only ++//! the suspend's start time is the client's alone. + //! + //! # Teardown never raises, and the log group is last + //! +@@ -270,8 +266,7 @@ pub struct RunRequest { + pub shell: bool, + /// `idlePolicy.maxIdleDurationSeconds`. + pub max_idle_sec: u32, +- /// `idlePolicy.suspendedDurationSeconds`: how long a VM the idle policy suspended lasts +- /// before the service terminates it. An explicit suspend isn't held to it (STATE-12). ++ /// `idlePolicy.suspendedDurationSeconds` — the window STATE-12 refuses past. + pub suspended_sec: u32, + /// `idlePolicy.autoResumeEnabled`. + pub auto_resume: bool, +@@ -409,8 +404,7 @@ impl RunRequest { + self + } + +- /// Sets `idlePolicy.suspendedDurationSeconds`, how long a VM the idle policy suspended +- /// lasts before the service terminates it. See [`RunRequest::suspended_sec`]. ++ /// Sets the suspended window this sandbox will refuse a resume past (STATE-12). + #[must_use] + pub fn with_suspended_sec(mut self, seconds: u32) -> Self { + self.suspended_sec = seconds; +@@ -624,8 +618,7 @@ impl TokenMinter for ControlPlaneMinter { + + /// One MicroVM's whole life. + /// +-/// See the module docs for the state machine, the resume of an ended VM, and why teardown is +-/// explicit. ++/// See the module docs for the state machine, the window, and why teardown is explicit. + pub struct Sandbox { + control: Arc, + image: Option, +@@ -651,8 +644,8 @@ pub struct Sandbox { + was_terminated: bool, + bootstrap_count: u32, + +- /// The suspended window from *our own* `RunMicrovm` request, or `None` for a sandbox that +- /// launched nothing. `GetMicrovm` reports the service's in `idlePolicy`. ++ /// The window from *our own* `RunMicrovm` request. `GetMicrovm` also reports it (in ++ /// `idlePolicy`), which is the fallback when this sandbox has no request of its own. + suspended_window: Option, + /// Whether the accepted launch carried a caller client token, and so may have adopted + /// a VM an earlier attempt launched (#195). +@@ -666,6 +659,8 @@ pub struct Sandbox { + /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence + /// is checked against it. + idle_window: Option, ++ /// The clock reading when the suspend call was accepted, or `None` when not suspended. ++ suspended_at: Option, + /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down + /// one. + torn_down: bool, +@@ -754,10 +749,10 @@ impl Sandbox { + /// `Sandbox::from_name`, which resolve credentials for a region, are in + /// `microvms_core::prelude`. + /// +- /// The plane's own clock is what every wait here sleeps on, rather than a second one ++ /// The plane's own clock is what times the suspended window, rather than a second one + /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil +- /// tests record: a deadline measured on one clock while the poll loop sleeps on another +- /// reads the wrong elapsed time. ++ /// tests record: a window measured on one clock while the poll loop sleeps on another ++ /// calls a closed window open. + pub fn with_control_plane(control: ControlPlane) -> Self { + Self { + control: Arc::new(control), +@@ -777,6 +772,7 @@ impl Sandbox { + launch_posture: None, + pending_agent_token: None, + idle_window: None, ++ suspended_at: None, + torn_down: false, + adopted: false, + detached: false, +@@ -1355,6 +1351,12 @@ impl Sandbox { + /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once + /// when the service reports RUNNING, the same as a launch this sandbox made itself. + /// ++ /// # The suspended window (STATE-12) ++ /// ++ /// The window comes from the `idlePolicy` `GetMicrovm` reports. A VM adopted while ++ /// already SUSPENDED has no suspend time this client observed, so the local check has ++ /// nothing to measure and the service answers the resume instead. ++ /// + /// `endpoint` must agree with the service's when the service reports one: a mismatch + /// means the identifiers came from two different records. `agent_token` is the bearer + /// credential the VM was launched with; it never appears in `Debug` or an error. +@@ -1603,6 +1605,10 @@ impl Sandbox { + // neither suspend nor resume accepts, bricking the handle over one bad request. + self.control.suspend(&id).await?; + self.set_lifecycle(Lifecycle::Suspending); ++ // Stamped after the call and before the wait, not after the wait: the idlePolicy's ++ // window starts when the platform begins suspending, so timing it from SUSPENDED ++ // would under-count the transition and call a closed window open. ++ self.suspended_at = Some(self.control.clock().elapsed()); + + let settled = self + .control +@@ -1649,16 +1655,12 @@ impl Sandbox { + /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be + /// refused — a 409 that reads like a broken VM. + /// +- /// # The service decides, and a VM it ended is `WindowClosed` (STATE-12) ++ /// # The window is checked first (STATE-12) + /// +- /// No resume is refused here for the time the VM has spent suspended. The service ends a +- /// VM the idle policy suspended once `suspendedDurationSeconds` passes, and it kept a VM +- /// this client suspended resumable 1283 s into a 60 s window (measured 2026-10-12, +- /// us-east-1, #597). A resume the service refuses because the VM ended, whatever ended +- /// it, is [`ErrorKind::WindowClosed`] ([`ControlPlane::resume`]), and this records the VM +- /// as terminated, so a second resume is refused here with no call (STATE-11). A VM the +- /// idle policy suspended is still RUNNING here, so its resume is refused before any call +- /// (STATE-7); see the module docs. ++ /// Before any wire call, because the answer is already known and calling costs the poll ++ /// timeout to learn something worse. The falsification is a service reporting ++ /// TERMINATED: without this check the resume burns the full deadline and then reports a ++ /// state the client could have named at once. + /// + /// # The proxy token is dropped (STATE-8) + /// +@@ -1671,7 +1673,7 @@ impl Sandbox { + let id = self.require_microvm("resume")?; + + // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal +- // comes before any call. ++ // comes before the window check and before any call. + if self.was_terminated || self.lifecycle == Lifecycle::Terminated { + return Err(Error::invalid_arg(format!( + "microvm {id} was terminated, and a terminated VM never returns to RUNNING \ +@@ -1686,23 +1688,15 @@ impl Sandbox { + ))); + } + +- // STATE-12: a `WindowClosed` refusal is a VM that ended before this resume. It's +- // recorded as TERMINATED whichever of TERMINATING or TERMINATED `GetMicrovm` read, +- // because the service has said the VM's state can't change, and as a termination, so +- // the next resume is refused before any call (STATE-11). Any other refusal leaves the +- // sandbox SUSPENDED, and a retry goes to the service again. +- if let Err(refused) = self.control.resume(&id).await { +- if refused.kind() == ErrorKind::WindowClosed { +- self.was_terminated = true; +- self.set_lifecycle(Lifecycle::Terminated); +- } +- return Err(refused); +- } ++ // STATE-12, first and locally. ++ self.require_open_suspended_window(&id)?; ++ ++ self.control.resume(&id).await?; + // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the +- // state this call was made from, so failing on it would fail every resume. A VM that +- // terminates once the resume is accepted never reaches RUNNING, and waiting only for +- // RUNNING there burns the full timeout and then reports a timeout message hiding a +- // cause the service had already stated in `stateReason`. ++ // state this call was made from, so failing on it would fail every resume. A VM the ++ // idlePolicy terminated during suspension never reaches RUNNING, and waiting only ++ // for RUNNING there burns the full timeout and then reports a timeout message ++ // hiding a cause the service had already stated in `stateReason`. + let running = self + .control + .wait_for_state( +@@ -1722,6 +1716,10 @@ impl Sandbox { + if let Some(session) = self.session.as_mut() { + session.rebind(endpoint); + } ++ // Cleared on success so the next cycle's window is measured from the next suspend. ++ // Leaving it set would accumulate every suspension's elapsed time into one total and ++ // reject a resume whose own window is wide open. ++ self.suspended_at = None; + + self.session.as_mut().ok_or_else(|| { + Error::new( +@@ -1732,6 +1730,41 @@ impl Sandbox { + }) + } + ++ /// Rejects a resume the launch-time `idlePolicy` has already made impossible (STATE-12). ++ /// ++ /// The message names the elapsed time, the window, and the `idlePolicy` finding, because ++ /// "cannot resume" alone sends a reader looking for the flag that reopens it. ++ fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> { ++ let reported = self ++ .microvm ++ .as_ref() ++ .and_then(|vm| vm.idle_policy.as_ref()) ++ .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds))); ++ let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at) ++ else { ++ // No window or no suspend time means this sandbox cannot know how long the VM ++ // has been suspended, and guessing would refuse a resume the service would honour. ++ return Ok(()); ++ }; ++ let elapsed = self.control.clock().elapsed().saturating_sub(since); ++ if elapsed <= window { ++ return Ok(()); ++ } ++ Err(Error::new( ++ ErrorKind::WindowClosed, ++ format!( ++ "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \ ++ window set at launch — the idlePolicy terminates a suspended VM once that window \ ++ passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \ ++ Refused before ResumeMicrovm, because calling would cost the full poll timeout \ ++ to learn the same thing less clearly. A longer window has to be set at launch \ ++ on the next VM; there is no call that extends this one.", ++ elapsed.as_secs(), ++ window.as_secs(), ++ ), ++ )) ++ } ++ + // ── terminate (STATE-9, STATE-10) ──────────────────────────────────────── + + /// Tears down, best-effort, never erroring. +@@ -1874,14 +1907,11 @@ impl Sandbox { + let vm = self.microvm.as_ref()?; + self.lifecycle.is_live().then(|| { + format!( +- "warning: the Sandbox for microvm {id} was dropped in {state} without \ +- terminate() or detach(). Nothing was torn down — Drop cannot await, so a \ +- teardown here would deadlock inside a runtime. {state} is the state this \ +- sandbox last saw, and the platform may have suspended or ended the VM since. A \ +- VM that's still live bills until it ends, at its maximumDurationInSeconds \ +- ceiling at the latest: terminate it with `microvm terminate {id}`.", +- id = vm.id, +- state = self.lifecycle, ++ "warning: the Sandbox for microvm {} was dropped in {} without terminate() or \ ++ detach(). Nothing was torn down — Drop cannot await, so a teardown here would \ ++ deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \ ++ ceiling: terminate it with `microvm terminate {}`.", ++ vm.id, self.lifecycle, vm.id, + ) + }) + } +diff --git a/crates/microvms-cli/src/cli.rs b/crates/microvms-cli/src/cli.rs +index 25b94f0..f3a820e 100644 +--- a/crates/microvms-cli/src/cli.rs ++++ b/crates/microvms-cli/src/cli.rs +@@ -301,10 +301,8 @@ pub enum Command { + + /// Thaw a suspended MicroVM and report its endpoint. + /// +- /// The launch-time `suspendedDurationSeconds` window terminates a VM the idle policy +- /// suspended once it passes. VMs suspended with `microvm suspend` resumed 900 s and more +- /// into a 60 s window (2026-10-12), which AWS doesn't promise, so the service decides. A +- /// resume of a VM that ended, whatever ended it, exits `ERR_WINDOW_CLOSED`; launch a new one. ++ /// The launch-time `suspendedDurationSeconds` window terminates a suspended VM once it ++ /// passes, so "resume later" silently stops working and the VM is gone rather than slow. + Resume(ResumeArgs), + + /// Tear down a MicroVM, and optionally its image and build log group. +@@ -1056,8 +1054,7 @@ pub struct RunArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +@@ -2506,8 +2503,7 @@ pub struct AgentUpArgs { + #[arg(long, default_value_t = 600)] + pub max_idle_sec: u32, + +- /// Terminate a VM the idle policy suspended once it's been suspended this long. An explicit +- /// suspend outlasted it when measured (2026-10-12), which AWS doesn't promise. ++ /// Terminate the VM after this long suspended. A resume past it cannot work. + #[arg(long, default_value_t = 600)] + pub suspended_sec: u32, + +diff --git a/crates/microvms-cli/src/commands/lifecycle.rs b/crates/microvms-cli/src/commands/lifecycle.rs +index c4c0b8b..ef6a544 100644 +--- a/crates/microvms-cli/src/commands/lifecycle.rs ++++ b/crates/microvms-cli/src/commands/lifecycle.rs +@@ -2084,12 +2084,10 @@ pub async fn suspend( + /// Thaws a suspended MicroVM and reports its endpoint. + /// + /// The suspended window is **not** checked here, and that is correct rather than a gap: the +-/// service ends a VM the idle policy suspended once `suspendedDurationSeconds` passes and keeps +-/// an explicitly suspended one resumable past it (measured 2026-10-12, us-east-1), so only the +-/// service can answer. Core's `ControlPlane::resume` reports its refusal of a VM that ended +-/// before the resume as `ERR_WINDOW_CLOSED`. A VM that dies after the resume is +-/// accepted fails the wait at once through `fail_on: DEAD_STATES`, with `stateReason`, instead +-/// of burning the poll timeout. ++/// window is in `GetMicrovm`'s `idlePolicy`, but when the suspend began is known only to the ++/// process that issued it. Guessing would either reject an open window or accept a closed ++/// one, both worse than letting the service answer — which is what `fail_on: DEAD_STATES` ++/// does, failing fast with `stateReason` instead of burning the poll timeout. + pub async fn resume( + ctx: &mut Ctx<'_, O, E>, + args: &ResumeArgs, +diff --git a/crates/microvms-cli/src/config.rs b/crates/microvms-cli/src/config.rs +index 9693850..6020ef2 100644 +--- a/crates/microvms-cli/src/config.rs ++++ b/crates/microvms-cli/src/config.rs +@@ -83,8 +83,7 @@ pub struct ProjectConfig { + pub auto_resume: Option, + /// `run --max-idle-sec`: suspend after this much inbound-traffic idleness. + pub max_idle_sec: Option, +- /// `run --suspended-sec`: terminate a VM the idle policy suspended once it's been suspended +- /// this long. ++ /// `run --suspended-sec`: terminate after this long suspended. + pub suspended_sec: Option, + /// `run --max-duration-sec`: hard ceiling on the VM's life. + pub max_duration_sec: Option, +diff --git a/crates/microvms-cli/src/exit.rs b/crates/microvms-cli/src/exit.rs +index 84342b9..4bba5a5 100644 +--- a/crates/microvms-cli/src/exit.rs ++++ b/crates/microvms-cli/src/exit.rs +@@ -254,8 +254,7 @@ pub const EXIT_TABLE: [ExitRow; 17] = [ + ExitRow { + exit: Exit::WindowClosed, + code: Some("ERR_WINDOW_CLOSED"), +- meaning: "the MicroVM ended before the resume (the suspended window, the maximum \ +- duration, or a terminate), and it can't be resumed; launch a new one", ++ meaning: "the launch-time suspended window passed, so there is nothing to resume", + finding: "`idlePolicy`", + }, + ExitRow { +@@ -443,7 +442,9 @@ pub fn classify(error: &Error) -> CliError { + (Exit::LaunchDied, _) => { + &["`microvm logs ` names the build log group the hook wrote to"] + } +- (Exit::WindowClosed, _) => &["the VM ended and no call brings it back: launch a new one"], ++ (Exit::WindowClosed, _) => { ++ &["a longer window is set at launch with --suspended-sec; no call extends this one"] ++ } + _ => &[], + }; + classified +diff --git a/crates/microvms-domain/src/error.rs b/crates/microvms-domain/src/error.rs +index bbbd0ed..15ed493 100644 +--- a/crates/microvms-domain/src/error.rs ++++ b/crates/microvms-domain/src/error.rs +@@ -130,9 +130,7 @@ pub enum ErrorKind { + BuildWedged, + /// The MicroVM reached a terminal state before RUNNING; read `stateReason`. + LaunchDied, +- /// The MicroVM ended before the resume (the suspended window, the maximum duration, or a +- /// terminate), and it can't be resumed; launch a new one. The service refused the resume, +- /// and `GetMicrovm` reports the VM TERMINATING or TERMINATED. ++ /// The launch-time suspended window passed, so there is nothing to resume. + WindowClosed, + /// A control-plane failure with no more specific class. + Platform, diff --git a/verify/guards/faults/model-conformance-client.toml b/verify/guards/faults/model-conformance-client.toml index 89c3bd1e..39c67c17 100644 --- a/verify/guards/faults/model-conformance-client.toml +++ b/verify/guards/faults/model-conformance-client.toml @@ -14,7 +14,7 @@ suite = "rust" id = "model-conformance-client-resume-resends-payload" note = "crates/model-conformance/tests/client_lifecycle.rs::every_walked_path_of_the_client_model_replays_against_the_sandbox" message = "run-hook payloads: sandbox 2, model 1" -transform = { file = "crates/microvms-app/src/sandbox.rs", replace = " self.control.resume(&id).await?;\n", with = " self.control.resume(&id).await?;\n let token = self.pending_agent_token.clone().unwrap_or_default();\n let payload = RunHookPayload::for_launch(&token, &Default::default())?;\n self.control.run_microvm(RunMicrovmRequest::new(&id, payload)).await?;\n" } +transform = { file = "crates/microvms-app/src/sandbox.rs", replace = " return Err(refused);\n }\n", with = " return Err(refused);\n }\n let token = self.pending_agent_token.clone().unwrap_or_default();\n let payload = RunHookPayload::for_launch(&token, &Default::default())?;\n self.control.run_microvm(RunMicrovmRequest::new(&id, payload)).await?;\n" } # A suspend that records SUSPENDING only once the wait settles. A wait that times out leaves the # sandbox RUNNING while the platform suspends, and the next suspend is issued outside RUNNING. @@ -97,11 +97,12 @@ message = "a path the recorder kept doesn't plan" transform = { file = "crates/model-conformance/tests/client_lifecycle.rs", replace = " Ok(steps)\n}\n", with = " if steps.len() > 6 {\n return Err(\"a completion arrives after another step\");\n }\n Ok(steps)\n}\n" } # The untaken-step paths the planner skips are pinned: a planner that skips another set, here -# one that stops skipping a closed window that opens again, fails before it replays anything. +# one that stops skipping a completion that arrives after another step, fails before it replays +# anything. [[family.fault]] id = "model-conformance-client-skips-moved" message = "the planner skips a different set of untaken-step paths than SKIPPED names" -transform = { file = "crates/model-conformance/tests/client_lifecycle.rs", replace = "(Action::ResumeRequested { .. }, Verdict::Issued) if window_closed => {", with = "(Action::ResumeRequested { .. }, Verdict::Issued) if false && window_closed => {" } +transform = { file = "crates/model-conformance/tests/client_lifecycle.rs", replace = " ) => return Err(\"a completion arrives after another step\"),\n", with = " ) => (Call::Nothing, 1),\n" } # A key that moves onto a property stating another rule: STATE-8 still starts some property's # name, so the claimed-key check alone passed, and `cargo test -p agentd-model` stays green. diff --git a/verify/guards/faults/resume-closed-window.toml b/verify/guards/faults/resume-closed-window.toml new file mode 100644 index 00000000..7ca49042 --- /dev/null +++ b/verify/guards/faults/resume-closed-window.toml @@ -0,0 +1,13 @@ +# ── a resume of a VM that ended: `ERR_WINDOW_CLOSED` from the service's refusal (#597) ───── + +# A closed window that drops the service's refusal (#601's review): the error carried the state +# and `stateReason` and nothing the service said, so a Rust caller couldn't reach the refusal's +# own text through `std::error::Error::source`. +[[fault]] +id = "resume-window-closed-drops-the-refusal" +guard = "control::microvm::tests::a_resume_refused_for_a_vm_a_terminate_ended_names_no_cause_and_keeps_the_refusal" +run = ["cargo", "test", "-p", "microvms-app", "--lib", "--", "--exact", "control::microvm::tests::a_resume_refused_for_a_vm_a_terminate_ended_names_no_cause_and_keeps_the_refusal"] +expect = "test-failed" +suite = "rust" +message = "the refusal is the source" +transform = { file = "crates/microvms-app/src/control/microvm.rs", replace = " .with_source(refused)\n", with = "" } diff --git a/verify/guards/faults/resume-window.toml b/verify/guards/faults/resume-window.toml new file mode 100644 index 00000000..6742c38c --- /dev/null +++ b/verify/guards/faults/resume-window.toml @@ -0,0 +1,55 @@ +# ── resume-window: FAIL_TO_PASS entries tools/fail-to-pass.py emitted ── + +# FAIL_TO_PASS: fails on f410ec2f4c80 with the fix taken out (tools/fail-to-pass.py). +[[fault]] +id = "f2p-a-resume-of-a-vm-the-service-ended-exits-window-closed" +guard = "guards::history::a_resume_of_a_vm_the_service_ended_exits_window_closed_and_records_no_resume" +run = ["cargo", "test", "-p", "microvms-cli", "--bin", "microvm", "--", "--exact", "guards::history::a_resume_of_a_vm_the_service_ended_exits_window_closed_and_records_no_resume"] +expect = "test-failed" +suite = "rust" +patch = "verify/guards/faults/f2p-a-resume-of-a-vm-the-service-ended-exits-window-closed.patch" + +# FAIL_TO_PASS: fails on f410ec2f4c80 with the fix taken out (tools/fail-to-pass.py). +[[fault]] +id = "f2p-every-walked-path-of-the-client-model-replays-against" +guard = "every_walked_path_of_the_client_model_replays_against_the_sandbox" +run = ["cargo", "test", "-p", "model-conformance", "--test", "client_lifecycle", "--", "--exact", "every_walked_path_of_the_client_model_replays_against_the_sandbox"] +expect = "test-failed" +suite = "rust" +patch = "verify/guards/faults/f2p-every-walked-path-of-the-client-model-replays-against.patch" + +# FAIL_TO_PASS: fails on f410ec2f4c80 with the fix taken out (tools/fail-to-pass.py). +[[fault]] +id = "f2p-a-resume-refused-for-a-vm-that-terminated-while" +guard = "control::microvm::tests::a_resume_refused_for_a_vm_that_terminated_while_suspended_is_window_closed" +run = ["cargo", "test", "-p", "microvms-app", "--lib", "--", "--exact", "control::microvm::tests::a_resume_refused_for_a_vm_that_terminated_while_suspended_is_window_closed"] +expect = "test-failed" +suite = "rust" +patch = "verify/guards/faults/f2p-a-resume-refused-for-a-vm-that-terminated-while.patch" + +# FAIL_TO_PASS: fails on f410ec2f4c80 with the fix taken out (tools/fail-to-pass.py). +[[fault]] +id = "f2p-an-adopted-vm-resumes-past-the-reported-window-through" +guard = "sandbox::tests::an_adopted_vm_resumes_past_the_reported_window_through_the_service" +run = ["cargo", "test", "-p", "microvms-app", "--lib", "--", "--exact", "sandbox::tests::an_adopted_vm_resumes_past_the_reported_window_through_the_service"] +expect = "test-failed" +suite = "rust" +patch = "verify/guards/faults/f2p-an-adopted-vm-resumes-past-the-reported-window-through.patch" + +# FAIL_TO_PASS: fails on f410ec2f4c80 with the fix taken out (tools/fail-to-pass.py). +[[fault]] +id = "f2p-a-resume-past-the-window-of-an-explicit-suspend-reaches" +guard = "sandbox::tests::a_resume_past_the_window_of_an_explicit_suspend_reaches_the_service_and_resumes" +run = ["cargo", "test", "-p", "microvms-app", "--lib", "--", "--exact", "sandbox::tests::a_resume_past_the_window_of_an_explicit_suspend_reaches_the_service_and_resumes"] +expect = "test-failed" +suite = "rust" +patch = "verify/guards/faults/f2p-a-resume-past-the-window-of-an-explicit-suspend-reaches.patch" + +# FAIL_TO_PASS: fails on f410ec2f4c80 with the fix taken out (tools/fail-to-pass.py). +[[fault]] +id = "f2p-a-resume-the-service-refuses-for-an-ended-vm-is-window" +guard = "sandbox::tests::a_resume_the_service_refuses_for_an_ended_vm_is_window_closed_and_recorded" +run = ["cargo", "test", "-p", "microvms-app", "--lib", "--", "--exact", "sandbox::tests::a_resume_the_service_refuses_for_an_ended_vm_is_window_closed_and_recorded"] +expect = "test-failed" +suite = "rust" +patch = "verify/guards/faults/f2p-a-resume-the-service-refuses-for-an-ended-vm-is-window.patch" diff --git a/verify/guards/unregistered.txt b/verify/guards/unregistered.txt index bb455eed..95a811ee 100644 --- a/verify/guards/unregistered.txt +++ b/verify/guards/unregistered.txt @@ -79,13 +79,11 @@ crates/microvms-app/src/sandbox.rs::a_suspend_from_a_non_running_state_reaches_n crates/microvms-app/src/sandbox.rs::a_suspend_whose_call_fails_stays_running_and_can_be_retried crates/microvms-app/src/sandbox.rs::adopt_refuses_bad_records_and_never_prints_the_token crates/microvms-app/src/sandbox.rs::an_adopted_sandbox_reports_the_services_idle_window_and_watches_its_lifecycle -crates/microvms-app/src/sandbox.rs::an_adopted_vm_refuses_a_resume_past_the_reported_window crates/microvms-app/src/sandbox.rs::an_adopted_vm_takes_its_lifecycle_from_the_service_and_refuses_run crates/microvms-app/src/sandbox.rs::an_over_budget_launch_env_is_refused_before_any_control_plane_call crates/microvms-app/src/sandbox.rs::asking_for_egress_and_the_deny_together_is_refused_before_any_call crates/microvms-app/src/sandbox.rs::ensure_image_builds_its_services_through_the_planes_adapters crates/microvms-app/src/sandbox.rs::the_advisory_deny_reaches_the_launch_environment_in_both_spellings -crates/microvms-app/src/sandbox.rs::the_lifecycle_watch_follows_every_transition crates/microvms-app/src/sandbox.rs::the_request_after_a_resume_mints_a_fresh_proxy_token crates/microvms-app/src/session/complete.rs::a_client_deadline_kills_then_acks_within_the_grace crates/microvms-app/src/session/complete.rs::a_cut_stream_falls_back_to_wait_and_ack diff --git a/verify/spec/core.symspec.json b/verify/spec/core.symspec.json index c5575f0a..204a3e44 100644 --- a/verify/spec/core.symspec.json +++ b/verify/spec/core.symspec.json @@ -841,14 +841,14 @@ "a17fdfa5-73a1-490b-b548-2afc37025055" ], "satisfies": [], - "sentence": "If a resume is attempted after the launch-time suspended window has elapsed, then the microvms-core shall reject the resume with an error naming the elapsed suspended window.", + "sentence": "If a resume is requested for a suspended MicroVM that the service terminated while it was suspended, then the microvms-core shall issue the resume to the service and report its refusal as a window-closed error naming the state and stateReason that GetMicrovm reports.", "status": "approved", "systemName": "microvms-core", - "systemResponse": "reject the resume with an error naming the elapsed suspended window", - "trigger": "a resume is attempted after the launch-time suspended window has elapsed", - "updatedAt": "2026-08-08T02:13:09.454Z", + "systemResponse": "issue the resume to the service and report its refusal as a window-closed error naming the state and stateReason that GetMicrovm reports", + "trigger": "a resume is requested for a suspended MicroVM that the service terminated while it was suspended", + "updatedAt": "2026-10-12T00:00:00.000Z", "verificationMethod": "test", - "verificationNote": "PLATFORM.md: idlePolicy terminates a suspended VM after suspendedDurationSeconds", + "verificationNote": "#597, measured 2026-10-12 in us-east-1: the service terminates a VM the idle policy suspended once suspendedDurationSeconds passes and refuses its resume with a ValidationException, and an explicit SuspendMicrovm resumed 1283 s into a 60 s window; the client issues every resume from SUSPENDED and leaves the window to the service", "verifies": [] }, "763ab89b-4db5-4daf-9a97-b9e40bcebda6": { diff --git a/verify/spec/traced/STATE.toml b/verify/spec/traced/STATE.toml index 32008fdc..6ccb6754 100644 --- a/verify/spec/traced/STATE.toml +++ b/verify/spec/traced/STATE.toml @@ -67,4 +67,4 @@ waive.live = "a refusal before any call never reaches the service; the model and issue = "#302" waive.gherkin = "the model walks every interleaving of caller requests and platform reports up to two cycles, and model-conformance's replay drives each path against `Sandbox`; a scenario would restate one of those paths" waive.fuzz = "the input is a sequence of requests and reports, which the model enumerates exhaustively and the replay drives; there is no byte stream to fuzz" -waive.live = "the window is launch-time `suspendedDurationSeconds`, and the refusal comes before any call once the client's clock passes it; the model, its replay and the app's tests drive that clock, which a live run can only wait out" +waive.live = "the service's two answers are the 2026-10-12 us-east-1 measurement in #597 (an idle-suspended VM refused once its window passed, an explicit suspend resumed 1283 s into a 60 s window), which the app's tests, the CLI's and the replay script verbatim; a live check holds a billed VM through an idle suspend and its whole window, which the suite doesn't run yet"