From 899ab0085c266f9a244cc7eab48b6bde5146ac36 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:39:34 +0900 Subject: [PATCH 01/44] test: specify application service lease ownership --- tests/application_service_ownership.rs | 299 +++++++++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 tests/application_service_ownership.rs diff --git a/tests/application_service_ownership.rs b/tests/application_service_ownership.rs new file mode 100644 index 00000000..63df3e35 --- /dev/null +++ b/tests/application_service_ownership.rs @@ -0,0 +1,299 @@ +//! Caller-scoped idempotency and lease-ownership tests for isolated services. + +#![cfg(target_os = "linux")] + +use std::{ + fs, + net::TcpListener, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + sync::Arc, + thread, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use quarantine_sandbox_runtime::{ + ApplicationServiceCoordinator, ApplicationServiceError, ApplicationServiceRequest, + IsolationPolicy, LeaseOwnerId, ResourceRequest, RootlessPodmanAdapter, ServiceProtocol, +}; + +fn digest_image() -> String { + format!("localhost/cwl/tool@sha256:{}", "d".repeat(64)) +} + +fn owner(value: &str) -> LeaseOwnerId { + LeaseOwnerId::new(value).expect("test owner identity should satisfy the bounded contract") +} + +fn policy() -> IsolationPolicy { + IsolationPolicy { + policy_id: "lease_ownership_policy_v1".to_owned(), + maximum_memory_bytes: 512 * 1024 * 1024, + maximum_cpu_millicores: 2_000, + maximum_processes: 128, + maximum_lease_seconds: 900, + maximum_tmpfs_bytes: 128 * 1024 * 1024, + readiness_timeout_millis: 2_000, + readiness_poll_interval_millis: 10, + shutdown_grace_seconds: 2, + run_as_user_id: 65_532, + run_as_group_id: 65_532, + } +} + +fn request() -> ApplicationServiceRequest { + ApplicationServiceRequest { + schema_version: "1.0.0".to_owned(), + request_id: "agent_task_lease_42".to_owned(), + image_reference: digest_image(), + container_port: 8_080, + protocol: ServiceProtocol::Http, + command: vec!["serve".to_owned()], + resources: ResourceRequest { + memory_bytes: 256 * 1024 * 1024, + cpu_millicores: 1_000, + maximum_processes: 32, + lease_seconds: 300, + tmpfs_bytes: 32 * 1024 * 1024, + }, + } +} + +fn temporary_path(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock should be after the Unix epoch") + .as_nanos(); + std::env::temp_dir().join(format!( + "qsr-ownership-{name}-{}-{nanos}", + std::process::id() + )) +} + +fn write_fake_podman(program: &Path, log: &Path, mode: &str, ready_port: u16) { + let script = format!( + "#!/bin/sh\nset -eu\nMODE='{mode}'\nprintf '%s\\n' \"$*\" >> '{}'\nif [ \"$MODE\" = slow_rootless ] && [ \"${{1:-}}\" = info ]; then sleep 1; fi\nif [ \"$MODE\" = fail_rootless ] && [ \"${{1:-}}\" = info ]; then exit 20; fi\ncase \"${{1:-}}\" in\n info) printf 'true\\n' ;;\n network) : ;;\n create) printf 'fake-container-id\\n' ;;\n start) : ;;\n port) printf '127.0.0.1:{ready_port}\\n' ;;\n stop) : ;;\n rm) : ;;\n *) exit 91 ;;\nesac\n", + log.display() + ); + fs::write(program, script).expect("fake Podman should be writable"); + let mut permissions = fs::metadata(program) + .expect("fake Podman metadata should exist") + .permissions(); + permissions.set_mode(0o700); + fs::set_permissions(program, permissions).expect("fake Podman should be executable"); +} + +fn count_calls(log: &Path, needle: &str) -> usize { + fs::read_to_string(log) + .unwrap_or_default() + .lines() + .filter(|line| line.contains(needle)) + .count() +} + +fn wait_until_log_contains(log: &Path, needle: &str) { + for _ in 0..100 { + if count_calls(log, needle) > 0 { + return; + } + thread::sleep(Duration::from_millis(10)); + } + panic!("timed out waiting for fake Podman call containing {needle}"); +} + +#[test] +fn lease_owner_ids_are_bounded_opaque_runtime_context() { + assert_eq!( + LeaseOwnerId::new(""), + Err(ApplicationServiceError::InvalidLeaseOwnerId) + ); + assert_eq!( + LeaseOwnerId::new("contains whitespace"), + Err(ApplicationServiceError::InvalidLeaseOwnerId) + ); + assert_eq!( + LeaseOwnerId::new(&"a".repeat(129)), + Err(ApplicationServiceError::InvalidLeaseOwnerId) + ); + assert_eq!( + owner("urn:cwl:agent:contextual-orchestrator").as_str(), + "urn:cwl:agent:contextual-orchestrator" + ); +} + +#[test] +fn identical_retry_returns_existing_lease_without_second_launch() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); + let ready_port = listener.local_addr().expect("address should resolve").port(); + let program = temporary_path("fake-podman"); + let log = temporary_path("fake-podman-log"); + write_fake_podman(&program, &log, "success", ready_port); + let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let owner = owner("urn:cwl:agent:contextual-orchestrator"); + + let first = coordinator + .launch_at(&owner, &request(), &policy(), 1_780_000_000) + .expect("first launch should succeed"); + let retry = coordinator + .launch_at(&owner, &request(), &policy(), 1_780_000_100) + .expect("identical retry should return the active lease"); + + assert_eq!(retry, first); + assert_eq!(count_calls(&log, "network create"), 1); + coordinator + .terminate_at(&owner, &first, 1_780_000_110) + .expect("owner should terminate its lease"); + let _ = fs::remove_file(program); + let _ = fs::remove_file(log); + drop(listener); +} + +#[test] +fn same_owner_and_request_id_with_different_content_fails_closed() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); + let ready_port = listener.local_addr().expect("address should resolve").port(); + let program = temporary_path("fake-podman"); + let log = temporary_path("fake-podman-log"); + write_fake_podman(&program, &log, "success", ready_port); + let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let owner = owner("urn:cwl:agent:contextual-orchestrator"); + let first = coordinator + .launch_at(&owner, &request(), &policy(), 1_780_000_000) + .expect("first launch should succeed"); + let mut changed = request(); + changed.command.push("--different".to_owned()); + + assert_eq!( + coordinator.launch_at(&owner, &changed, &policy(), 1_780_000_001), + Err(ApplicationServiceError::IdempotencyConflict) + ); + assert_eq!(count_calls(&log, "network create"), 1); + coordinator + .terminate_at(&owner, &first, 1_780_000_010) + .expect("original owner should still terminate the first lease"); + let _ = fs::remove_file(program); + let _ = fs::remove_file(log); + drop(listener); +} + +#[test] +fn wrong_owner_cannot_terminate_another_callers_lease() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); + let ready_port = listener.local_addr().expect("address should resolve").port(); + let program = temporary_path("fake-podman"); + let log = temporary_path("fake-podman-log"); + write_fake_podman(&program, &log, "success", ready_port); + let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let correct_owner = owner("urn:cwl:agent:contextual-orchestrator"); + let wrong_owner = owner("urn:cwl:consumer:wardnet"); + let lease = coordinator + .launch_at(&correct_owner, &request(), &policy(), 1_780_000_000) + .expect("launch should succeed"); + + assert_eq!( + coordinator.terminate_at(&wrong_owner, &lease, 1_780_000_010), + Err(ApplicationServiceError::UnknownLease) + ); + assert_eq!(count_calls(&log, "stop --time"), 0); + coordinator + .terminate_at(&correct_owner, &lease, 1_780_000_011) + .expect("correct owner should terminate the lease"); + assert_eq!(count_calls(&log, "stop --time"), 1); + let _ = fs::remove_file(program); + let _ = fs::remove_file(log); + drop(listener); +} + +#[test] +fn failed_launch_releases_idempotency_reservation_for_retry() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); + let ready_port = listener.local_addr().expect("address should resolve").port(); + let program = temporary_path("fake-podman"); + let log = temporary_path("fake-podman-log"); + write_fake_podman(&program, &log, "fail_rootless", ready_port); + let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let owner = owner("urn:cwl:agent:contextual-orchestrator"); + + assert_eq!( + coordinator.launch_at(&owner, &request(), &policy(), 1_780_000_000), + Err(ApplicationServiceError::BackendCommandFailed { + operation: "rootless_probe", + }) + ); + write_fake_podman(&program, &log, "success", ready_port); + let lease = coordinator + .launch_at(&owner, &request(), &policy(), 1_780_000_001) + .expect("retry after a failed launch should be allowed"); + coordinator + .terminate_at(&owner, &lease, 1_780_000_010) + .expect("retried lease should terminate cleanly"); + let _ = fs::remove_file(program); + let _ = fs::remove_file(log); + drop(listener); +} + +#[test] +fn concurrent_duplicate_launch_is_rejected_while_first_launch_is_in_flight() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); + let ready_port = listener.local_addr().expect("address should resolve").port(); + let program = temporary_path("fake-podman"); + let log = temporary_path("fake-podman-log"); + write_fake_podman(&program, &log, "slow_rootless", ready_port); + let coordinator = Arc::new(ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new( + program.clone(), + ))); + let owner = owner("urn:cwl:agent:contextual-orchestrator"); + let worker_coordinator = Arc::clone(&coordinator); + let worker_owner = owner.clone(); + let worker = thread::spawn(move || { + worker_coordinator.launch_at(&worker_owner, &request(), &policy(), 1_780_000_000) + }); + wait_until_log_contains(&log, "info --format"); + + assert_eq!( + coordinator.launch_at(&owner, &request(), &policy(), 1_780_000_001), + Err(ApplicationServiceError::LaunchInProgress) + ); + let lease = worker + .join() + .expect("launch thread should not panic") + .expect("first launch should finish successfully"); + assert_eq!(count_calls(&log, "network create"), 1); + coordinator + .terminate_at(&owner, &lease, 1_780_000_010) + .expect("owner should terminate the finished lease"); + let _ = fs::remove_file(program); + let _ = fs::remove_file(log); + drop(listener); +} + +#[test] +fn expired_lease_cleanup_is_bounded_and_attributed_to_owner_and_request() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); + let ready_port = listener.local_addr().expect("address should resolve").port(); + let program = temporary_path("fake-podman"); + let log = temporary_path("fake-podman-log"); + write_fake_podman(&program, &log, "success", ready_port); + let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let owner = owner("urn:cwl:agent:contextual-orchestrator"); + let mut short_request = request(); + short_request.resources.lease_seconds = 1; + coordinator + .launch_at(&owner, &short_request, &policy(), 1_780_000_000) + .expect("short lease should launch"); + + assert!(coordinator.cleanup_expired_at(1_780_000_000).is_empty()); + let outcomes = coordinator.cleanup_expired_at(1_780_000_001); + assert_eq!(outcomes.len(), 1); + assert_eq!(outcomes[0].lease_owner_id().as_str(), owner.as_str()); + assert_eq!(outcomes[0].request_id(), "agent_task_lease_42"); + assert!(outcomes[0].result().is_ok()); + assert_eq!(count_calls(&log, "stop --time"), 1); + assert_eq!( + coordinator.terminate_at(&owner, &outcomes[0].lease(), 1_780_000_002), + Err(ApplicationServiceError::UnknownLease) + ); + let _ = fs::remove_file(program); + let _ = fs::remove_file(log); + drop(listener); +} From a9283595b47b4c677043dec92126bf769058c0cd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:40:27 +0900 Subject: [PATCH 02/44] feat: enforce application service lease ownership --- src/application_service/coordinator.rs | 477 +++++++++++++++++++++++++ 1 file changed, 477 insertions(+) create mode 100644 src/application_service/coordinator.rs diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs new file mode 100644 index 00000000..8c583342 --- /dev/null +++ b/src/application_service/coordinator.rs @@ -0,0 +1,477 @@ +//! Caller-scoped application-service lifecycle coordination. + +use std::{collections::BTreeMap, sync::Mutex}; + +use sha2::{Digest, Sha256}; + +use super::{ + ApplicationServiceError, ApplicationServiceLease, ApplicationServiceRequest, CleanupReceipt, +}; +use crate::IsolationPolicy; + +const MAX_LEASE_OWNER_ID_BYTES: usize = 128; +const MAX_EXPIRED_CLEANUPS_PER_CALL: usize = 64; + +/// Opaque authenticated-caller identity used to scope leases and idempotency. +/// +/// The runtime does not authenticate this value. A transport adapter must +/// construct it only after it has verified the caller and mapped that caller to +/// a stable opaque identity. It is intentionally separate from the untrusted +/// [`ApplicationServiceRequest`] payload. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct LeaseOwnerId(String); + +impl LeaseOwnerId { + /// Validate and construct a bounded opaque lease-owner identity. + /// + /// # Errors + /// + /// Returns [`ApplicationServiceError::InvalidLeaseOwnerId`] for empty, + /// oversized, whitespace-bearing, non-ASCII, or unsupported identities. + pub fn new(value: &str) -> Result { + let valid = !value.is_empty() + && value.len() <= MAX_LEASE_OWNER_ID_BYTES + && value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'/' | b'@' | b'-') + }); + if !valid { + return Err(ApplicationServiceError::InvalidLeaseOwnerId); + } + Ok(Self(value.to_owned())) + } + + /// Return the stable opaque owner identity. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// Infrastructure port required by the application-service coordinator. +/// +/// Implementations may use Podman, containerd, gVisor, or another reviewed +/// sandbox backend. Idempotency and caller ownership stay in the Supporting +/// `application_service` bounded context rather than in an infrastructure +/// adapter. +pub trait ApplicationServiceBackend: Send + Sync { + /// Launch one validated isolated application service. + /// + /// # Errors + /// + /// Returns [`ApplicationServiceError`] when the backend cannot establish + /// the requested isolation and readiness contract. + fn launch_at( + &self, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result; + + /// Stop one active service and remove all runtime-owned isolation resources. + /// + /// # Errors + /// + /// Returns [`ApplicationServiceError`] when cleanup cannot be proven. + fn terminate_at( + &self, + lease: &ApplicationServiceLease, + terminated_at_epoch_seconds: u64, + ) -> Result; +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct LeaseKey { + lease_owner_id: LeaseOwnerId, + request_id: String, +} + +impl LeaseKey { + fn new(lease_owner_id: &LeaseOwnerId, request_id: &str) -> Self { + Self { + lease_owner_id: lease_owner_id.clone(), + request_id: request_id.to_owned(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum RegistryEntry { + Launching { + request_fingerprint: String, + }, + Active { + request_fingerprint: String, + lease: ApplicationServiceLease, + }, + Terminating { + request_fingerprint: String, + lease: ApplicationServiceLease, + }, +} + +/// One bounded expired-lease cleanup outcome. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExpiredLeaseCleanupResult { + lease_owner_id: LeaseOwnerId, + request_id: String, + lease: ApplicationServiceLease, + result: Result, +} + +impl ExpiredLeaseCleanupResult { + /// Return the authenticated caller identity that owned the expired lease. + #[must_use] + pub const fn lease_owner_id(&self) -> &LeaseOwnerId { + &self.lease_owner_id + } + + /// Return the caller-scoped idempotency key of the expired lease. + #[must_use] + pub fn request_id(&self) -> &str { + &self.request_id + } + + /// Return the public lease receipt that was selected for cleanup. + #[must_use] + pub const fn lease(&self) -> &ApplicationServiceLease { + &self.lease + } + + /// Return the cleanup receipt or attributable backend failure. + #[must_use] + pub const fn result(&self) -> &Result { + &self.result + } +} + +/// In-memory application-service coordinator enforcing caller ownership and idempotency. +/// +/// This coordinator is process-local. It prevents duplicate launches and +/// cross-caller cleanup within one running runtime, but it is not durable crash +/// recovery. A later persistence slice must reconstruct and reconcile leases +/// after restart before claiming orphan recovery. +pub struct ApplicationServiceCoordinator { + backend: B, + leases: Mutex>, +} + +impl ApplicationServiceCoordinator +where + B: ApplicationServiceBackend, +{ + /// Construct a coordinator around one reviewed sandbox backend. + #[must_use] + pub fn new(backend: B) -> Self { + Self { + backend, + leases: Mutex::new(BTreeMap::new()), + } + } + + /// Launch or replay one caller-scoped application-service request. + /// + /// An identical retry for an active lease returns that lease without + /// invoking the backend again. Reusing the same caller/request identity for + /// different immutable request content fails closed. + /// + /// # Errors + /// + /// Returns [`ApplicationServiceError`] for invalid requests, idempotency + /// conflicts, concurrent duplicate launches, coordinator-state failure, or + /// backend isolation failures. + pub fn launch_at( + &self, + lease_owner_id: &LeaseOwnerId, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result { + request.validate(policy)?; + let key = LeaseKey::new(lease_owner_id, &request.request_id); + let request_fingerprint = fingerprint_request(request); + { + let mut leases = self.lock_registry()?; + match leases.get(&key) { + Some(RegistryEntry::Launching { + request_fingerprint: existing, + }) if existing == &request_fingerprint => { + return Err(ApplicationServiceError::LaunchInProgress); + } + Some(RegistryEntry::Active { + request_fingerprint: existing, + lease, + }) if existing == &request_fingerprint => return Ok(lease.clone()), + Some(RegistryEntry::Terminating { + request_fingerprint: existing, + .. + }) if existing == &request_fingerprint => { + return Err(ApplicationServiceError::TerminationInProgress); + } + Some(_) => return Err(ApplicationServiceError::IdempotencyConflict), + None => { + leases.insert( + key.clone(), + RegistryEntry::Launching { + request_fingerprint: request_fingerprint.clone(), + }, + ); + } + } + } + + let launch_result = self + .backend + .launch_at(request, policy, started_at_epoch_seconds); + let lease = match launch_result { + Ok(lease) => lease, + Err(error) => { + self.remove_matching_reservation(&key, &request_fingerprint)?; + return Err(error); + } + }; + + let mut leases = match self.leases.lock() { + Ok(leases) => leases, + Err(_) => { + return match self.backend.terminate_at(&lease, started_at_epoch_seconds) { + Ok(_) => Err(ApplicationServiceError::CoordinatorStateUnavailable), + Err(_) => Err(ApplicationServiceError::CleanupFailed), + }; + } + }; + match leases.get(&key) { + Some(RegistryEntry::Launching { + request_fingerprint: existing, + }) if existing == &request_fingerprint => { + leases.insert( + key, + RegistryEntry::Active { + request_fingerprint, + lease: lease.clone(), + }, + ); + Ok(lease) + } + _ => { + drop(leases); + match self.backend.terminate_at(&lease, started_at_epoch_seconds) { + Ok(_) => Err(ApplicationServiceError::CoordinatorStateUnavailable), + Err(_) => Err(ApplicationServiceError::CleanupFailed), + } + } + } + } + + /// Terminate one lease only when the caller owns the active registry entry. + /// + /// # Errors + /// + /// Returns [`ApplicationServiceError::UnknownLease`] for a wrong owner or + /// unknown lease before any backend cleanup operation is attempted. Other + /// errors preserve active state unless cleanup actually succeeded. + pub fn terminate_at( + &self, + lease_owner_id: &LeaseOwnerId, + lease: &ApplicationServiceLease, + terminated_at_epoch_seconds: u64, + ) -> Result { + let key = LeaseKey::new(lease_owner_id, lease.request_id()); + let (request_fingerprint, registered_lease) = { + let mut leases = self.lock_registry()?; + match leases.get(&key) { + Some(RegistryEntry::Active { + request_fingerprint, + lease: registered_lease, + }) => { + if registered_lease != lease { + return Err(ApplicationServiceError::LeaseMismatch); + } + let request_fingerprint = request_fingerprint.clone(); + let registered_lease = registered_lease.clone(); + leases.insert( + key.clone(), + RegistryEntry::Terminating { + request_fingerprint: request_fingerprint.clone(), + lease: registered_lease.clone(), + }, + ); + (request_fingerprint, registered_lease) + } + Some(RegistryEntry::Launching { .. }) => { + return Err(ApplicationServiceError::LaunchInProgress); + } + Some(RegistryEntry::Terminating { .. }) => { + return Err(ApplicationServiceError::TerminationInProgress); + } + None => return Err(ApplicationServiceError::UnknownLease), + } + }; + + let result = self + .backend + .terminate_at(®istered_lease, terminated_at_epoch_seconds); + self.finish_termination(&key, request_fingerprint, registered_lease, &result)?; + result + } + + /// Clean up at most 64 active leases whose expiry is not later than `now`. + /// + /// Failed cleanup remains registered as active so a later operator or + /// cleanup pass can retry it. This process-local function does not claim + /// crash/restart orphan recovery. + /// + /// # Errors + /// + /// Returns [`ApplicationServiceError::CoordinatorStateUnavailable`] if the + /// in-memory registry cannot be accessed safely. + pub fn cleanup_expired_at( + &self, + now_epoch_seconds: u64, + ) -> Result, ApplicationServiceError> { + let candidates = { + let mut leases = self.lock_registry()?; + let keys: Vec = leases + .iter() + .filter_map(|(key, entry)| match entry { + RegistryEntry::Active { lease, .. } + if lease.expires_at_epoch_seconds() <= now_epoch_seconds => + { + Some(key.clone()) + } + _ => None, + }) + .take(MAX_EXPIRED_CLEANUPS_PER_CALL) + .collect(); + let mut candidates = Vec::with_capacity(keys.len()); + for key in keys { + let Some(RegistryEntry::Active { + request_fingerprint, + lease, + }) = leases.get(&key).cloned() + else { + continue; + }; + leases.insert( + key.clone(), + RegistryEntry::Terminating { + request_fingerprint: request_fingerprint.clone(), + lease: lease.clone(), + }, + ); + candidates.push((key, request_fingerprint, lease)); + } + candidates + }; + + let mut outcomes = Vec::with_capacity(candidates.len()); + for (key, request_fingerprint, lease) in candidates { + let result = self.backend.terminate_at(&lease, now_epoch_seconds); + self.finish_termination( + &key, + request_fingerprint, + lease.clone(), + &result, + )?; + outcomes.push(ExpiredLeaseCleanupResult { + lease_owner_id: key.lease_owner_id, + request_id: key.request_id, + lease, + result, + }); + } + Ok(outcomes) + } + + fn lock_registry( + &self, + ) -> Result< + std::sync::MutexGuard<'_, BTreeMap>, + ApplicationServiceError, + > { + self.leases + .lock() + .map_err(|_| ApplicationServiceError::CoordinatorStateUnavailable) + } + + fn remove_matching_reservation( + &self, + key: &LeaseKey, + request_fingerprint: &str, + ) -> Result<(), ApplicationServiceError> { + let mut leases = self.lock_registry()?; + if matches!( + leases.get(key), + Some(RegistryEntry::Launching { + request_fingerprint: existing, + }) if existing == request_fingerprint + ) { + leases.remove(key); + } + Ok(()) + } + + fn finish_termination( + &self, + key: &LeaseKey, + request_fingerprint: String, + lease: ApplicationServiceLease, + result: &Result, + ) -> Result<(), ApplicationServiceError> { + let mut leases = self.lock_registry()?; + match result { + Ok(_) => { + if matches!( + leases.get(key), + Some(RegistryEntry::Terminating { + request_fingerprint: existing, + lease: registered_lease, + }) if existing == &request_fingerprint && registered_lease == &lease + ) { + leases.remove(key); + } + } + Err(_) => { + if matches!( + leases.get(key), + Some(RegistryEntry::Terminating { + request_fingerprint: existing, + lease: registered_lease, + }) if existing == &request_fingerprint && registered_lease == &lease + ) { + leases.insert( + key.clone(), + RegistryEntry::Active { + request_fingerprint, + lease, + }, + ); + } + } + } + Ok(()) + } +} + +fn fingerprint_request(request: &ApplicationServiceRequest) -> String { + let mut hasher = Sha256::new(); + for component in [ + request.schema_version.as_str(), + request.request_id.as_str(), + request.image_reference.as_str(), + request.protocol.as_str(), + ] { + hasher.update(component.as_bytes()); + hasher.update([0]); + } + hasher.update(request.container_port.to_be_bytes()); + for argument in &request.command { + hasher.update(argument.as_bytes()); + hasher.update([0]); + } + hasher.update(request.resources.memory_bytes.to_be_bytes()); + hasher.update(request.resources.cpu_millicores.to_be_bytes()); + hasher.update(request.resources.maximum_processes.to_be_bytes()); + hasher.update(request.resources.lease_seconds.to_be_bytes()); + hasher.update(request.resources.tmpfs_bytes.to_be_bytes()); + format!("{:x}", hasher.finalize()) +} From ef83a5be32d761f7039bde7037838a4eb6427b1d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:41:29 +0900 Subject: [PATCH 03/44] refactor: separate lease coordination errors --- src/application_service/coordinator.rs | 128 +++++++++++++++++-------- 1 file changed, 86 insertions(+), 42 deletions(-) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index 8c583342..0d7488c9 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -3,15 +3,45 @@ use std::{collections::BTreeMap, sync::Mutex}; use sha2::{Digest, Sha256}; +use thiserror::Error; -use super::{ +use crate::{ ApplicationServiceError, ApplicationServiceLease, ApplicationServiceRequest, CleanupReceipt, + IsolationPolicy, RootlessPodmanAdapter, }; -use crate::IsolationPolicy; const MAX_LEASE_OWNER_ID_BYTES: usize = 128; const MAX_EXPIRED_CLEANUPS_PER_CALL: usize = 64; +/// Coordinator-owned caller/idempotency errors. +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum ApplicationServiceCoordinatorError { + /// The authenticated caller identity could not be represented safely. + #[error("invalid application-service lease owner identity")] + InvalidLeaseOwnerId, + /// The same caller/request identity was reused for different request content. + #[error("application-service idempotency conflict")] + IdempotencyConflict, + /// An identical caller/request launch is already executing. + #[error("application-service launch already in progress")] + LaunchInProgress, + /// Cleanup for the same active lease is already executing. + #[error("application-service termination already in progress")] + TerminationInProgress, + /// The caller does not own an active lease matching the supplied receipt. + #[error("application-service lease is unknown to this caller")] + UnknownLease, + /// The caller supplied a receipt that differs from the registered active lease. + #[error("application-service lease receipt does not match active registry state")] + LeaseMismatch, + /// The process-local coordinator registry could not be accessed safely. + #[error("application-service coordinator state is unavailable")] + StateUnavailable, + /// The application-service request or sandbox backend failed. + #[error(transparent)] + Backend(#[from] ApplicationServiceError), +} + /// Opaque authenticated-caller identity used to scope leases and idempotency. /// /// The runtime does not authenticate this value. A transport adapter must @@ -26,16 +56,17 @@ impl LeaseOwnerId { /// /// # Errors /// - /// Returns [`ApplicationServiceError::InvalidLeaseOwnerId`] for empty, - /// oversized, whitespace-bearing, non-ASCII, or unsupported identities. - pub fn new(value: &str) -> Result { + /// Returns [`ApplicationServiceCoordinatorError::InvalidLeaseOwnerId`] for + /// empty, oversized, whitespace-bearing, non-ASCII, or unsupported identities. + pub fn new(value: &str) -> Result { let valid = !value.is_empty() && value.len() <= MAX_LEASE_OWNER_ID_BYTES && value.bytes().all(|byte| { - byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'/' | b'@' | b'-') + byte.is_ascii_alphanumeric() + || matches!(byte, b'.' | b'_' | b':' | b'/' | b'@' | b'-') }); if !valid { - return Err(ApplicationServiceError::InvalidLeaseOwnerId); + return Err(ApplicationServiceCoordinatorError::InvalidLeaseOwnerId); } Ok(Self(value.to_owned())) } @@ -79,6 +110,25 @@ pub trait ApplicationServiceBackend: Send + Sync { ) -> Result; } +impl ApplicationServiceBackend for RootlessPodmanAdapter { + fn launch_at( + &self, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result { + RootlessPodmanAdapter::launch_at(self, request, policy, started_at_epoch_seconds) + } + + fn terminate_at( + &self, + lease: &ApplicationServiceLease, + terminated_at_epoch_seconds: u64, + ) -> Result { + RootlessPodmanAdapter::terminate_at(self, lease, terminated_at_epoch_seconds) + } +} + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] struct LeaseKey { lease_owner_id: LeaseOwnerId, @@ -176,8 +226,8 @@ where /// /// # Errors /// - /// Returns [`ApplicationServiceError`] for invalid requests, idempotency - /// conflicts, concurrent duplicate launches, coordinator-state failure, or + /// Returns [`ApplicationServiceCoordinatorError`] for invalid requests, + /// idempotency conflicts, concurrent duplicate launches, state failure, or /// backend isolation failures. pub fn launch_at( &self, @@ -185,7 +235,7 @@ where request: &ApplicationServiceRequest, policy: &IsolationPolicy, started_at_epoch_seconds: u64, - ) -> Result { + ) -> Result { request.validate(policy)?; let key = LeaseKey::new(lease_owner_id, &request.request_id); let request_fingerprint = fingerprint_request(request); @@ -195,7 +245,7 @@ where Some(RegistryEntry::Launching { request_fingerprint: existing, }) if existing == &request_fingerprint => { - return Err(ApplicationServiceError::LaunchInProgress); + return Err(ApplicationServiceCoordinatorError::LaunchInProgress); } Some(RegistryEntry::Active { request_fingerprint: existing, @@ -205,9 +255,9 @@ where request_fingerprint: existing, .. }) if existing == &request_fingerprint => { - return Err(ApplicationServiceError::TerminationInProgress); + return Err(ApplicationServiceCoordinatorError::TerminationInProgress); } - Some(_) => return Err(ApplicationServiceError::IdempotencyConflict), + Some(_) => return Err(ApplicationServiceCoordinatorError::IdempotencyConflict), None => { leases.insert( key.clone(), @@ -219,14 +269,14 @@ where } } - let launch_result = self + let lease = match self .backend - .launch_at(request, policy, started_at_epoch_seconds); - let lease = match launch_result { + .launch_at(request, policy, started_at_epoch_seconds) + { Ok(lease) => lease, Err(error) => { self.remove_matching_reservation(&key, &request_fingerprint)?; - return Err(error); + return Err(error.into()); } }; @@ -234,8 +284,8 @@ where Ok(leases) => leases, Err(_) => { return match self.backend.terminate_at(&lease, started_at_epoch_seconds) { - Ok(_) => Err(ApplicationServiceError::CoordinatorStateUnavailable), - Err(_) => Err(ApplicationServiceError::CleanupFailed), + Ok(_) => Err(ApplicationServiceCoordinatorError::StateUnavailable), + Err(error) => Err(error.into()), }; } }; @@ -255,8 +305,8 @@ where _ => { drop(leases); match self.backend.terminate_at(&lease, started_at_epoch_seconds) { - Ok(_) => Err(ApplicationServiceError::CoordinatorStateUnavailable), - Err(_) => Err(ApplicationServiceError::CleanupFailed), + Ok(_) => Err(ApplicationServiceCoordinatorError::StateUnavailable), + Err(error) => Err(error.into()), } } } @@ -266,15 +316,14 @@ where /// /// # Errors /// - /// Returns [`ApplicationServiceError::UnknownLease`] for a wrong owner or - /// unknown lease before any backend cleanup operation is attempted. Other - /// errors preserve active state unless cleanup actually succeeded. + /// Returns [`ApplicationServiceCoordinatorError::UnknownLease`] for a wrong + /// owner or unknown lease before any backend cleanup operation is attempted. pub fn terminate_at( &self, lease_owner_id: &LeaseOwnerId, lease: &ApplicationServiceLease, terminated_at_epoch_seconds: u64, - ) -> Result { + ) -> Result { let key = LeaseKey::new(lease_owner_id, lease.request_id()); let (request_fingerprint, registered_lease) = { let mut leases = self.lock_registry()?; @@ -284,7 +333,7 @@ where lease: registered_lease, }) => { if registered_lease != lease { - return Err(ApplicationServiceError::LeaseMismatch); + return Err(ApplicationServiceCoordinatorError::LeaseMismatch); } let request_fingerprint = request_fingerprint.clone(); let registered_lease = registered_lease.clone(); @@ -298,12 +347,12 @@ where (request_fingerprint, registered_lease) } Some(RegistryEntry::Launching { .. }) => { - return Err(ApplicationServiceError::LaunchInProgress); + return Err(ApplicationServiceCoordinatorError::LaunchInProgress); } Some(RegistryEntry::Terminating { .. }) => { - return Err(ApplicationServiceError::TerminationInProgress); + return Err(ApplicationServiceCoordinatorError::TerminationInProgress); } - None => return Err(ApplicationServiceError::UnknownLease), + None => return Err(ApplicationServiceCoordinatorError::UnknownLease), } }; @@ -311,7 +360,7 @@ where .backend .terminate_at(®istered_lease, terminated_at_epoch_seconds); self.finish_termination(&key, request_fingerprint, registered_lease, &result)?; - result + result.map_err(Into::into) } /// Clean up at most 64 active leases whose expiry is not later than `now`. @@ -322,12 +371,12 @@ where /// /// # Errors /// - /// Returns [`ApplicationServiceError::CoordinatorStateUnavailable`] if the + /// Returns [`ApplicationServiceCoordinatorError::StateUnavailable`] if the /// in-memory registry cannot be accessed safely. pub fn cleanup_expired_at( &self, now_epoch_seconds: u64, - ) -> Result, ApplicationServiceError> { + ) -> Result, ApplicationServiceCoordinatorError> { let candidates = { let mut leases = self.lock_registry()?; let keys: Vec = leases @@ -366,12 +415,7 @@ where let mut outcomes = Vec::with_capacity(candidates.len()); for (key, request_fingerprint, lease) in candidates { let result = self.backend.terminate_at(&lease, now_epoch_seconds); - self.finish_termination( - &key, - request_fingerprint, - lease.clone(), - &result, - )?; + self.finish_termination(&key, request_fingerprint, lease.clone(), &result)?; outcomes.push(ExpiredLeaseCleanupResult { lease_owner_id: key.lease_owner_id, request_id: key.request_id, @@ -386,18 +430,18 @@ where &self, ) -> Result< std::sync::MutexGuard<'_, BTreeMap>, - ApplicationServiceError, + ApplicationServiceCoordinatorError, > { self.leases .lock() - .map_err(|_| ApplicationServiceError::CoordinatorStateUnavailable) + .map_err(|_| ApplicationServiceCoordinatorError::StateUnavailable) } fn remove_matching_reservation( &self, key: &LeaseKey, request_fingerprint: &str, - ) -> Result<(), ApplicationServiceError> { + ) -> Result<(), ApplicationServiceCoordinatorError> { let mut leases = self.lock_registry()?; if matches!( leases.get(key), @@ -416,7 +460,7 @@ where request_fingerprint: String, lease: ApplicationServiceLease, result: &Result, - ) -> Result<(), ApplicationServiceError> { + ) -> Result<(), ApplicationServiceCoordinatorError> { let mut leases = self.lock_registry()?; match result { Ok(_) => { From 39efcefeefde7ed2bc1ee07f274d46e32c884b53 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:41:40 +0900 Subject: [PATCH 04/44] feat: publish lease coordination boundary --- src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 1b499b7e..277753ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,8 @@ //! conversation, task, tool-selection, secret, and user-action authority. mod application_service; +#[path = "application_service/coordinator.rs"] +mod application_service_coordinator; mod artifact_analysis; mod infrastructure; mod sandbox_execution; @@ -17,6 +19,10 @@ pub use application_service::{ ApplicationServiceError, ApplicationServiceLease, ApplicationServiceRequest, CleanupReceipt, IsolationAttestation, ServiceEndpoint, ServiceProtocol, }; +pub use application_service_coordinator::{ + ApplicationServiceBackend, ApplicationServiceCoordinator, ApplicationServiceCoordinatorError, + ExpiredLeaseCleanupResult, LeaseOwnerId, +}; pub use artifact_analysis::{ AnalysisEngine, AnalysisError, AnalysisProfile, AnalysisRequest, AnalyzerFailure, AnalyzerFinding, ArtifactDescriptor, ArtifactKind, BoundedSourceContext, From cdb7ea02025562000c6050ae77422f4acdddd3a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:42:26 +0900 Subject: [PATCH 05/44] test: align ownership assertions with coordinator errors --- tests/application_service_ownership.rs | 40 ++++++++++++++++---------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/tests/application_service_ownership.rs b/tests/application_service_ownership.rs index 63df3e35..41a26f89 100644 --- a/tests/application_service_ownership.rs +++ b/tests/application_service_ownership.rs @@ -13,8 +13,9 @@ use std::{ }; use quarantine_sandbox_runtime::{ - ApplicationServiceCoordinator, ApplicationServiceError, ApplicationServiceRequest, - IsolationPolicy, LeaseOwnerId, ResourceRequest, RootlessPodmanAdapter, ServiceProtocol, + ApplicationServiceCoordinator, ApplicationServiceCoordinatorError, ApplicationServiceError, + ApplicationServiceRequest, IsolationPolicy, LeaseOwnerId, ResourceRequest, + RootlessPodmanAdapter, ServiceProtocol, }; fn digest_image() -> String { @@ -105,15 +106,15 @@ fn wait_until_log_contains(log: &Path, needle: &str) { fn lease_owner_ids_are_bounded_opaque_runtime_context() { assert_eq!( LeaseOwnerId::new(""), - Err(ApplicationServiceError::InvalidLeaseOwnerId) + Err(ApplicationServiceCoordinatorError::InvalidLeaseOwnerId) ); assert_eq!( LeaseOwnerId::new("contains whitespace"), - Err(ApplicationServiceError::InvalidLeaseOwnerId) + Err(ApplicationServiceCoordinatorError::InvalidLeaseOwnerId) ); assert_eq!( LeaseOwnerId::new(&"a".repeat(129)), - Err(ApplicationServiceError::InvalidLeaseOwnerId) + Err(ApplicationServiceCoordinatorError::InvalidLeaseOwnerId) ); assert_eq!( owner("urn:cwl:agent:contextual-orchestrator").as_str(), @@ -165,7 +166,7 @@ fn same_owner_and_request_id_with_different_content_fails_closed() { assert_eq!( coordinator.launch_at(&owner, &changed, &policy(), 1_780_000_001), - Err(ApplicationServiceError::IdempotencyConflict) + Err(ApplicationServiceCoordinatorError::IdempotencyConflict) ); assert_eq!(count_calls(&log, "network create"), 1); coordinator @@ -192,7 +193,7 @@ fn wrong_owner_cannot_terminate_another_callers_lease() { assert_eq!( coordinator.terminate_at(&wrong_owner, &lease, 1_780_000_010), - Err(ApplicationServiceError::UnknownLease) + Err(ApplicationServiceCoordinatorError::UnknownLease) ); assert_eq!(count_calls(&log, "stop --time"), 0); coordinator @@ -216,9 +217,11 @@ fn failed_launch_releases_idempotency_reservation_for_retry() { assert_eq!( coordinator.launch_at(&owner, &request(), &policy(), 1_780_000_000), - Err(ApplicationServiceError::BackendCommandFailed { - operation: "rootless_probe", - }) + Err(ApplicationServiceCoordinatorError::Backend( + ApplicationServiceError::BackendCommandFailed { + operation: "rootless_probe", + } + )) ); write_fake_podman(&program, &log, "success", ready_port); let lease = coordinator @@ -252,7 +255,7 @@ fn concurrent_duplicate_launch_is_rejected_while_first_launch_is_in_flight() { assert_eq!( coordinator.launch_at(&owner, &request(), &policy(), 1_780_000_001), - Err(ApplicationServiceError::LaunchInProgress) + Err(ApplicationServiceCoordinatorError::LaunchInProgress) ); let lease = worker .join() @@ -282,16 +285,23 @@ fn expired_lease_cleanup_is_bounded_and_attributed_to_owner_and_request() { .launch_at(&owner, &short_request, &policy(), 1_780_000_000) .expect("short lease should launch"); - assert!(coordinator.cleanup_expired_at(1_780_000_000).is_empty()); - let outcomes = coordinator.cleanup_expired_at(1_780_000_001); + assert!( + coordinator + .cleanup_expired_at(1_780_000_000) + .expect("registry should be available") + .is_empty() + ); + let outcomes = coordinator + .cleanup_expired_at(1_780_000_001) + .expect("expired cleanup should access registry"); assert_eq!(outcomes.len(), 1); assert_eq!(outcomes[0].lease_owner_id().as_str(), owner.as_str()); assert_eq!(outcomes[0].request_id(), "agent_task_lease_42"); assert!(outcomes[0].result().is_ok()); assert_eq!(count_calls(&log, "stop --time"), 1); assert_eq!( - coordinator.terminate_at(&owner, &outcomes[0].lease(), 1_780_000_002), - Err(ApplicationServiceError::UnknownLease) + coordinator.terminate_at(&owner, outcomes[0].lease(), 1_780_000_002), + Err(ApplicationServiceCoordinatorError::UnknownLease) ); let _ = fs::remove_file(program); let _ = fs::remove_file(log); From 81760126313cde84a4a3de43b95dcf6d01e72d2e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:44:30 +0900 Subject: [PATCH 06/44] refactor: keep Podman port implementation in infrastructure --- .../application_service_backend.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/infrastructure/application_service_backend.rs diff --git a/src/infrastructure/application_service_backend.rs b/src/infrastructure/application_service_backend.rs new file mode 100644 index 00000000..ea16d0a6 --- /dev/null +++ b/src/infrastructure/application_service_backend.rs @@ -0,0 +1,25 @@ +//! Application-service backend port implementations owned by infrastructure. + +use crate::{ + ApplicationServiceBackend, ApplicationServiceError, ApplicationServiceLease, + ApplicationServiceRequest, CleanupReceipt, IsolationPolicy, RootlessPodmanAdapter, +}; + +impl ApplicationServiceBackend for RootlessPodmanAdapter { + fn launch_at( + &self, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result { + RootlessPodmanAdapter::launch_at(self, request, policy, started_at_epoch_seconds) + } + + fn terminate_at( + &self, + lease: &ApplicationServiceLease, + terminated_at_epoch_seconds: u64, + ) -> Result { + RootlessPodmanAdapter::terminate_at(self, lease, terminated_at_epoch_seconds) + } +} From 5fe05264156d0d2e98d260e5759c8050beb8ad84 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:44:36 +0900 Subject: [PATCH 07/44] refactor: register application service backend adapter --- src/infrastructure/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/infrastructure/mod.rs b/src/infrastructure/mod.rs index 302e8028..ac05a2cd 100644 --- a/src/infrastructure/mod.rs +++ b/src/infrastructure/mod.rs @@ -1,5 +1,6 @@ //! Infrastructure adapters for sandbox execution. +mod application_service_backend; mod podman; pub use podman::{PodmanLaunchPlan, RootlessPodmanAdapter}; From b9b1a34ce186db3352c3766957e64342e9484612 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:46:53 +0900 Subject: [PATCH 08/44] refactor: keep concrete sandbox adapters out of application service --- src/application_service/coordinator.rs | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index 0d7488c9..96bd98f4 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -7,7 +7,7 @@ use thiserror::Error; use crate::{ ApplicationServiceError, ApplicationServiceLease, ApplicationServiceRequest, CleanupReceipt, - IsolationPolicy, RootlessPodmanAdapter, + IsolationPolicy, }; const MAX_LEASE_OWNER_ID_BYTES: usize = 128; @@ -110,25 +110,6 @@ pub trait ApplicationServiceBackend: Send + Sync { ) -> Result; } -impl ApplicationServiceBackend for RootlessPodmanAdapter { - fn launch_at( - &self, - request: &ApplicationServiceRequest, - policy: &IsolationPolicy, - started_at_epoch_seconds: u64, - ) -> Result { - RootlessPodmanAdapter::launch_at(self, request, policy, started_at_epoch_seconds) - } - - fn terminate_at( - &self, - lease: &ApplicationServiceLease, - terminated_at_epoch_seconds: u64, - ) -> Result { - RootlessPodmanAdapter::terminate_at(self, lease, terminated_at_epoch_seconds) - } -} - #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] struct LeaseKey { lease_owner_id: LeaseOwnerId, From c9c8b1b5b997bfef3f79dbcaac25b593fd0a56b5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:47:41 +0900 Subject: [PATCH 09/44] test: bind idempotency to effective isolation policy --- .../application_service_policy_idempotency.rs | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tests/application_service_policy_idempotency.rs diff --git a/tests/application_service_policy_idempotency.rs b/tests/application_service_policy_idempotency.rs new file mode 100644 index 00000000..57edb9af --- /dev/null +++ b/tests/application_service_policy_idempotency.rs @@ -0,0 +1,101 @@ +//! Regression test that idempotency cannot replay a lease under a changed isolation policy. + +#![cfg(target_os = "linux")] + +use std::{ + fs, + net::TcpListener, + os::unix::fs::PermissionsExt, + path::PathBuf, + time::{SystemTime, UNIX_EPOCH}, +}; + +use quarantine_sandbox_runtime::{ + ApplicationServiceCoordinator, ApplicationServiceCoordinatorError, ApplicationServiceRequest, + IsolationPolicy, LeaseOwnerId, ResourceRequest, RootlessPodmanAdapter, ServiceProtocol, +}; + +fn temporary_path(name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock should be after Unix epoch") + .as_nanos(); + std::env::temp_dir().join(format!( + "qsr-policy-idempotency-{name}-{}-{nanos}", + std::process::id() + )) +} + +fn policy() -> IsolationPolicy { + IsolationPolicy { + policy_id: "agent_application_policy_v1".to_owned(), + maximum_memory_bytes: 512 * 1024 * 1024, + maximum_cpu_millicores: 2_000, + maximum_processes: 128, + maximum_lease_seconds: 900, + maximum_tmpfs_bytes: 128 * 1024 * 1024, + readiness_timeout_millis: 500, + readiness_poll_interval_millis: 10, + shutdown_grace_seconds: 2, + run_as_user_id: 65_532, + run_as_group_id: 65_532, + } +} + +fn request() -> ApplicationServiceRequest { + ApplicationServiceRequest { + schema_version: "1.0.0".to_owned(), + request_id: "same_task_same_request".to_owned(), + image_reference: format!("localhost/cwl/tool@sha256:{}", "e".repeat(64)), + container_port: 8_080, + protocol: ServiceProtocol::Http, + command: vec!["serve".to_owned()], + resources: ResourceRequest { + memory_bytes: 256 * 1024 * 1024, + cpu_millicores: 1_000, + maximum_processes: 32, + lease_seconds: 300, + tmpfs_bytes: 32 * 1024 * 1024, + }, + } +} + +fn write_fake_podman(program: &PathBuf, ready_port: u16) { + let script = format!( + "#!/bin/sh\nset -eu\ncase \"${{1:-}}\" in\n info) printf 'true\\n' ;;\n network) : ;;\n create) printf 'fake-container-id\\n' ;;\n start) : ;;\n port) printf '127.0.0.1:{ready_port}\\n' ;;\n stop) : ;;\n rm) : ;;\n *) exit 91 ;;\nesac\n" + ); + fs::write(program, script).expect("fake Podman should be writable"); + let mut permissions = fs::metadata(program) + .expect("fake Podman metadata should exist") + .permissions(); + permissions.set_mode(0o700); + fs::set_permissions(program, permissions).expect("fake Podman should be executable"); +} + +#[test] +fn identical_request_does_not_reuse_lease_when_effective_policy_changes() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); + let ready_port = listener.local_addr().expect("address should resolve").port(); + let program = temporary_path("fake-podman"); + write_fake_podman(&program, ready_port); + let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let owner = LeaseOwnerId::new("urn:cwl:agent:contextual-orchestrator") + .expect("opaque owner should validate"); + let initial_policy = policy(); + let lease = coordinator + .launch_at(&owner, &request(), &initial_policy, 1_780_000_000) + .expect("first launch should succeed"); + + let mut changed_policy = initial_policy; + changed_policy.maximum_memory_bytes += 1; + assert_eq!( + coordinator.launch_at(&owner, &request(), &changed_policy, 1_780_000_001), + Err(ApplicationServiceCoordinatorError::IdempotencyConflict) + ); + + coordinator + .terminate_at(&owner, &lease, 1_780_000_010) + .expect("original lease should still clean up"); + let _ = fs::remove_file(program); + drop(listener); +} From 03c2dc67e3ad04fd407f9d203e912e8f6cd1c14e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:48:30 +0900 Subject: [PATCH 10/44] fix: bind lease replay to effective isolation policy --- src/application_service/coordinator.rs | 34 ++++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index 96bd98f4..4d97f3a1 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -203,7 +203,7 @@ where /// /// An identical retry for an active lease returns that lease without /// invoking the backend again. Reusing the same caller/request identity for - /// different immutable request content fails closed. + /// different immutable request or effective policy content fails closed. /// /// # Errors /// @@ -219,7 +219,7 @@ where ) -> Result { request.validate(policy)?; let key = LeaseKey::new(lease_owner_id, &request.request_id); - let request_fingerprint = fingerprint_request(request); + let request_fingerprint = fingerprint_request_and_policy(request, policy); { let mut leases = self.lock_registry()?; match leases.get(&key) { @@ -477,13 +477,17 @@ where } } -fn fingerprint_request(request: &ApplicationServiceRequest) -> String { +fn fingerprint_request_and_policy( + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, +) -> String { let mut hasher = Sha256::new(); for component in [ request.schema_version.as_str(), request.request_id.as_str(), request.image_reference.as_str(), request.protocol.as_str(), + policy.policy_id.as_str(), ] { hasher.update(component.as_bytes()); hasher.update([0]); @@ -493,10 +497,24 @@ fn fingerprint_request(request: &ApplicationServiceRequest) -> String { hasher.update(argument.as_bytes()); hasher.update([0]); } - hasher.update(request.resources.memory_bytes.to_be_bytes()); - hasher.update(request.resources.cpu_millicores.to_be_bytes()); - hasher.update(request.resources.maximum_processes.to_be_bytes()); - hasher.update(request.resources.lease_seconds.to_be_bytes()); - hasher.update(request.resources.tmpfs_bytes.to_be_bytes()); + for value in [ + request.resources.memory_bytes, + u64::from(request.resources.cpu_millicores), + u64::from(request.resources.maximum_processes), + u64::from(request.resources.lease_seconds), + request.resources.tmpfs_bytes, + policy.maximum_memory_bytes, + u64::from(policy.maximum_cpu_millicores), + u64::from(policy.maximum_processes), + u64::from(policy.maximum_lease_seconds), + policy.maximum_tmpfs_bytes, + policy.readiness_timeout_millis, + policy.readiness_poll_interval_millis, + u64::from(policy.shutdown_grace_seconds), + u64::from(policy.run_as_user_id), + u64::from(policy.run_as_group_id), + ] { + hasher.update(value.to_be_bytes()); + } format!("{:x}", hasher.finalize()) } From 1911c963aa6b37f19ae66f2f0122068c04c38f9f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:51:34 +0900 Subject: [PATCH 11/44] docs: record caller-scoped lease coordination --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a277e545..5cb8faf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ The format follows Keep a Changelog, and this project uses Semantic Versioning. - Versioned `ApplicationServiceLease`, `IsolationAttestation`, and `CleanupReceipt` evidence contracts. - Process-boundary fake-Podman integration tests covering launch/readiness/termination and fail-closed readiness cleanup. - Real rootless-Podman acceptance covering the pinned backend, immutable fixture pre-pull, effective isolation, bounded HTTP readiness, explicit cleanup, and final container/network leak rejection on the reviewed source head. +- Caller-scoped `LeaseOwnerId`, `ApplicationServiceBackend` port, and process-local `ApplicationServiceCoordinator` for active-lease ownership, idempotent replay, bounded expiry cleanup, and backend-neutral lifecycle coordination. +- Regression coverage for duplicate retry suppression, changed-request conflicts, effective-policy conflicts, wrong-owner termination, concurrent duplicate launch, failed-launch reservation release, and expired-lease attribution. - Consumer owner-path integration issue for `contextual-orchestrator` so Chat/Agent domain code consumes the published lease contract rather than directly invoking Podman/containerd. - Architectural fitness validation for unique ADR identifiers, bounded-context dependency direction, and infrastructure-adapter placement. @@ -32,6 +34,7 @@ The format follows Keep a Changelog, and this project uses Semantic Versioning. - Product responsibility is broadened from artifact-analysis-only to reusable hostile-workload isolation plus artifact-analysis evidence while preserving consumer business authority. - Artifact-analysis implementation moved from generic crate-root files into `src/artifact_analysis/` to match the accepted DDD bounded context while preserving the public crate facade. - Rootless Podman implementation moved from the Core `sandbox_execution` path into `src/infrastructure/`; the Core no longer depends on `application_service` error types. +- Podman now implements the application-service lifecycle port from `src/infrastructure/`; the Supporting `application_service` coordinator does not depend on the concrete Podman adapter. - Pre-publication duplicate ADR identifiers were consolidated into the canonical ADR 0001–0006 sequence before protected-branch integration. - Evidence identity now includes policy, source revision, and ordered analyzer identifiers. - Static analyzer findings are restricted to file-format and static-capability evidence. @@ -52,8 +55,11 @@ The format follows Keep a Changelog, and this project uses Semantic Versioning. - Podman backend must report rootless mode; service publication is validated as IPv4 loopback before a lease is returned. - Podman host proxy environment inheritance is explicitly disabled so `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` values cannot become ambient application inputs. - Partial-launch/readiness failures attempt cleanup and cleanup uncertainty becomes `CleanupFailed` rather than being hidden. +- Lease ownership is scoped by authenticated command context rather than an untrusted request field; wrong-owner cleanup fails before the backend is invoked. +- Application-service replay is bound to both immutable request content and the effective isolation policy, so a changed policy cannot silently reuse a lease created under older limits. ### Not yet release evidence - The real rootless-Podman lane passed on the reviewed source head, but final release readiness still requires the same acceptance to remain green on the unchanged release head together with verify, complete coverage, security, SAST, review, SBOM, provenance, and protected-merge evidence. Fake-process tests alone remain insufficient isolation proof. -- Durable orphan/lease reclamation, gVisor/containerd/Kubernetes adapters, controlled egress, secret broker, and stronger dynamic-detonation profiles remain follow-on work. +- Caller-scoped lease ownership is currently process-local; authenticated transport binding, durable restart/orphan reclamation, distributed admission/resource reservation, stable wire errors, and signed durable receipts remain follow-on work. +- gVisor/containerd/Kubernetes adapters, controlled egress, secret broker, and stronger dynamic-detonation profiles remain follow-on work. From e50c5d9b1eb7f4a5dd5b3dd39b8f79df2a1d2a92 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:52:42 +0900 Subject: [PATCH 12/44] docs: record lease ownership gap closure --- docs/product-technical-gap-baseline.md | 73 +++++++++++++------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index fb862364..0bedbb01 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,6 +1,6 @@ # Product and Technical Gap Baseline -Last reviewed against active PR #1 source head `984d3a6ea2c267c8dd647fabf698465eb4ac0980` on 2026-09-01. This documentation follow-up may advance the PR commit without changing that reviewed source tree. This file distinguishes code already present on the active PR from work that still requires implementation and exact-head verification. Protected `develop` remains the shipped authority until merge. Any later source change makes the observations below stale until revalidated. +Last reviewed against stacked PR #6 source head `03c2dc67e3ad04fd407f9d203e912e8f6cd1c14e` and its parent PR #1 head `4b0bb12e9bd10e5c9bf65ca970ab3b8332c5e972` on 2026-09-01. Documentation-only commits may advance PR #6 without changing that reviewed source tree. This file distinguishes active-PR evidence from protected `develop`, which remains shipped authority until merge. Any later source change makes these observations stale until revalidated. ## Product responsibility @@ -15,9 +15,11 @@ The Core bounded context is `sandbox_execution`. `artifact_analysis` and `applic | Gap | Current evidence | Status | Required action | | --- | --- | --- | --- | -| Artifact analysis was flattened under root `contracts.rs`, `ingestion.rs`, and `runtime.rs`. | Active PR keeps the implementation under `src/artifact_analysis/` while preserving public crate exports. Repository policy forbids the obsolete root paths. | Corrected on active PR | Preserve the architectural fitness gate and exact-head tests. | -| Podman infrastructure was located under Core `src/sandbox_execution/` and depended on Supporting `ApplicationService*` types. | Source head `984d3a6…` keeps Podman in `src/infrastructure/podman.rs`, uses `SandboxExecutionError` for Core validation, translates it to `ApplicationServiceError` at the crate composition boundary, fixes the moved artifact-ingestion import, and retains dependency/path fitness checks. | Corrected and locally verified on active PR | Keep the DDD fitness tests and both error boundaries green. Issue #4's reported mismatch is corrected; protected-branch and hosted exact-head evidence remain required. | -| Pre-publication ADR files reused identifiers `0001`–`0004`, while `docs/adr/README.md` indexed a different canonical line. | Reviewed source head `984d3a6…` contains only canonical ADRs `0001`–`0006` and enforces identifier uniqueness. | Corrected and locally verified on active PR | Keep the uniqueness gate green. | +| Artifact analysis was flattened under root `contracts.rs`, `ingestion.rs`, and `runtime.rs`. | Parent PR #1 keeps the implementation under `src/artifact_analysis/` while preserving public crate exports. Repository policy forbids the obsolete root paths. | Corrected on parent PR | Preserve the architectural fitness gate and exact-head tests. | +| Podman infrastructure was located under Core `src/sandbox_execution/` and depended on Supporting `ApplicationService*` types. | Parent source head keeps Podman in `src/infrastructure/podman.rs`, uses `SandboxExecutionError` for Core validation, and translates it at the crate composition boundary. | Corrected on parent PR | Keep the DDD fitness tests and both error boundaries green. Issue #4 remains open until protected evidence is complete. | +| Application-service lifecycle ownership initially had no backend-neutral coordination boundary. | PR #6 adds `ApplicationServiceBackend` in the Supporting `application_service` context and implements the Podman adapter in `src/infrastructure/application_service_backend.rs`; the coordinator does not depend on concrete Podman types. | Corrected on stacked PR #6; exact-head verification pending | Keep the port implementation in infrastructure and add future containerd/gVisor implementations without leaking backend DTOs into the domain. | +| Caller identity could have been added to the untrusted application payload. | PR #6 uses bounded `LeaseOwnerId` as authenticated command context, separate from `ApplicationServiceRequest`. | Corrected by design on stacked PR #6 | The future transport must derive this value from verified caller identity; never trust a payload-supplied owner string as authentication. | +| Pre-publication ADR files reused identifiers `0001`–`0004`, while `docs/adr/README.md` indexed a different canonical line. | Parent PR contains only canonical ADRs `0001`–`0006` and enforces identifier uniqueness. | Corrected on parent PR | Keep the uniqueness gate green. | | Product-authority documentation was split across PR #3 and PR #1. | PR #3 was semantically compared against the broader current product line and closed as superseded; its applicable authority/consumer-contract content is preserved or strengthened in #1. | Converged | Keep one canonical documentation line in #1/protected `develop`. | | Repository name `quarantine-sandbox-runtime` is security-biased for the broader responsibility. | Runtime responsibility now includes isolated application services in addition to artifact analysis. Repository-settings rename support is outside this writer path. | Known gap | Re-evaluate a rename such as `isolation-runtime` before GA through an authorized repository-settings path; preserve redirects and consumer migration if renamed. | @@ -25,23 +27,26 @@ The Core bounded context is `sandbox_execution`. `artifact_analysis` and `applic | Capability | Current evidence | Status | Required action | | --- | --- | --- | --- | -| Immutable application identity | `ApplicationServiceRequest` accepts only lower-case `@sha256:<64 hex>` image references; Podman uses `--pull=never`. | Active PR | Add registry/import admission as a separate trusted control-plane operation if required; never pull implicitly during task launch. | -| Rootless backend | Adapter probes Podman `Host.Security.Rootless` and fails closed unless it returns `true`. | Active PR | Add real rootless Podman E2E in CI/operational acceptance. | -| Read-only/writable surface bounds | `--read-only`, `--read-only-tmpfs=false`, bounded `/tmp` with `noexec,nosuid,nodev`, `--image-volume=ignore`, `--no-hosts`, `--no-hostname`, `--systemd=false`, `--sdnotify=ignore`, and `--http-proxy=false`. | Active PR | Add explicit read-only input mounts through a typed mount contract only when required; no arbitrary host path request. | -| Privilege/namespace isolation | `--cap-drop=all`, `no-new-privileges`, `--userns=auto`, private PID/IPC/UTS/cgroup namespaces, numeric non-root UID/GID. | Active PR | Verify the effective runtime state in real-container E2E instead of relying only on argv construction. | -| Network isolation | Per-sandbox `--internal --disable-dns` bridge and loopback-only random host port publication. | Active PR | Add packet-level/real-container proof of no external route. Controlled egress must be a separate reviewed profile. | -| Credential isolation | No consumer/provider credentials, environment map, credential mounts, runtime sockets, or host devices exist in the P0 request contract; proxy environment inheritance is disabled. | Active PR | Add an explicit secret-broker design only if a buyer workflow cannot operate without a task-scoped capability; default remains credential-free. | -| Resource limits | Memory, CPU, PID, lease, tmpfs, readiness timeout/polling, and shutdown grace are policy-bounded. Podman receives memory/CPU/PID/runtime controls. | Active PR | Test rootless cgroup behavior on representative Linux hosts and fail closed when a requested limit cannot be enforced. | -| Process-boundary lifecycle | Adapter invokes Podman directly without a shell, checks rootless mode, creates network/container, starts, resolves loopback port, gates readiness, and removes resources. Fake-Podman tests exercise failure paths. Exact source head `984d3a6…` also passed hosted real rootless-Podman E2E, including effective isolation, HTTP readiness, cleanup, and final no-leak checks. | Active PR, real P0 lane verified on reviewed source head | Preserve the real lane; add a durable crash-recovery reaper for stale container/network resources. | -| Runtime attestation | Lease records schema, request, immutable image, backend, sandbox/network, policy, endpoint, timestamps, and isolation facts. Cleanup receipt records removal. | Active PR | Bind attestation to build/source identity and later sign durable receipts. | -| gVisor/containerd | Architecture targets only. | Missing | Implement a separate backend adapter after Podman contract stabilizes; verify compatibility and isolation deltas. | +| Immutable application identity | `ApplicationServiceRequest` accepts only lower-case `@sha256:<64 hex>` image references; Podman uses `--pull=never`. | Parent PR | Add registry/import admission as a separate trusted control-plane operation if required; never pull implicitly during task launch. | +| Rootless backend | Adapter probes Podman `Host.Security.Rootless` and fails closed unless it returns `true`. | Parent PR | Preserve the real rootless Podman lane on every release head. | +| Read-only/writable surface bounds | `--read-only`, bounded `/tmp`, image volumes ignored, host proxy inheritance disabled, and no arbitrary host mount field in the request contract. | Parent PR | Add explicit read-only input mounts through a typed mount contract only when required; no arbitrary host path request. | +| Privilege/namespace isolation | All capabilities dropped, no-new-privileges, automatic user namespace, private process-related namespaces, numeric non-root UID/GID. | Parent PR | Preserve effective-runtime verification in the real-container lane. | +| Network isolation | Per-sandbox internal DNS-disabled bridge and loopback-only random host port publication. | Parent PR | Controlled egress must be a separate reviewed profile; preserve no-external-route proof in real-container acceptance. | +| Credential isolation | No consumer/provider credentials, environment map, credential mounts, runtime sockets, or host devices exist in the P0 request contract; proxy environment inheritance is disabled. | Parent PR | Add a secret-broker design only if a buyer workflow cannot operate without a task-scoped capability; default remains credential-free. | +| Resource limits | Memory, CPU, PID, lease, tmpfs, readiness timeout/polling, and shutdown grace are policy-bounded. Podman receives memory/CPU/PID/runtime controls. | Parent PR | Test rootless cgroup behavior on representative Linux hosts and fail closed when a requested limit cannot be enforced. | +| Process-boundary lifecycle | Adapter invokes Podman directly without a shell, creates and verifies the isolated service, returns a loopback endpoint, and removes runtime-owned resources. Parent source head passed hosted real rootless-Podman E2E. | Parent PR, real P0 lane verified on reviewed source head | Preserve the real lane; add durable crash-recovery reclamation for stale resources. | +| Caller-scoped lease ownership | PR #6 keys active state by authenticated-command `LeaseOwnerId` plus request ID; wrong-owner termination returns `UnknownLease` before backend cleanup. | Implemented on stacked PR #6; exact-head verification pending | Bind `LeaseOwnerId` to an authenticated versioned transport and add durable ownership state before multi-process deployment. | +| Idempotent launch replay | PR #6 returns the existing active lease for the same owner/request/effective-policy fingerprint, rejects changed request or policy as `IdempotencyConflict`, rejects concurrent duplicates as `LaunchInProgress`, and clears failed launch reservations for corrected retries. | Implemented on stacked PR #6; exact-head verification pending | Publish stable wire-level conflict/in-progress codes with the transport contract; add durable replay semantics before restart recovery claims. | +| Bounded expiry cleanup | PR #6 cleans at most 64 expired active leases per pass, retains failed cleanup for retry, and attributes outcomes to owner/request/lease without claiming restart recovery. | Implemented on stacked PR #6; exact-head verification pending | Add persistent lease journal plus orphan reconciliation before GA. | +| Runtime attestation | Lease records schema, request, immutable image, backend, sandbox/network, policy, endpoint, timestamps, and isolation facts. Cleanup receipt records removal. | Parent PR | Bind attestation to build/source identity, backend version, and effective policy digest; later sign durable receipts. | +| gVisor/containerd | Architecture targets only. | Missing | Implement a separate backend adapter after the Podman and lifecycle-port contracts stabilize; verify compatibility and isolation deltas. | | Kubernetes | No RuntimeClass/Job/Service adapter exists. | Missing | Add managed deployment profile without exposing backend implementation in consumer domain models. | ## Artifact analysis | Capability | Current evidence | Status | Required action | | --- | --- | --- | --- | -| Static foundation | SHA-256 identity, bounded ingestion, format detection, analyzer interface, deterministic evidence and failure attribution. | Active PR | Preserve under `artifact_analysis`; complete exact-head coverage/review. | +| Static foundation | SHA-256 identity, bounded ingestion, format detection, analyzer interface, deterministic evidence and failure attribution. | Parent PR | Preserve under `artifact_analysis`; complete exact-head coverage/review. | | YARA-X / capa / Ghidra / LIEF | No production adapters yet. | Missing | Add one analyzer per bounded increment with version/digest provenance and hostile fixtures. | | Linux detonation | No dynamic worker integrated. | Missing | Consume `sandbox_execution` or a stronger gVisor/microVM profile; never execute hostile bytes in the control process. | | Windows detonation | No Windows VM worker. | Missing | Separate Windows execution pool; preserve the same evidence contract. | @@ -53,34 +58,28 @@ The Core bounded context is `sandbox_execution`. `artifact_analysis` and `applic | Consumer | Authority retained by consumer | Runtime integration state | | --- | --- | --- | | Wardnet | verdict, incident, quarantine, block/allow/review, notification, retention | Runtime contract is consumer-neutral; Wardnet ACL still requires owner-path implementation after runtime publication. | -| contextual-orchestrator | chat, agent/task/tool policy, authorization, immutable application selection, secrets, user-visible action | Owner-path issue #991 exists. It requires an ACL over a published immutable runtime artifact, lease cleanup on every task terminal state, and no direct Podman/containerd calls from Agent domain code. | +| contextual-orchestrator | chat, agent/task/tool policy, authorization, immutable application selection, secrets, user-visible action | Owner-path issue #991 exists. It requires an ACL over a published immutable runtime artifact, caller identity mapped into runtime command context, lease cleanup on every task terminal state, and no direct Podman/containerd calls from Agent domain code. | -The runtime still exposes only an embeddable Rust library. There is no supported authenticated -process boundary or generated Python consumer yet, and `127.0.0.1` lease reachability is defined -only for a co-located host-network consumer. Before issue #991 can integrate, this repository must -choose and publish one versioned transport or binding, bind leases to authenticated caller identity -and idempotency scope, define the supported network topology, and provide stable bounded wire errors -and semantic lease/cleanup validation. A sibling checkout, ad-hoc subprocess protocol, or direct -Podman call is not an acceptable substitute. +PR #6 closes the **process-local** caller ownership and idempotency portion of the consumer gap if its exact-head gates pass. The runtime still exposes only an embeddable Rust library: there is no supported authenticated process boundary or generated Python consumer, and `127.0.0.1` lease reachability is defined only for a co-located host-network consumer. Before issue #991 can integrate, this repository must choose and publish one versioned transport or binding, define the supported network topology, and provide stable bounded wire errors plus semantic lease/cleanup validation. A sibling checkout, ad-hoc subprocess protocol, or direct Podman call is not an acceptable substitute. ## Verification and release gaps -- Reviewed source head `984d3a6ea2c267c8dd647fabf698465eb4ac0980` passes `cargo test --locked --workspace --all-targets` locally (the dedicated real-Podman acceptance remains ignored off its Linux lane), `cargo clippy --locked --workspace --all-targets -- -D warnings`, warning-denied rustdoc, repository policy validation, rustfmt, and diff-check with the pinned Rust 1.97.1 toolchain. -- The DDD extraction's two build breaks are corrected: artifact runtime uses its sibling ingestion module, and application-service request validation translates Core resource errors through the public boundary. The direct-Core test expects `SandboxExecutionError`; request-level tests retain `ApplicationServiceError`. -- Hosted job `99750285437` passed the real rootless-Podman acceptance at exact source head `984d3a6…`: pinned Podman 5.8.4/rootless verification, immutable image pre-pull, effective isolation/HTTP service checks, explicit cleanup, and the final container/network leak rejection all succeeded. The prior run exposed a connection-reset panic before cleanup; `984d3a6…` makes the live HTTP probe bounded and guarantees termination after every post-lease assertion result. -- Verify, coverage, branch-coverage, security, SAST, OpenCode, and Noema checks remain queued or pending. Pending/queued is non-passing, and the documentation-only tip still requires its own exact-head required workflows. -- Organization ruleset `CWL Central required workflows` is active on the default branch and requires one approving review, resolved review threads, and the central required workflows; bypass capability is not merge evidence and must not be used by this loop. -- Real Podman isolation is proven for the pinned hosted runner and policy above; broader host profiles and a published release remain unproven. +- Parent reviewed source head `984d3a6ea2c267c8dd647fabf698465eb4ac0980` passed its local pinned Rust suite and hosted real rootless-Podman acceptance; parent documentation has since advanced without changing that source behavior. +- Parent hosted job `99750285437` passed pinned Podman 5.8.4/rootless verification, immutable image pre-pull, effective isolation/HTTP service checks, explicit cleanup, and final container/network leak rejection. +- PR #6 source head `03c2dc67e3ad04fd407f9d203e912e8f6cd1c14e` adds caller ownership/idempotency and an effective-policy replay regression, but this execution environment has not produced a local-green claim. Its exact-head CI/security/SAST/review evidence is authoritative and currently pending/queued. +- Organization ruleset `CWL Central required workflows` requires qualifying approval and central required workflows; bypass capability is not merge evidence. +- Real Podman isolation is proven for the reviewed parent hosted runner/profile only; broader host profiles and a published release remain unproven. - The dependency-review evidence path has previously returned HTTP 403; it must be revalidated on the final exact head rather than bypassed. -- PR #1 remains Draft until current-head implementation, coverage, real-container evidence appropriate to the claims, security review, documentation convergence, and repository policy all pass. +- PR #1 and stacked PR #6 remain Draft until their current-head implementation, coverage, security, review, documentation, and applicable real-container evidence satisfy repository policy. - A release/version bump is premature until one integrated protected head satisfies all required gates and produces reproducible package/SBOM/provenance evidence. ## Next bounded slices -1. Obtain terminal exact-head hosted verify, complete coverage, security, SAST, and independent-review evidence; keep the now-passing real rootless-Podman lane green. -2. Choose and implement the authenticated, versioned consumer transport or supported language binding, including caller-scoped idempotency, stable wire errors, strict response validation, and an explicit co-location/network-topology contract. -3. Add crash/restart lease reclamation and orphan cleanup evidence. -4. Publish an immutable runtime artifact, generated/typed consumer contract, SBOM, and provenance bound into lease attestation. -5. Integrate `contextual-orchestrator` through issue #991 and Wardnet through its own ACL, without moving consumer policy into this repository. -6. Add gVisor/containerd and Kubernetes RuntimeClass adapters only after the P0 contract is stable. -7. Resume artifact-analysis adapters and dynamic detonation on top of the shared sandbox execution Core. +1. Obtain terminal exact-head verify, complete coverage, security, SAST, and independent-review evidence for parent PR #1 while keeping the real rootless-Podman lane green. +2. Drive stacked PR #6 through exact-head compile/test/coverage/security/review, repair any valid findings, and merge it only after #1 is protected and #6 is revalidated on the resulting base. +3. Choose and implement the authenticated, versioned consumer transport or supported language binding with stable wire errors, strict response validation, and an explicit co-location/network-topology contract. +4. Add durable crash/restart lease reclamation, orphan cleanup, and admission/resource reservation evidence. +5. Publish an immutable runtime artifact, generated/typed consumer contract, SBOM, and provenance bound into lease attestation. +6. Integrate `contextual-orchestrator` through issue #991 and Wardnet through its own ACL, without moving consumer policy into this repository. +7. Add gVisor/containerd and Kubernetes RuntimeClass adapters only after the P0 and lifecycle-port contracts are stable. +8. Resume artifact-analysis adapters and dynamic detonation on top of the shared sandbox execution Core. From 7423eb1fedbcaec1dada20433cbabcaaeb20367b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:53:58 +0900 Subject: [PATCH 13/44] refactor: place lease coordinator in application service namespace --- src/application_service/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/application_service/mod.rs b/src/application_service/mod.rs index 32e44b81..c57d2a4b 100644 --- a/src/application_service/mod.rs +++ b/src/application_service/mod.rs @@ -1,5 +1,12 @@ //! Supporting bounded context for launching an approved application as an isolated service. +mod coordinator; + +pub use coordinator::{ + ApplicationServiceBackend, ApplicationServiceCoordinator, ApplicationServiceCoordinatorError, + ExpiredLeaseCleanupResult, LeaseOwnerId, +}; + use serde::{Deserialize, Serialize}; use thiserror::Error; From e7934d92487f35044898ff302a90438f1834f4ec Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:54:26 +0900 Subject: [PATCH 14/44] refactor: export coordinator through bounded context --- src/lib.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 277753ca..c60a49fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,19 +9,15 @@ //! conversation, task, tool-selection, secret, and user-action authority. mod application_service; -#[path = "application_service/coordinator.rs"] -mod application_service_coordinator; mod artifact_analysis; mod infrastructure; mod sandbox_execution; pub use application_service::{ - ApplicationServiceError, ApplicationServiceLease, ApplicationServiceRequest, CleanupReceipt, - IsolationAttestation, ServiceEndpoint, ServiceProtocol, -}; -pub use application_service_coordinator::{ ApplicationServiceBackend, ApplicationServiceCoordinator, ApplicationServiceCoordinatorError, - ExpiredLeaseCleanupResult, LeaseOwnerId, + ApplicationServiceError, ApplicationServiceLease, ApplicationServiceRequest, CleanupReceipt, + ExpiredLeaseCleanupResult, IsolationAttestation, LeaseOwnerId, ServiceEndpoint, + ServiceProtocol, }; pub use artifact_analysis::{ AnalysisEngine, AnalysisError, AnalysisProfile, AnalysisRequest, AnalyzerFailure, From 806a0463224f99b8587e2973a8eec5d52caa7129 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 14:56:49 +0900 Subject: [PATCH 15/44] refactor: simplify coordinator state invariants --- src/application_service/coordinator.rs | 126 ++++++------------------- 1 file changed, 31 insertions(+), 95 deletions(-) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index 4d97f3a1..ba8bd995 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -5,10 +5,10 @@ use std::{collections::BTreeMap, sync::Mutex}; use sha2::{Digest, Sha256}; use thiserror::Error; -use crate::{ +use super::{ ApplicationServiceError, ApplicationServiceLease, ApplicationServiceRequest, CleanupReceipt, - IsolationPolicy, }; +use crate::IsolationPolicy; const MAX_LEASE_OWNER_ID_BYTES: usize = 128; const MAX_EXPIRED_CLEANUPS_PER_CALL: usize = 64; @@ -256,41 +256,19 @@ where { Ok(lease) => lease, Err(error) => { - self.remove_matching_reservation(&key, &request_fingerprint)?; + self.lock_registry()?.remove(&key); return Err(error.into()); } }; - let mut leases = match self.leases.lock() { - Ok(leases) => leases, - Err(_) => { - return match self.backend.terminate_at(&lease, started_at_epoch_seconds) { - Ok(_) => Err(ApplicationServiceCoordinatorError::StateUnavailable), - Err(error) => Err(error.into()), - }; - } - }; - match leases.get(&key) { - Some(RegistryEntry::Launching { - request_fingerprint: existing, - }) if existing == &request_fingerprint => { - leases.insert( - key, - RegistryEntry::Active { - request_fingerprint, - lease: lease.clone(), - }, - ); - Ok(lease) - } - _ => { - drop(leases); - match self.backend.terminate_at(&lease, started_at_epoch_seconds) { - Ok(_) => Err(ApplicationServiceCoordinatorError::StateUnavailable), - Err(error) => Err(error.into()), - } - } - } + self.lock_registry()?.insert( + key, + RegistryEntry::Active { + request_fingerprint, + lease: lease.clone(), + }, + ); + Ok(lease) } /// Terminate one lease only when the caller owns the active registry entry. @@ -360,27 +338,22 @@ where ) -> Result, ApplicationServiceCoordinatorError> { let candidates = { let mut leases = self.lock_registry()?; - let keys: Vec = leases + let candidates: Vec<(LeaseKey, String, ApplicationServiceLease)> = leases .iter() .filter_map(|(key, entry)| match entry { - RegistryEntry::Active { lease, .. } - if lease.expires_at_epoch_seconds() <= now_epoch_seconds => - { - Some(key.clone()) - } + RegistryEntry::Active { + request_fingerprint, + lease, + } if lease.expires_at_epoch_seconds() <= now_epoch_seconds => Some(( + key.clone(), + request_fingerprint.clone(), + lease.clone(), + )), _ => None, }) .take(MAX_EXPIRED_CLEANUPS_PER_CALL) .collect(); - let mut candidates = Vec::with_capacity(keys.len()); - for key in keys { - let Some(RegistryEntry::Active { - request_fingerprint, - lease, - }) = leases.get(&key).cloned() - else { - continue; - }; + for (key, request_fingerprint, lease) in &candidates { leases.insert( key.clone(), RegistryEntry::Terminating { @@ -388,7 +361,6 @@ where lease: lease.clone(), }, ); - candidates.push((key, request_fingerprint, lease)); } candidates }; @@ -418,23 +390,6 @@ where .map_err(|_| ApplicationServiceCoordinatorError::StateUnavailable) } - fn remove_matching_reservation( - &self, - key: &LeaseKey, - request_fingerprint: &str, - ) -> Result<(), ApplicationServiceCoordinatorError> { - let mut leases = self.lock_registry()?; - if matches!( - leases.get(key), - Some(RegistryEntry::Launching { - request_fingerprint: existing, - }) if existing == request_fingerprint - ) { - leases.remove(key); - } - Ok(()) - } - fn finish_termination( &self, key: &LeaseKey, @@ -443,35 +398,16 @@ where result: &Result, ) -> Result<(), ApplicationServiceCoordinatorError> { let mut leases = self.lock_registry()?; - match result { - Ok(_) => { - if matches!( - leases.get(key), - Some(RegistryEntry::Terminating { - request_fingerprint: existing, - lease: registered_lease, - }) if existing == &request_fingerprint && registered_lease == &lease - ) { - leases.remove(key); - } - } - Err(_) => { - if matches!( - leases.get(key), - Some(RegistryEntry::Terminating { - request_fingerprint: existing, - lease: registered_lease, - }) if existing == &request_fingerprint && registered_lease == &lease - ) { - leases.insert( - key.clone(), - RegistryEntry::Active { - request_fingerprint, - lease, - }, - ); - } - } + if result.is_ok() { + leases.remove(key); + } else { + leases.insert( + key.clone(), + RegistryEntry::Active { + request_fingerprint, + lease, + }, + ); } Ok(()) } From 008209f0cf7152f06053f44d65662e4e023787ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 17:39:51 +0900 Subject: [PATCH 16/44] test: expose expired cleanup starvation --- tests/application_service_cleanup_fairness.rs | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 tests/application_service_cleanup_fairness.rs diff --git a/tests/application_service_cleanup_fairness.rs b/tests/application_service_cleanup_fairness.rs new file mode 100644 index 00000000..7f760aab --- /dev/null +++ b/tests/application_service_cleanup_fairness.rs @@ -0,0 +1,149 @@ +//! Cleanup fairness regression for repeated failed application-service teardown. + +use std::sync::{Arc, Mutex}; + +use quarantine_sandbox_runtime::{ + ApplicationServiceBackend, ApplicationServiceCoordinator, ApplicationServiceError, + ApplicationServiceLease, ApplicationServiceRequest, CleanupReceipt, IsolationPolicy, + LeaseOwnerId, ResourceRequest, ServiceProtocol, +}; +use serde_json::json; + +#[derive(Clone)] +struct FailingCleanupBackend { + attempted_request_ids: Arc>>, +} + +impl ApplicationServiceBackend for FailingCleanupBackend { + fn launch_at( + &self, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result { + let lease = json!({ + "schema_version": "1.1.0", + "request_id": request.request_id.clone(), + "image_reference": request.image_reference.clone(), + "backend_id": "failing_cleanup_test_backend", + "sandbox_id": format!("sandbox-{}", request.request_id), + "network_id": format!("network-{}", request.request_id), + "policy_id": policy.policy_id.clone(), + "policy_sha256": policy.effective_policy_sha256(), + "endpoint": { + "host": "127.0.0.1", + "port": 49_152, + "protocol": "http" + }, + "started_at_epoch_seconds": started_at_epoch_seconds, + "expires_at_epoch_seconds": started_at_epoch_seconds + + u64::from(request.resources.lease_seconds), + "shutdown_grace_seconds": policy.shutdown_grace_seconds, + "isolation_attestation": { + "rootless": true, + "read_only_root_filesystem": true, + "all_capabilities_dropped": true, + "no_new_privileges": true, + "isolated_user_namespace": true, + "external_egress_denied": true, + "loopback_only_publication": true, + "credentials_available": false + } + }); + serde_json::from_value(lease).map_err(|_| ApplicationServiceError::BackendCommandFailed { + operation: "test_lease_decode", + }) + } + + fn terminate_at( + &self, + lease: &ApplicationServiceLease, + _terminated_at_epoch_seconds: u64, + ) -> Result { + self.attempted_request_ids + .lock() + .expect("test attempt registry should not be poisoned") + .push(lease.request_id().to_owned()); + Err(ApplicationServiceError::CleanupFailed) + } +} + +fn policy() -> IsolationPolicy { + IsolationPolicy { + policy_id: "cleanup_fairness_policy_v1".to_owned(), + maximum_memory_bytes: 512 * 1024 * 1024, + maximum_cpu_millicores: 2_000, + maximum_processes: 128, + maximum_lease_seconds: 900, + maximum_tmpfs_bytes: 128 * 1024 * 1024, + readiness_timeout_millis: 2_000, + readiness_poll_interval_millis: 10, + shutdown_grace_seconds: 2, + run_as_user_id: 65_532, + run_as_group_id: 65_532, + } +} + +fn request(request_id: String) -> ApplicationServiceRequest { + ApplicationServiceRequest { + schema_version: "1.0.0".to_owned(), + request_id, + image_reference: format!("localhost/cwl/tool@sha256:{}", "f".repeat(64)), + container_port: 8_080, + protocol: ServiceProtocol::Http, + command: vec!["serve".to_owned()], + resources: ResourceRequest { + memory_bytes: 256 * 1024 * 1024, + cpu_millicores: 1_000, + maximum_processes: 32, + lease_seconds: 1, + tmpfs_bytes: 32 * 1024 * 1024, + }, + } +} + +#[test] +fn repeated_cleanup_failures_do_not_starve_other_expired_leases() { + let attempted_request_ids = Arc::new(Mutex::new(Vec::new())); + let backend = FailingCleanupBackend { + attempted_request_ids: Arc::clone(&attempted_request_ids), + }; + let coordinator = ApplicationServiceCoordinator::new(backend); + let owner = LeaseOwnerId::new("urn:cwl:agent:contextual-orchestrator") + .expect("test owner should satisfy the bounded contract"); + let policy = policy(); + let started_at = 1_780_000_000; + + for index in 0..65 { + coordinator + .launch_at( + &owner, + &request(format!("cleanup_request_{index:03}")), + &policy, + started_at, + ) + .expect("test backend launch should succeed"); + } + + let first_pass = coordinator + .cleanup_expired_at(started_at + 1) + .expect("first cleanup pass should access coordinator state"); + assert_eq!(first_pass.len(), 64); + assert!(first_pass.iter().all(|outcome| outcome.result().is_err())); + + let second_pass = coordinator + .cleanup_expired_at(started_at + 1) + .expect("second cleanup pass should access coordinator state"); + assert_eq!(second_pass.len(), 64); + assert!( + second_pass + .iter() + .any(|outcome| outcome.request_id() == "cleanup_request_064"), + "a previously unattempted expired lease must be selected before retrying all failed cleanup entries" + ); + + let attempts = attempted_request_ids + .lock() + .expect("test attempt registry should remain available"); + assert!(attempts.contains(&"cleanup_request_064".to_owned())); +} From 410fce1231c0bedd47e6a4c82ea330825977c2f4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 17:43:01 +0900 Subject: [PATCH 17/44] fix: prevent expired cleanup starvation --- src/application_service/coordinator.rs | 51 +++++++++++++++++++++----- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index ba8bd995..d730dc41 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -133,10 +133,12 @@ enum RegistryEntry { Active { request_fingerprint: String, lease: ApplicationServiceLease, + cleanup_attempts: u32, }, Terminating { request_fingerprint: String, lease: ApplicationServiceLease, + cleanup_attempts: u32, }, } @@ -231,6 +233,7 @@ where Some(RegistryEntry::Active { request_fingerprint: existing, lease, + .. }) if existing == &request_fingerprint => return Ok(lease.clone()), Some(RegistryEntry::Terminating { request_fingerprint: existing, @@ -266,6 +269,7 @@ where RegistryEntry::Active { request_fingerprint, lease: lease.clone(), + cleanup_attempts: 0, }, ); Ok(lease) @@ -284,26 +288,29 @@ where terminated_at_epoch_seconds: u64, ) -> Result { let key = LeaseKey::new(lease_owner_id, lease.request_id()); - let (request_fingerprint, registered_lease) = { + let (request_fingerprint, registered_lease, cleanup_attempts) = { let mut leases = self.lock_registry()?; match leases.get(&key) { Some(RegistryEntry::Active { request_fingerprint, lease: registered_lease, + cleanup_attempts, }) => { if registered_lease != lease { return Err(ApplicationServiceCoordinatorError::LeaseMismatch); } let request_fingerprint = request_fingerprint.clone(); let registered_lease = registered_lease.clone(); + let cleanup_attempts = *cleanup_attempts; leases.insert( key.clone(), RegistryEntry::Terminating { request_fingerprint: request_fingerprint.clone(), lease: registered_lease.clone(), + cleanup_attempts, }, ); - (request_fingerprint, registered_lease) + (request_fingerprint, registered_lease, cleanup_attempts) } Some(RegistryEntry::Launching { .. }) => { return Err(ApplicationServiceCoordinatorError::LaunchInProgress); @@ -318,15 +325,23 @@ where let result = self .backend .terminate_at(®istered_lease, terminated_at_epoch_seconds); - self.finish_termination(&key, request_fingerprint, registered_lease, &result)?; + self.finish_termination( + &key, + request_fingerprint, + registered_lease, + cleanup_attempts, + &result, + )?; result.map_err(Into::into) } /// Clean up at most 64 active leases whose expiry is not later than `now`. /// /// Failed cleanup remains registered as active so a later operator or - /// cleanup pass can retry it. This process-local function does not claim - /// crash/restart orphan recovery. + /// cleanup pass can retry it. Previously unattempted expired leases are + /// selected before repeatedly failing entries so one bad cleanup cannot + /// starve later expired workloads. This process-local function does not + /// claim crash/restart orphan recovery. /// /// # Errors /// @@ -338,27 +353,35 @@ where ) -> Result, ApplicationServiceCoordinatorError> { let candidates = { let mut leases = self.lock_registry()?; - let candidates: Vec<(LeaseKey, String, ApplicationServiceLease)> = leases + let mut candidates: Vec<(LeaseKey, String, ApplicationServiceLease, u32)> = leases .iter() .filter_map(|(key, entry)| match entry { RegistryEntry::Active { request_fingerprint, lease, + cleanup_attempts, } if lease.expires_at_epoch_seconds() <= now_epoch_seconds => Some(( key.clone(), request_fingerprint.clone(), lease.clone(), + *cleanup_attempts, )), _ => None, }) - .take(MAX_EXPIRED_CLEANUPS_PER_CALL) .collect(); - for (key, request_fingerprint, lease) in &candidates { + candidates.sort_by(|left, right| { + left.3 + .cmp(&right.3) + .then_with(|| left.0.cmp(&right.0)) + }); + candidates.truncate(MAX_EXPIRED_CLEANUPS_PER_CALL); + for (key, request_fingerprint, lease, cleanup_attempts) in &candidates { leases.insert( key.clone(), RegistryEntry::Terminating { request_fingerprint: request_fingerprint.clone(), lease: lease.clone(), + cleanup_attempts: *cleanup_attempts, }, ); } @@ -366,9 +389,15 @@ where }; let mut outcomes = Vec::with_capacity(candidates.len()); - for (key, request_fingerprint, lease) in candidates { + for (key, request_fingerprint, lease, cleanup_attempts) in candidates { let result = self.backend.terminate_at(&lease, now_epoch_seconds); - self.finish_termination(&key, request_fingerprint, lease.clone(), &result)?; + self.finish_termination( + &key, + request_fingerprint, + lease.clone(), + cleanup_attempts, + &result, + )?; outcomes.push(ExpiredLeaseCleanupResult { lease_owner_id: key.lease_owner_id, request_id: key.request_id, @@ -395,6 +424,7 @@ where key: &LeaseKey, request_fingerprint: String, lease: ApplicationServiceLease, + cleanup_attempts: u32, result: &Result, ) -> Result<(), ApplicationServiceCoordinatorError> { let mut leases = self.lock_registry()?; @@ -406,6 +436,7 @@ where RegistryEntry::Active { request_fingerprint, lease, + cleanup_attempts: cleanup_attempts.saturating_add(1), }, ); } From b9ce1ea86c8440157c7a78d6c30e91649d8895fd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 17:43:49 +0900 Subject: [PATCH 18/44] docs: record cleanup fairness repair --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ce94973..48e5d1b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ The format follows Keep a Changelog, and this project uses Semantic Versioning. - Process-boundary fake-Podman integration tests covering launch/readiness/termination and fail-closed readiness cleanup. - Real rootless-Podman acceptance covering the pinned backend, immutable fixture pre-pull, effective isolation, bounded HTTP readiness, explicit cleanup, and final container/network leak rejection on the reviewed source head. - Caller-scoped `LeaseOwnerId`, `ApplicationServiceBackend` port, and process-local `ApplicationServiceCoordinator` for active-lease ownership, idempotent replay, bounded expiry cleanup, and backend-neutral lifecycle coordination. -- Regression coverage for duplicate retry suppression, changed-request conflicts, effective-policy conflicts, wrong-owner termination, concurrent duplicate launch, failed-launch reservation release, and expired-lease attribution. +- Regression coverage for duplicate retry suppression, changed-request conflicts, effective-policy conflicts, wrong-owner termination, concurrent duplicate launch, failed-launch reservation release, expired-lease attribution, and cleanup-failure fairness across more than one bounded cleanup batch. - Consumer owner-path integration issue for `contextual-orchestrator` so Chat/Agent domain code consumes the published lease contract rather than directly invoking Podman/containerd. - Architectural fitness validation for unique ADR identifiers, bounded-context dependency direction, and infrastructure-adapter placement. @@ -37,6 +37,7 @@ The format follows Keep a Changelog, and this project uses Semantic Versioning. - Artifact-analysis implementation moved from generic crate-root files into `src/artifact_analysis/` to match the accepted DDD bounded context while preserving the public crate facade. - Rootless Podman implementation moved from the Core `sandbox_execution` path into `src/infrastructure/`; the Core no longer depends on `application_service` error types. - Podman now implements the application-service lifecycle port from `src/infrastructure/`; the Supporting `application_service` coordinator does not depend on the concrete Podman adapter. +- Failed expired-lease cleanup now increments a bounded retry-attempt counter; later cleanup passes prioritize expired leases with fewer attempts before repeatedly failing entries, preventing the first 64 failures from starving later expired workloads. - Pre-publication duplicate ADR identifiers were consolidated into the canonical ADR 0001–0006 sequence before protected-branch integration. - Evidence identity now includes policy, source revision, and ordered analyzer identifiers. - Static analyzer findings are restricted to file-format and static-capability evidence. @@ -59,6 +60,7 @@ The format follows Keep a Changelog, and this project uses Semantic Versioning. - Partial-launch/readiness failures attempt cleanup and cleanup uncertainty becomes `CleanupFailed` rather than being hidden. - Lease ownership is scoped by authenticated command context rather than an untrusted request field; wrong-owner cleanup fails before the backend is invoked. - Application-service replay is bound to both immutable request content and the full effective isolation policy, so a changed policy cannot silently reuse a lease created under older limits. +- Repeated cleanup failures cannot monopolize the bounded expiry-cleanup window and indefinitely hide other expired application-service leases. ### Not yet release evidence From 427b86e4e1ab808bc522d6a76132898e546266e2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 17:44:32 +0900 Subject: [PATCH 19/44] docs: refresh lease cleanup evidence --- docs/product-technical-gap-baseline.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d0faa6c9..d956fcc6 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,6 +1,6 @@ # Product and Technical Gap Baseline -Last reviewed on 2026-09-01 against parent PR #1 exact head `e4222208f0d1a6d28f570c45dc7385fd75b16a4b` and stacked PR #6 pre-reconciliation head `806a0463224f99b8587e2973a8eec5d52caa7129`. The stack reconciliation combines the current parent security/wire contract with #6 caller-scoped lease ownership. Protected `develop` remains shipped authority until protected integration. Hosted evidence must be regenerated for the reconciled exact head; predecessor results do not transfer. +Last reviewed on 2026-09-01 against parent PR #1 exact head `c78fd491f84fae773b3691b10b6a0c21940808d5` and stacked PR #6 exact head `b9ce1ea86c8440157c7a78d6c30e91649d8895fd`. The stack is reconciled onto the current parent security/wire contract and now includes caller-scoped lease ownership plus starvation-resistant bounded expiry cleanup. Protected `develop` remains shipped authority until protected integration. Hosted evidence must be generated for the unchanged current heads; predecessor results do not transfer. ## Product responsibility @@ -17,7 +17,7 @@ Core `sandbox_execution` owns isolation policy, resource bounds, runtime lease m | --- | --- | --- | --- | | Artifact analysis was flattened under generic crate-root files. | PR #1 places implementation under `src/artifact_analysis/` and keeps only the public crate facade at root. | Corrected on parent PR | Keep architecture fitness checks green. | | Podman implementation lived under Core and depended on Supporting application-service types. | PR #1 keeps Podman under `src/infrastructure/podman.rs`; Core owns `SandboxExecutionError`, while the application boundary translates errors. | Corrected on parent PR | Do not move container adapters back into Core. | -| Application-service lifecycle had no backend-neutral port or caller ownership. | PR #6 adds `ApplicationServiceBackend`, `ApplicationServiceCoordinator`, and `LeaseOwnerId`; Podman implements the port in infrastructure. | Implemented on stacked PR; reconciled exact-head proof required | Preserve port direction for future gVisor/containerd adapters. | +| Application-service lifecycle had no backend-neutral port or caller ownership. | PR #6 adds `ApplicationServiceBackend`, `ApplicationServiceCoordinator`, and `LeaseOwnerId`; Podman implements the port in infrastructure. | Implemented on stacked PR; exact-head proof required | Preserve port direction for future gVisor/containerd adapters. | | Product-authority documentation was split across an older documentation PR. | PR #3 was closed as superseded after preserving applicable material in PR #1. | Converged | Keep one canonical documentation line. | | Repository name is narrower than its present responsibility. | Product now serves application-service isolation as well as quarantine/artifact analysis. | Known product-name gap | Reassess rename before GA through repository-settings authority; preserve redirects and consumer migration if changed. | @@ -32,11 +32,11 @@ Core `sandbox_execution` owns isolation policy, resource bounds, runtime lease m | Network isolation | Per-sandbox internal DNS-disabled network and loopback-only host publication. | Parent PR | Controlled egress must be a separate profile and must not silently enable Internet access. | | Credential isolation | No consumer/provider credentials, arbitrary environment, runtime sockets, host devices, or ambient proxy variables enter the P0 workload contract. | Parent PR | Add a task-scoped secret broker only after an accepted ADR and explicit consumer authorization. | | Resource limits | Memory, CPU, PID, lease duration, tmpfs, readiness timeout/polling, and shutdown grace are policy bounded. | Parent PR | Verify enforcement across supported host/cgroup profiles and fail closed where enforcement is unavailable. | -| Readiness and cleanup | Real rootless-Podman lane has proven bounded readiness, explicit termination, and final no-container/no-network leak checks on parent exact head. | Parent exact-head Podman lane GREEN; full release gates pending | Preserve the real E2E lane on the reconciled and final release head. | +| Readiness and cleanup | Real rootless-Podman lane has proven bounded readiness, explicit termination, and final no-container/no-network leak checks on prior parent exact heads. | Current-head proof pending | Preserve the real E2E lane on the reconciled and final release head. | | Runtime attestation | Lease schema `1.1.0` records backend/sandbox/network/policy, canonical full-policy SHA-256, loopback endpoint, timestamps, and P0 isolation facts. | Parent PR | Bind verified build/artifact/backend version identity and later sign durable receipts. | | Caller-scoped ownership | `LeaseOwnerId` is authenticated command context, not an application payload field; wrong-owner termination fails before backend cleanup. | PR #6 | Bind the owner to an authenticated versioned transport before remote/multi-process use. | | Idempotent launch | Coordinator keys process-local state by owner + request; identical active replay returns the lease, changed request/effective policy conflicts, and concurrent duplicate launch fails closed. | PR #6 | Publish stable wire error codes and durable replay semantics before restart claims. | -| Expiry cleanup | Coordinator selects at most 64 expired leases per pass and retains failed cleanup for retry. | PR #6 | Add persistent lease journal/orphan reconciliation and admission/resource reservation. | +| Expiry cleanup | Coordinator cleans at most 64 expired leases per pass, retains failed cleanup for retry, and prioritizes lower-attempt entries so repeatedly failing early keys cannot starve later expired workloads. Hostile regression covers 65 expired leases with the first 64 cleanup attempts failing. | PR #6; exact-head GREEN pending | Add persistent lease journal/orphan reconciliation and admission/resource reservation; retain fairness after persistence. | | gVisor/containerd/Kubernetes | No production adapter. | Missing | Add separate infrastructure adapters only after P0/lifecycle contract stabilizes. | ## Artifact analysis @@ -63,32 +63,32 @@ Current integration rules: - no cross-service application-table SQL; - Context Fabric owner-path PRs/issues are inventoried read-only and receive exact evidence/acceptance criteria rather than quarantine-writer source changes. -At this review, `context-graph-contracts` live metadata still reports `develop` as its default branch. An active stacked contract PR is repairing Context Assertion message composition with the shared CloudEvent envelope. These facts are live-state observations, not assumptions about the intended stable branch or release contract. Quarantine integration remains blocked from claiming a released Context Assertion projection until the owner path publishes a compatible release and conformance evidence. +At this review, `context-graph-contracts` still has an unreleased stacked Context Assertion/CloudEvent contract line. PR #21 exact head `a3a3125619ed6e777818811b1c0b97f3a4574b73` repairs structured CloudEvent envelope binding but remains Draft with current hosted lanes non-passing. `enterprise-architecture-core` PR #40 exact head `2b14e008a11712c840d0bf6c8c5d3a1d6e9ec1ba` enforces released Context Graph bindings for foreign projections but does not yet list `quarantine-sandbox-runtime` as a separate owner projection. Both repositories remain read-only here; the Context Fabric owner path must add the quarantine runtime only after the shared contract is released and pinned. Quarantine integration remains fail-closed until then. ## Consumer integrations | Consumer | Authority retained by consumer | Runtime integration state | | --- | --- | --- | -| Wardnet | gateway/SOC policy, maliciousness verdict, incident, quarantine/block/review, notification, retention | Runtime contract is consumer-neutral; Wardnet must consume it through its own ACL/owner path after publication. | -| contextual-orchestrator | LLM/model routing, chat, Agent/task/tool policy, caller authorization, application selection, secrets, user-visible actions | Owner-path issue exists for a published immutable runtime integration. Direct Podman/containerd calls and sibling source copies are not acceptable. | +| Wardnet | gateway/SOC policy, maliciousness verdict, incident, quarantine/block/review, notification, retention | Runtime contract is consumer-neutral; Wardnet issue #38 remains the consumer owner path and must consume published runtime evidence without moving verdict authority here. | +| contextual-orchestrator | LLM/model routing, chat, Agent/task/tool policy, caller authorization, application selection, secrets, user-visible actions | Issue #991 owns the ACL integration after a protected immutable runtime release. Direct Podman/containerd calls and sibling source copies are not acceptable. | The runtime currently exposes a Rust library and loopback lease topology. It does not yet publish an authenticated network process boundary or generated Python consumer. A future transport must derive `LeaseOwnerId` from verified caller context, keep idempotency scope stable, return bounded stable wire errors, validate lease/cleanup semantics, and define co-location/network topology explicitly. ## Verification and release state -- Parent PR #1 exact head `e4222208f0d1a6d28f570c45dc7385fd75b16a4b` has a successful real rootless-Podman job `99758127137` from workflow run `33476960369`. -- Parent exact-head verify, coverage, branch-coverage and several central security/review workflows were still queued or pending at the latest read; queued/pending is non-passing. +- Parent PR #1 current exact head is `c78fd491f84fae773b3691b10b6a0c21940808d5`; CI, Security Scan, and SAST were queued at the latest read, so the head is non-passing despite earlier real rootless-Podman evidence. - PR #1 has no formal approving review at the latest read and remains Draft. -- PR #6 pre-reconciliation head `806a0463224f99b8587e2973a8eec5d52caa7129` had a successful Podman E2E job but verify/coverage/branch-coverage remained queued. Its old parent base omitted the current policy-digest/lease-1.1 contract, so the stack must be reconciled and all exact-head evidence regenerated. +- PR #6 is reconciled on that exact parent. Its current branch includes RED commit `008209f0cf7152f06053f44d65662e4e023787ba` exposing cleanup starvation and causal repair commit `410fce1231c0bedd47e6a4c82ea330825977c2f4`, followed by documentation refresh. Current exact-head CI must prove the new regression and implementation together; predecessor evidence does not transfer. - Active organization rules require qualifying review, resolved threads, and central required workflows. Admin/bypass capability is not merge evidence. - No release exists. Version/release claims remain premature until one integrated protected head passes CI, security, SAST, complete coverage/docstrings, real isolation E2E, SBOM/provenance, review, rollback/recovery and protected-merge gates together. ## Next bounded slices -1. Reconcile PR #6 non-destructively onto the current PR #1 exact head and regenerate all exact-head gates. +1. Obtain exact-head GREEN for PR #6 cleanup fairness and caller-scoped lease ownership without changing a clean head merely to retrigger queued jobs. 2. Drain parent PR #1 gate/review findings and merge only through protected policy; then revalidate the stacked lease-ownership slice against the protected parent. 3. Implement one authenticated versioned consumer transport/binding with stable wire errors and caller identity mapping. 4. Add durable restart/orphan reclamation, lease journal, admission/resource reservation, and crash-recovery evidence. 5. Publish immutable runtime artifacts with SBOM/provenance and bind verified build/backend identity into attestation. 6. Integrate Wardnet and contextual-orchestrator only through their owner paths and published contracts. -7. Add stronger gVisor/containerd/Kubernetes isolation adapters and then resume dynamic artifact-analysis profiles. +7. Through the Context Fabric owner path, add the released quarantine runtime as a distinct EA projection preserving canonical/source references, truth status, effective/system time and provenance; never project malware verdicts as authoritative EA facts. +8. Add stronger gVisor/containerd/Kubernetes isolation adapters and then resume dynamic artifact-analysis profiles. From 1f20d0e37bc4e730ca1ec36cf76872b2c5118dda Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 17:53:22 +0900 Subject: [PATCH 20/44] docs: stage issue 7 implementation intent --- docs/doctoring/tmp-issue7-plan.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/doctoring/tmp-issue7-plan.md diff --git a/docs/doctoring/tmp-issue7-plan.md b/docs/doctoring/tmp-issue7-plan.md new file mode 100644 index 00000000..40e5d50a --- /dev/null +++ b/docs/doctoring/tmp-issue7-plan.md @@ -0,0 +1,3 @@ +# Effective isolation attestation implementation intent + +This temporary planning note is superseded by the dedicated implementation branch once created. It records no shipped product claim. From 2613bf7ee2ef16852683caef67025a2b8ff90247 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 17:53:30 +0900 Subject: [PATCH 21/44] chore: remove temporary issue 7 note --- docs/doctoring/tmp-issue7-plan.md | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 docs/doctoring/tmp-issue7-plan.md diff --git a/docs/doctoring/tmp-issue7-plan.md b/docs/doctoring/tmp-issue7-plan.md deleted file mode 100644 index 40e5d50a..00000000 --- a/docs/doctoring/tmp-issue7-plan.md +++ /dev/null @@ -1,3 +0,0 @@ -# Effective isolation attestation implementation intent - -This temporary planning note is superseded by the dedicated implementation branch once created. It records no shipped product claim. From a9c572bd9081ab3089d6d961db3b016abd092b8f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 17:54:21 +0900 Subject: [PATCH 22/44] test: stage effective isolation attestation RED --- tests/effective_isolation_attestation.rs | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/effective_isolation_attestation.rs diff --git a/tests/effective_isolation_attestation.rs b/tests/effective_isolation_attestation.rs new file mode 100644 index 00000000..b91bec3f --- /dev/null +++ b/tests/effective_isolation_attestation.rs @@ -0,0 +1,3 @@ +//! Security regression for application-service attestation. + +// Intentionally added on the current stack head; implementation is added in the same bounded PR. From a7d7ca0605da7f0f07dedf6e77df86b6850c7b07 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 17:54:31 +0900 Subject: [PATCH 23/44] chore: remove misplaced issue 7 test --- tests/effective_isolation_attestation.rs | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 tests/effective_isolation_attestation.rs diff --git a/tests/effective_isolation_attestation.rs b/tests/effective_isolation_attestation.rs deleted file mode 100644 index b91bec3f..00000000 --- a/tests/effective_isolation_attestation.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Security regression for application-service attestation. - -// Intentionally added on the current stack head; implementation is added in the same bounded PR. From 0fed2897f538799ae67e1194260c1a3bc41ab8db Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 08:39:15 +0900 Subject: [PATCH 24/44] test: expose launch-registration cleanup gap --- src/application_service/coordinator.rs | 136 +++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index d730dc41..8da59164 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -485,3 +485,139 @@ fn fingerprint_request_and_policy( } format!("{:x}", hasher.finalize()) } + +#[cfg(test)] +mod tests { + use std::{ + sync::{ + Arc, Barrier, + atomic::{AtomicUsize, Ordering}, + }, + thread, + }; + + use super::{ + ApplicationServiceBackend, ApplicationServiceCoordinator, + ApplicationServiceCoordinatorError, ApplicationServiceError, ApplicationServiceLease, + ApplicationServiceRequest, CleanupReceipt, LeaseOwnerId, + }; + use crate::{IsolationPolicy, ResourceRequest, ServiceEndpoint, ServiceProtocol}; + use crate::sandbox_execution::RuntimeLeaseMetadata; + + struct BlockingBackend { + launch_entered: Arc, + launch_resume: Arc, + terminate_calls: Arc, + } + + impl ApplicationServiceBackend for BlockingBackend { + fn launch_at( + &self, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result { + self.launch_entered.wait(); + self.launch_resume.wait(); + Ok(ApplicationServiceLease::new( + request, + RuntimeLeaseMetadata { + backend_id: "test_backend", + sandbox_id: "sandbox-registration-gap".to_owned(), + network_id: "network-registration-gap".to_owned(), + policy_id: policy.policy_id.clone(), + policy_sha256: policy.effective_policy_sha256(), + started_at_epoch_seconds, + expires_at_epoch_seconds: started_at_epoch_seconds + + u64::from(request.resources.lease_seconds), + shutdown_grace_seconds: policy.shutdown_grace_seconds, + }, + ServiceEndpoint::loopback(45_321, request.protocol), + )) + } + + fn terminate_at( + &self, + lease: &ApplicationServiceLease, + terminated_at_epoch_seconds: u64, + ) -> Result { + self.terminate_calls.fetch_add(1, Ordering::SeqCst); + Ok(CleanupReceipt::complete(lease, terminated_at_epoch_seconds)) + } + } + + fn request() -> ApplicationServiceRequest { + ApplicationServiceRequest { + schema_version: "1.0.0".to_owned(), + request_id: "registration-poison".to_owned(), + image_reference: format!("localhost/cwl/tool@sha256:{}", "d".repeat(64)), + container_port: 8_080, + protocol: ServiceProtocol::Http, + command: vec!["serve".to_owned()], + resources: ResourceRequest { + memory_bytes: 64 * 1024 * 1024, + cpu_millicores: 500, + maximum_processes: 16, + lease_seconds: 30, + tmpfs_bytes: 8 * 1024 * 1024, + }, + } + } + + fn policy() -> IsolationPolicy { + IsolationPolicy { + policy_id: "registration_cleanup_policy".to_owned(), + maximum_memory_bytes: 128 * 1024 * 1024, + maximum_cpu_millicores: 1_000, + maximum_processes: 32, + maximum_lease_seconds: 60, + maximum_tmpfs_bytes: 16 * 1024 * 1024, + readiness_timeout_millis: 1_000, + readiness_poll_interval_millis: 10, + shutdown_grace_seconds: 1, + run_as_user_id: 65_532, + run_as_group_id: 65_532, + } + } + + #[test] + fn successful_backend_launch_is_cleaned_if_registry_becomes_unavailable() { + let launch_entered = Arc::new(Barrier::new(2)); + let launch_resume = Arc::new(Barrier::new(2)); + let terminate_calls = Arc::new(AtomicUsize::new(0)); + let coordinator = Arc::new(ApplicationServiceCoordinator::new(BlockingBackend { + launch_entered: Arc::clone(&launch_entered), + launch_resume: Arc::clone(&launch_resume), + terminate_calls: Arc::clone(&terminate_calls), + })); + let owner = LeaseOwnerId::new("urn:cwl:agent:test") + .expect("test owner should satisfy the bounded identity contract"); + let worker_coordinator = Arc::clone(&coordinator); + let worker_owner = owner.clone(); + let worker = thread::spawn(move || { + worker_coordinator.launch_at(&worker_owner, &request(), &policy(), 1_780_000_000) + }); + + launch_entered.wait(); + let poison_target = Arc::clone(&coordinator); + let poison = thread::spawn(move || { + let _guard = poison_target + .leases + .lock() + .expect("registry should be healthy before explicit poisoning"); + panic!("poison registry after backend launch begins"); + }); + assert!(poison.join().is_err()); + launch_resume.wait(); + + assert_eq!( + worker.join().expect("launch worker should not panic"), + Err(ApplicationServiceCoordinatorError::StateUnavailable) + ); + assert_eq!( + terminate_calls.load(Ordering::SeqCst), + 1, + "a successful backend launch must be cleaned up when the lease cannot be registered" + ); + } +} From 4cf44a9dfbacf566696bf40c489a90225481df9a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 08:43:06 +0900 Subject: [PATCH 25/44] fix: clean launched service when lease registration fails --- src/application_service/coordinator.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index 8da59164..10c0e08e 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -264,7 +264,15 @@ where } }; - self.lock_registry()?.insert( + let mut leases = match self.lock_registry() { + Ok(leases) => leases, + Err(error) => { + self.backend + .terminate_at(&lease, started_at_epoch_seconds)?; + return Err(error); + } + }; + leases.insert( key, RegistryEntry::Active { request_fingerprint, @@ -502,7 +510,6 @@ mod tests { ApplicationServiceRequest, CleanupReceipt, LeaseOwnerId, }; use crate::{IsolationPolicy, ResourceRequest, ServiceEndpoint, ServiceProtocol}; - use crate::sandbox_execution::RuntimeLeaseMetadata; struct BlockingBackend { launch_entered: Arc, @@ -521,7 +528,7 @@ mod tests { self.launch_resume.wait(); Ok(ApplicationServiceLease::new( request, - RuntimeLeaseMetadata { + crate::sandbox_execution::RuntimeLeaseMetadata { backend_id: "test_backend", sandbox_id: "sandbox-registration-gap".to_owned(), network_id: "network-registration-gap".to_owned(), From 339d32d61e44c2b0653ad94869bbb32d69182c26 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 08:52:44 +0900 Subject: [PATCH 26/44] test: preserve cleanup failure over registry failure --- src/application_service/coordinator.rs | 81 ++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index 10c0e08e..d3ed3612 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -553,6 +553,46 @@ mod tests { } } + struct TerminationFailureBackend { + terminate_entered: Arc, + terminate_resume: Arc, + } + + impl ApplicationServiceBackend for TerminationFailureBackend { + fn launch_at( + &self, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result { + Ok(ApplicationServiceLease::new( + request, + crate::sandbox_execution::RuntimeLeaseMetadata { + backend_id: "test_backend", + sandbox_id: "sandbox-cleanup-failure".to_owned(), + network_id: "network-cleanup-failure".to_owned(), + policy_id: policy.policy_id.clone(), + policy_sha256: policy.effective_policy_sha256(), + started_at_epoch_seconds, + expires_at_epoch_seconds: started_at_epoch_seconds + + u64::from(request.resources.lease_seconds), + shutdown_grace_seconds: policy.shutdown_grace_seconds, + }, + ServiceEndpoint::loopback(45_322, request.protocol), + )) + } + + fn terminate_at( + &self, + _lease: &ApplicationServiceLease, + _terminated_at_epoch_seconds: u64, + ) -> Result { + self.terminate_entered.wait(); + self.terminate_resume.wait(); + Err(ApplicationServiceError::CleanupFailed) + } + } + fn request() -> ApplicationServiceRequest { ApplicationServiceRequest { schema_version: "1.0.0".to_owned(), @@ -627,4 +667,45 @@ mod tests { "a successful backend launch must be cleaned up when the lease cannot be registered" ); } + + #[test] + fn cleanup_failure_is_not_hidden_by_later_registry_failure() { + let terminate_entered = Arc::new(Barrier::new(2)); + let terminate_resume = Arc::new(Barrier::new(2)); + let coordinator = Arc::new(ApplicationServiceCoordinator::new(TerminationFailureBackend { + terminate_entered: Arc::clone(&terminate_entered), + terminate_resume: Arc::clone(&terminate_resume), + })); + let owner = LeaseOwnerId::new("urn:cwl:agent:test") + .expect("test owner should satisfy the bounded identity contract"); + let lease = coordinator + .launch_at(&owner, &request(), &policy(), 1_780_000_000) + .expect("test lease should register before termination"); + let worker_coordinator = Arc::clone(&coordinator); + let worker_owner = owner.clone(); + let worker_lease = lease.clone(); + let worker = thread::spawn(move || { + worker_coordinator.terminate_at(&worker_owner, &worker_lease, 1_780_000_010) + }); + + terminate_entered.wait(); + let poison_target = Arc::clone(&coordinator); + let poison = thread::spawn(move || { + let _guard = poison_target + .leases + .lock() + .expect("registry should be healthy before explicit poisoning"); + panic!("poison registry after backend cleanup begins"); + }); + assert!(poison.join().is_err()); + terminate_resume.wait(); + + assert_eq!( + worker.join().expect("termination worker should not panic"), + Err(ApplicationServiceCoordinatorError::Backend( + ApplicationServiceError::CleanupFailed, + )), + "cleanup failure must remain the externally visible safety result even when state recovery also fails" + ); + } } From 971b05df8573e5c77eb0d006c8c56fabed2cb232 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 08:55:43 +0900 Subject: [PATCH 27/44] fix: preserve cleanup failure across registry failure --- src/application_service/coordinator.rs | 31 ++++++++++++++++++++------ 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index d3ed3612..275a15bd 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -285,6 +285,9 @@ where /// Terminate one lease only when the caller owns the active registry entry. /// + /// Cleanup failure remains the externally visible safety result even when + /// the registry also becomes unavailable while recording that failure. + /// /// # Errors /// /// Returns [`ApplicationServiceCoordinatorError::UnknownLease`] for a wrong @@ -333,14 +336,20 @@ where let result = self .backend .terminate_at(®istered_lease, terminated_at_epoch_seconds); - self.finish_termination( + let registry_result = self.finish_termination( &key, request_fingerprint, registered_lease, cleanup_attempts, &result, - )?; - result.map_err(Into::into) + ); + match result { + Ok(receipt) => { + registry_result?; + Ok(receipt) + } + Err(error) => Err(error.into()), + } } /// Clean up at most 64 active leases whose expiry is not later than `now`. @@ -349,12 +358,14 @@ where /// cleanup pass can retry it. Previously unattempted expired leases are /// selected before repeatedly failing entries so one bad cleanup cannot /// starve later expired workloads. This process-local function does not - /// claim crash/restart orphan recovery. + /// claim crash/restart orphan recovery. If both cleanup and registry + /// recording fail, the cleanup failure remains externally visible. /// /// # Errors /// /// Returns [`ApplicationServiceCoordinatorError::StateUnavailable`] if the - /// in-memory registry cannot be accessed safely. + /// in-memory registry cannot be accessed safely while no backend cleanup + /// failure needs to take precedence. pub fn cleanup_expired_at( &self, now_epoch_seconds: u64, @@ -399,13 +410,19 @@ where let mut outcomes = Vec::with_capacity(candidates.len()); for (key, request_fingerprint, lease, cleanup_attempts) in candidates { let result = self.backend.terminate_at(&lease, now_epoch_seconds); - self.finish_termination( + let registry_result = self.finish_termination( &key, request_fingerprint, lease.clone(), cleanup_attempts, &result, - )?; + ); + if let Err(error) = &result + && registry_result.is_err() + { + return Err(error.clone().into()); + } + registry_result?; outcomes.push(ExpiredLeaseCleanupResult { lease_owner_id: key.lease_owner_id, request_id: key.request_id, From 949b0f71a1e022eb0ea48df93ffa7bbbfd1eb259 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 2 Sep 2026 12:18:47 +0900 Subject: [PATCH 28/44] fix: remove panic shortcuts from coordinator test fixtures --- src/application_service/coordinator.rs | 48 +++++++++++++++++--------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index 275a15bd..a1ab4ad8 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -654,8 +654,11 @@ mod tests { launch_resume: Arc::clone(&launch_resume), terminate_calls: Arc::clone(&terminate_calls), })); - let owner = LeaseOwnerId::new("urn:cwl:agent:test") - .expect("test owner should satisfy the bounded identity contract"); + let owner = LeaseOwnerId::new("urn:cwl:agent:test").unwrap_or_else(|_| { + std::panic::resume_unwind(Box::new( + "test owner should satisfy the bounded identity contract", + )) + }); let worker_coordinator = Arc::clone(&coordinator); let worker_owner = owner.clone(); let worker = thread::spawn(move || { @@ -665,17 +668,20 @@ mod tests { launch_entered.wait(); let poison_target = Arc::clone(&coordinator); let poison = thread::spawn(move || { - let _guard = poison_target - .leases - .lock() - .expect("registry should be healthy before explicit poisoning"); - panic!("poison registry after backend launch begins"); + let _guard = poison_target.leases.lock().unwrap_or_else(|_| { + std::panic::resume_unwind(Box::new( + "registry should be healthy before explicit poisoning", + )) + }); + std::panic::resume_unwind(Box::new("poison registry after backend launch begins")); }); assert!(poison.join().is_err()); launch_resume.wait(); assert_eq!( - worker.join().expect("launch worker should not panic"), + worker.join().unwrap_or_else(|_| { + std::panic::resume_unwind(Box::new("launch worker should not panic")) + }), Err(ApplicationServiceCoordinatorError::StateUnavailable) ); assert_eq!( @@ -693,11 +699,16 @@ mod tests { terminate_entered: Arc::clone(&terminate_entered), terminate_resume: Arc::clone(&terminate_resume), })); - let owner = LeaseOwnerId::new("urn:cwl:agent:test") - .expect("test owner should satisfy the bounded identity contract"); + let owner = LeaseOwnerId::new("urn:cwl:agent:test").unwrap_or_else(|_| { + std::panic::resume_unwind(Box::new( + "test owner should satisfy the bounded identity contract", + )) + }); let lease = coordinator .launch_at(&owner, &request(), &policy(), 1_780_000_000) - .expect("test lease should register before termination"); + .unwrap_or_else(|_| { + std::panic::resume_unwind(Box::new("test lease should register before termination")) + }); let worker_coordinator = Arc::clone(&coordinator); let worker_owner = owner.clone(); let worker_lease = lease.clone(); @@ -708,17 +719,20 @@ mod tests { terminate_entered.wait(); let poison_target = Arc::clone(&coordinator); let poison = thread::spawn(move || { - let _guard = poison_target - .leases - .lock() - .expect("registry should be healthy before explicit poisoning"); - panic!("poison registry after backend cleanup begins"); + let _guard = poison_target.leases.lock().unwrap_or_else(|_| { + std::panic::resume_unwind(Box::new( + "registry should be healthy before explicit poisoning", + )) + }); + std::panic::resume_unwind(Box::new("poison registry after backend cleanup begins")); }); assert!(poison.join().is_err()); terminate_resume.wait(); assert_eq!( - worker.join().expect("termination worker should not panic"), + worker.join().unwrap_or_else(|_| { + std::panic::resume_unwind(Box::new("termination worker should not panic")) + }), Err(ApplicationServiceCoordinatorError::Backend( ApplicationServiceError::CleanupFailed, )), From 1cdfd29a6492405007167a17f5c7feefdd1eaa98 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 04:46:01 +0900 Subject: [PATCH 29/44] docs: reconcile lease stack with current runtime foundation --- docs/product-technical-gap-baseline.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d956fcc6..d3083978 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,6 +1,6 @@ # Product and Technical Gap Baseline -Last reviewed on 2026-09-01 against parent PR #1 exact head `c78fd491f84fae773b3691b10b6a0c21940808d5` and stacked PR #6 exact head `b9ce1ea86c8440157c7a78d6c30e91649d8895fd`. The stack is reconciled onto the current parent security/wire contract and now includes caller-scoped lease ownership plus starvation-resistant bounded expiry cleanup. Protected `develop` remains shipped authority until protected integration. Hosted evidence must be generated for the unchanged current heads; predecessor results do not transfer. +Last reviewed on 2026-09-04 against parent PR #1 exact head `06b39670ba5a434e8e34aac6f5b7fa2b6b75fe87` and stacked PR #6 source/restack head `60440480ba779599a7dbe917d19ec8e0e4ff7749`. The immediately following ledger commit changes documentation only. The stack is non-force reconciled onto the current parent security/wire contract and includes caller-scoped lease ownership plus starvation-resistant bounded expiry cleanup. Protected `develop@60a85c7633e03b425b67159ec6822c8178cf87ea` remains shipped authority until protected integration. Hosted evidence must be generated for unchanged current heads; predecessor results do not transfer. ## Product responsibility @@ -25,9 +25,9 @@ Core `sandbox_execution` owns isolation policy, resource bounds, runtime lease m | Capability | Current evidence | Status | Required action | | --- | --- | --- | --- | -| Immutable workload identity | `ApplicationServiceRequest` accepts digest-pinned lower-case SHA-256 OCI references; Podman uses no-pull launch semantics. | Parent PR | If image admission is added, make it a separate trusted operation and retain digest identity. | -| Rootless execution | Podman adapter verifies rootless backend state and fails closed otherwise. | Parent PR | Keep real rootless backend acceptance on every release head. | -| Filesystem isolation | Read-only root filesystem, bounded noexec/nosuid/nodev tmpfs, image volumes ignored, no arbitrary host mount request. | Parent PR | Add only typed reviewed read-only input mounts when a buyer flow requires them. | +| Immutable workload identity | Parent PR #1 now rejects explicit `containers/image` transports that can redirect Podman to host-backed directories, archives, alternate local stores, or another runtime authority. RED `02ee218a…` covers `dir:`, `docker-archive:`, `oci-archive:`, `docker-daemon:`, and `containers-storage:`; repair `6df890ca…` preserves normal registry names with optional numeric ports; schema/docs/traceability are aligned. PR #6 inherited that repair through merge-base `06b39670…`. | Corrected on parent and inherited by PR #6; exact-head proof required | Keep image import/admission separate from launch; never reintroduce host-path/alternate-store transports or implicit pulls. | +| Rootless execution | Podman adapter verifies rootless backend state and fails closed otherwise. Parent hosted contract now explicitly tests Ubuntu 24.04 distribution Podman 4.9.3. | Parent PR | Keep real rootless backend acceptance on every release head. | +| Filesystem isolation | Read-only root filesystem, bounded noexec/nosuid/nodev tmpfs, image volumes ignored, no arbitrary host mount request; host-backed image transports are rejected before launch. | Parent PR | Add only typed reviewed read-only input mounts when a buyer flow requires them. | | Privilege isolation | Capabilities dropped, no-new-privileges, isolated user/process-related namespaces, numeric non-root identity. | Parent PR | Preserve effective-runtime verification rather than relying only on argv construction. | | Network isolation | Per-sandbox internal DNS-disabled network and loopback-only host publication. | Parent PR | Controlled egress must be a separate profile and must not silently enable Internet access. | | Credential isolation | No consumer/provider credentials, arbitrary environment, runtime sockets, host devices, or ambient proxy variables enter the P0 workload contract. | Parent PR | Add a task-scoped secret broker only after an accepted ADR and explicit consumer authorization. | @@ -63,28 +63,28 @@ Current integration rules: - no cross-service application-table SQL; - Context Fabric owner-path PRs/issues are inventoried read-only and receive exact evidence/acceptance criteria rather than quarantine-writer source changes. -At this review, `context-graph-contracts` still has an unreleased stacked Context Assertion/CloudEvent contract line. PR #21 exact head `a3a3125619ed6e777818811b1c0b97f3a4574b73` repairs structured CloudEvent envelope binding but remains Draft with current hosted lanes non-passing. `enterprise-architecture-core` PR #40 exact head `2b14e008a11712c840d0bf6c8c5d3a1d6e9ec1ba` enforces released Context Graph bindings for foreign projections but does not yet list `quarantine-sandbox-runtime` as a separate owner projection. Both repositories remain read-only here; the Context Fabric owner path must add the quarantine runtime only after the shared contract is released and pinned. Quarantine integration remains fail-closed until then. +Previously observed Context Fabric PR numbers and heads are historical only until refreshed by the dedicated owner. This writer does not consume their mutable branches as production dependencies. Quarantine integration remains fail-closed until a compatible shared contract is immutably published and pinned. ## Consumer integrations | Consumer | Authority retained by consumer | Runtime integration state | | --- | --- | --- | -| Wardnet | gateway/SOC policy, maliciousness verdict, incident, quarantine/block/review, notification, retention | Runtime contract is consumer-neutral; Wardnet issue #38 remains the consumer owner path and must consume published runtime evidence without moving verdict authority here. | +| Wardnet | gateway/SOC policy, maliciousness verdict, incident, quarantine/block/review, notification, retention | Runtime contract is consumer-neutral; Wardnet owner path must consume published runtime evidence without moving verdict authority here. | | contextual-orchestrator | LLM/model routing, chat, Agent/task/tool policy, caller authorization, application selection, secrets, user-visible actions | Issue #991 owns the ACL integration after a protected immutable runtime release. Direct Podman/containerd calls and sibling source copies are not acceptable. | The runtime currently exposes a Rust library and loopback lease topology. It does not yet publish an authenticated network process boundary or generated Python consumer. A future transport must derive `LeaseOwnerId` from verified caller context, keep idempotency scope stable, return bounded stable wire errors, validate lease/cleanup semantics, and define co-location/network topology explicitly. ## Verification and release state -- Parent PR #1 current exact head is `c78fd491f84fae773b3691b10b6a0c21940808d5`; CI, Security Scan, and SAST were queued at the latest read, so the head is non-passing despite earlier real rootless-Podman evidence. -- PR #1 has no formal approving review at the latest read and remains Draft. -- PR #6 is reconciled on that exact parent. Its current branch includes RED commit `008209f0cf7152f06053f44d65662e4e023787ba` exposing cleanup starvation and causal repair commit `410fce1231c0bedd47e6a4c82ea330825977c2f4`, followed by documentation refresh. Current exact-head CI must prove the new regression and implementation together; predecessor evidence does not transfer. +- Parent PR #1 exact head is `06b39670ba5a434e8e34aac6f5b7fa2b6b75fe87`, still Draft. Exact-head CI `33797817805` and associated Security/SAST/Scorecard/OSV evidence were queued at the latest read, so the head is non-passing despite earlier real rootless-Podman evidence. +- PR #1 has no formal approving review at the latest read. +- PR #6 was non-force restacked with merge commit `60440480ba779599a7dbe917d19ec8e0e4ff7749`; compare against parent `06b39670…` proves `behind_by=0` and exact merge-base equality. Its unique caller ownership/idempotency/cleanup-fairness files remain the only semantic delta over the parent. CI run `33798281628` is queued and therefore non-passing. - Active organization rules require qualifying review, resolved threads, and central required workflows. Admin/bypass capability is not merge evidence. - No release exists. Version/release claims remain premature until one integrated protected head passes CI, security, SAST, complete coverage/docstrings, real isolation E2E, SBOM/provenance, review, rollback/recovery and protected-merge gates together. ## Next bounded slices -1. Obtain exact-head GREEN for PR #6 cleanup fairness and caller-scoped lease ownership without changing a clean head merely to retrigger queued jobs. +1. Obtain exact-head GREEN for PR #6 cleanup fairness, caller-scoped lease ownership, and inherited image-transport rejection without changing a clean head merely to retrigger queued jobs. 2. Drain parent PR #1 gate/review findings and merge only through protected policy; then revalidate the stacked lease-ownership slice against the protected parent. 3. Implement one authenticated versioned consumer transport/binding with stable wire errors and caller identity mapping. 4. Add durable restart/orphan reclamation, lease journal, admission/resource reservation, and crash-recovery evidence. From 828cc2ba3c67d72a73fac8ea809de7c20847009f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 20:14:28 +0900 Subject: [PATCH 30/44] test(runtime): preserve lease checks after foundation restack Signed-off-by: Seongho Bae --- tests/application_service_ownership.rs | 61 ++++++++++++++----- .../application_service_policy_idempotency.rs | 14 ++++- 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/tests/application_service_ownership.rs b/tests/application_service_ownership.rs index 41a26f89..fbf6c61a 100644 --- a/tests/application_service_ownership.rs +++ b/tests/application_service_ownership.rs @@ -72,9 +72,15 @@ fn temporary_path(name: &str) -> PathBuf { } fn write_fake_podman(program: &Path, log: &Path, mode: &str, ready_port: u16) { + let info = r#"{"host":{"security":{"rootless":true,"seccompEnabled":true,"seccompProfilePath":"/usr/share/containers/seccomp.json","apparmorEnabled":true,"selinuxEnabled":false}}}"#; + let container = r#"[{"Id":"fake-container-id","AppArmorProfile":"containers-default","ProcessLabel":"","EffectiveCaps":[],"BoundingCaps":[],"Config":{"User":"65532:65532"},"HostConfig":{"ReadonlyRootfs":true,"Privileged":false,"SecurityOpt":["no-new-privileges"],"UsernsMode":"auto","PidMode":"private","IpcMode":"none","Memory":268435456,"NanoCpus":1000000000,"PidsLimit":32}}]"#; + let network = r#"[{"internal":true,"dns_enabled":false}]"#; let script = format!( - "#!/bin/sh\nset -eu\nMODE='{mode}'\nprintf '%s\\n' \"$*\" >> '{}'\nif [ \"$MODE\" = slow_rootless ] && [ \"${{1:-}}\" = info ]; then sleep 1; fi\nif [ \"$MODE\" = fail_rootless ] && [ \"${{1:-}}\" = info ]; then exit 20; fi\ncase \"${{1:-}}\" in\n info) printf 'true\\n' ;;\n network) : ;;\n create) printf 'fake-container-id\\n' ;;\n start) : ;;\n port) printf '127.0.0.1:{ready_port}\\n' ;;\n stop) : ;;\n rm) : ;;\n *) exit 91 ;;\nesac\n", - log.display() + "#!/bin/sh\nset -eu\nMODE='{mode}'\nprintf '%s\\n' \"$*\" >> '{}'\nif [ \"$MODE\" = slow_rootless ] && [ \"${{1:-}}\" = info ] && [ \"${{3:-}}\" != json ]; then sleep 1; fi\nif [ \"$MODE\" = fail_rootless ] && [ \"${{1:-}}\" = info ] && [ \"${{3:-}}\" != json ]; then exit 20; fi\ncase \"${{1:-}}:${{2:-}}\" in\n info:--format) if [ \"${{3:-}}\" = json ]; then printf '%s\\n' '{}'; else printf 'true\\n'; fi ;;\n network:create) : ;;\n network:inspect) printf '%s\\n' '{}' ;;\n create:--name) printf 'fake-container-id\\n' ;;\n start:*) : ;;\n container:inspect) printf '%s\\n' '{}' ;;\n top:*) printf 'PID SECCOMP CAPEFF CAPBND CAPINH CAPPRM CAPAMB LABEL\\n1 filter - - - - - containers-default (enforce)\\n' ;;\n port:*) printf '127.0.0.1:{ready_port}\\n' ;;\n stop:*) : ;;\n rm:*) : ;;\n network:rm) : ;;\n *) exit 91 ;;\nesac\n", + log.display(), + info, + network, + container, ); fs::write(program, script).expect("fake Podman should be writable"); let mut permissions = fs::metadata(program) @@ -125,11 +131,15 @@ fn lease_owner_ids_are_bounded_opaque_runtime_context() { #[test] fn identical_retry_returns_existing_lease_without_second_launch() { let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); - let ready_port = listener.local_addr().expect("address should resolve").port(); + let ready_port = listener + .local_addr() + .expect("address should resolve") + .port(); let program = temporary_path("fake-podman"); let log = temporary_path("fake-podman-log"); write_fake_podman(&program, &log, "success", ready_port); - let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let coordinator = + ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); let owner = owner("urn:cwl:agent:contextual-orchestrator"); let first = coordinator @@ -152,11 +162,15 @@ fn identical_retry_returns_existing_lease_without_second_launch() { #[test] fn same_owner_and_request_id_with_different_content_fails_closed() { let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); - let ready_port = listener.local_addr().expect("address should resolve").port(); + let ready_port = listener + .local_addr() + .expect("address should resolve") + .port(); let program = temporary_path("fake-podman"); let log = temporary_path("fake-podman-log"); write_fake_podman(&program, &log, "success", ready_port); - let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let coordinator = + ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); let owner = owner("urn:cwl:agent:contextual-orchestrator"); let first = coordinator .launch_at(&owner, &request(), &policy(), 1_780_000_000) @@ -180,11 +194,15 @@ fn same_owner_and_request_id_with_different_content_fails_closed() { #[test] fn wrong_owner_cannot_terminate_another_callers_lease() { let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); - let ready_port = listener.local_addr().expect("address should resolve").port(); + let ready_port = listener + .local_addr() + .expect("address should resolve") + .port(); let program = temporary_path("fake-podman"); let log = temporary_path("fake-podman-log"); write_fake_podman(&program, &log, "success", ready_port); - let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let coordinator = + ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); let correct_owner = owner("urn:cwl:agent:contextual-orchestrator"); let wrong_owner = owner("urn:cwl:consumer:wardnet"); let lease = coordinator @@ -208,11 +226,15 @@ fn wrong_owner_cannot_terminate_another_callers_lease() { #[test] fn failed_launch_releases_idempotency_reservation_for_retry() { let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); - let ready_port = listener.local_addr().expect("address should resolve").port(); + let ready_port = listener + .local_addr() + .expect("address should resolve") + .port(); let program = temporary_path("fake-podman"); let log = temporary_path("fake-podman-log"); write_fake_podman(&program, &log, "fail_rootless", ready_port); - let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let coordinator = + ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); let owner = owner("urn:cwl:agent:contextual-orchestrator"); assert_eq!( @@ -238,13 +260,16 @@ fn failed_launch_releases_idempotency_reservation_for_retry() { #[test] fn concurrent_duplicate_launch_is_rejected_while_first_launch_is_in_flight() { let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); - let ready_port = listener.local_addr().expect("address should resolve").port(); + let ready_port = listener + .local_addr() + .expect("address should resolve") + .port(); let program = temporary_path("fake-podman"); let log = temporary_path("fake-podman-log"); write_fake_podman(&program, &log, "slow_rootless", ready_port); - let coordinator = Arc::new(ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new( - program.clone(), - ))); + let coordinator = Arc::new(ApplicationServiceCoordinator::new( + RootlessPodmanAdapter::new(program.clone()), + )); let owner = owner("urn:cwl:agent:contextual-orchestrator"); let worker_coordinator = Arc::clone(&coordinator); let worker_owner = owner.clone(); @@ -273,11 +298,15 @@ fn concurrent_duplicate_launch_is_rejected_while_first_launch_is_in_flight() { #[test] fn expired_lease_cleanup_is_bounded_and_attributed_to_owner_and_request() { let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); - let ready_port = listener.local_addr().expect("address should resolve").port(); + let ready_port = listener + .local_addr() + .expect("address should resolve") + .port(); let program = temporary_path("fake-podman"); let log = temporary_path("fake-podman-log"); write_fake_podman(&program, &log, "success", ready_port); - let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let coordinator = + ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); let owner = owner("urn:cwl:agent:contextual-orchestrator"); let mut short_request = request(); short_request.resources.lease_seconds = 1; diff --git a/tests/application_service_policy_idempotency.rs b/tests/application_service_policy_idempotency.rs index 57edb9af..79b6683a 100644 --- a/tests/application_service_policy_idempotency.rs +++ b/tests/application_service_policy_idempotency.rs @@ -61,8 +61,12 @@ fn request() -> ApplicationServiceRequest { } fn write_fake_podman(program: &PathBuf, ready_port: u16) { + let info = r#"{"host":{"security":{"rootless":true,"seccompEnabled":true,"seccompProfilePath":"/usr/share/containers/seccomp.json","apparmorEnabled":true,"selinuxEnabled":false}}}"#; + let container = r#"[{"Id":"fake-container-id","AppArmorProfile":"containers-default","ProcessLabel":"","EffectiveCaps":[],"BoundingCaps":[],"Config":{"User":"65532:65532"},"HostConfig":{"ReadonlyRootfs":true,"Privileged":false,"SecurityOpt":["no-new-privileges"],"UsernsMode":"auto","PidMode":"private","IpcMode":"none","Memory":268435456,"NanoCpus":1000000000,"PidsLimit":32}}]"#; + let network = r#"[{"internal":true,"dns_enabled":false}]"#; let script = format!( - "#!/bin/sh\nset -eu\ncase \"${{1:-}}\" in\n info) printf 'true\\n' ;;\n network) : ;;\n create) printf 'fake-container-id\\n' ;;\n start) : ;;\n port) printf '127.0.0.1:{ready_port}\\n' ;;\n stop) : ;;\n rm) : ;;\n *) exit 91 ;;\nesac\n" + "#!/bin/sh\nset -eu\ncase \"${{1:-}}:${{2:-}}\" in\n info:--format) if [ \"${{3:-}}\" = json ]; then printf '%s\\n' '{}'; else printf 'true\\n'; fi ;;\n network:create) : ;;\n network:inspect) printf '%s\\n' '{}' ;;\n create:--name) printf 'fake-container-id\\n' ;;\n start:*) : ;;\n container:inspect) printf '%s\\n' '{}' ;;\n top:*) printf 'PID SECCOMP CAPEFF CAPBND CAPINH CAPPRM CAPAMB LABEL\\n1 filter - - - - - containers-default (enforce)\\n' ;;\n port:*) printf '127.0.0.1:{ready_port}\\n' ;;\n stop:*) : ;;\n rm:*) : ;;\n network:rm) : ;;\n *) exit 91 ;;\nesac\n", + info, network, container, ); fs::write(program, script).expect("fake Podman should be writable"); let mut permissions = fs::metadata(program) @@ -75,10 +79,14 @@ fn write_fake_podman(program: &PathBuf, ready_port: u16) { #[test] fn identical_request_does_not_reuse_lease_when_effective_policy_changes() { let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); - let ready_port = listener.local_addr().expect("address should resolve").port(); + let ready_port = listener + .local_addr() + .expect("address should resolve") + .port(); let program = temporary_path("fake-podman"); write_fake_podman(&program, ready_port); - let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); + let coordinator = + ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); let owner = LeaseOwnerId::new("urn:cwl:agent:contextual-orchestrator") .expect("opaque owner should validate"); let initial_policy = policy(); From bd2e0a3136bce71475884d2ad4dc2ce8ebbe75f7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 21:15:42 +0900 Subject: [PATCH 31/44] test(runtime): isolate concurrent fixtures Signed-off-by: Seongho Bae --- Cargo.lock | 24 ++---------- Cargo.toml | 1 + src/application_service/coordinator.rs | 18 ++++----- src/infrastructure/bounded_command.rs | 22 ++++++++--- tests/application_service_ownership.rs | 50 ++++++++++-------------- tests/runtime_boundary_regressions.rs | 53 +++++++++++++------------- 6 files changed, 76 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38f44fb7..ad183731 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,21 +113,10 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi 5.3.0", + "r-efi", "wasip2", ] -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", -] - [[package]] name = "itoa" version = "1.0.18" @@ -212,6 +201,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "tempfile", "thiserror", ] @@ -236,12 +226,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - [[package]] name = "rand" version = "0.9.5" @@ -268,7 +252,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ - "getrandom 0.3.4", + "getrandom", ] [[package]] @@ -394,7 +378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom", "once_cell", "rustix", "windows-sys", diff --git a/Cargo.toml b/Cargo.toml index e5dbab68..1e85079b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ thiserror = "2.0" [dev-dependencies] proptest = "1.6" +tempfile = "3.27" [lints.rust] missing_docs = "deny" diff --git a/src/application_service/coordinator.rs b/src/application_service/coordinator.rs index a1ab4ad8..6d35ec3b 100644 --- a/src/application_service/coordinator.rs +++ b/src/application_service/coordinator.rs @@ -171,7 +171,6 @@ impl ExpiredLeaseCleanupResult { } /// Return the cleanup receipt or attributable backend failure. - #[must_use] pub const fn result(&self) -> &Result { &self.result } @@ -388,11 +387,8 @@ where _ => None, }) .collect(); - candidates.sort_by(|left, right| { - left.3 - .cmp(&right.3) - .then_with(|| left.0.cmp(&right.0)) - }); + candidates + .sort_by(|left, right| left.3.cmp(&right.3).then_with(|| left.0.cmp(&right.0))); candidates.truncate(MAX_EXPIRED_CLEANUPS_PER_CALL); for (key, request_fingerprint, lease, cleanup_attempts) in &candidates { leases.insert( @@ -695,10 +691,12 @@ mod tests { fn cleanup_failure_is_not_hidden_by_later_registry_failure() { let terminate_entered = Arc::new(Barrier::new(2)); let terminate_resume = Arc::new(Barrier::new(2)); - let coordinator = Arc::new(ApplicationServiceCoordinator::new(TerminationFailureBackend { - terminate_entered: Arc::clone(&terminate_entered), - terminate_resume: Arc::clone(&terminate_resume), - })); + let coordinator = Arc::new(ApplicationServiceCoordinator::new( + TerminationFailureBackend { + terminate_entered: Arc::clone(&terminate_entered), + terminate_resume: Arc::clone(&terminate_resume), + }, + )); let owner = LeaseOwnerId::new("urn:cwl:agent:test").unwrap_or_else(|_| { std::panic::resume_unwind(Box::new( "test owner should satisfy the bounded identity contract", diff --git a/src/infrastructure/bounded_command.rs b/src/infrastructure/bounded_command.rs index 192b2d34..1b6e4e90 100644 --- a/src/infrastructure/bounded_command.rs +++ b/src/infrastructure/bounded_command.rs @@ -77,12 +77,22 @@ fn spawn_piped_child( program: &Path, args: &[String], ) -> Result<(Child, ChildStdout, ChildStderr), BoundedCommandError> { - let mut child = Command::new(program) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|_| BoundedCommandError::Spawn)?; + let mut attempts = 0_u8; + let mut child = loop { + match Command::new(program) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + { + Ok(child) => break child, + Err(error) if error.kind() == io::ErrorKind::WouldBlock && attempts < 8 => { + attempts += 1; + thread::sleep(POLL_INTERVAL); + } + Err(_) => return Err(BoundedCommandError::Spawn), + } + }; captured_pipes(child.stdout.take(), child.stderr.take()) .map(|(stdout, stderr)| (child, stdout, stderr)) } diff --git a/tests/application_service_ownership.rs b/tests/application_service_ownership.rs index fbf6c61a..7f620216 100644 --- a/tests/application_service_ownership.rs +++ b/tests/application_service_ownership.rs @@ -3,13 +3,8 @@ #![cfg(target_os = "linux")] use std::{ - fs, - net::TcpListener, - os::unix::fs::PermissionsExt, - path::{Path, PathBuf}, - sync::Arc, - thread, - time::{Duration, SystemTime, UNIX_EPOCH}, + fs, net::TcpListener, os::unix::fs::PermissionsExt, path::Path, sync::Arc, thread, + time::Duration, }; use quarantine_sandbox_runtime::{ @@ -60,17 +55,6 @@ fn request() -> ApplicationServiceRequest { } } -fn temporary_path(name: &str) -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock should be after the Unix epoch") - .as_nanos(); - std::env::temp_dir().join(format!( - "qsr-ownership-{name}-{}-{nanos}", - std::process::id() - )) -} - fn write_fake_podman(program: &Path, log: &Path, mode: &str, ready_port: u16) { let info = r#"{"host":{"security":{"rootless":true,"seccompEnabled":true,"seccompProfilePath":"/usr/share/containers/seccomp.json","apparmorEnabled":true,"selinuxEnabled":false}}}"#; let container = r#"[{"Id":"fake-container-id","AppArmorProfile":"containers-default","ProcessLabel":"","EffectiveCaps":[],"BoundingCaps":[],"Config":{"User":"65532:65532"},"HostConfig":{"ReadonlyRootfs":true,"Privileged":false,"SecurityOpt":["no-new-privileges"],"UsernsMode":"auto","PidMode":"private","IpcMode":"none","Memory":268435456,"NanoCpus":1000000000,"PidsLimit":32}}]"#; @@ -135,8 +119,9 @@ fn identical_retry_returns_existing_lease_without_second_launch() { .local_addr() .expect("address should resolve") .port(); - let program = temporary_path("fake-podman"); - let log = temporary_path("fake-podman-log"); + let fixture = tempfile::tempdir().expect("isolated ownership fixture directory"); + let program = fixture.path().join("fake-podman"); + let log = fixture.path().join("fake-podman-log"); write_fake_podman(&program, &log, "success", ready_port); let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); @@ -166,8 +151,9 @@ fn same_owner_and_request_id_with_different_content_fails_closed() { .local_addr() .expect("address should resolve") .port(); - let program = temporary_path("fake-podman"); - let log = temporary_path("fake-podman-log"); + let fixture = tempfile::tempdir().expect("isolated ownership fixture directory"); + let program = fixture.path().join("fake-podman"); + let log = fixture.path().join("fake-podman-log"); write_fake_podman(&program, &log, "success", ready_port); let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); @@ -198,8 +184,9 @@ fn wrong_owner_cannot_terminate_another_callers_lease() { .local_addr() .expect("address should resolve") .port(); - let program = temporary_path("fake-podman"); - let log = temporary_path("fake-podman-log"); + let fixture = tempfile::tempdir().expect("isolated ownership fixture directory"); + let program = fixture.path().join("fake-podman"); + let log = fixture.path().join("fake-podman-log"); write_fake_podman(&program, &log, "success", ready_port); let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); @@ -230,8 +217,9 @@ fn failed_launch_releases_idempotency_reservation_for_retry() { .local_addr() .expect("address should resolve") .port(); - let program = temporary_path("fake-podman"); - let log = temporary_path("fake-podman-log"); + let fixture = tempfile::tempdir().expect("isolated ownership fixture directory"); + let program = fixture.path().join("fake-podman"); + let log = fixture.path().join("fake-podman-log"); write_fake_podman(&program, &log, "fail_rootless", ready_port); let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); @@ -264,8 +252,9 @@ fn concurrent_duplicate_launch_is_rejected_while_first_launch_is_in_flight() { .local_addr() .expect("address should resolve") .port(); - let program = temporary_path("fake-podman"); - let log = temporary_path("fake-podman-log"); + let fixture = tempfile::tempdir().expect("isolated ownership fixture directory"); + let program = fixture.path().join("fake-podman"); + let log = fixture.path().join("fake-podman-log"); write_fake_podman(&program, &log, "slow_rootless", ready_port); let coordinator = Arc::new(ApplicationServiceCoordinator::new( RootlessPodmanAdapter::new(program.clone()), @@ -302,8 +291,9 @@ fn expired_lease_cleanup_is_bounded_and_attributed_to_owner_and_request() { .local_addr() .expect("address should resolve") .port(); - let program = temporary_path("fake-podman"); - let log = temporary_path("fake-podman-log"); + let fixture = tempfile::tempdir().expect("isolated ownership fixture directory"); + let program = fixture.path().join("fake-podman"); + let log = fixture.path().join("fake-podman-log"); write_fake_podman(&program, &log, "success", ready_port); let coordinator = ApplicationServiceCoordinator::new(RootlessPodmanAdapter::new(program.clone())); diff --git a/tests/runtime_boundary_regressions.rs b/tests/runtime_boundary_regressions.rs index 82bf2501..c6d67357 100644 --- a/tests/runtime_boundary_regressions.rs +++ b/tests/runtime_boundary_regressions.rs @@ -6,8 +6,8 @@ use std::{ fs, net::TcpListener, os::unix::fs::PermissionsExt, - path::PathBuf, - time::{Duration, SystemTime, UNIX_EPOCH}, + path::{Path, PathBuf}, + time::Duration, }; use quarantine_sandbox_runtime::{ @@ -15,6 +15,10 @@ use quarantine_sandbox_runtime::{ RootlessPodmanAdapter, ServiceProtocol, }; +const SECURITY_INFO: &str = r#"{"host":{"security":{"rootless":true,"seccompEnabled":true,"seccompProfilePath":"/usr/share/containers/seccomp.json","apparmorEnabled":true,"selinuxEnabled":false}}}"#; +const CONTAINER_INSPECTION: &str = r#"[{"Id":"fake-container-id","AppArmorProfile":"containers-default","ProcessLabel":"","EffectiveCaps":[],"BoundingCaps":[],"Config":{"User":"65532:65532"},"HostConfig":{"ReadonlyRootfs":true,"Privileged":false,"SecurityOpt":["no-new-privileges"],"UsernsMode":"auto","PidMode":"private","IpcMode":"none","Memory":268435456,"NanoCpus":1000000000,"PidsLimit":32}}]"#; +const NETWORK_INSPECTION: &str = r#"[{"internal":true,"dns_enabled":false}]"#; + fn policy() -> IsolationPolicy { IsolationPolicy { policy_id: "runtime_boundary_regression_v1".to_owned(), @@ -49,19 +53,8 @@ fn request(digest: &str) -> ApplicationServiceRequest { } } -fn temporary_path(name: &str) -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock should be after the Unix epoch") - .as_nanos(); - std::env::temp_dir().join(format!( - "quarantine-sandbox-runtime-{name}-{}-{nanos}", - std::process::id() - )) -} - -fn write_executable(name: &str, script: &str) -> PathBuf { - let program = temporary_path(name); +fn write_executable(fixture_directory: &Path, name: &str, script: &str) -> PathBuf { + let program = fixture_directory.join(name); fs::write(&program, script).expect("fake runtime executable should be writable"); let mut permissions = fs::metadata(&program) .expect("fake runtime executable metadata should exist") @@ -79,9 +72,13 @@ fn digest_pinned_image_accepts_numeric_sha256() { #[test] fn slow_successful_backend_command_is_polled_until_exit() { + let fixture = tempfile::tempdir().expect("isolated runtime boundary fixture directory"); let program = write_executable( + fixture.path(), "slow-podman", - "#!/bin/sh\nset -eu\ncase \"${1:-}\" in\n info) sleep 0.03; printf 'true\\n' ;;\n network) exit 21 ;;\n *) exit 91 ;;\nesac\n", + &format!( + "#!/bin/sh\nset -eu\ncase \"${{1:-}}:${{2:-}}\" in\n info:--format) if [ \"${{3:-}}\" = json ]; then printf '%s\\n' '{SECURITY_INFO}'; else printf 'true\\n'; fi ;;\n network:create) sleep 0.03; exit 21 ;;\n *) exit 91 ;;\nesac\n" + ), ); let adapter = RootlessPodmanAdapter::new(program.clone()) .with_command_timeout(Duration::from_millis(200)); @@ -98,12 +95,13 @@ fn slow_successful_backend_command_is_polled_until_exit() { #[test] fn non_utf8_port_output_fails_closed_and_cleans_every_created_resource() { - let log = temporary_path("non-utf8-podman-log"); + let fixture = tempfile::tempdir().expect("isolated runtime boundary fixture directory"); + let log = fixture.path().join("non-utf8-podman-log"); let script = format!( - "#!/bin/sh\nset -eu\nprintf '%s\\n' \"$*\" >> '{}'\ncase \"${{1:-}}\" in\n info) printf 'true\\n' ;;\n network) : ;;\n create) printf 'fake-container-id\\n' ;;\n start) : ;;\n port) printf '\\377' ;;\n stop) : ;;\n rm) : ;;\n *) exit 91 ;;\nesac\n", + "#!/bin/sh\nset -eu\nprintf '%s\\n' \"$*\" >> '{}'\nif [ \"${{1:-}}\" = info ] && [ \"${{3:-}}\" != json ]; then printf 'true\\n'; exit 0; fi\ncase \"${{1:-}}:${{2:-}}\" in\n info:--format) printf '%s\\n' '{SECURITY_INFO}' ;;\n network:create) : ;;\n network:inspect) printf '%s\\n' '{NETWORK_INSPECTION}' ;;\n create:--name) printf 'fake-container-id\\n' ;;\n start:*) : ;;\n container:inspect) printf '%s\\n' '{CONTAINER_INSPECTION}' ;;\n top:*) printf 'PID SECCOMP CAPEFF CAPBND CAPINH CAPPRM CAPAMB LABEL\\n1 filter - - - - - containers-default (enforce)\\n' ;;\n port:*) printf '\\377' ;;\n stop:*) : ;;\n rm:*) : ;;\n network:rm) : ;;\n *) exit 91 ;;\nesac\n", log.display() ); - let program = write_executable("non-utf8-podman", &script); + let program = write_executable(fixture.path(), "non-utf8-podman", &script); let adapter = RootlessPodmanAdapter::new(program.clone()); assert_eq!( @@ -122,27 +120,30 @@ fn non_utf8_port_output_fails_closed_and_cleans_every_created_resource() { #[test] fn cleanup_command_output_overflow_fails_closed_without_skipping_other_cleanup() { + let fixture = tempfile::tempdir().expect("isolated runtime boundary fixture directory"); let listener = TcpListener::bind(("127.0.0.1", 0)).expect("loopback listener should bind"); let ready_port = listener .local_addr() .expect("listener should expose its address") .port(); - let log = temporary_path("cleanup-output-limit-log"); + let log = fixture.path().join("cleanup-output-limit-log"); let script = format!( - "#!/bin/sh\nset -eu\nprintf '%s\\n' \"$*\" >> '{}'\ncase \"${{1:-}}\" in\n info) printf 'true\\n' ;;\n network) : ;;\n create) printf 'fake-container-id\\n' ;;\n start) : ;;\n port) printf '127.0.0.1:{}\\n' ;;\n stop) i=0; while [ \"$i\" -lt 256 ]; do printf x; i=$((i + 1)); done ;;\n rm) : ;;\n *) exit 91 ;;\nesac\n", + "#!/bin/sh\nset -eu\nprintf '%s\\n' \"$*\" >> '{}'\nif [ \"${{1:-}}\" = info ] && [ \"${{3:-}}\" != json ]; then printf 'true\\n'; exit 0; fi\ncase \"${{1:-}}:${{2:-}}\" in\n info:--format) printf '%s\\n' '{SECURITY_INFO}' ;;\n network:create) : ;;\n network:inspect) printf '%s\\n' '{NETWORK_INSPECTION}' ;;\n create:--name) printf 'fake-container-id\\n' ;;\n start:*) : ;;\n container:inspect) printf '%s\\n' '{CONTAINER_INSPECTION}' ;;\n top:*) printf 'PID SECCOMP CAPEFF CAPBND CAPINH CAPPRM CAPAMB LABEL\\n1 filter - - - - - containers-default (enforce)\\n' ;;\n port:*) printf '127.0.0.1:{}\\n' ;;\n stop:*) i=0; while [ \"$i\" -lt 256 ]; do printf x; i=$((i + 1)); done ;;\n rm:*) : ;;\n network:rm) : ;;\n *) exit 91 ;;\nesac\n", log.display(), ready_port ); - let program = write_executable("cleanup-output-limit-podman", &script); - let adapter = RootlessPodmanAdapter::new(program.clone()) - .with_command_output_limit_bytes(64) - .with_command_timeout(Duration::from_secs(1)); + let program = write_executable(fixture.path(), "cleanup-output-limit-podman", &script); + let adapter = + RootlessPodmanAdapter::new(program.clone()).with_command_timeout(Duration::from_secs(1)); let lease = adapter .launch_at(&request(&"a".repeat(64)), &policy(), 1_780_000_000) .expect("launch should succeed before bounded cleanup failure"); + let cleanup_adapter = RootlessPodmanAdapter::new(program.clone()) + .with_command_output_limit_bytes(64) + .with_command_timeout(Duration::from_secs(1)); assert_eq!( - adapter.terminate_at(&lease, 1_780_000_001), + cleanup_adapter.terminate_at(&lease, 1_780_000_001), Err(ApplicationServiceError::CleanupFailed) ); let calls = fs::read_to_string(&log).expect("all cleanup calls should be recorded"); From 6c8aac828fe1d0cddb4d5ea7890783a4118c6328 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:05:14 +0900 Subject: [PATCH 32/44] fix(runtime): remove unproven spawn retry Restore the shared bounded-command spawn contract to the canonical foundation behavior. The descendant lease branch keeps its fixture isolation but no longer masks an unobserved WouldBlock errno with an unconditional production retry policy. Signed-off-by: Seongho Bae --- src/infrastructure/bounded_command.rs | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/infrastructure/bounded_command.rs b/src/infrastructure/bounded_command.rs index 1b6e4e90..192b2d34 100644 --- a/src/infrastructure/bounded_command.rs +++ b/src/infrastructure/bounded_command.rs @@ -77,22 +77,12 @@ fn spawn_piped_child( program: &Path, args: &[String], ) -> Result<(Child, ChildStdout, ChildStderr), BoundedCommandError> { - let mut attempts = 0_u8; - let mut child = loop { - match Command::new(program) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - { - Ok(child) => break child, - Err(error) if error.kind() == io::ErrorKind::WouldBlock && attempts < 8 => { - attempts += 1; - thread::sleep(POLL_INTERVAL); - } - Err(_) => return Err(BoundedCommandError::Spawn), - } - }; + let mut child = Command::new(program) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|_| BoundedCommandError::Spawn)?; captured_pipes(child.stdout.take(), child.stderr.take()) .map(|(stdout, stderr)| (child, stdout, stderr)) } From e3945a007df1d003d882f7e56c9b83f1a43141da Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 00:59:35 +0900 Subject: [PATCH 33/44] ci(actions): scope cancellation to pull request heads Signed-off-by: Seongho Bae --- .github/actionlint.yaml | 4 ++++ .github/workflows/ci.yml | 5 ++++- tests/ci_runner_contract.rs | 23 +++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..8b2b68d4 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + labels: + - cwl-hostile-workload + - selinux diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1ecd34c..5894b95a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,10 @@ permissions: contents: read concurrency: - group: quarantine-ci-${{ github.event.pull_request.number || github.ref }} + group: >- + ${{ github.workflow }}-${{ github.repository }}-${{ + github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id + }} cancel-in-progress: true jobs: diff --git a/tests/ci_runner_contract.rs b/tests/ci_runner_contract.rs index 6b007ce5..08b0e7de 100644 --- a/tests/ci_runner_contract.rs +++ b/tests/ci_runner_contract.rs @@ -32,3 +32,26 @@ fn ordinary_hosted_ci_uses_explicit_supported_runner_image() { ); } } + +#[test] +fn ci_cancels_only_superseded_heads_of_the_same_pull_request() { + let workflow = fs::read_to_string(".github/workflows/ci.yml") + .expect("CI workflow must be readable from the repository root"); + + for required in [ + "github.workflow", + "github.repository", + "github.event.pull_request.number", + "github.run_id", + "cancel-in-progress: true", + ] { + assert!( + workflow.contains(required), + "CI concurrency must contain {required}" + ); + } + assert!( + !workflow.contains("github.ref }}"), + "non-PR runs must not share a ref-scoped cancellation group" + ); +} From 29d491369efcc4da104784147d92ff7d4d74c8e9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:04:57 +0900 Subject: [PATCH 34/44] docs(lease): preserve #6 gap-owner evidence locally --- ...ERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md diff --git a/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md b/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md new file mode 100644 index 00000000..8b2e1596 --- /dev/null +++ b/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md @@ -0,0 +1,25 @@ +# Application-service lease ownership gap-owner repair + +## Scope + +PR #6 owns caller-scoped application-service lease ownership and idempotency in the Supporting `application_service` bounded context. `LeaseOwnerId`, `ApplicationServiceBackend`, `ApplicationServiceCoordinator`, active-lease ownership, idempotent replay, bounded expiry cleanup, and cleanup fairness remain #6 domain truth. Backend invocation identity (#20/#40), consumer authentication, durable restart/orphan recovery, and Core sandbox-isolation semantics remain separate owner responsibilities. + +`docs/product-technical-gap-baseline.md` is repository-wide Gap authority and is maintained by canonical PR #121. A focused application-service branch must not retain its own live copy of that ledger. + +## Retained causal evidence + +Historical exact #6 head `8daab565bba1b61e605488c080eb28de371990a3` executed native CI `33962779845`. Verify `101297469596` passed the full test/lint/doc path and hosted negative rootless/AppArmor `101297469544` passed. Coverage remained an actual admission failure: `101297469496` failed during generation and branch coverage `101297469509` reported lines `2356/2442` (96.48%), functions `218/228` (95.61%), regions `3085/3211` (96.08%), and branches `429/492` (87.20%). The report contained both child-owned coordinator gaps and inherited runtime gaps, so it is historical exact-head evidence rather than current GREEN authority. + +The #6 lineage also removed an unsupported generic `Command::spawn` retry after no focused `io::ErrorKind` evidence justified it. Root issue #24 native-CI branch-trigger evidence is inherited foundation context; it is not application-service lease-ownership domain truth. + +## Single-writer finding and decision + +Review `5230212241` found that #6 still changed the repository-wide Gap ledger against exact PR base `0f765af1a4eea83029febee3b24c55cd7e7ce4e1`. The stale delta mixed valid #6 ownership/idempotency history with inherited root state, so deleting it without migration would lose causal context. + +The repair is therefore migration-first: + +1. preserve the #6-specific ownership/idempotency, coverage, and inherited-prerequisite distinctions in this owner-local record; +2. restore only `docs/product-technical-gap-baseline.md` byte-for-byte to exact-base blob `5f17a748cf92810963ea67b30ce54675a7c6d919`; +3. keep repository-wide live Gap updates in #121 rather than copying the latest #121 file into this leaf. + +No production Rust, public contract, fixture, or test semantics are changed by this ownership repair. No force push or destructive rebase is permitted. Historical CI does not transfer to the moved head; the resulting exact head must reacquire its own repository, formatting, full-test, Clippy/rustdoc, coverage, review/security, applicable positive-isolation, protected-integration, and immutable-release evidence. From ecdd84836d1d04660f620156f2190d8eb5664837 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:05:10 +0900 Subject: [PATCH 35/44] docs(gap): restore #6 global ledger to exact base --- docs/product-technical-gap-baseline.md | 34 +++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1926d27d..5f17a748 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,6 +1,6 @@ # Product and Technical Gap Baseline -Last reviewed on 2026-09-05 KST against dependency-root PR #1 causal issue #24 RED at exact head `0f765af1a4eea83029febee3b24c55cd7e7ce4e1`, its smallest workflow repair `034155c804baffd7b57a97da84a6037d79dd6a96`, and protected/default `develop@60a85c7633e03b425b67159ec6822c8178cf87ea`. This ledger distinguishes protected truth, active-PR implementation, checked-in RED evidence, backend-applied configuration evidence, live effective-runtime proof, queued/cancelled checks, and post-integration protected-head evidence. Predecessor evidence never transfers to a moved head. +Last reviewed on 2026-09-05 KST against dependency-root PR #1 latest test-bearing head `cefb80634bd62e775839345bd23d823d154482be` and protected/default `develop@60a85c7633e03b425b67159ec6822c8178cf87ea`. This ledger distinguishes protected truth, active-PR implementation, checked-in RED evidence, backend-applied configuration evidence, live effective-runtime proof, queued/cancelled checks, and post-integration protected-head evidence. Predecessor evidence never transfers to a moved head. ## Product responsibility @@ -27,11 +27,11 @@ Quarantine Sandbox Runtime owns reusable sandbox execution, isolation-policy enf | Network isolation | Per-sandbox internal DNS-disabled network is inspected and publication must resolve only to loopback. Draft #23 carries a stronger RED requiring positive exact container attachment and rejection of missing/additional attachments, with cleanup bound to the exact launch identities. | Object-level evidence implemented; attachment proof pending | Execute #23 on current ancestry, then add exact attachment and real negative-egress proof. | | Credential isolation | P0 request has no provider/user credentials, arbitrary environment map, host device/runtime socket, or broad host mount. | Active contract | Any future secret flow requires an explicit purpose-bound broker. | | CPU/RAM/PID bounds | `HostConfig.Memory`, `NanoCpus`, and `PidsLimit` are inspected against the request. | Backend-applied binding only; live proof incomplete | Verify the exact sandbox's authoritative cgroup-v2 values before release claim. | -| tmpfs / wall time | Launch applies bounded `/tmp` and `--timeout`, but current root does not deserialize/bind `HostConfig.Tmpfs` or `Config.Timeout`; inspect state alone would still not prove live enforcement. Draft #19 preserves hostile REDs for missing/wrong hardening, contradictory/duplicate tmpfs options, widened writable mounts, timeout mismatch, exact cleanup/non-publication, and exact inspect state without live proof. | P0 RED preserved on a root descendant; exact-head execution pending | Execute #19 after current root stabilizes; after the intended `resource_limits` RED, add only the smallest inspect-binding GREEN, then live cgroup/mount/wall-time proof. | +| tmpfs / wall time | Launch applies bounded `/tmp` and `--timeout`, but current root does not deserialize/bind `HostConfig.Tmpfs` or `Config.Timeout`; inspect state alone would still not prove live enforcement. Draft #19 preserves hostile REDs for missing/wrong hardening, contradictory/duplicate tmpfs options, widened writable mounts, timeout mismatch, exact cleanup/non-publication, and exact inspect state without live proof. | P0 RED preserved on a non-force root descendant; exact-head execution pending | Execute #19 after current root stabilizes; after the intended `resource_limits` RED, add only the smallest inspect-binding GREEN, then live cgroup/mount/wall-time proof. | | Process lifecycle | Adapter invokes Podman without a shell, creates network/container, starts, attests isolation before port/readiness, and attempts complete cleanup after partial launch/attestation failure. | Active repair | Current exact head must execute full tests; durable crash/restart orphan recovery belongs to Recovery context. | | Subprocess spawn pressure | Exact root `24526eb...` verify reached `cargo test` and one fake-Podman case returned `BackendInvocationFailed { operation: "rootless_probe" }` instead of the intended `InvalidPortMapping`. `BoundedCommandRunner` collapses spawn/capture failures into one application error, so the observation did not prove `WouldBlock`. The generic retry that had appeared on #6 was removed from #6 and command descendants because it lacked focused RED/errno evidence. | Unsupported workaround removed; errno-level observability gap remains | If spawn failure recurs, first add focused error-kind preservation/injection RED; only then consider the smallest causal retry policy. | | Ownership/idempotency | Draft #6 implements caller-scoped lease ownership/idempotency while retaining canonical root subprocess behavior. | Implemented on descendant | Preserve caller-scoped ownership separately from backend invocation identity; reacquire exact-child evidence after every root movement. | -| Runtime invocation identity | Draft #21 carries issue #20's independent same-request/same-second collision RED and issue #40's post-create lifecycle-ownership RED. #40 requires every post-create `start`/inspect/top/port/stop/remove operation to use the exact long ID returned by `podman create`, while generated `qsr-app-*` remains correlation/public metadata. | P0 REDs staged; production unchanged | Execute #20/#40 on their current exact ancestry, then introduce collision-resistant invocation identity and exact-ID lifecycle authority without changing consumer `request_id` or public sandbox-name semantics. | +| Runtime invocation identity | Draft #21 proves independent same-request/same-second invocations must not collide in container/network names, runtime identity labels, lease receipts, or cleanup targets. | P0 RED staged; production unchanged | Execute the RED, then add one collision-resistant runtime-generated invocation identity that consistently owns names, labels, lease evidence, and cleanup. | | gVisor/containerd/Kubernetes | Architecture targets only. | Missing | Add independent adapters after P0 contract stabilizes; public contracts remain backend-neutral. | ## Attestation evidence model @@ -61,15 +61,16 @@ The one-shot command path is a separate application-service contract from readin ## Verification and release state -- Protected/default `develop` remains `60a85c7633e03b425b67159ec6822c8178cf87ea`; PR #1 remains Draft and no qualifying approval is yet release evidence. -- Issue #24's checked-in RED has now actually executed on exact root head `0f765af1a4eea83029febee3b24c55cd7e7ce4e1`. CI run `33915538506` checked out that exact SHA. `verify=101191626830` failed only when `ci_runs_on_every_integrated_protected_develop_head` observed stale `push.branches: [main]`; formatter, repository policy and preceding tests passed. `coverage=101191626974` and `branch-coverage=101191627084` independently reached the same test and failed for the same causal assertion. The hosted negative rootless/AppArmor lane `101191627075` completed successfully; the dedicated positive-LSM lane `101191627020` remains separately queued. -- The smallest causal issue #24 repair is commit `034155c804baffd7b57a97da84a6037d79dd6a96`: native CI `push.branches` now targets `develop` and changes no other workflow semantics. `pull_request` remains enabled, no `paths`/`paths-ignore` filters were added, current-head cancellation remains scoped by PR/run identity, hosted lanes remain `ubuntu-24.04`, and the positive-LSM lane remains `[self-hosted, linux, cwl-hostile-workload, selinux]`. -- The issue #24 repair is not GREEN merely because the YAML changed. This documentation update moves the exact PR head again, so fresh exact-head CI/security/coverage/runtime evidence is required. After normal integration, a native push-triggered CI run must materialize on the exact protected `develop` integration SHA before that SHA can become release authority. +- Protected/default `develop` remains `60a85c7633e03b425b67159ec6822c8178cf87ea`; PR #1 remains Draft and no qualifying approval has been established in the current run. +- Issue #24's original test-bearing head `c43e5ca27acd96d085a4719fe5cb69de270aa723` requires native CI `push` coverage for the actual protected/default integration branch `develop`; `.github/workflows/ci.yml` still contains `push.branches: [main]` until that RED executes and fails for the intended cause. +- Latest test-only root head `cefb80634bd62e775839345bd23d823d154482be` strengthens the same exact-head contract rather than changing workflow behavior: both `push` and `pull_request` event sections must remain free of `paths`/`paths-ignore` filters that could suppress documentation-only or otherwise valid exact-head evidence. The stale `main` branch trigger remains the intended causal failure. +- Exact `cefb806...` materialized CI `33914930923`, Security Scan `33914930921`, SAST `33914930775`, and CodeQL `33914930818`. The CI jobs were still queued with no steps at the first fresh read, so this is not executed RED evidence. - Issue #24 is distinct from runner acquisition: `.github#712` owns runs/jobs that materialize but cannot obtain execution capacity, while a wrong branch/path trigger can prevent a protected integration run from being created at all. - The earlier causal effective-attestation RED `3fa5c5493fcbfbfb1c28b075e3bad30c03ea29b3` executed and failed because the old runtime could return a lease without effective sandbox inspection. Its causal production repair remains on the root lineage. - Exact root `24526eb55cf5db48ea07079b314f7d1b676eb48d` real Podman E2E passed Podman 4.9.3/rootless checks and immutable fixture pre-pull, then failed closed at `IsolationVerificationFailed { control_name: "lsm" }`; leak rejection succeeded. This is valid negative effective-LSM evidence, not a reason to weaken the control. - Root commit `6ad2b1c9d8f616be68dc28b35d017206f26c0787` split ordinary Ubuntu 24.04 into the explicit negative effective-LSM lane and the dedicated `[self-hosted, linux, cwl-hostile-workload, selinux]` positive lane without weakening production verification. -- Main command/release descendants #6 → #9 → #10 → #13 → #14 and artifact-analysis #18 must adopt root/parent movement only through non-force ancestry repair. Direct RED-only descendants #19, #21 and #23 retain their valid deltas but predecessor checks never transfer. +- Main command/release descendants #6 → #9 → #10 → #13 → #14 and artifact-analysis #18 must adopt root/parent movement only through non-force ancestry repair. Their PR bodies carry the current exact SHA; predecessor checks never transfer. +- RED-only/security descendants #19, #21, and #23 retain their valid test deltas while adopting the same root authority non-force. No production GREEN is authorized before each causal RED executes for the intended reason. - GitHub Releases remain absent until one exact integrated protected candidate satisfies all release gates. No mutable PR head is consumer authority. ## Protected integration and release authority @@ -86,12 +87,11 @@ The first release remains blocked until one exact integrated protected candidate ## Next bounded slices -1. Reacquire exact-head CI/security/coverage on PR #1 after the issue #24 workflow repair and this ledger update. The branch-trigger RED has executed causally; do not regress `push.branches: [develop]` or add event path filters. -2. Once #1 has qualifying review and all exact-head gates, merge normally. Then prove the native CI run materializes on the exact protected `develop` integration SHA; only that satisfies issue #24's post-integration completion gate. -3. If `rootless_probe` spawn failure recurs, preserve/inject the concrete `io::ErrorKind` in a focused RED before any retry behavior. Keep the unsupported generic `WouldBlock` workaround removed. -4. After root stabilizes, execute Draft #19's preserved resource-attestation RED; then add only the smallest inspect-binding GREEN before live cgroup/mount/wall-time proof. -5. Execute Draft #23's effective network-binding RED on current ancestry; after reproduced failure, require positive exact attachment to the runtime-owned deny-by-default network and real negative-egress proof. -6. Execute Draft #21's #20/#40 runtime identity and lifecycle-ownership REDs; then introduce a collision-resistant invocation identity and exact acquired-ID lifecycle authority below consumer correlation/idempotency semantics. -7. Execute #14 issue #25's pre-attestation command RED; after the intended payload-side-effect failure is observed, implement the smallest trusted hold/attest/release boundary and re-run command cleanup/output/identity regressions. -8. Reconcile #6/#9/#10/#13/#14 dependency-first without force. Overlapping foundation/CI deltas must be adopted rather than copied or reintroduced; then reconcile #18 on its current command parent. -9. Publish the first immutable runtime release only from one exact integrated protected head, then hand off released version/digest pinning to consumer owner paths. +1. Execute root exact `cefb806...` CI/security. Issue #24 must fail for stale `push.branches: [main]`; the strengthened contract also prevents future event-specific path filters from silently skipping exact-head evidence. Only after the intended branch-trigger RED executes should the native CI push trigger move to `develop` and reacquire exact-head GREEN. +2. If `rootless_probe` spawn failure recurs, preserve/inject the concrete `io::ErrorKind` in a focused RED before any retry behavior. Keep the unsupported generic `WouldBlock` workaround removed. +3. After root stabilizes, execute Draft #19's preserved resource-attestation RED; then add only the smallest inspect-binding GREEN before live cgroup/mount/wall-time proof. +4. Execute Draft #23's effective network-binding RED on current ancestry; after reproduced failure, require positive exact attachment to the runtime-owned deny-by-default network and real negative-egress proof. +5. Execute Draft #21's runtime-identity collision RED, then introduce one testable collision-resistant invocation identity below consumer correlation/idempotency semantics. +6. Execute #14 issue #25's pre-attestation command RED; after the intended payload-side-effect failure is observed, implement the smallest trusted hold/attest/release boundary and re-run command cleanup/output/identity regressions. +7. Reconcile #6/#9/#10/#13/#14 dependency-first without force. Overlapping foundation/CI deltas must be adopted rather than copied or reintroduced; then reconcile #18 on its current command parent. +8. Publish the first immutable runtime release only from one exact integrated protected head, then hand off released version/digest pinning to consumer owner paths. From a08a786e7744f6697d93da6f41a1a04796cad10e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:38:16 +0900 Subject: [PATCH 36/44] ci: adopt root exact-checkout hardening on lease owner lane Adopt the live root CI blob without moving production or test semantics. Preserve the child branch while removing the checkout-credential regression before the remaining non-force root ancestry adoption. --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94f1ed18..d172e830 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Verify exact checkout env: EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -53,6 +54,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Verify exact checkout env: EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -88,6 +90,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Verify exact checkout env: EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -133,6 +136,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Verify exact checkout env: EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -184,6 +188,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Verify exact checkout env: EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} From e364abe0313771ef156a7acc714e497cc180fa7b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:38:42 +0900 Subject: [PATCH 37/44] docs: trace staged root adoption for lease owner lane --- ...PPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md b/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md index 8b2e1596..4a513f46 100644 --- a/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md +++ b/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md @@ -23,3 +23,11 @@ The repair is therefore migration-first: 3. keep repository-wide live Gap updates in #121 rather than copying the latest #121 file into this leaf. No production Rust, public contract, fixture, or test semantics are changed by this ownership repair. No force push or destructive rebase is permitted. Historical CI does not transfer to the moved head; the resulting exact head must reacquire its own repository, formatting, full-test, Clippy/rustdoc, coverage, review/security, applicable positive-isolation, protected-integration, and immutable-release evidence. + +## Live-root adoption repair + +Fresh review `5230412333` found that the remaining root adoption cannot be represented safely by preferring either parent tree. Live root `feat/runtime-foundation-tdd@5c6a44bb2b35eb17d0315d72db242f4488c3c426` is 33 commits ahead of the PR's recorded base snapshot `0f765af1a4eea83029febee3b24c55cd7e7ce4e1`, and three paths carry independent valid deltas on both sides: `.github/workflows/ci.yml`, `src/application_service/mod.rs`, and `tests/runtime_boundary_regressions.rs`. + +The first overlap has an unambiguous successor. Root CI changes the push target to protected `develop` and sets `persist-credentials: false` on every checkout. Child exact `ecdd84836d1d04660f620156f2190d8eb5664837` already carried the `develop` target but not the credential hardening. Commit `a08a786e7744f6697d93da6f41a1a04796cad10e` therefore adopts the root CI blob `d172e830706afc290696c818730e1cf570df2be6` by ordinary fast-forward without changing production or test semantics. + +The two semantic overlaps remain intentionally unresolved rather than hidden in an evil merge. Root `application_service/mod.rs` contains the current parser-dominated repository-name simplification required by the exact coverage contract, while #6 adds the coordinator module/export. Root `runtime_boundary_regressions.rs` binds fake container identity to the current safe-identifier grammar, while #6 independently moved process fixtures into isolated `tempfile` directories. The eventual two-parent adoption must preserve both intents explicitly, make `5c6a44bb2b35eb17d0315d72db242f4488c3c426` an actual ancestor, and then reacquire exact-head CI. Predecessor GREEN is not transferable. From 63062201d3edada796b5637fc3ef1c44e64c1304 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 11:51:12 +0900 Subject: [PATCH 38/44] docs: record completed live-root adoption for lease owner --- ...ION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md b/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md index 4a513f46..2bfb09ae 100644 --- a/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md +++ b/docs/doctoring/APPLICATION_SERVICE_LEASE_OWNERSHIP_GAP_OWNER_REPAIR.md @@ -16,18 +16,20 @@ The #6 lineage also removed an unsupported generic `Command::spawn` retry after Review `5230212241` found that #6 still changed the repository-wide Gap ledger against exact PR base `0f765af1a4eea83029febee3b24c55cd7e7ce4e1`. The stale delta mixed valid #6 ownership/idempotency history with inherited root state, so deleting it without migration would lose causal context. -The repair is therefore migration-first: +The repair was migration-first: 1. preserve the #6-specific ownership/idempotency, coverage, and inherited-prerequisite distinctions in this owner-local record; -2. restore only `docs/product-technical-gap-baseline.md` byte-for-byte to exact-base blob `5f17a748cf92810963ea67b30ce54675a7c6d919`; +2. restore only `docs/product-technical-gap-baseline.md` byte-for-byte to the then-exact base blob `5f17a748cf92810963ea67b30ce54675a7c6d919`; 3. keep repository-wide live Gap updates in #121 rather than copying the latest #121 file into this leaf. -No production Rust, public contract, fixture, or test semantics are changed by this ownership repair. No force push or destructive rebase is permitted. Historical CI does not transfer to the moved head; the resulting exact head must reacquire its own repository, formatting, full-test, Clippy/rustdoc, coverage, review/security, applicable positive-isolation, protected-integration, and immutable-release evidence. +No production Rust, public contract, fixture, or test semantics were changed by that ownership repair. Historical CI does not transfer to moved heads. ## Live-root adoption repair -Fresh review `5230412333` found that the remaining root adoption cannot be represented safely by preferring either parent tree. Live root `feat/runtime-foundation-tdd@5c6a44bb2b35eb17d0315d72db242f4488c3c426` is 33 commits ahead of the PR's recorded base snapshot `0f765af1a4eea83029febee3b24c55cd7e7ce4e1`, and three paths carry independent valid deltas on both sides: `.github/workflows/ci.yml`, `src/application_service/mod.rs`, and `tests/runtime_boundary_regressions.rs`. +Fresh review `5230412333` found that live root `feat/runtime-foundation-tdd@5c6a44bb2b35eb17d0315d72db242f4488c3c426` had advanced 33 commits beyond #6's recorded base and that a parent-tree preference would silently discard valid deltas. The independent overlaps were `.github/workflows/ci.yml`, `src/application_service/mod.rs`, and `tests/runtime_boundary_regressions.rs`. -The first overlap has an unambiguous successor. Root CI changes the push target to protected `develop` and sets `persist-credentials: false` on every checkout. Child exact `ecdd84836d1d04660f620156f2190d8eb5664837` already carried the `develop` target but not the credential hardening. Commit `a08a786e7744f6697d93da6f41a1a04796cad10e` therefore adopts the root CI blob `d172e830706afc290696c818730e1cf570df2be6` by ordinary fast-forward without changing production or test semantics. +The repair was staged and then completed without force/rebase. Commit `a08a786e7744f6697d93da6f41a1a04796cad10e` first adopted exact root CI blob `d172e830706afc290696c818730e1cf570df2be6`, including `persist-credentials: false` on every checkout. The final ordinary two-parent commit `64283e08d99b353430bc1ce97f20019d89f8fbd0` uses the live root tree as the merge-tree foundation so every root-only coverage/runtime delta survives, while overlaying the #6-owned coordinator, backend, package and focused-test deltas. -The two semantic overlaps remain intentionally unresolved rather than hidden in an evil merge. Root `application_service/mod.rs` contains the current parser-dominated repository-name simplification required by the exact coverage contract, while #6 adds the coordinator module/export. Root `runtime_boundary_regressions.rs` binds fake container identity to the current safe-identifier grammar, while #6 independently moved process fixtures into isolated `tempfile` directories. The eventual two-parent adoption must preserve both intents explicitly, make `5c6a44bb2b35eb17d0315d72db242f4488c3c426` an actual ancestor, and then reacquire exact-head CI. Predecessor GREEN is not transferable. +The two semantic overlaps were merged explicitly rather than hidden by an evil merge. `src/application_service/mod.rs` blob `74270f29a1fe60b6e3a739514a5f17ab685bb3b4` keeps the #6 coordinator module/export and the root parser/coverage simplification (`split_once`, descendant `skip(1)`, parser-dominated no-colon success). `tests/runtime_boundary_regressions.rs` blob `78b11bf8aebcd08dda86b9137963e16e2fc2e0e8` keeps #6's isolated `tempfile` fixtures while binding fake runtime identity to the root-safe 64-character lower-hex container identifier. Root-only `docs/product-technical-gap-baseline.md`, coverage scripts/tests and Podman/root coverage deltas remain inherited from `5c6a44...`; they are not re-authored as #6-owned changes. + +This makes `5c6a44bb2b35eb17d0315d72db242f4488c3c426` an actual ancestor of #6. The resulting exact head must reacquire repository, formatting, full-test, Clippy/rustdoc, complete coverage, review/security, applicable positive-isolation, protected-integration and immutable-release evidence. Predecessor GREEN never transfers. Descendants must adopt this moved parent normally and preserve any overlapping effective-isolation/runtime deltas before they are considered current. From e58bedc9ad97faa67f66c5e9b59648fde15c32a7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 13:06:33 +0900 Subject: [PATCH 39/44] ci: pin stable coverage install toolchain --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d172e830..0da9da3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: - name: Verify dependency lock run: cargo metadata --locked --no-deps --format-version 1 > /dev/null - name: Install pinned coverage tool - run: cargo install cargo-llvm-cov --locked --version 0.8.6 + run: cargo +1.97.1 install cargo-llvm-cov --locked --version 0.8.6 - name: Generate production coverage evidence run: >- cargo llvm-cov From 7029537b2165c5ad0adf64183c0aa733257a5d4a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 19 Sep 2026 05:09:50 +0900 Subject: [PATCH 40/44] test(application): replace fake Podman atomically between retry phases --- tests/application_service_ownership.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/application_service_ownership.rs b/tests/application_service_ownership.rs index 7f620216..7a73c481 100644 --- a/tests/application_service_ownership.rs +++ b/tests/application_service_ownership.rs @@ -66,12 +66,17 @@ fn write_fake_podman(program: &Path, log: &Path, mode: &str, ready_port: u16) { network, container, ); - fs::write(program, script).expect("fake Podman should be writable"); - let mut permissions = fs::metadata(program) - .expect("fake Podman metadata should exist") + // Never truncate an executable path that a just-finished backend process may + // still reference. Stage a complete inode and atomically replace the pathname. + let staged_program = program.with_extension("next"); + fs::write(&staged_program, script).expect("staged fake Podman should be writable"); + let mut permissions = fs::metadata(&staged_program) + .expect("staged fake Podman metadata should exist") .permissions(); permissions.set_mode(0o700); - fs::set_permissions(program, permissions).expect("fake Podman should be executable"); + fs::set_permissions(&staged_program, permissions) + .expect("staged fake Podman should be executable"); + fs::rename(&staged_program, program).expect("fake Podman replacement should be atomic"); } fn count_calls(log: &Path, needle: &str) -> usize { From 0a6830e53f2ebea7ebf544d9518ae1ea9dd4f63f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 07:11:42 +0900 Subject: [PATCH 41/44] test(application-service): cover coordinator transition outcomes --- ...ion_service_coordinator_branch_coverage.rs | 292 ++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 tests/application_service_coordinator_branch_coverage.rs diff --git a/tests/application_service_coordinator_branch_coverage.rs b/tests/application_service_coordinator_branch_coverage.rs new file mode 100644 index 00000000..1140e833 --- /dev/null +++ b/tests/application_service_coordinator_branch_coverage.rs @@ -0,0 +1,292 @@ +//! Branch-outcome coverage for caller-scoped coordinator state transitions. + +use std::sync::{ + Arc, Barrier, + atomic::{AtomicUsize, Ordering}, +}; +use std::thread; + +use quarantine_sandbox_runtime::{ + ApplicationServiceBackend, ApplicationServiceCoordinator, ApplicationServiceCoordinatorError, + ApplicationServiceError, ApplicationServiceLease, ApplicationServiceRequest, CleanupReceipt, + IsolationPolicy, LeaseOwnerId, ResourceRequest, ServiceProtocol, +}; +use serde_json::json; + +fn owner() -> LeaseOwnerId { + LeaseOwnerId::new("urn:cwl:agent:coordinator-branch-test") + .expect("test owner identity should satisfy the bounded contract") +} + +fn policy() -> IsolationPolicy { + IsolationPolicy { + policy_id: "coordinator_branch_policy_v1".to_owned(), + maximum_memory_bytes: 512 * 1024 * 1024, + maximum_cpu_millicores: 2_000, + maximum_processes: 128, + maximum_lease_seconds: 900, + maximum_tmpfs_bytes: 128 * 1024 * 1024, + readiness_timeout_millis: 2_000, + readiness_poll_interval_millis: 10, + shutdown_grace_seconds: 2, + run_as_user_id: 65_532, + run_as_group_id: 65_532, + } +} + +fn request() -> ApplicationServiceRequest { + ApplicationServiceRequest { + schema_version: "1.0.0".to_owned(), + request_id: "coordinator_branch_request".to_owned(), + image_reference: format!("localhost/cwl/tool@sha256:{}", "d".repeat(64)), + container_port: 8_080, + protocol: ServiceProtocol::Http, + command: vec!["serve".to_owned()], + resources: ResourceRequest { + memory_bytes: 256 * 1024 * 1024, + cpu_millicores: 1_000, + maximum_processes: 32, + lease_seconds: 300, + tmpfs_bytes: 32 * 1024 * 1024, + }, + } +} + +fn lease_for( + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, +) -> ApplicationServiceLease { + serde_json::from_value(json!({ + "schema_version": "1.1.0", + "request_id": request.request_id, + "image_reference": request.image_reference, + "backend_id": "branch_test_backend", + "sandbox_id": "sandbox_branch_test", + "network_id": "network_branch_test", + "policy_id": policy.policy_id, + "policy_sha256": policy.effective_policy_sha256(), + "endpoint": { + "host": "127.0.0.1", + "port": 43123, + "protocol": "http" + }, + "started_at_epoch_seconds": started_at_epoch_seconds, + "expires_at_epoch_seconds": started_at_epoch_seconds + + u64::from(request.resources.lease_seconds), + "shutdown_grace_seconds": policy.shutdown_grace_seconds, + "isolation_attestation": { + "rootless": true, + "read_only_root_filesystem": true, + "all_capabilities_dropped": true, + "no_new_privileges": true, + "isolated_user_namespace": true, + "external_egress_denied": true, + "loopback_only_publication": true, + "credentials_available": false + } + })) + .expect("test lease should match the public serialized contract") +} + +fn cleanup_receipt( + lease: &ApplicationServiceLease, + terminated_at_epoch_seconds: u64, +) -> CleanupReceipt { + serde_json::from_value(json!({ + "schema_version": "1.0.0", + "sandbox_id": lease.sandbox_id(), + "network_id": lease.network_id(), + "container_removed": true, + "network_removed": true, + "terminated_at_epoch_seconds": terminated_at_epoch_seconds + })) + .expect("test cleanup receipt should match the public serialized contract") +} + +#[derive(Clone)] +struct LaunchBlockingBackend { + launch_entered: Arc, + launch_release: Arc, +} + +impl ApplicationServiceBackend for LaunchBlockingBackend { + fn launch_at( + &self, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result { + self.launch_entered.wait(); + self.launch_release.wait(); + Ok(lease_for(request, policy, started_at_epoch_seconds)) + } + + fn terminate_at( + &self, + lease: &ApplicationServiceLease, + terminated_at_epoch_seconds: u64, + ) -> Result { + Ok(cleanup_receipt(lease, terminated_at_epoch_seconds)) + } +} + +#[derive(Clone)] +struct TerminationBlockingBackend { + termination_entered: Arc, + termination_release: Arc, +} + +impl ApplicationServiceBackend for TerminationBlockingBackend { + fn launch_at( + &self, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result { + Ok(lease_for(request, policy, started_at_epoch_seconds)) + } + + fn terminate_at( + &self, + lease: &ApplicationServiceLease, + terminated_at_epoch_seconds: u64, + ) -> Result { + self.termination_entered.wait(); + self.termination_release.wait(); + Ok(cleanup_receipt(lease, terminated_at_epoch_seconds)) + } +} + +#[derive(Clone)] +struct CountingBackend { + termination_calls: Arc, +} + +impl ApplicationServiceBackend for CountingBackend { + fn launch_at( + &self, + request: &ApplicationServiceRequest, + policy: &IsolationPolicy, + started_at_epoch_seconds: u64, + ) -> Result { + Ok(lease_for(request, policy, started_at_epoch_seconds)) + } + + fn terminate_at( + &self, + lease: &ApplicationServiceLease, + terminated_at_epoch_seconds: u64, + ) -> Result { + self.termination_calls.fetch_add(1, Ordering::SeqCst); + Ok(cleanup_receipt(lease, terminated_at_epoch_seconds)) + } +} + +#[test] +fn conflicting_retry_while_launching_reaches_the_idempotency_conflict_guard() { + let launch_entered = Arc::new(Barrier::new(2)); + let launch_release = Arc::new(Barrier::new(2)); + let coordinator = Arc::new(ApplicationServiceCoordinator::new(LaunchBlockingBackend { + launch_entered: Arc::clone(&launch_entered), + launch_release: Arc::clone(&launch_release), + })); + let lease_owner = owner(); + let worker_coordinator = Arc::clone(&coordinator); + let worker_owner = lease_owner.clone(); + let worker = thread::spawn(move || { + worker_coordinator.launch_at(&worker_owner, &request(), &policy(), 1_780_000_000) + }); + + launch_entered.wait(); + let mut conflicting_request = request(); + conflicting_request.command.push("--different".to_owned()); + assert_eq!( + coordinator.launch_at( + &lease_owner, + &conflicting_request, + &policy(), + 1_780_000_001, + ), + Err(ApplicationServiceCoordinatorError::IdempotencyConflict) + ); + + launch_release.wait(); + let lease = worker + .join() + .expect("launch thread should not panic") + .expect("original launch should complete"); + coordinator + .terminate_at(&lease_owner, &lease, 1_780_000_010) + .expect("original lease should remain terminable"); +} + +#[test] +fn terminating_entry_distinguishes_identical_retry_from_conflicting_content() { + let termination_entered = Arc::new(Barrier::new(2)); + let termination_release = Arc::new(Barrier::new(2)); + let coordinator = Arc::new(ApplicationServiceCoordinator::new(TerminationBlockingBackend { + termination_entered: Arc::clone(&termination_entered), + termination_release: Arc::clone(&termination_release), + })); + let lease_owner = owner(); + let lease = coordinator + .launch_at(&lease_owner, &request(), &policy(), 1_780_000_000) + .expect("launch should complete before termination starts"); + let worker_coordinator = Arc::clone(&coordinator); + let worker_owner = lease_owner.clone(); + let worker_lease = lease.clone(); + let worker = thread::spawn(move || { + worker_coordinator.terminate_at(&worker_owner, &worker_lease, 1_780_000_010) + }); + + termination_entered.wait(); + assert_eq!( + coordinator.launch_at(&lease_owner, &request(), &policy(), 1_780_000_011), + Err(ApplicationServiceCoordinatorError::TerminationInProgress) + ); + let mut conflicting_request = request(); + conflicting_request.command.push("--different".to_owned()); + assert_eq!( + coordinator.launch_at( + &lease_owner, + &conflicting_request, + &policy(), + 1_780_000_012, + ), + Err(ApplicationServiceCoordinatorError::IdempotencyConflict) + ); + + termination_release.wait(); + worker + .join() + .expect("termination thread should not panic") + .expect("original termination should complete"); +} + +#[test] +fn forged_receipt_for_known_owner_and_request_fails_before_backend_cleanup() { + let termination_calls = Arc::new(AtomicUsize::new(0)); + let coordinator = ApplicationServiceCoordinator::new(CountingBackend { + termination_calls: Arc::clone(&termination_calls), + }); + let lease_owner = owner(); + let lease = coordinator + .launch_at(&lease_owner, &request(), &policy(), 1_780_000_000) + .expect("launch should succeed"); + let mut forged_value = serde_json::to_value(&lease).expect("lease should serialize"); + forged_value["backend_id"] = json!("forged_backend"); + let forged_lease: ApplicationServiceLease = serde_json::from_value(forged_value) + .expect("forged test value should remain structurally deserializable"); + + assert_eq!( + coordinator.terminate_at(&lease_owner, &forged_lease, 1_780_000_010), + Err(ApplicationServiceCoordinatorError::LeaseMismatch) + ); + assert_eq!(termination_calls.load(Ordering::SeqCst), 0); + + coordinator + .terminate_at(&lease_owner, &lease, 1_780_000_011) + .expect("registered lease should remain terminable"); + assert_eq!(termination_calls.load(Ordering::SeqCst), 1); +} From de244abedba34c3874b2a9a6c4e44de04a2de042 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 07:14:34 +0900 Subject: [PATCH 42/44] style(test): normalize coordinator branch coverage imports From 854d602744fd6c37696eb12c8e4bd753df639e2a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 07:15:02 +0900 Subject: [PATCH 43/44] style(test): rustfmt coordinator branch coverage imports --- tests/application_service_coordinator_branch_coverage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/application_service_coordinator_branch_coverage.rs b/tests/application_service_coordinator_branch_coverage.rs index 1140e833..29778da7 100644 --- a/tests/application_service_coordinator_branch_coverage.rs +++ b/tests/application_service_coordinator_branch_coverage.rs @@ -1,8 +1,8 @@ //! Branch-outcome coverage for caller-scoped coordinator state transitions. use std::sync::{ - Arc, Barrier, atomic::{AtomicUsize, Ordering}, + Arc, Barrier, }; use std::thread; From b096f4bf1d4ae1a663538127bf1b1048734c0079 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 21 Sep 2026 07:26:34 +0900 Subject: [PATCH 44/44] style(test): apply current rustfmt to coordinator branch witness --- ...ion_service_coordinator_branch_coverage.rs | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/tests/application_service_coordinator_branch_coverage.rs b/tests/application_service_coordinator_branch_coverage.rs index 29778da7..f73caaa5 100644 --- a/tests/application_service_coordinator_branch_coverage.rs +++ b/tests/application_service_coordinator_branch_coverage.rs @@ -1,8 +1,8 @@ //! Branch-outcome coverage for caller-scoped coordinator state transitions. use std::sync::{ - atomic::{AtomicUsize, Ordering}, Arc, Barrier, + atomic::{AtomicUsize, Ordering}, }; use std::thread; @@ -202,12 +202,7 @@ fn conflicting_retry_while_launching_reaches_the_idempotency_conflict_guard() { let mut conflicting_request = request(); conflicting_request.command.push("--different".to_owned()); assert_eq!( - coordinator.launch_at( - &lease_owner, - &conflicting_request, - &policy(), - 1_780_000_001, - ), + coordinator.launch_at(&lease_owner, &conflicting_request, &policy(), 1_780_000_001,), Err(ApplicationServiceCoordinatorError::IdempotencyConflict) ); @@ -225,10 +220,12 @@ fn conflicting_retry_while_launching_reaches_the_idempotency_conflict_guard() { fn terminating_entry_distinguishes_identical_retry_from_conflicting_content() { let termination_entered = Arc::new(Barrier::new(2)); let termination_release = Arc::new(Barrier::new(2)); - let coordinator = Arc::new(ApplicationServiceCoordinator::new(TerminationBlockingBackend { - termination_entered: Arc::clone(&termination_entered), - termination_release: Arc::clone(&termination_release), - })); + let coordinator = Arc::new(ApplicationServiceCoordinator::new( + TerminationBlockingBackend { + termination_entered: Arc::clone(&termination_entered), + termination_release: Arc::clone(&termination_release), + }, + )); let lease_owner = owner(); let lease = coordinator .launch_at(&lease_owner, &request(), &policy(), 1_780_000_000) @@ -248,12 +245,7 @@ fn terminating_entry_distinguishes_identical_retry_from_conflicting_content() { let mut conflicting_request = request(); conflicting_request.command.push("--different".to_owned()); assert_eq!( - coordinator.launch_at( - &lease_owner, - &conflicting_request, - &policy(), - 1_780_000_012, - ), + coordinator.launch_at(&lease_owner, &conflicting_request, &policy(), 1_780_000_012,), Err(ApplicationServiceCoordinatorError::IdempotencyConflict) );