fix(codebuild): sweep only build containers whose owning process is gone

crates/fakecloud-codebuild/src/runtime.rs

@@ -226,11 +226,12 @@ fn current_instance_label() -> String {
 /// build containers survive: `reconcile_builds` flips the persisted
 /// `IN_PROGRESS` builds to `FAILED`, but nothing reaps the containers, so the
 /// daemon slowly leaks them. Sweep every container tagged `fakecloud-codebuild`
-/// whose `fakecloud-instance` label is not the current process. Best-effort:
-/// any daemon error is ignored (the backend stays usable). Containers owned by
-/// the live process are left untouched so a concurrent build is never killed.
+/// whose owning process (its `fakecloud-instance` label) is gone. Containers
+/// owned by any *live* fakecloud process are left untouched: several can share
+/// one daemon, and removing another running server's build container fails
+/// that build mid-provisioning. Best-effort: any daemon error is ignored (the
+/// backend stays usable).
 pub async fn sweep_orphan_containers(cli: &str) {
-    let current = current_instance_label();
     let out = Command::new(cli)
         .args([
             "ps",
@@ -254,7 +255,10 @@ pub async fn sweep_orphan_containers(cli: &str) {
             continue;
         };
         let instance = parts.next().unwrap_or("");
-        if instance != current {
+        if fakecloud_core::container_net::owned_by_dead_process(
+            instance,
+            fakecloud_core::container_net::pid_alive,
+        ) {
             orphans.push(id.to_string());
         }
     }

crates/fakecloud-core/Cargo.toml

@@ -29,3 +29,6 @@ tokio = { workspace = true }
 tokio-util = { workspace = true }
 tracing = { workspace = true }
 uuid = { workspace = true }
+
+[target.'cfg(unix)'.dependencies]
+libc = "0.2"

crates/fakecloud-core/src/container_net.rs

@@ -158,6 +158,47 @@ pub fn bounded_status(cli: &str, args: &[String]) -> bool {
     wait_bounded(&mut child) && child.wait().map(|s| s.success()).unwrap_or(false)
 }
 
+/// True if the given PID is a live process on this host.
+///
+/// On Unix this is `kill(pid, 0)`: it returns 0 if the process exists
+/// (including zombies), or sets `errno` to `ESRCH` if not. On non-Unix
+/// platforms it conservatively returns `true`, so a caller never removes a
+/// resource it can't prove is orphaned.
+#[cfg(unix)]
+pub fn pid_alive(pid: u32) -> bool {
+    // SAFETY: `kill` with signal 0 is a liveness probe; it does not
+    // actually deliver a signal. Any PID value is safe to pass.
+    let rc = unsafe { libc::kill(pid as libc::pid_t, 0) };
+    if rc == 0 {
+        return true;
+    }
+    // errno == EPERM means the process exists but we can't signal it —
+    // still alive from our perspective.
+    std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
+}
+
+#[cfg(not(unix))]
+pub fn pid_alive(_pid: u32) -> bool {
+    true
+}
+
+/// Whether a container or network labelled `fakecloud-instance=<label>` was
+/// left behind by a fakecloud process that is gone. The label is
+/// `fakecloud-<pid>`; an object is orphaned only when that PID is neither the
+/// current process nor alive. Several fakecloud processes can share one
+/// daemon (parallel test servers, side-by-side installs), so an object owned
+/// by *another live* process is never an orphan. A label that doesn't parse is
+/// not treated as an orphan either -- nothing proves its owner is gone.
+pub fn owned_by_dead_process(label: &str, is_alive: impl Fn(u32) -> bool) -> bool {
+    let Some(pid) = label
+        .strip_prefix("fakecloud-")
+        .and_then(|p| p.parse::<u32>().ok())
+    else {
+        return false;
+    };
+    pid != std::process::id() && !is_alive(pid)
+}
+
 /// True when `cli` is podman or a podman-compatible binary. Matches on the
 /// filename component so absolute paths (`/opt/homebrew/bin/podman`) and
 /// wrappers (`podman-remote`) both register as podman. Docker Desktop's
@@ -607,6 +648,31 @@ mod tests {
         );
     }
 
+    #[test]
+    fn only_objects_of_a_dead_owner_are_orphans() {
+        let me = std::process::id();
+        let alive = |pid: u32| pid == 4242;
+        // Another live fakecloud process: never an orphan.
+        assert!(!owned_by_dead_process("fakecloud-4242", alive));
+        // Its owner is gone: an orphan.
+        assert!(owned_by_dead_process("fakecloud-777", alive));
+        // The current process, even if the probe says otherwise.
+        assert!(!owned_by_dead_process(&format!("fakecloud-{me}"), |_| {
+            false
+        }));
+        // Nothing proves an unparseable owner is gone.
+        for label in ["", "fakecloud-", "fakecloud-abc", "other-777"] {
+            assert!(!owned_by_dead_process(label, alive), "{label:?}");
+        }
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn pid_alive_probes_real_processes() {
+        assert!(pid_alive(std::process::id()));
+        assert!(!pid_alive(u32::MAX - 1));
+    }
+
     #[test]
     fn push_add_host_args_noop_for_podman() {
         let net = HostNetworking {

crates/fakecloud-e2e/tests/codebuild_real_execution.rs

@@ -259,6 +259,63 @@ async fn restart_fails_in_flight_build_instead_of_zombie() {
     );
 }
 
+#[tokio::test]
+async fn another_server_starting_does_not_kill_a_running_build() {
+    if !require_docker_or_skip("another_server_starting_does_not_kill_a_running_build") {
+        return;
+    }
+    // Two fakecloud processes share one daemon (here: parallel test servers).
+    // A server sweeps leaked build containers at startup; it must only remove
+    // those whose owning process is gone, never a live server's in-flight build.
+    let s = TestServer::start().await;
+    let cb = aws_sdk_codebuild::Client::new(&s.aws_config().await);
+    let spec =
+        "version: 0.2\nphases:\n  build:\n    commands:\n      - sleep 10\n      - echo survived\n";
+    create_project(&cb, "e2e-shared-daemon", spec).await;
+    let build_id = cb
+        .start_build()
+        .project_name("e2e-shared-daemon")
+        .send()
+        .await
+        .expect("start build")
+        .build_value()
+        .unwrap()
+        .id()
+        .unwrap()
+        .to_string();
+
+    // Wait until the build is past provisioning, i.e. its container exists.
+    for _ in 0..120 {
+        let out = cb.batch_get_builds().ids(&build_id).send().await.unwrap();
+        let phase = out
+            .builds()
+            .first()
+            .and_then(|b| b.current_phase())
+            .unwrap_or("");
+        if !matches!(phase, "" | "SUBMITTED" | "QUEUED" | "PROVISIONING") {
+            break;
+        }
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+
+    // A second server starts and runs its startup sweep while the build runs.
+    let other = TestServer::start().await;
+    let other_cb = aws_sdk_codebuild::Client::new(&other.aws_config().await);
+    other_cb
+        .list_projects()
+        .send()
+        .await
+        .expect("second server serves");
+
+    let build = wait_complete(&cb, &build_id).await;
+    assert_eq!(
+        build.build_status(),
+        Some(&StatusType::Succeeded),
+        "the second server's sweep must not remove this server's build container; phases: {:?}",
+        build.phases()
+    );
+}
+
 #[tokio::test]
 async fn cross_phase_shell_state_persists() {
     if !require_docker_or_skip("cross_phase_shell_state_persists") {

crates/fakecloud-e2e/tests/ec2_instance_runtime.rs

@@ -124,7 +124,19 @@ async fn run_instances_boots_real_container_with_user_data() {
     let container = container_for(&instance_id);
     assert!(!container.is_empty(), "no backing container found");
     let running = docker(&["inspect", "-f", "{{.State.Running}}", &container]);
-    assert_eq!(running, "true", "container should be running");
+    assert_eq!(
+        running,
+        "true",
+        "container should be running; state: {}; logs: {:?}",
+        docker(&[
+            "inspect",
+            "-f",
+            "status={{.State.Status}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}} \
+             error={{.State.Error}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}",
+            &container,
+        ]),
+        docker(&["logs", "--tail", "50", &container]),
+    );
 
     // User-data ran at boot (it executes asynchronously, so poll briefly).
     let mut marker = String::new();

crates/fakecloud-server/src/reaper.rs

@@ -68,20 +68,16 @@ fn reap_orphans(cli: &str, list_args: &[&str], remove_argv: impl Fn(&str) -> Vec
         return 0;
     };
 
-    let self_pid = std::process::id();
     let mut reaped = 0usize;
 
     for line in listing.lines() {
         let Some((id, label)) = line.split_once(' ') else {
             continue;
         };
-        let Some(pid_str) = label.strip_prefix("fakecloud-") else {
-            continue;
-        };
-        let Ok(pid) = pid_str.parse::<u32>() else {
-            continue;
-        };
-        if pid == self_pid || pid_alive(pid) {
+        if !fakecloud_core::container_net::owned_by_dead_process(
+            label,
+            fakecloud_core::container_net::pid_alive,
+        ) {
             continue;
         }
         let removed = fakecloud_core::container_net::bounded_status(cli, &remove_argv(id));
@@ -93,33 +89,9 @@ fn reap_orphans(cli: &str, list_args: &[&str], remove_argv: impl Fn(&str) -> Vec
     reaped
 }
 
-/// True if the given PID is a live process on this host.
-///
-/// On Unix we use `kill(pid, 0)`: it returns 0 if the process exists
-/// (including zombies), or sets `errno` to `ESRCH` if not. On non-Unix
-/// platforms we conservatively return `true` so the reaper never removes
-/// a container it can't prove is orphaned.
-#[cfg(unix)]
-pub fn pid_alive(pid: u32) -> bool {
-    // SAFETY: `kill` with signal 0 is a liveness probe; it does not
-    // actually deliver a signal. Any PID value is safe to pass.
-    let rc = unsafe { libc::kill(pid as libc::pid_t, 0) };
-    if rc == 0 {
-        return true;
-    }
-    // errno == EPERM means the process exists but we can't signal it —
-    // still alive from our perspective.
-    std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
-}
-
-#[cfg(not(unix))]
-pub fn pid_alive(_pid: u32) -> bool {
-    true
-}
-
 #[cfg(all(test, unix))]
 mod tests {
-    use super::pid_alive;
+    use fakecloud_core::container_net::pid_alive;
 
     #[test]
     fn self_is_alive() {