From 263236fd8219ffe4e98bfca690dfbd6eadff78c4 Mon Sep 17 00:00:00 2001
From: Lucas Vieira <vieiralucas4@gmail.com>
Date: Mon, 14 Sep 2026 02:02:13 -0300
Subject: [PATCH 1/2] fix(codebuild): sweep only build containers whose owning
 process is gone

At startup CodeBuild removed every `fakecloud-codebuild` container whose
`fakecloud-instance` label was not the current process -- including those of
other fakecloud processes that are still running and sharing the daemon.
Any second server starting (parallel e2e test servers, side-by-side
installs) killed the first server's in-flight build container, failing the
build in PROVISIONING with "failed to create /codebuild/build: " (the
`docker exec` into a container that had just been removed). The
codebuild_real_execution e2e tests hit exactly that when run in parallel.

The shared startup reaper already got this right by checking whether the
owning PID is alive. Move that check to fakecloud_core::container_net
(`pid_alive` plus `owned_by_dead_process`) and use it from both, so a
container is swept only when its owner is neither this process nor alive.

Also: ec2_instance_runtime's "container should be running" assertion now
reports the container's status, exit code, OOM flag, error, start/finish
time and logs, so its remaining intermittent failure names its cause.

Tests: owned_by_dead_process unit test; a new e2e starts a second server
while a build runs and asserts the build still succeeds (fails with the
build FAILED under the old sweep).
---
 crates/fakecloud-codebuild/src/runtime.rs     | 14 ++--
 crates/fakecloud-core/Cargo.toml              |  3 +
 crates/fakecloud-core/src/container_net.rs    | 66 +++++++++++++++++++
 .../tests/codebuild_real_execution.rs         | 57 ++++++++++++++++
 .../tests/ec2_instance_runtime.rs             | 14 +++-
 crates/fakecloud-server/src/reaper.rs         | 38 ++---------
 6 files changed, 153 insertions(+), 39 deletions(-)

diff --git a/crates/fakecloud-codebuild/src/runtime.rs b/crates/fakecloud-codebuild/src/runtime.rs
index 7dfd4a3d9..8a5e88f6a 100644
--- a/crates/fakecloud-codebuild/src/runtime.rs
+++ b/crates/fakecloud-codebuild/src/runtime.rs
@@ -226,11 +226,12 @@ fn current_instance_label() -> String {
 /// build containers survive: `reconcile_builds` flips the persisted
 /// `IN_PROGRESS` builds to `FAILED`, but nothing reaps the containers, so the
 /// daemon slowly leaks them. Sweep every container tagged `fakecloud-codebuild`
-/// whose `fakecloud-instance` label is not the current process. Best-effort:
-/// any daemon error is ignored (the backend stays usable). Containers owned by
-/// the live process are left untouched so a concurrent build is never killed.
+/// whose owning process (its `fakecloud-instance` label) is gone. Containers
+/// owned by any *live* fakecloud process are left untouched: several can share
+/// one daemon, and removing another running server's build container fails
+/// that build mid-provisioning. Best-effort: any daemon error is ignored (the
+/// backend stays usable).
 pub async fn sweep_orphan_containers(cli: &str) {
-    let current = current_instance_label();
     let out = Command::new(cli)
         .args([
             "ps",
@@ -254,7 +255,10 @@ pub async fn sweep_orphan_containers(cli: &str) {
             continue;
         };
         let instance = parts.next().unwrap_or("");
-        if instance != current {
+        if fakecloud_core::container_net::owned_by_dead_process(
+            instance,
+            fakecloud_core::container_net::pid_alive,
+        ) {
             orphans.push(id.to_string());
         }
     }
diff --git a/crates/fakecloud-core/Cargo.toml b/crates/fakecloud-core/Cargo.toml
index d52efa7ab..0b27c2438 100644
--- a/crates/fakecloud-core/Cargo.toml
+++ b/crates/fakecloud-core/Cargo.toml
@@ -29,3 +29,6 @@ tokio = { workspace = true }
 tokio-util = { workspace = true }
 tracing = { workspace = true }
 uuid = { workspace = true }
+
+[target.'cfg(unix)'.dependencies]
+libc = "0.2"
diff --git a/crates/fakecloud-core/src/container_net.rs b/crates/fakecloud-core/src/container_net.rs
index f4bf39be7..6d099f498 100644
--- a/crates/fakecloud-core/src/container_net.rs
+++ b/crates/fakecloud-core/src/container_net.rs
@@ -158,6 +158,47 @@ pub fn bounded_status(cli: &str, args: &[String]) -> bool {
     wait_bounded(&mut child) && child.wait().map(|s| s.success()).unwrap_or(false)
 }
 
+/// True if the given PID is a live process on this host.
+///
+/// On Unix this is `kill(pid, 0)`: it returns 0 if the process exists
+/// (including zombies), or sets `errno` to `ESRCH` if not. On non-Unix
+/// platforms it conservatively returns `true`, so a caller never removes a
+/// resource it can't prove is orphaned.
+#[cfg(unix)]
+pub fn pid_alive(pid: u32) -> bool {
+    // SAFETY: `kill` with signal 0 is a liveness probe; it does not
+    // actually deliver a signal. Any PID value is safe to pass.
+    let rc = unsafe { libc::kill(pid as libc::pid_t, 0) };
+    if rc == 0 {
+        return true;
+    }
+    // errno == EPERM means the process exists but we can't signal it —
+    // still alive from our perspective.
+    std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
+}
+
+#[cfg(not(unix))]
+pub fn pid_alive(_pid: u32) -> bool {
+    true
+}
+
+/// Whether a container or network labelled `fakecloud-instance=<label>` was
+/// left behind by a fakecloud process that is gone. The label is
+/// `fakecloud-<pid>`; an object is orphaned only when that PID is neither the
+/// current process nor alive. Several fakecloud processes can share one
+/// daemon (parallel test servers, side-by-side installs), so an object owned
+/// by *another live* process is never an orphan. A label that doesn't parse is
+/// not treated as an orphan either -- nothing proves its owner is gone.
+pub fn owned_by_dead_process(label: &str, is_alive: impl Fn(u32) -> bool) -> bool {
+    let Some(pid) = label
+        .strip_prefix("fakecloud-")
+        .and_then(|p| p.parse::<u32>().ok())
+    else {
+        return false;
+    };
+    pid != std::process::id() && !is_alive(pid)
+}
+
 /// True when `cli` is podman or a podman-compatible binary. Matches on the
 /// filename component so absolute paths (`/opt/homebrew/bin/podman`) and
 /// wrappers (`podman-remote`) both register as podman. Docker Desktop's
@@ -607,6 +648,31 @@ mod tests {
         );
     }
 
+    #[test]
+    fn only_objects_of_a_dead_owner_are_orphans() {
+        let me = std::process::id();
+        let alive = |pid: u32| pid == 4242;
+        // Another live fakecloud process: never an orphan.
+        assert!(!owned_by_dead_process("fakecloud-4242", alive));
+        // Its owner is gone: an orphan.
+        assert!(owned_by_dead_process("fakecloud-777", alive));
+        // The current process, even if the probe says otherwise.
+        assert!(!owned_by_dead_process(&format!("fakecloud-{me}"), |_| {
+            false
+        }));
+        // Nothing proves an unparseable owner is gone.
+        for label in ["", "fakecloud-", "fakecloud-abc", "other-777"] {
+            assert!(!owned_by_dead_process(label, alive), "{label:?}");
+        }
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn pid_alive_probes_real_processes() {
+        assert!(pid_alive(std::process::id()));
+        assert!(!pid_alive(u32::MAX - 1));
+    }
+
     #[test]
     fn push_add_host_args_noop_for_podman() {
         let net = HostNetworking {
diff --git a/crates/fakecloud-e2e/tests/codebuild_real_execution.rs b/crates/fakecloud-e2e/tests/codebuild_real_execution.rs
index 75ef5f962..5acf6a0dd 100644
--- a/crates/fakecloud-e2e/tests/codebuild_real_execution.rs
+++ b/crates/fakecloud-e2e/tests/codebuild_real_execution.rs
@@ -259,6 +259,63 @@ async fn restart_fails_in_flight_build_instead_of_zombie() {
     );
 }
 
+#[tokio::test]
+async fn another_server_starting_does_not_kill_a_running_build() {
+    if !require_docker_or_skip("another_server_starting_does_not_kill_a_running_build") {
+        return;
+    }
+    // Two fakecloud processes share one daemon (here: parallel test servers).
+    // A server sweeps leaked build containers at startup; it must only remove
+    // those whose owning process is gone, never a live server's in-flight build.
+    let s = TestServer::start().await;
+    let cb = aws_sdk_codebuild::Client::new(&s.aws_config().await);
+    let spec =
+        "version: 0.2\nphases:\n  build:\n    commands:\n      - sleep 10\n      - echo survived\n";
+    create_project(&cb, "e2e-shared-daemon", spec).await;
+    let build_id = cb
+        .start_build()
+        .project_name("e2e-shared-daemon")
+        .send()
+        .await
+        .expect("start build")
+        .build_value()
+        .unwrap()
+        .id()
+        .unwrap()
+        .to_string();
+
+    // Wait until the build is past provisioning, i.e. its container exists.
+    for _ in 0..120 {
+        let out = cb.batch_get_builds().ids(&build_id).send().await.unwrap();
+        let phase = out
+            .builds()
+            .first()
+            .and_then(|b| b.current_phase())
+            .unwrap_or("");
+        if !matches!(phase, "" | "SUBMITTED" | "QUEUED" | "PROVISIONING") {
+            break;
+        }
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+
+    // A second server starts and runs its startup sweep while the build runs.
+    let other = TestServer::start().await;
+    let other_cb = aws_sdk_codebuild::Client::new(&other.aws_config().await);
+    other_cb
+        .list_projects()
+        .send()
+        .await
+        .expect("second server serves");
+
+    let build = wait_complete(&cb, &build_id).await;
+    assert_eq!(
+        build.build_status(),
+        Some(&StatusType::Succeeded),
+        "the second server's sweep must not remove this server's build container; phases: {:?}",
+        build.phases()
+    );
+}
+
 #[tokio::test]
 async fn cross_phase_shell_state_persists() {
     if !require_docker_or_skip("cross_phase_shell_state_persists") {
diff --git a/crates/fakecloud-e2e/tests/ec2_instance_runtime.rs b/crates/fakecloud-e2e/tests/ec2_instance_runtime.rs
index bdd32d1dd..4ea87a350 100644
--- a/crates/fakecloud-e2e/tests/ec2_instance_runtime.rs
+++ b/crates/fakecloud-e2e/tests/ec2_instance_runtime.rs
@@ -124,7 +124,19 @@ async fn run_instances_boots_real_container_with_user_data() {
     let container = container_for(&instance_id);
     assert!(!container.is_empty(), "no backing container found");
     let running = docker(&["inspect", "-f", "{{.State.Running}}", &container]);
-    assert_eq!(running, "true", "container should be running");
+    assert_eq!(
+        running,
+        "true",
+        "container should be running; state: {}; logs: {:?}",
+        docker(&[
+            "inspect",
+            "-f",
+            "status={{.State.Status}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}} \
+             error={{.State.Error}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}",
+            &container,
+        ]),
+        docker(&["logs", "--tail", "50", &container]),
+    );
 
     // User-data ran at boot (it executes asynchronously, so poll briefly).
     let mut marker = String::new();
diff --git a/crates/fakecloud-server/src/reaper.rs b/crates/fakecloud-server/src/reaper.rs
index 43063c703..9ff385e57 100644
--- a/crates/fakecloud-server/src/reaper.rs
+++ b/crates/fakecloud-server/src/reaper.rs
@@ -68,20 +68,16 @@ fn reap_orphans(cli: &str, list_args: &[&str], remove_argv: impl Fn(&str) -> Vec
         return 0;
     };
 
-    let self_pid = std::process::id();
     let mut reaped = 0usize;
 
     for line in listing.lines() {
         let Some((id, label)) = line.split_once(' ') else {
             continue;
         };
-        let Some(pid_str) = label.strip_prefix("fakecloud-") else {
-            continue;
-        };
-        let Ok(pid) = pid_str.parse::<u32>() else {
-            continue;
-        };
-        if pid == self_pid || pid_alive(pid) {
+        if !fakecloud_core::container_net::owned_by_dead_process(
+            label,
+            fakecloud_core::container_net::pid_alive,
+        ) {
             continue;
         }
         let removed = fakecloud_core::container_net::bounded_status(cli, &remove_argv(id));
@@ -93,33 +89,9 @@ fn reap_orphans(cli: &str, list_args: &[&str], remove_argv: impl Fn(&str) -> Vec
     reaped
 }
 
-/// True if the given PID is a live process on this host.
-///
-/// On Unix we use `kill(pid, 0)`: it returns 0 if the process exists
-/// (including zombies), or sets `errno` to `ESRCH` if not. On non-Unix
-/// platforms we conservatively return `true` so the reaper never removes
-/// a container it can't prove is orphaned.
-#[cfg(unix)]
-pub fn pid_alive(pid: u32) -> bool {
-    // SAFETY: `kill` with signal 0 is a liveness probe; it does not
-    // actually deliver a signal. Any PID value is safe to pass.
-    let rc = unsafe { libc::kill(pid as libc::pid_t, 0) };
-    if rc == 0 {
-        return true;
-    }
-    // errno == EPERM means the process exists but we can't signal it —
-    // still alive from our perspective.
-    std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
-}
-
-#[cfg(not(unix))]
-pub fn pid_alive(_pid: u32) -> bool {
-    true
-}
-
 #[cfg(all(test, unix))]
 mod tests {
-    use super::pid_alive;
+    use fakecloud_core::container_net::pid_alive;
 
     #[test]
     fn self_is_alive() {

From 774b45921226bfb572094d86f4beef426e11872b Mon Sep 17 00:00:00 2001
From: Lucas Vieira <vieiralucas4@gmail.com>
Date: Mon, 14 Sep 2026 02:03:10 -0300
Subject: [PATCH 2/2] chore: record libc as a fakecloud-core dependency in
 Cargo.lock

---
 Cargo.lock | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Cargo.lock b/Cargo.lock
index 7c713629b..1ef5afd06 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5312,6 +5312,7 @@ dependencies = [
  "flate2",
  "http 1.4.0",
  "http-body-util",
+ "libc",
  "md-5 0.10.6",
  "parking_lot",
  "serde",