From 794013777b4b3f70d7d37e457155443cf44b6bbe Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 29 Sep 2026 20:10:46 -0300 Subject: [PATCH] fix(ecs,lambda): pull fakecloud's ECR registry over plain HTTP under podman fakecloud rewrites an AWS ECR image URI to its own OCI registry at 127.0.0.1:, which serves plain HTTP. Docker treats a loopback registry as insecure on its own, but podman insists on TLS for every registry, so ECS tasks and image-based Lambda functions failed to pull with "server gave HTTP response to HTTPS client" (#2585). - container_image::pull_image takes a RegistryTransport; a pull of a rewritten ECR reference is PlainHttp, and podman then gets --tls-verify=false. Upstream registries keep TLS; docker is unchanged (its pull has no such flag). - ECS task launch and the Lambda docker backend (start + prepull) pass the transport from whether the URI was rewritten. - New ecs_podman_ecr e2e pushes to fakecloud ECR and runs an ECS task under podman; routed to the podman E2E partition. - ECS docs note the podman behavior. Closes #2585 --- crates/fakecloud-core/src/container_image.rs | 144 +++++++++++- crates/fakecloud-core/src/ecr_uri.rs | 12 +- .../src/bin/e2e_nextest_partitions.rs | 24 +- crates/fakecloud-e2e/tests/ecs_podman_ecr.rs | 214 ++++++++++++++++++ .../src/runtime/task_lifecycle.rs | 3 + crates/fakecloud-lambda/src/runtime/docker.rs | 6 + website/content/docs/services/ecs.md | 2 +- 7 files changed, 383 insertions(+), 22 deletions(-) create mode 100644 crates/fakecloud-e2e/tests/ecs_podman_ecr.rs diff --git a/crates/fakecloud-core/src/container_image.rs b/crates/fakecloud-core/src/container_image.rs index 5b2c91585..aee331e6c 100644 --- a/crates/fakecloud-core/src/container_image.rs +++ b/crates/fakecloud-core/src/container_image.rs @@ -32,6 +32,32 @@ const MAX_PULL_ATTEMPTS: u32 = 5; /// Delay before the first retry; doubles on each further retry. const BASE_RETRY_DELAY: Duration = Duration::from_secs(1); +/// How the registry a pulled reference names is reached. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RegistryTransport { + /// TLS, as every upstream registry (ECR Public, Docker Hub, ...) serves. + Https, + /// Plain HTTP: fakecloud's own OCI registry, which an AWS ECR URI is + /// rewritten to. Docker treats a loopback registry as insecure on its + /// own, but Podman insists on TLS for every registry unless told + /// otherwise, so its pull fails with "server gave HTTP response to HTTPS + /// client" without `--tls-verify=false`. + PlainHttp, +} + +impl RegistryTransport { + /// The transport for a pull whose reference was (`true`) or was not + /// rewritten to fakecloud's registry by + /// [`crate::ecr_uri::translate_to_local_at`]. + pub fn for_local_rewrite(rewritten: bool) -> Self { + if rewritten { + Self::PlainHttp + } else { + Self::Https + } + } +} + /// How an image became available for a launch. #[derive(Debug, Clone, PartialEq, Eq)] pub enum PulledImage { @@ -48,15 +74,17 @@ pub enum PulledImage { /// falls back to a locally cached copy, or is retried with backoff when /// nothing is cached; any other failure is returned at once. `docker_config` /// is exported as `DOCKER_CONFIG` for the pull so registry credentials -/// resolve. +/// resolve. `transport` says whether the registry is fakecloud's own +/// plain-HTTP one. /// /// Returns the pull's stderr as the error. pub async fn pull_image( cli: &str, docker_config: Option<&Path>, reference: &str, + transport: RegistryTransport, ) -> Result { - pull_image_with(cli, docker_config, reference, BASE_RETRY_DELAY).await + pull_image_with(cli, docker_config, reference, transport, BASE_RETRY_DELAY).await } /// Make `reference` available locally, pulling it only when it is not @@ -81,13 +109,21 @@ async fn ensure_image_with( if image_cached(cli, docker_config, reference).await { return Ok(PulledImage::Present); } - pull_image_with(cli, docker_config, reference, base_delay).await + pull_image_with( + cli, + docker_config, + reference, + RegistryTransport::Https, + base_delay, + ) + .await } async fn pull_image_with( cli: &str, docker_config: Option<&Path>, reference: &str, + transport: RegistryTransport, base_delay: Duration, ) -> Result { let mut delay = base_delay; @@ -98,7 +134,7 @@ async fn pull_image_with( cmd.env("DOCKER_CONFIG", p); } let out = cmd - .args(["pull", reference]) + .args(pull_args(cli, reference, transport)) .output() .await .map_err(|e| format!("{cli} pull: {e}"))?; @@ -132,6 +168,18 @@ async fn pull_image_with( } } +/// The `pull` arguments for `cli`. Only Podman needs telling that a +/// plain-HTTP registry is one: Docker accepts it for a loopback registry, and +/// its `pull` has no `--tls-verify` flag at all. +fn pull_args(cli: &str, reference: &str, transport: RegistryTransport) -> Vec { + let mut args = vec!["pull".to_string()]; + if transport == RegistryTransport::PlainHttp && crate::container_net::is_podman_binary(cli) { + args.push("--tls-verify=false".to_string()); + } + args.push(reference.to_string()); + args +} + /// Whether `reference` resolves to an image in the local cache. Runs with /// the same `DOCKER_CONFIG` as the pull: the config also selects the Docker /// context, so without it the check could consult a different daemon than @@ -276,10 +324,17 @@ mod tests { /// when `cached`. Every invocation is appended to `calls.log`. struct FakeCli { dir: tempfile::TempDir, + name: String, } impl FakeCli { fn new(pull_failures: u32, pull_stderr: &str, cached: bool) -> Self { + Self::named("cli", pull_failures, pull_stderr, cached) + } + + /// A fake installed under `name`, which is what decides whether + /// fakecloud drives it as Podman. + fn named(name: &str, pull_failures: u32, pull_stderr: &str, cached: bool) -> Self { let dir = tempfile::tempdir().unwrap(); let script = format!( r#"#!/bin/sh @@ -304,7 +359,7 @@ exit 2 "#, dir = dir.path().display(), ); - let path = dir.path().join("cli"); + let path = dir.path().join(name); std::fs::write(&path, script).unwrap(); std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); // Executing a file that another process holds open for writing @@ -327,11 +382,14 @@ exit 2 } } let _ = std::fs::remove_file(dir.path().join("calls.log")); - Self { dir } + Self { + dir, + name: name.to_string(), + } } fn cli(&self) -> String { - self.dir.path().join("cli").display().to_string() + self.dir.path().join(&self.name).display().to_string() } fn calls(&self) -> Vec { @@ -347,7 +405,22 @@ exit 2 } async fn pull_ref(&self, reference: &str) -> Result { - pull_image_with(&self.cli(), None, reference, Duration::from_millis(1)).await + self.pull_via(reference, RegistryTransport::Https).await + } + + async fn pull_via( + &self, + reference: &str, + transport: RegistryTransport, + ) -> Result { + pull_image_with( + &self.cli(), + None, + reference, + transport, + Duration::from_millis(1), + ) + .await } async fn ensure(&self) -> Result { @@ -384,6 +457,61 @@ exit 2 assert_eq!(cli.ensure().await, Err(missing.to_string())); } + #[tokio::test] + async fn podman_pulls_fakecloud_registry_over_plain_http() { + // Podman defaults to TLS even for a loopback registry, and fakecloud's + // serves plain HTTP (issue #2585). + let cli = FakeCli::named("podman", 0, "", false); + let got = cli + .pull_via("127.0.0.1:4566/test:healthy", RegistryTransport::PlainHttp) + .await; + assert_eq!(got, Ok(PulledImage::Pulled)); + assert_eq!( + cli.calls(), + ["pull --tls-verify=false 127.0.0.1:4566/test:healthy"] + ); + } + + #[tokio::test] + async fn podman_keeps_tls_for_upstream_registries() { + let cli = FakeCli::named("podman", 0, "", false); + let got = cli + .pull_via( + "public.ecr.aws/docker/library/alpine:3.20", + RegistryTransport::Https, + ) + .await; + assert_eq!(got, Ok(PulledImage::Pulled)); + assert_eq!( + cli.calls(), + ["pull public.ecr.aws/docker/library/alpine:3.20"] + ); + } + + #[tokio::test] + async fn docker_pulls_fakecloud_registry_without_a_tls_flag() { + // `docker pull` has no --tls-verify; the daemon already treats a + // loopback registry as insecure. + let cli = FakeCli::named("docker", 0, "", false); + let got = cli + .pull_via("127.0.0.1:4566/test:healthy", RegistryTransport::PlainHttp) + .await; + assert_eq!(got, Ok(PulledImage::Pulled)); + assert_eq!(cli.calls(), ["pull 127.0.0.1:4566/test:healthy"]); + } + + #[test] + fn transport_follows_the_local_rewrite() { + assert_eq!( + RegistryTransport::for_local_rewrite(true), + RegistryTransport::PlainHttp + ); + assert_eq!( + RegistryTransport::for_local_rewrite(false), + RegistryTransport::Https + ); + } + #[tokio::test] async fn a_successful_pull_needs_no_cache_check() { let cli = FakeCli::new(0, "", false); diff --git a/crates/fakecloud-core/src/ecr_uri.rs b/crates/fakecloud-core/src/ecr_uri.rs index 0f87929ea..81cb03f21 100644 --- a/crates/fakecloud-core/src/ecr_uri.rs +++ b/crates/fakecloud-core/src/ecr_uri.rs @@ -5,9 +5,11 @@ //! fakecloud can't pull from AWS — there is no AWS account. Instead we //! translate the URI to `127.0.0.1:/:` and pull //! from fakecloud's own OCI v2 registry (which is just another route on -//! the same HTTP server). Docker treats `127.0.0.1:` as an insecure -//! registry automatically on both Linux and Docker Desktop, so no daemon -//! config is required. +//! the same HTTP server), which serves plain HTTP. Docker treats +//! `127.0.0.1:` as an insecure registry automatically on both Linux and +//! Docker Desktop, so no daemon config is required; Podman does not, so its +//! pulls are told explicitly (see +//! [`crate::container_image::RegistryTransport`]). /// Detect whether `image` is an AWS private-ECR URI. Match shape: /// `.dkr.ecr..amazonaws.com/[:|@sha256:]`. @@ -27,7 +29,9 @@ pub fn is_aws_ecr_uri(image: &str) -> bool { /// go straight to the upstream daemon. /// /// Docker's localhost-registry behaviour means the daemon on both Linux -/// and macOS Docker Desktop accepts `127.0.0.1:` over plain HTTP. +/// and macOS Docker Desktop accepts `127.0.0.1:` over plain HTTP; +/// Podman needs `--tls-verify=false`, which +/// [`crate::container_image::pull_image`] adds for a rewritten reference. pub fn translate_to_local(image: &str, server_port: u16) -> Option { translate_to_local_at(image, "127.0.0.1", server_port) } diff --git a/crates/fakecloud-e2e/src/bin/e2e_nextest_partitions.rs b/crates/fakecloud-e2e/src/bin/e2e_nextest_partitions.rs index e8ffd6f07..1c388c13b 100644 --- a/crates/fakecloud-e2e/src/bin/e2e_nextest_partitions.rs +++ b/crates/fakecloud-e2e/src/bin/e2e_nextest_partitions.rs @@ -7,6 +7,12 @@ use serde_json::{json, Value}; const PACKAGE: &str = "fakecloud-e2e"; const USAGE: &str = "usage: e2e_nextest_partitions [matrix|check]"; +// Everything outside the Lambda binaries and the podman-only ECS binary, which +// needs the podman install only the podman partition pays for. +const GENERAL_FILTER: &str = concat!( + "package(fakecloud-e2e) and not binary(lambda) and not binary(lambda_invoke) ", + "and not binary(ecs_podman_ecr)", +); const LAMBDA_RUNTIME_FAMILY_PARTITIONS: [&str; 6] = [ "lambda-runtimes-python", "lambda-runtimes-nodejs", @@ -90,49 +96,49 @@ struct Partition { const PARTITIONS: [Partition; 19] = [ Partition { name: "general-1", - filter: "package(fakecloud-e2e) and not binary(lambda) and not binary(lambda_invoke)", + filter: GENERAL_FILTER, partition: Some("hash:1/8"), install_podman: false, }, Partition { name: "general-2", - filter: "package(fakecloud-e2e) and not binary(lambda) and not binary(lambda_invoke)", + filter: GENERAL_FILTER, partition: Some("hash:2/8"), install_podman: false, }, Partition { name: "general-3", - filter: "package(fakecloud-e2e) and not binary(lambda) and not binary(lambda_invoke)", + filter: GENERAL_FILTER, partition: Some("hash:3/8"), install_podman: false, }, Partition { name: "general-4", - filter: "package(fakecloud-e2e) and not binary(lambda) and not binary(lambda_invoke)", + filter: GENERAL_FILTER, partition: Some("hash:4/8"), install_podman: false, }, Partition { name: "general-5", - filter: "package(fakecloud-e2e) and not binary(lambda) and not binary(lambda_invoke)", + filter: GENERAL_FILTER, partition: Some("hash:5/8"), install_podman: false, }, Partition { name: "general-6", - filter: "package(fakecloud-e2e) and not binary(lambda) and not binary(lambda_invoke)", + filter: GENERAL_FILTER, partition: Some("hash:6/8"), install_podman: false, }, Partition { name: "general-7", - filter: "package(fakecloud-e2e) and not binary(lambda) and not binary(lambda_invoke)", + filter: GENERAL_FILTER, partition: Some("hash:7/8"), install_podman: false, }, Partition { name: "general-8", - filter: "package(fakecloud-e2e) and not binary(lambda) and not binary(lambda_invoke)", + filter: GENERAL_FILTER, partition: Some("hash:8/8"), install_podman: false, }, @@ -199,7 +205,7 @@ const PARTITIONS: [Partition; 19] = [ }, Partition { name: "lambda-container-podman", - filter: "binary(lambda) and test(lambda_invoke_podman)", + filter: "(binary(lambda) and test(lambda_invoke_podman)) or binary(ecs_podman_ecr)", partition: None, install_podman: true, }, diff --git a/crates/fakecloud-e2e/tests/ecs_podman_ecr.rs b/crates/fakecloud-e2e/tests/ecs_podman_ecr.rs new file mode 100644 index 000000000..b1160c350 --- /dev/null +++ b/crates/fakecloud-e2e/tests/ecs_podman_ecr.rs @@ -0,0 +1,214 @@ +//! ECR -> ECS under the podman backend (issue #2585). fakecloud rewrites an +//! AWS ECR URI to its own plain-HTTP registry at `127.0.0.1:`. Docker +//! treats a loopback registry as insecure on its own; podman does not, so the +//! task's pull failed with "server gave HTTP response to HTTPS client" until +//! the runtime passed `--tls-verify=false` for fakecloud's registry. +//! +//! Runs in the podman E2E partition, which installs podman. Per the project's +//! no-silent-skip rule it hard-fails when podman is unavailable. + +mod helpers; + +use std::process::Stdio; +use std::time::Duration; + +use aws_sdk_ecs::types::ContainerDefinition; +use base64::Engine; +use helpers::TestServer; +use tokio::process::Command; + +const SEED_IMAGE: &str = "public.ecr.aws/docker/library/alpine:3.20"; +const AWS_ACCOUNT: &str = "123456789012"; +const AWS_REGION: &str = "us-east-1"; + +fn require_podman() { + let ok = std::process::Command::new("podman") + .arg("info") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false); + assert!( + ok, + "podman is required for this test but `podman info` failed" + ); +} + +/// The host fakecloud's registry is reached at from podman, and the server +/// env that makes the runtime rewrite ECR URIs to that host. On Linux podman +/// pulls on the host network, so `127.0.0.1` works. On macOS it pulls inside +/// the podman machine VM, where only `host.containers.internal` reaches the +/// host -- the rewrite fakecloud applies when told it runs in a container. +fn registry_host_and_env() -> (&'static str, Vec<(&'static str, &'static str)>) { + let mut env = vec![("FAKECLOUD_CONTAINER_CLI", "podman")]; + if cfg!(target_os = "linux") { + ("127.0.0.1", env) + } else { + env.push(("FAKECLOUD_IN_CONTAINER", "1")); + ("host.containers.internal", env) + } +} + +async fn podman(args: &[&str]) -> std::process::Output { + Command::new("podman") + .args(args) + .output() + .await + .expect("spawn podman") +} + +async fn podman_ok(args: &[&str]) { + let out = podman(args).await; + assert!( + out.status.success(), + "podman {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); +} + +/// Push the seed image to fakecloud ECR as `repo:tag` and return the AWS URI +/// a task definition references it by. +async fn seed_image(registry_host: &str, port: u16, repo: &str, tag: &str) -> String { + let local_uri = format!("{registry_host}:{port}/{repo}:{tag}"); + + let mut pulled = false; + for attempt in 0..5u64 { + if podman(&["image", "exists", SEED_IMAGE]) + .await + .status + .success() + || podman(&["pull", "-q", SEED_IMAGE]).await.status.success() + { + pulled = true; + break; + } + tokio::time::sleep(Duration::from_secs(5 * (attempt + 1))).await; + } + assert!(pulled, "could not pull seed image {SEED_IMAGE}"); + podman_ok(&["tag", SEED_IMAGE, &local_uri]).await; + + let auth_dir = tempfile::tempdir().expect("tempdir"); + let auth_file = auth_dir.path().join("auth.json"); + let auth = base64::engine::general_purpose::STANDARD.encode("AWS:fakecloud-seed"); + let config = serde_json::json!({ + "auths": { format!("{registry_host}:{port}"): { "auth": auth } } + }); + std::fs::write(&auth_file, config.to_string()).expect("write auth file"); + podman_ok(&[ + "push", + "--tls-verify=false", + "--authfile", + auth_file.to_str().unwrap(), + &local_uri, + ]) + .await; + + // Drop the local name so the task can only get the image from the + // registry, which is the path under test. + podman_ok(&["rmi", &local_uri]).await; + + format!("{AWS_ACCOUNT}.dkr.ecr.{AWS_REGION}.amazonaws.com/{repo}:{tag}") +} + +fn port_from_endpoint(endpoint: &str) -> u16 { + endpoint + .rsplit(':') + .next() + .and_then(|p| p.trim_end_matches('/').parse().ok()) + .expect("port from endpoint") +} + +#[tokio::test] +async fn ecs_task_pulls_ecr_image_with_podman() { + require_podman(); + let (registry_host, env) = registry_host_and_env(); + let server = TestServer::start_with_env(&env).await; + let port = port_from_endpoint(server.endpoint()); + + server + .ecr_client() + .await + .create_repository() + .repository_name("podman-pull") + .send() + .await + .expect("create_repository"); + let aws_uri = seed_image(registry_host, port, "podman-pull", "v1").await; + + let ecs = server.ecs_client().await; + ecs.create_cluster() + .cluster_name("podman-ecr") + .send() + .await + .expect("create_cluster"); + ecs.register_task_definition() + .family("podman-ecr-task") + .container_definitions( + ContainerDefinition::builder() + .name("app") + .image(&aws_uri) + .essential(true) + .entry_point("/bin/sh") + .command("-c") + .command("echo from-podman-ecr && exit 0") + .build(), + ) + .send() + .await + .expect("register_task_definition"); + + let run = ecs + .run_task() + .cluster("podman-ecr") + .task_definition("podman-ecr-task") + .send() + .await + .expect("run_task"); + let arn = run.tasks()[0].task_arn().unwrap().to_string(); + let task_id = arn.rsplit('/').next().unwrap().to_string(); + + let mut stopped = None; + for _ in 0..240 { + let desc = ecs + .describe_tasks() + .cluster("podman-ecr") + .tasks(&arn) + .send() + .await + .expect("describe_tasks"); + let task = desc.tasks()[0].clone(); + if task.last_status() == Some("STOPPED") { + stopped = Some(task); + break; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + let task = stopped.expect("task did not reach STOPPED"); + assert_ne!( + task.stop_code().map(|c| c.as_str()), + Some("TaskFailedToStart"), + "task failed to start: {:?}", + task.stopped_reason() + ); + + let logs: serde_json::Value = reqwest::Client::new() + .get(format!( + "{}/_fakecloud/ecs/tasks/{task_id}/logs", + server.endpoint() + )) + .send() + .await + .expect("fetch task logs") + .json() + .await + .expect("task logs json"); + assert!( + logs["logs"] + .as_str() + .unwrap_or_default() + .contains("from-podman-ecr"), + "task logs: {logs}" + ); + assert_eq!(logs["exitCode"].as_i64(), Some(0), "task logs: {logs}"); +} diff --git a/crates/fakecloud-ecs/src/runtime/task_lifecycle.rs b/crates/fakecloud-ecs/src/runtime/task_lifecycle.rs index f8043d06a..67ca34e52 100644 --- a/crates/fakecloud-ecs/src/runtime/task_lifecycle.rs +++ b/crates/fakecloud-ecs/src/runtime/task_lifecycle.rs @@ -116,6 +116,9 @@ impl EcsRuntime { &self.cli, self.docker_config_path().as_deref(), pull_uri, + fakecloud_core::container_image::RegistryTransport::for_local_rewrite( + local_pull_uri.is_some(), + ), ) .await .map_err(RuntimeError::ImagePull)?; diff --git a/crates/fakecloud-lambda/src/runtime/docker.rs b/crates/fakecloud-lambda/src/runtime/docker.rs index a5d33906a..177d9fff7 100644 --- a/crates/fakecloud-lambda/src/runtime/docker.rs +++ b/crates/fakecloud-lambda/src/runtime/docker.rs @@ -128,6 +128,9 @@ impl DockerBackend { &self.cli, self.docker_config_path().as_deref(), pull_uri, + fakecloud_core::container_image::RegistryTransport::for_local_rewrite( + local_pull_uri.is_some(), + ), ) .await .map_err(|e| RuntimeError::ContainerStartFailed(format!("docker pull failed: {e}")))?; @@ -499,6 +502,9 @@ impl LambdaBackend for DockerBackend { &self.cli, self.docker_config_path().as_deref(), pull_uri, + fakecloud_core::container_image::RegistryTransport::for_local_rewrite( + local_uri.is_some(), + ), ) .await .map_err(|e| { diff --git a/website/content/docs/services/ecs.md b/website/content/docs/services/ecs.md index 06d4775a0..36cd7692a 100644 --- a/website/content/docs/services/ecs.md +++ b/website/content/docs/services/ecs.md @@ -85,7 +85,7 @@ Without a container runtime (docker/podman missing), `RunTask` still returns tas ### Pulling from fakecloud ECR -Tasks that reference AWS private-ECR URIs (`.dkr.ecr..amazonaws.com/:`) are resolved against fakecloud's own OCI v2 endpoint. The runtime pulls from `127.0.0.1:/:`, retags to the AWS URI, and runs the container under the user-visible image name. On Linux this is transparent because the docker daemon auto-treats `127.0.0.1` as an insecure registry. On Docker Desktop for macOS or Windows, the daemon runs in a VM and `127.0.0.1` maps to the VM itself, not the host — for full fidelity there, add `127.0.0.0/8` to Docker Desktop's insecure-registries and ensure fakecloud is reachable from the VM (e.g. via `host.docker.internal` forwarding). +Tasks that reference AWS private-ECR URIs (`.dkr.ecr..amazonaws.com/:`) are resolved against fakecloud's own OCI v2 endpoint. The runtime pulls from `127.0.0.1:/:`, retags to the AWS URI, and runs the container under the user-visible image name. The registry serves plain HTTP. On Linux this is transparent because the docker daemon auto-treats `127.0.0.1` as an insecure registry. Podman does not, so under the podman backend fakecloud pulls its own registry with `--tls-verify=false` (only for rewritten ECR URIs; upstream registries keep TLS verification). On Docker Desktop for macOS or Windows, the daemon runs in a VM and `127.0.0.1` maps to the VM itself, not the host — for full fidelity there, add `127.0.0.0/8` to Docker Desktop's insecure-registries and ensure fakecloud is reachable from the VM (e.g. via `host.docker.internal` forwarding). Same resolution path applies to Lambda functions deployed with `PackageType=Image`: `Code.ImageUri` pointing at a fakecloud ECR URI is pulled and run on invoke.