diff --git a/crates/fakecloud-cloudfront/src/dataplane.rs b/crates/fakecloud-cloudfront/src/dataplane.rs index fa191d47a..9702e1f24 100644 --- a/crates/fakecloud-cloudfront/src/dataplane.rs +++ b/crates/fakecloud-cloudfront/src/dataplane.rs @@ -375,19 +375,35 @@ fn select_target_origin<'a>(cfg: &'a DistributionConfig, path: &str) -> &'a str &cfg.default_cache_behavior.target_origin_id } -/// An S3 static-website endpoint (`bucket.s3-website-.amazonaws.com` or -/// `bucket.s3-website..amazonaws.com`). Matched precisely (`.s3-website` -/// label plus the `.amazonaws.com` suffix) so a custom origin that merely -/// contains the substring — e.g. `my.s3-website.example.com` — is NOT rerouted -/// to the local fakecloud port. -fn is_s3_website(domain: &str) -> bool { - domain.contains(".s3-website") && domain.ends_with(".amazonaws.com") +/// An S3 origin this process serves: a virtual-hosted S3 endpoint naming a +/// bucket, in any of the forms AWS publishes. That covers the REST endpoints +/// an `S3OriginConfig` origin carries (`.s3..amazonaws.com`, +/// CDK's `S3BucketOrigin` emitting the bucket's `RegionalDomainName`; the +/// legacy `.s3.amazonaws.com`; the dash-separated +/// `.s3-.amazonaws.com`; dualstack and FIPS) and the +/// static-website endpoints (`.s3-website-.amazonaws.com`, +/// `.s3-website..amazonaws.com`), under every partition's DNS +/// suffix. +/// +/// Delegates to the shared `Host` parser, so a domain is recognized exactly +/// when the S3 front door can resolve its bucket from `Host` (dotted bucket +/// names, case-insensitivity and the LocalStack hostname convention included). +/// A bucket is required: a path-style `s3..amazonaws.com` has nothing +/// to serve, and a look-alike such as `my.s3-website.example.com` is not an +/// AWS hostname at all, so neither is rerouted. +fn is_s3_origin(domain: &str) -> bool { + fakecloud_core::protocol::parse_routing_host(domain) + .is_some_and(|h| h.service == "s3" && h.bucket.is_some()) } /// Resolve an [`crate::model::Origin`] to the upstream to connect to. /// -/// - S3-website origins are served by this same fakecloud process, so connect to -/// its own port while preserving the website domain in `Host`. +/// - S3 origins (REST or static-website endpoints naming a bucket) are served by +/// this same fakecloud process, so connect to its own port while preserving +/// the bucket domain in `Host`, where the S3 front door reads the bucket from. +/// This holds even when the origin is declared with a `CustomOriginConfig` +/// (website endpoints always are, and a REST endpoint may be): the bucket +/// lives here, and the real hostname would reach real AWS. /// - Custom origins honor `CustomOriginConfig`: an `https-only` protocol policy /// is fetched over HTTPS (else HTTP), and the configured `HTTPPort`/`HTTPSPort` /// is appended UNLESS the `domain_name` already carries an explicit `:port` @@ -395,7 +411,7 @@ fn is_s3_website(domain: &str) -> bool { /// - Bare origins (no config) are reached over HTTP at their domain verbatim. fn upstream_for(origin: &crate::model::Origin, s3_endpoint: &str) -> UpstreamTarget { let domain = &origin.domain_name; - if is_s3_website(domain) { + if is_s3_origin(domain) { return UpstreamTarget { url_base: format!("http://{s3_endpoint}"), host_header: domain.clone(), @@ -589,13 +605,21 @@ mod tests { } #[test] - fn s3_website_detection_is_precise() { - assert!(is_s3_website("b.s3-website-us-east-1.amazonaws.com")); - assert!(is_s3_website("b.s3-website.us-east-1.amazonaws.com")); + fn s3_origin_detection_is_precise() { + assert!(is_s3_origin("b.s3-website-us-east-1.amazonaws.com")); + assert!(is_s3_origin("b.s3-website.us-east-1.amazonaws.com")); + assert!(is_s3_origin("b.s3.us-east-1.amazonaws.com")); // A custom origin that merely contains the substring must NOT match. - assert!(!is_s3_website("my.s3-website.example.com")); - assert!(!is_s3_website("api.example.com")); - assert!(!is_s3_website("127.0.0.1:8080")); + assert!(!is_s3_origin("my.s3-website.example.com")); + assert!(!is_s3_origin("b.s3.us-east-1.amazonaws.com.example.com")); + assert!(!is_s3_origin("api.example.com")); + assert!(!is_s3_origin("127.0.0.1:8080")); + // Path-style endpoints name no bucket, so there is nothing to serve. + assert!(!is_s3_origin("s3.us-east-1.amazonaws.com")); + assert!(!is_s3_origin("s3.amazonaws.com")); + // Other AWS service hostnames are not S3 origins. + assert!(!is_s3_origin("abc.execute-api.us-east-1.amazonaws.com")); + assert!(!is_s3_origin("my-lb-1.us-east-1.elb.amazonaws.com")); } #[test] @@ -608,6 +632,53 @@ mod tests { assert_eq!(up.host_header, "b.s3-website-us-east-1.amazonaws.com"); } + #[test] + fn s3_rest_origin_routes_to_local_port() { + // What an `S3OriginConfig` origin carries (CDK's `S3BucketOrigin` emits the + // bucket's `RegionalDomainName`). These resolve in real DNS, so without + // rerouting they are proxied to real AWS S3, which answers `NoSuchBucket`. + for domain in [ + "b.s3.us-east-1.amazonaws.com", + "b.s3.amazonaws.com", + "b.s3-us-east-1.amazonaws.com", + "my.dotted.bucket.s3.eu-west-2.amazonaws.com", + "b.s3.dualstack.eu-west-2.amazonaws.com", + "b.s3-fips.us-gov-west-1.amazonaws.com", + "b.s3.cn-north-1.amazonaws.com.cn", + "B.S3.US-EAST-1.AMAZONAWS.COM", + ] { + let up = upstream_for(&origin(domain, None), "127.0.0.1:4566"); + assert_eq!(up.url_base, "http://127.0.0.1:4566", "{domain}"); + assert_eq!(up.host_header, domain, "{domain}"); + } + } + + #[test] + fn s3_lookalike_origin_is_not_rerouted() { + // Not an AWS hostname: a real custom origin that must keep its own domain. + let up = upstream_for(&origin("s3.example.com", None), "127.0.0.1:4566"); + assert_eq!(up.url_base, "http://s3.example.com"); + } + + #[test] + fn s3_origin_with_custom_origin_config_is_still_served_locally() { + // A bucket REST endpoint may be declared as a custom origin (e.g. CDK's + // `HttpOrigin(bucket.bucketRegionalDomainName)`), and a website endpoint + // always is. The bucket lives in this process either way; honoring the + // config would send the fetch to real AWS S3. + for domain in [ + "b.s3.us-east-1.amazonaws.com", + "b.s3-website.eu-central-1.amazonaws.com", + ] { + let up = upstream_for( + &origin(domain, Some(custom("https-only", 80, 8443))), + "127.0.0.1:4566", + ); + assert_eq!(up.url_base, "http://127.0.0.1:4566", "{domain}"); + assert_eq!(up.host_header, domain, "{domain}"); + } + } + #[test] fn https_only_custom_origin_uses_https_and_port() { let up = upstream_for( diff --git a/crates/fakecloud-core/src/dispatch.rs b/crates/fakecloud-core/src/dispatch.rs index f62344c20..b6c1bdb82 100644 --- a/crates/fakecloud-core/src/dispatch.rs +++ b/crates/fakecloud-core/src/dispatch.rs @@ -1088,22 +1088,15 @@ impl DispatchConfig { /// bucket-level operation was dispatched unbuffered, so the handler and IAM /// enforcement saw an empty body. /// -/// Known limitation: under virtual-hosted addressing the whole path is the -/// key on real S3, so an object key that begins with `/` (or a key -/// that IS the bucket name) is not representable here — the passthrough reads -/// it as a client that put the bucket in the path, and the prefix is dropped. -/// Nothing in a single request distinguishes the two, so this resolves the -/// ambiguity in favor of the mixed-addressing client. Keys shaped like their -/// own bucket name are the cost. +/// Under virtual-hosted addressing the whole wire path is the object key, as +/// on real S3, so the bucket is always prefixed: on +/// `docs.s3..amazonaws.com`, `GET /docs/intro.html` addresses the key +/// `docs/intro.html`, not `intro.html`. fn s3_routing_path(wire_path: &str, host_bucket: Option<&str>) -> String { let Some(bucket) = host_bucket else { return wire_path.to_string(); }; - let prefix_with_slash = format!("/{bucket}/"); - let is_bucket_root = wire_path.trim_end_matches('/') == format!("/{bucket}"); - if wire_path.starts_with(&prefix_with_slash) || is_bucket_root { - wire_path.to_string() - } else if wire_path == "/" || wire_path.is_empty() { + if wire_path == "/" || wire_path.is_empty() { format!("/{bucket}") } else { format!("/{bucket}{wire_path}") @@ -2060,6 +2053,26 @@ mod tests { ); } + #[test] + fn s3_routing_path_prefixes_the_host_bucket() { + assert_eq!(s3_routing_path("/", Some("b")), "/b"); + assert_eq!(s3_routing_path("", Some("b")), "/b"); + assert_eq!(s3_routing_path("/k.txt", Some("b")), "/b/k.txt"); + assert_eq!(s3_routing_path("/dir/k", Some("a.b")), "/a.b/dir/k"); + assert_eq!(s3_routing_path("/b/k", None), "/b/k"); + } + + #[test] + fn s3_routing_path_keeps_a_key_that_starts_with_the_bucket_name() { + // The wire path is the whole key on a virtual-hosted request, so a key + // whose first segment equals the bucket name must keep it. + assert_eq!( + s3_routing_path("/docs/intro.html", Some("docs")), + "/docs/docs/intro.html" + ); + assert_eq!(s3_routing_path("/docs", Some("docs")), "/docs/docs"); + } + #[test] fn streaming_route_path_style_create_bucket_with_trailing_slash_skipped() { // The AWS SDKs send CreateBucket as `PUT //`. The trailing @@ -2127,36 +2140,23 @@ mod tests { } #[test] - fn streaming_route_virtual_hosted_bucket_in_path_skipped() { - // `Host: my-bucket.s3...` with the bucket ALSO in the path is the - // router's bucket-root shape (dispatch collapses it to CreateBucket), - // so it must not stream -- with or without the trailing slash. + fn streaming_route_virtual_hosted_path_naming_the_bucket_streams() { + // Under virtual-hosted addressing the whole path is the key, as on + // real S3: `PUT /my-bucket` on `Host: my-bucket.s3...` uploads the + // object `my-bucket`, not a CreateBucket, so every one of these + // streams. let mut headers = s3_sigv4_headers(); headers.insert( "host", "my-bucket.s3.us-east-1.amazonaws.com".parse().unwrap(), ); - assert_eq!( - streaming_route(&http::Method::PUT, "/my-bucket", &headers, &HashMap::new()), - None, - ); - assert_eq!( - streaming_route(&http::Method::PUT, "/my-bucket/", &headers, &HashMap::new()), - None, - ); - // ...but a path the router reads as a key under that bucket still - // streams. (Per the `s3_routing_path` limitation, a virtual-hosted key - // that genuinely begins with `my-bucket/` is indistinguishable from - // this and is routed the same way.) - assert_eq!( - streaming_route( - &http::Method::PUT, - "/my-bucket/key.txt", - &headers, - &HashMap::new(), - ), - Some(("s3", "")), - ); + for path in ["/my-bucket", "/my-bucket/", "/my-bucket/key.txt"] { + assert_eq!( + streaming_route(&http::Method::PUT, path, &headers, &HashMap::new()), + Some(("s3", "")), + "{path}", + ); + } } #[test] diff --git a/crates/fakecloud-core/src/protocol.rs b/crates/fakecloud-core/src/protocol.rs index 3a7ff0d92..b356dc488 100644 --- a/crates/fakecloud-core/src/protocol.rs +++ b/crates/fakecloud-core/src/protocol.rs @@ -337,8 +337,9 @@ pub fn detect_service( /// `.s3..localhost.localstack.cloud[:port]`) and real AWS /// service hostnames (`..amazonaws.com`, S3 path-style /// and virtual-hosted-style including the legacy no-region -/// `s3.amazonaws.com` / `.s3.amazonaws.com` forms and the older -/// dash-separated `s3-.amazonaws.com` form). +/// `s3.amazonaws.com` / `.s3.amazonaws.com` forms, the older +/// dash-separated `s3-.amazonaws.com` form, the dualstack, FIPS and +/// static-website endpoints), under every AWS partition's DNS suffix. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RoutingHost { pub service: String, @@ -348,7 +349,17 @@ pub struct RoutingHost { } const LOCALSTACK_SUFFIX: &str = ".localhost.localstack.cloud"; -const AWS_SUFFIX: &str = ".amazonaws.com"; +/// The DNS suffix of every AWS partition: commercial and GovCloud +/// (`amazonaws.com`), China (`amazonaws.com.cn`) and the isolated partitions. +/// Endpoint hostnames share one shape across them, only the suffix differs. +const AWS_SUFFIXES: &[&str] = &[ + ".amazonaws.com", + ".amazonaws.com.cn", + ".c2s.ic.gov", + ".sc2s.sgov.gov", + ".cloud.adc-e.uk", + ".csp.hci.ic.gov", +]; /// Parse a `Host` header value for a LocalStack- or AWS-shaped hostname. /// Returns `None` for anything that doesn't match — callers fall through @@ -362,13 +373,10 @@ pub fn parse_routing_host(host: &str) -> Option { if let Some(prefix) = hostname.strip_suffix(LOCALSTACK_SUFFIX) { return parse_localstack_prefix(prefix); } - if hostname == "amazonaws.com" { - return None; - } - if let Some(prefix) = hostname.strip_suffix(AWS_SUFFIX) { - return parse_aws_prefix(prefix); - } - None + AWS_SUFFIXES + .iter() + .find_map(|suffix| hostname.strip_suffix(suffix)) + .and_then(parse_aws_prefix) } /// Pull the `Host` header and parse it with [`parse_routing_host`]. @@ -416,7 +424,7 @@ fn parse_localstack_prefix(prefix: &str) -> Option { } } -/// Parse the prefix before `.amazonaws.com`. +/// Parse the prefix before an AWS partition DNS suffix (`.amazonaws.com`, ...). /// /// Handles every variant AWS has shipped for the common REST/Query services: /// @@ -427,6 +435,10 @@ fn parse_localstack_prefix(prefix: &str) -> Option { /// - `.s3` — legacy virtual-hosted S3 (implicitly `us-east-1`). /// - `s3-` — older dash-separated path-style S3. /// - `.s3-` — older dash-separated virtual-hosted S3. +/// - `[.]s3.dualstack.`, `[.]s3-fips[.dualstack].` +/// — dualstack and FIPS S3 endpoints. +/// - `.s3-website-` / `.s3-website.` — S3 +/// static-website endpoints. fn parse_aws_prefix(prefix: &str) -> Option { if prefix.is_empty() { return None; @@ -437,6 +449,19 @@ fn parse_aws_prefix(prefix: &str) -> Option { } let last = *labels.last()?; + // `.s3-website-`: dash-separated S3 static-website + // endpoint. Checked before the generic `s3-` form below, which + // would otherwise read the region as `website-`. + if let Some(region) = last.strip_prefix("s3-website-") { + if !region.is_empty() && labels.len() >= 2 { + return Some(RoutingHost { + service: "s3".to_string(), + region: region.to_string(), + bucket: Some(labels[..labels.len() - 1].join(".")), + }); + } + } + // `s3-` as the last label: dash-separated S3. Bucket, if any, // is whatever precedes it. if let Some(region) = last.strip_prefix("s3-") { @@ -505,23 +530,33 @@ fn parse_aws_prefix(prefix: &str) -> Option { }); } - match labels.len() { - // `.` — the common case. Covers `s3.` - // path-style S3 too, since the service label falls through here. - 2 => Some(RoutingHost { - service: labels[0].to_string(), - region: labels[1].to_string(), + // Region-last S3 endpoints: `[.].` where the + // endpoint is `s3`, `s3-fips`, either of those followed by `.dualstack`, + // or `s3-website` (the dot-separated static-website endpoint). Whatever + // precedes the endpoint is the bucket (dotted names included). + let before_region = &labels[..labels.len() - 1]; + let (endpoint_labels, s3_endpoint) = match before_region { + [rest @ .., s3, "dualstack"] if matches!(*s3, "s3" | "s3-fips") => (rest, true), + [rest @ .., endpoint] if matches!(*endpoint, "s3" | "s3-fips" | "s3-website") => { + (rest, true) + } + _ => (before_region, false), + }; + if s3_endpoint { + return Some(RoutingHost { + service: "s3".to_string(), + region: last.to_string(), + bucket: (!endpoint_labels.is_empty()).then(|| endpoint_labels.join(".")), + }); + } + + // `.` — the common case for every other service. + match labels.as_slice() { + [service, region] => Some(RoutingHost { + service: service.to_string(), + region: region.to_string(), bucket: None, }), - // `.s3.` — modern virtual-hosted S3. - n if n >= 3 && labels[n - 2] == "s3" => { - let bucket = labels[..n - 2].join("."); - Some(RoutingHost { - service: "s3".to_string(), - region: labels[n - 1].to_string(), - bucket: Some(bucket), - }) - } _ => None, } } @@ -1563,6 +1598,67 @@ mod tests { assert_eq!(h.bucket.as_deref(), Some("my-bucket")); } + #[test] + fn parse_routing_host_aws_s3_dualstack_and_fips() { + for (host, bucket) in [ + ( + "my-bucket.s3.dualstack.us-east-1.amazonaws.com", + Some("my-bucket"), + ), + ("a.b.s3.dualstack.eu-west-2.amazonaws.com", Some("a.b")), + ("s3.dualstack.eu-west-2.amazonaws.com", None), + ( + "my-bucket.s3-fips.us-gov-west-1.amazonaws.com", + Some("my-bucket"), + ), + ( + "my-bucket.s3-fips.dualstack.us-east-1.amazonaws.com", + Some("my-bucket"), + ), + ] { + let h = parse_routing_host(host).unwrap(); + assert_eq!(h.service, "s3", "{host}"); + assert_eq!(h.bucket.as_deref(), bucket, "{host}"); + } + let h = parse_routing_host("b.s3.dualstack.eu-west-2.amazonaws.com").unwrap(); + assert_eq!(h.region, "eu-west-2"); + } + + #[test] + fn parse_routing_host_aws_s3_website_endpoints() { + // Dash form: the region is what follows `s3-website-`, not + // `website-` as the generic `s3-` rule would read it. + let h = parse_routing_host("site.s3-website-us-west-2.amazonaws.com").unwrap(); + assert_eq!(h.service, "s3"); + assert_eq!(h.region, "us-west-2"); + assert_eq!(h.bucket.as_deref(), Some("site")); + + // Dot form (every region launched since 2014). + let h = parse_routing_host("my.site.s3-website.eu-central-1.amazonaws.com").unwrap(); + assert_eq!(h.service, "s3"); + assert_eq!(h.region, "eu-central-1"); + assert_eq!(h.bucket.as_deref(), Some("my.site")); + } + + #[test] + fn parse_routing_host_other_partition_suffixes() { + let h = parse_routing_host("my-bucket.s3.cn-north-1.amazonaws.com.cn").unwrap(); + assert_eq!(h.service, "s3"); + assert_eq!(h.region, "cn-north-1"); + assert_eq!(h.bucket.as_deref(), Some("my-bucket")); + + let h = parse_routing_host("sqs.cn-northwest-1.amazonaws.com.cn").unwrap(); + assert_eq!(h.service, "sqs"); + assert_eq!(h.region, "cn-northwest-1"); + + let h = parse_routing_host("b.s3.us-iso-east-1.c2s.ic.gov").unwrap(); + assert_eq!(h.region, "us-iso-east-1"); + assert_eq!(h.bucket.as_deref(), Some("b")); + + assert!(parse_routing_host("amazonaws.com.cn").is_none()); + assert!(parse_routing_host(".amazonaws.com.cn").is_none()); + } + #[test] fn parse_routing_host_rejects_plain_localhost() { assert!(parse_routing_host("localhost:4566").is_none()); diff --git a/crates/fakecloud-e2e/tests/cloudfront_dataplane.rs b/crates/fakecloud-e2e/tests/cloudfront_dataplane.rs index d65d40c1f..086ed0da2 100644 --- a/crates/fakecloud-e2e/tests/cloudfront_dataplane.rs +++ b/crates/fakecloud-e2e/tests/cloudfront_dataplane.rs @@ -483,6 +483,80 @@ async fn serves_static_from_s3_website_origin_and_routes_api() { assert_eq!(r.text().await.unwrap(), "ECHO /api/orders"); } +/// Regression: an `S3OriginConfig` origin carries the bucket's REST domain +/// (`.s3..amazonaws.com` — what CDK's `S3BucketOrigin` emits), +/// which resolves in real DNS. Before the fix the data plane proxied it to real +/// AWS S3 instead of this process, so the distribution never served the bucket. +#[tokio::test] +async fn serves_static_from_s3_rest_origin() { + let server = TestServer::start().await; + let s3 = server.s3_client().await; + s3.create_bucket() + .bucket("restsite") + .send() + .await + .expect("create_bucket"); + put_object( + &s3, + "restsite", + "assets/app.js", + "application/javascript", + b"APPJS", + ) + .await; + + let cf = server.cloudfront_client().await; + let dist = make_spa_distribution(&cf, "restsite.s3.us-east-1.amazonaws.com", None).await; + assert!(wait_for_served(&server, dist.id(), Duration::from_secs(10)).await); + + let r = viewer_get(&server, dist.domain_name(), "/assets/app.js").await; + assert_eq!(r.status(), 200); + assert_eq!(r.text().await.unwrap(), "APPJS"); +} + +/// The other S3 REST domain forms (here the bucket's `DualStackDomainName`) +/// are served locally too, and the viewer path is the whole object key even +/// when its first segment repeats the bucket name: `/docs/intro.html` on the +/// `docs` bucket is the key `docs/intro.html`, not `intro.html`. A REST origin +/// naming a bucket that does not exist gets the local S3's 404, not a fetch +/// against real AWS. +#[tokio::test] +async fn serves_s3_rest_origin_dualstack_key_prefixed_by_bucket_name() { + let server = TestServer::start().await; + let s3 = server.s3_client().await; + s3.create_bucket() + .bucket("docs") + .send() + .await + .expect("create_bucket"); + put_object(&s3, "docs", "docs/intro.html", "text/html", b"NESTED").await; + put_object(&s3, "docs", "intro.html", "text/html", b"TOP").await; + + let cf = server.cloudfront_client().await; + let dist = make_spa_distribution(&cf, "docs.s3.dualstack.eu-west-2.amazonaws.com", None).await; + assert!(wait_for_served(&server, dist.id(), Duration::from_secs(10)).await); + + let r = viewer_get(&server, dist.domain_name(), "/docs/intro.html").await; + assert_eq!(r.status(), 200); + assert_eq!(r.text().await.unwrap(), "NESTED"); + let r = viewer_get(&server, dist.domain_name(), "/intro.html").await; + assert_eq!(r.status(), 200); + assert_eq!(r.text().await.unwrap(), "TOP"); + + let missing = + make_spa_distribution(&cf, "no-such-bucket.s3.us-east-1.amazonaws.com", None).await; + assert!(wait_for_served(&server, missing.id(), Duration::from_secs(10)).await); + let r = viewer_get(&server, missing.domain_name(), "/x.html").await; + // The local S3 answers the missing bucket with a 404 S3 error document + // (GetObject reports it as `NoSuchKey`), not a proxy error or real AWS. + assert_eq!(r.status(), 404); + let body = r.text().await.unwrap(); + assert!( + body.contains("") && body.contains("x.html"), + "{body}" + ); +} + #[tokio::test] async fn stays_served_after_restart_persistent() { let tmp = tempfile::tempdir().unwrap();