diff --git a/README.md b/README.md index 01d22a4a25..5bd7df07e2 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,10 @@ kubectl port-forward -n ate-system svc/atenet-router 8000:80 3. In a **separate terminal**, send an HTTP request to increment the counter: ```shell -curl -X POST -H "Host: my-counter-1.ate-demo-counter.actors.resources.substrate.ate.dev" -i http://localhost:8000/ +curl -X POST \ + -H "X-Ate-Actor-Name: my-counter-1" \ + -H "X-Ate-Atespace: ate-demo-counter" \ + -i http://localhost:8000/ ``` Worker capacity is versioned: the dataplane (the atelet DaemonSet and the @@ -234,7 +237,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * `cmd/ateapi`: The core control plane API server exposing gRPC endpoints to manage actor and worker lifecycles. * `cmd/atelet`: A node-level DaemonSet that supervises physical worker pods, coordinates snapshotting, and manages state transfers. * `cmd/atecontroller`: A Kubernetes controller that reconciles WorkerPool custom resources. -* `cmd/atenet`: A combined networking controller providing DNS, Envoy routing, and proxy sidecars. +* `cmd/atenet`: A combined networking controller providing Envoy routing and proxy sidecars. * `cmd/ateom-gvisor`: An interior-pod helper running inside sandboxed worker pods to execute `runsc` checkpoint and restore commands. * `cmd/ateom-microvm`: The micro-VM peer of `ateom-gvisor`, running actors as cloud-hypervisor VMs. * `cmd/podcertcontroller`: A "polyfill" that provides Pod Certificate signers that diff --git a/benchmarking/automation/testtypes/nighthawk_ingress.py b/benchmarking/automation/testtypes/nighthawk_ingress.py index 1245cee476..624904e821 100644 --- a/benchmarking/automation/testtypes/nighthawk_ingress.py +++ b/benchmarking/automation/testtypes/nighthawk_ingress.py @@ -102,7 +102,7 @@ def validate(test: dict[str, Any]) -> None: if not re.fullmatch(r"[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?", nh["atespace"]): raise ValueError( f"nighthawk-ingress test {name!r} atespace {nh['atespace']!r} " - f"must be a DNS label (it is routed as a Host-header subdomain)" + f"must be a DNS label" ) diff --git a/benchmarking/locust/tests/counter_demo.py b/benchmarking/locust/tests/counter_demo.py index c99343ace4..81759c3e5d 100644 --- a/benchmarking/locust/tests/counter_demo.py +++ b/benchmarking/locust/tests/counter_demo.py @@ -48,11 +48,9 @@ tracer = get_tracer(__name__) -# Atenet router fronts all actor traffic. Actors are addressed by setting -# the HTTP Host header to ..actors.resources.substrate.ate.dev; -# the router resolves that to the actor's current worker pod. +# Atenet router fronts all actor traffic. The actor routing headers select +# the actor whose current worker pod the router resolves. ROUTER_URL = "http://atenet-router.ate-system.svc.cluster.local" -ACTOR_DOMAIN = "actors.resources.substrate.ate.dev" class CounterUser(User): @@ -99,12 +97,9 @@ def on_start(self) -> None: except Exception as e: logger.error(f"Failed to create actor {self.actor_name}: {e}") - # One HTTP session per user, talking to the router. The Host header - # pins each request to this actor regardless of which worker pod - # hosts it after a resume. + # One HTTP session per user, talking to the router. self.http_session = requests.Session() self.run_url = f"{ROUTER_URL}/" - self.host_header = f"{self.actor_name}.{ATESPACE}.{ACTOR_DOMAIN}" def on_stop(self) -> None: update_user_count(-1, self.__class__.__name__) @@ -139,7 +134,10 @@ def run_and_suspend(self) -> None: # 2. Run/Increment (HTTP via atenet-router) start_time = time.time() with tracer.start_as_current_span("RunCounter") as span: - headers = {"Host": self.host_header} + headers = { + "X-Ate-Actor-Name": self.actor_name, + "X-Ate-Atespace": ATESPACE, + } inject(headers) try: response = self.http_session.post(self.run_url, headers=headers) diff --git a/benchmarking/nighthawk-ingress/README.md b/benchmarking/nighthawk-ingress/README.md index 6b19eaa1eb..923d7c75d7 100644 --- a/benchmarking/nighthawk-ingress/README.md +++ b/benchmarking/nighthawk-ingress/README.md @@ -30,14 +30,14 @@ just don't steer the search.* ## What it measures Every request exercises the **full production routing path** — created and -warmed sandboxed actors, Host-header routing, the ext_proc routing +warmed sandboxed actors, identity-header routing, the ext_proc routing decision, and the mTLS hop to the worker: ```mermaid flowchart LR subgraph job["nighthawk runner Job"] alc["nighthawk_adaptive_load_client
exponential ramp + binary search"] - svc["nighthawk_service
16 event loops, Host header
rotated across all actors"] + svc["nighthawk_service
16 event loops, actor headers
rotated across all actors"] alc -->|gRPC| svc end @@ -54,7 +54,7 @@ flowchart LR atunnel["atunnel :443"] --> glutton["glutton actor
POST /ping :80"] end - svc -->|"HTTP :80, Host:
actor-N.benchmark.actors..."| envoy + svc -->|"HTTP :80
X-Ate-Actor-Name + X-Ate-Atespace"| envoy extproc -->|ResumeActor| ateapi envoy -->|"mTLS :443"| atunnel ``` @@ -84,9 +84,9 @@ One Kubernetes Job per `type: nighthawk-ingress` tests.yaml entry, driven by down afterwards, so nothing leaks. 2. **Create + warm actors.** The runner creates one glutton actor per WorkerPool worker (the entry's `workerCount`) via ateapi and POSTs - `/ping` through the router with each actor's Host header until it + `/ping` through the router with each actor's routing headers until it answers 200. -3. **Adaptive search.** Open-loop traffic with the Host header rotated +3. **Adaptive search.** Open-loop traffic with the actor routing headers rotated across all actors; `clientConcurrency` event loops (default 16, decoupled from `envoyCpu`) and large per-loop pools so the harness is never the bottleneck. Exponential ramp → binary search → a 60s @@ -174,7 +174,7 @@ Start with `capacity.json`, drill into `stats.jsonl`: The fleet size is the entry's top-level `workerCount` (required): the benchmark warms one actor per worker, so it is also the number of glutton -actors receiving rotated-Host traffic. Everything else lives in the +actors receiving rotated identity-header traffic. Everything else lives in the `nighthawk-ingress:` block: | Knob | Default | Meaning | diff --git a/benchmarking/nighthawk-ingress/actors.py b/benchmarking/nighthawk-ingress/actors.py index 90dbc8bc23..808fa2da29 100644 --- a/benchmarking/nighthawk-ingress/actors.py +++ b/benchmarking/nighthawk-ingress/actors.py @@ -110,11 +110,10 @@ def _warm_actor( ) -> None: """Bring one created actor to serving, so the load ladder never measures a cold start: resume the actor, then poll POST /ping through - the router (addressed by the actor's Host header) until it answers + the router until it answers 200 or the deadline expires. Resume errors are retried: ateapi returns FailedPrecondition/Unavailable until a worker frees up.""" ref = ateapi_pb2.ObjectRef(atespace=atespace, name=name) - host = spec_mod.actor_host(name, atespace) session = requests.Session() last_err: str = "not attempted" while time.time() < deadline: @@ -125,7 +124,10 @@ def _warm_actor( try: resp = session.post( f"{router_url.rstrip('/')}/ping", - headers={"Host": host}, + headers={ + "X-Ate-Actor-Name": name, + "X-Ate-Atespace": atespace, + }, data=b"", timeout=10, ) diff --git a/benchmarking/nighthawk-ingress/runner.py b/benchmarking/nighthawk-ingress/runner.py index f5abf09bf8..921673f8fc 100644 --- a/benchmarking/nighthawk-ingress/runner.py +++ b/benchmarking/nighthawk-ingress/runner.py @@ -268,12 +268,10 @@ def main() -> None: log=lambda m: tee(logs, m), ) - hosts = [ - spec_mod.actor_host(n, args.atespace) for n in actor_names - ] spec_dict = spec_mod.build_spec_dict( uri=f"{args.router_url.rstrip('/')}/ping", - hosts=hosts, + actor_names=actor_names, + atespace=args.atespace, client_concurrency=args.client_concurrency, connections=args.connections, max_pending_requests=args.max_pending, diff --git a/benchmarking/nighthawk-ingress/spec.py b/benchmarking/nighthawk-ingress/spec.py index 32ebd742ce..29edc6fc41 100644 --- a/benchmarking/nighthawk-ingress/spec.py +++ b/benchmarking/nighthawk-ingress/spec.py @@ -58,11 +58,6 @@ ) -def actor_host(actor: str, atespace: str) -> str: - """Host header the router routes on (internal/resources/actor.go).""" - return f"{actor}.{atespace}.actors.resources.substrate.ate.dev" - - def _metric_spec(name: str) -> dict: return {"metric_name": name, "metrics_plugin_name": BUILTIN_METRICS_PLUGIN} @@ -91,7 +86,8 @@ def _binary_threshold( def build_spec_dict( *, uri: str, - hosts: list[str], + actor_names: list[str], + atespace: str, client_concurrency: int, connections: int, max_pending_requests: int, @@ -117,12 +113,16 @@ def build_spec_dict( "request_method": "POST", "request_headers": [ { - "header": {"key": "host", "value": host}, + "header": {"key": "x-ate-actor-name", "value": actor_name}, + "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", + }, + { + "header": {"key": "x-ate-atespace", "value": atespace}, "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", - } + }, ], } - for host in hosts + for actor_name in actor_names ] # Threshold roles: tail latency (mean+2stdev, ~p95 proxy — no true # percentiles in the builtin adaptive metrics) is the SLO bound; diff --git a/benchmarking/nighthawk-ingress/tests/test_spec.py b/benchmarking/nighthawk-ingress/tests/test_spec.py index e2b565cb8c..fb754adf2f 100644 --- a/benchmarking/nighthawk-ingress/tests/test_spec.py +++ b/benchmarking/nighthawk-ingress/tests/test_spec.py @@ -33,9 +33,8 @@ def build(**overrides): kwargs = dict( uri="http://atenet-router.ate-system.svc.cluster.local:80/ping", - hosts=[ - spec_mod.actor_host(f"sb-{i}", "benchmark") for i in range(3) - ], + actor_names=[f"sb-{i}" for i in range(3)], + atespace="benchmark", client_concurrency=4, connections=1000, max_pending_requests=10000, @@ -67,14 +66,23 @@ def test_traffic_template_shape(): } -def test_host_rotation_covers_all_actors(): - hosts = [spec_mod.actor_host(f"sb-{i}", "benchmark") for i in range(5)] - spec = build(hosts=hosts) +def test_identity_rotation_covers_all_actors(): + actor_names = [f"sb-{i}" for i in range(5)] + spec = build(actor_names=actor_names) plugin = spec["nighthawk_traffic_template"]["request_source_plugin_config"] assert plugin["name"] == spec_mod.REQUEST_SOURCE_PLUGIN options = plugin["typed_config"]["options_list"]["options"] - got = [o["request_headers"][0]["header"]["value"] for o in options] - assert got == hosts + got = [ + {header["header"]["key"]: header["header"]["value"] for header in o["request_headers"]} + for o in options + ] + assert got == [ + { + "x-ate-actor-name": actor_name, + "x-ate-atespace": "benchmark", + } + for actor_name in actor_names + ] assert all(o["request_method"] == "POST" for o in options) # 0 = loop the list indefinitely. assert plugin["typed_config"]["num_requests"] == 0 @@ -116,13 +124,6 @@ def test_tail_latency_slo_threshold(): assert len(build(tail_latency_slo_ms=None)["metric_thresholds"]) == 2 -def test_actor_host_format(): - assert ( - spec_mod.actor_host("sb-1", "benchmark") - == "sb-1.benchmark.actors.resources.substrate.ate.dev" - ) - - def test_spec_round_trips_through_real_protos(): """Round-trip through the real Nighthawk protos; self-skips outside the runner image (no FileDescriptorSet).""" diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 0d67133b6c..3f8d95f151 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -75,7 +75,7 @@ var deployControllerCmd = &cobra.Command{ var deployAtenetCmd = &cobra.Command{ Use: "atenet", - Short: "Deploy the atenet dataplane only: router, egress, and DNS", + Short: "Deploy the atenet dataplane only: router and egress", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return env.DeployAtenet(cmd.Context()) diff --git a/cmd/ate-setup/internal/steps/delete.go b/cmd/ate-setup/internal/steps/delete.go index 73317c0e96..8d0f6d2554 100644 --- a/cmd/ate-setup/internal/steps/delete.go +++ b/cmd/ate-setup/internal/steps/delete.go @@ -75,7 +75,6 @@ func (e *Env) DeleteAtenet(ctx context.Context) error { // other does not. {"atenet-egress.yaml"}, {"atenet-egress-with-sdsmint.yaml"}, - {"atenet-dns.yaml"}, } { if err := e.Kube.DeletePath(ctx, e.Cfg.Manifest(path...)); err != nil { return err diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 9c450adef5..d3e8a720c8 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -289,7 +289,7 @@ func (e *Env) DeployAtelet(ctx context.Context) error { return e.Kube.RolloutStatus(ctx, kube.KindDaemonSet, NamespaceAteSystem, ateletName, e.Cfg.RolloutTimeout) } -// DeployAtenet redeploys the atenet dataplane: router, egress, and DNS. +// DeployAtenet redeploys the atenet dataplane: router and egress. func (e *Env) DeployAtenet(ctx context.Context) error { log.Step("deploy_atenet") @@ -316,11 +316,8 @@ func (e *Env) DeployAtenet(ctx context.Context) error { if err := e.applyAtenetEgress(ctx); err != nil { return err } - if err := e.ResolveAndApply(ctx, e.Cfg.Manifest("atenet-dns.yaml")); err != nil { - return err - } - for _, name := range []string{"atenet-router", "atenet-egress", "dns"} { + for _, name := range []string{"atenet-router", "atenet-egress"} { if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, NamespaceAteSystem, name, e.Cfg.RolloutTimeout); err != nil { return err } diff --git a/cmd/atenet/README.md b/cmd/atenet/README.md index f2ab3f1a2b..bf3917e015 100644 --- a/cmd/atenet/README.md +++ b/cmd/atenet/README.md @@ -2,7 +2,6 @@ atenet is a combined daemon for all networking functionality. -* DNS server for ATE Actor resolution: `atenet dns` * Envoy control plane for programming ATE resolution. `atenet router` This is built as a single binary for convenience in the prototyping. @@ -31,8 +30,8 @@ likely be split in the future for better scalability.) a separate process on the worker pod, not part of Envoy -- so the port to reach on the actor itself (its default port, or an arbitrary one for CONNECT) still travels as a real header, `atunnel.TargetPortHeader`. - `:authority`/`Host` reaches atunnel unmodified either way, so it authorizes - the actor by its own DNS name. + `X-Ate-Actor-Name` and `X-Ate-Atespace` identify the Actor independently of + `:authority`/`Host`. * Termination: the router drains gracefully on SIGTERM (readiness flip → endpoint propagation → Envoy admin-API drain → ext_proc drain), and the Envoy container's `preStop` hook waits for the router's drain-complete @@ -45,15 +44,6 @@ likely be split in the future for better scalability.) RBAC permissions: * get, list, watch on ate-system EndpointSlices -### dns - -* `atenet dns` will be deployed as: - * Deployment - * Service exposing tcp and udp 53 - -* read, list on kube-system services -* read, list on ate-system services - ## testing Run the package tests with `go test ./cmd/atenet/...`. Cluster e2e diff --git a/cmd/atenet/internal/dns.go b/cmd/atenet/internal/dns.go deleted file mode 100644 index 85798fa73f..0000000000 --- a/cmd/atenet/internal/dns.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "context" - "fmt" - "log/slog" - "os" - "os/signal" - "syscall" - "time" - - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/spf13/cobra" - "k8s.io/client-go/tools/clientcmd" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" - - "github.com/agent-substrate/substrate/cmd/atenet/internal/dns" -) - -type DnsConfig struct { - LogLevel string - Kubeconfig string - ReconcileInterval time.Duration - CorefilePath string -} - -func NewDnsCmd() *cobra.Command { - var cfg DnsConfig - - cmd := &cobra.Command{ - Use: "dns", - Short: "Orchestrates CoreDNS and GKE stub resolver configuration", - RunE: func(cmd *cobra.Command, args []string) error { - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() - - serverboot.InitLogger() - if err := serverboot.SetLogLevel(cfg.LogLevel); err != nil { - return err - } - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-sigChan - cancel() - }() - - k8sCfg, err := config.GetConfig() - if err != nil { - if cfg.Kubeconfig != "" { - k8sCfg, err = clientcmd.BuildConfigFromFlags("", cfg.Kubeconfig) - if err != nil { - return fmt.Errorf("failed to read config from path %s: %w", cfg.Kubeconfig, err) - } - } else { - return fmt.Errorf("unable to establish Kubernetes configuration parameters: %w", err) - } - } - - k8sClient, err := client.New(k8sCfg, client.Options{}) - if err != nil { - return fmt.Errorf("failed to initialize cluster client: %w", err) - } - - dnsController := &dns.Controller{ - Client: k8sClient, - Interval: cfg.ReconcileInterval, - CorefilePath: cfg.CorefilePath, - Reloader: dns.NewConfigReloader(), - } - - slog.InfoContext(ctx, "Starting DNS Controller subsystem") - return dnsController.Run(ctx) - }, - } - - cmd.Flags().StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error") - cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") - cmd.Flags().DurationVar(&cfg.ReconcileInterval, "interval", 10*time.Second, "Interval for reconciling DNS configurations") - cmd.Flags().StringVar(&cfg.CorefilePath, "corefile-path", "/etc/coredns/Corefile", "Path to the local Corefile configuration on shared volume") - - return cmd -} diff --git a/cmd/atenet/internal/dns/README.md b/cmd/atenet/internal/dns/README.md deleted file mode 100644 index a134e829ab..0000000000 --- a/cmd/atenet/internal/dns/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# DNS Controller - -The DNS Controller orchestrates the configuration needed to setup the ATE routing. - -We want to resolve requests for ..actors.resources.substrate.ate.dev to the router service address. - -* Stub resolver mode: orchestrate running a CoreDNS instance with the actor name mapped to the atenet-router service address. - -Cluster resources: - -* Deployment `ate-system:dns`. Label: app=dns -* Service `ate-system:dns`. -* ConfigMap `ate-system:dns`. - -These are defined in manifests/ate-install/atenet-dns.yaml. - -## Stub resolver mode - -* Ensure stub resolver CoreDNS is running as: - * Deployment `ate-system:dns`. - * Service `ate-system:dns` pointing to the Deployment. - -ConfigMap `ate-system:dns`: - -``` -# Match any 'A' query for an actor name + atespace pattern under actors.resources.substrate.ate.dev - template IN A actors.resources.substrate.ate.dev { - match "^[a-z0-9]([-a-z0-9]*[a-z0-9])?\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?\\.actors\\.resources\\.substrate\\.ate\\.dev\\.$" - answer "{{ .Name }} 60 IN A " - } -``` - -## Integration - -* CoreDNS: Update CoreDNS ConfigMap to add the stub resolver. -* GKE DNS: Update the GKE DNS ConfigMap to add the stub resolver. diff --git a/cmd/atenet/internal/dns/corefile.go b/cmd/atenet/internal/dns/corefile.go deleted file mode 100644 index 0b301e7e29..0000000000 --- a/cmd/atenet/internal/dns/corefile.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "fmt" - "strings" - "time" - - "github.com/agent-substrate/substrate/internal/resources" -) - -// corefileTemplate is a Sprintf template for the CoreDNS configuration. -var corefileTemplate string - -func init() { - corefileTemplate = buildTemplate() -} - -func buildTemplate() string { - // Build up the corefileTemplate programmatically to make it easier to understand. - var directives []string - // Plugins to enable. - directives = append(directives, "log") - directives = append(directives, "errors") - directives = append(directives, "health :8080") - directives = append(directives, "ready :8181") - directives = append(directives, "reload") - - // Construct match pattern for ... Both the - // actor name and the atespace are DNS-1123 labels (same regex). - directives = append(directives, fmt.Sprintf("template IN A %s {", resources.ActorDNSSuffix)) - // Escape the suffix's dots so they match literally; the final \. matches the FQDN's trailing dot. - escapedSuffix := strings.ReplaceAll(resources.ActorDNSSuffix, ".", `\.`) - directives = append(directives, fmt.Sprintf(` match "^%s\.%s\.%s\.$"`, resources.ResourceNameRegexPattern, resources.ResourceNameRegexPattern, escapedSuffix)) - // Note the %s -- this will be filled with the router IP. - directives = append(directives, ` answer "{{ .Name }} 60 IN A %s"`) - directives = append(directives, "}") - - // Generate the template. - b := strings.Builder{} - fmt.Fprintf(&b, "# Generated at %s\n", time.Now()) - fmt.Fprintf(&b, "%s:53 {\n ", resources.ActorDNSSuffix) - fmt.Fprint(&b, strings.Join(directives, "\n ")) - fmt.Fprint(&b, "\n}\n") - - return b.String() -} - -func makeCoreFile(routerIP string) string { - return fmt.Sprintf(corefileTemplate, routerIP) -} diff --git a/cmd/atenet/internal/dns/corefile_test.go b/cmd/atenet/internal/dns/corefile_test.go deleted file mode 100644 index f13429e475..0000000000 --- a/cmd/atenet/internal/dns/corefile_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "strings" - "testing" - - "github.com/agent-substrate/substrate/internal/resources" -) - -func TestMakeCoreFile(t *testing.T) { - tests := []struct { - name string - routerIP string - expected []string - }{ - { - name: "standard local IP", - routerIP: "10.240.0.10", - expected: []string{ - "actors.resources.substrate.ate.dev:53 {", - "log", - "errors", - "health :8080", - "ready :8181", - "reload", - "template IN A actors.resources.substrate.ate.dev {", - `match "^` + resources.ResourceNameRegexPattern + `\.` + resources.ResourceNameRegexPattern + `\.actors\.resources\.substrate\.ate\.dev\.$"`, - `answer "{{ .Name }} 60 IN A 10.240.0.10"`, - }, - }, - { - name: "different IP", - routerIP: "192.168.1.1", - expected: []string{ - "actors.resources.substrate.ate.dev:53 {", - `answer "{{ .Name }} 60 IN A 192.168.1.1"`, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := makeCoreFile(tc.routerIP) - for _, exp := range tc.expected { - if !strings.Contains(got, exp) { - t.Errorf("makeCoreFile(%q) missing expected substring %q\nGot:\n%s", tc.routerIP, exp, got) - } - } - }) - } -} diff --git a/cmd/atenet/internal/dns/dns.go b/cmd/atenet/internal/dns/dns.go deleted file mode 100644 index cf2db99b69..0000000000 --- a/cmd/atenet/internal/dns/dns.go +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "os" - "path/filepath" - "strconv" - "strings" - "syscall" - "time" - - "github.com/agent-substrate/substrate/internal/resources" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - // serviceName is the name of the CoreDNS service. - serviceName = "dns" - systemNamespace = "ate-system" -) - -// Controller manages the DNS configuration for the ATE. -type Controller struct { - Client client.Client - Interval time.Duration - CorefilePath string - Reloader ConfigReloader -} - -// Run the DNS orchestration loop until ctx is canceled. -func (c *Controller) Run(ctx context.Context) error { - slog.InfoContext(ctx, "DNS Controller started", slog.Duration("interval", c.Interval), slog.String("corefile", c.CorefilePath)) - slog.InfoContext(ctx, "Using template", "template", corefileTemplate) - - ticker := time.NewTicker(c.Interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - slog.InfoContext(ctx, "DNS Controller stopped") - return nil - case <-ticker.C: - if err := c.reconcile(ctx); err != nil { - slog.ErrorContext(ctx, "Error during DNS reconciliation", slog.Any("error", err)) - } - } - } -} - -func (c *Controller) reconcile(ctx context.Context) error { - slog.DebugContext(ctx, "Reconciling DNS orchestration configuration...") - - // 1. Get the ClusterIP of atenet-router in ate-system namespace - routerSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: "atenet-router", Namespace: systemNamespace}, routerSvc); err != nil { - if errors.IsNotFound(err) { - slog.WarnContext(ctx, "atenet-router service not found, skipping until it is available") - return nil - } - return fmt.Errorf("failed to get atenet-router service: %w", err) - } - - routerIP := routerSvc.Spec.ClusterIP - if routerIP == "" || routerIP == "None" { - slog.WarnContext(ctx, "atenet-router service has no ClusterIP yet, waiting...") - return nil - } - - // 2. Get the ClusterIP of dns service in ate-system namespace - dnsSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: systemNamespace}, dnsSvc); err != nil { - if errors.IsNotFound(err) { - slog.WarnContext(ctx, "dns service not found, skipping until it is available") - return nil - } - return fmt.Errorf("failed to get dns service: %w", err) - } - - dnsIP := dnsSvc.Spec.ClusterIP - if dnsIP == "" || dnsIP == "None" { - slog.WarnContext(ctx, "dns service has no ClusterIP yet, waiting...") - return nil - } - - // 3. Reconcile CoreDNS Corefile on shared volume - if err := c.reconcileCoreDNSConfig(ctx, routerIP); err != nil { - return fmt.Errorf("failed to reconcile CoreDNS config file: %w", err) - } - - // 4. Reconcile GKE kube-dns ConfigMap with dns service IP - if err := c.reconcileKubeDNSConfig(ctx, dnsIP); err != nil { - return fmt.Errorf("failed to reconcile kube-dns configmap: %w", err) - } - - return nil -} - -func (c *Controller) reconcileCoreDNSConfig(ctx context.Context, routerIP string) error { - expectedCorefile := makeCoreFile(routerIP) - - // Read Corefile from local shared volume path to see if it needs updating - corefileBytes, err := os.ReadFile(c.CorefilePath) - if err == nil && string(corefileBytes) == expectedCorefile { - slog.DebugContext(ctx, "CoreDNS Corefile is up-to-date", slog.String("routerIP", routerIP)) - return nil - } - - // Write updated Corefile back to shared volume in its entirety - if err := os.WriteFile(c.CorefilePath, []byte(expectedCorefile), 0644); err != nil { - return fmt.Errorf("failed to write updated Corefile to %s: %w", c.CorefilePath, err) - } - slog.InfoContext(ctx, "CoreDNS Corefile updated", slog.String("routerIP", routerIP)) - - // Signal CoreDNS process to reload - if err := c.Reloader.Reload(ctx); err != nil { - return fmt.Errorf("failed to reload CoreDNS: %w", err) - } - - return nil -} - -// reconcileKubeDNSConfig ensures that the kube-dns ConfigMap has a stub domain for ate-system. -func (c *Controller) reconcileKubeDNSConfig(ctx context.Context, dnsIP string) error { - cm := &corev1.ConfigMap{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: "kube-dns", Namespace: "kube-system"}, cm); err != nil { - if errors.IsNotFound(err) { - slog.WarnContext(ctx, "kube-dns ConfigMap not found in kube-system namespace, skipping stub resolver configuration") - return nil - } - return fmt.Errorf("failed to retrieve kube-dns ConfigMap: %w", err) - } - - if cm.Data == nil { - cm.Data = make(map[string]string) - } - - stubDomainsStr := cm.Data["stubDomains"] - var stubDomains map[string][]string - - if stubDomainsStr != "" { - if err := json.Unmarshal([]byte(stubDomainsStr), &stubDomains); err != nil { - return fmt.Errorf("failed to parse stubDomains JSON: %w", err) - } - } else { - stubDomains = make(map[string][]string) - } - - ips, exists := stubDomains[resources.ActorDNSSuffix] - if exists && len(ips) == 1 && ips[0] == dnsIP { - slog.DebugContext(ctx, "kube-dns stubDomains are already up-to-date", slog.String("dnsIP", dnsIP)) - return nil - } - - stubDomains[resources.ActorDNSSuffix] = []string{dnsIP} - - newStubDomainsBytes, err := json.Marshal(stubDomains) - if err != nil { - return fmt.Errorf("failed to marshal stubDomains JSON: %w", err) - } - - cm.Data["stubDomains"] = string(newStubDomainsBytes) - if err := c.Client.Update(ctx, cm); err != nil { - return fmt.Errorf("failed to update kube-dns ConfigMap: %w", err) - } - - slog.InfoContext(ctx, "kube-dns stubDomains successfully updated with custom DNS IP", slog.String("dnsIP", dnsIP)) - return nil -} - -// ConfigReloader defines an interface for dynamically signaling CoreDNS to reload its configuration. -type ConfigReloader interface { - Reload(ctx context.Context) error -} - -type procConfigReloader struct{} - -func NewConfigReloader() ConfigReloader { - return &procConfigReloader{} -} - -func (r *procConfigReloader) Reload(ctx context.Context) error { - pid, err := findPID("coredns") - if err != nil { - slog.ErrorContext(ctx, "findPID error", slog.Any("error", err)) - return nil - } - - process, err := os.FindProcess(pid) - if err != nil { - return fmt.Errorf("FindProcess %d: %w", pid, err) - } - - // CoreDNS catches SIGUSR1 or SIGHUP to trigger dynamic reload of Corefile - if err := process.Signal(syscall.SIGUSR1); err != nil { - return fmt.Errorf("SendProcess SIGUSR1 %d: %w", pid, err) - } - - slog.InfoContext(ctx, "Successfully signaled reload", slog.Int("pid", pid)) - return nil -} - -func findPID(cmdName string) (int, error) { - files, err := os.ReadDir("/proc") - if err != nil { - return 0, fmt.Errorf("ReadDir /proc: %w", err) - } - - for _, file := range files { - if !file.IsDir() { - continue - } - - pid, err := strconv.Atoi(file.Name()) - if err != nil { - continue - } - - commBytes, err := os.ReadFile(filepath.Join("/proc", file.Name(), "comm")) - if err != nil { - // Process might have terminated - continue - } - - commGot := strings.TrimSpace(string(commBytes)) - if commGot == cmdName { - return pid, nil - } - } - - return 0, fmt.Errorf("%q process not found", cmdName) -} diff --git a/cmd/atenet/internal/dns/dns_test.go b/cmd/atenet/internal/dns/dns_test.go deleted file mode 100644 index 34116db284..0000000000 --- a/cmd/atenet/internal/dns/dns_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -type mockConfigReloader struct { - reloaded bool -} - -func (m *mockConfigReloader) Reload(ctx context.Context) error { - m.reloaded = true - return nil -} - -func TestReconcile(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - - // 1. Create mock services - routerSvc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "atenet-router", - Namespace: "ate-system", - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.0.0.1", - }, - } - - dnsSvc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dns", - Namespace: "ate-system", - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.0.0.2", - }, - } - - initialCorefile := ` -.:53 { - errors -} -` - - // 2. Set up a temporary local Corefile on disk - tempDir := t.TempDir() - corefilePath := filepath.Join(tempDir, "Corefile") - if err := os.WriteFile(corefilePath, []byte(initialCorefile), 0644); err != nil { - t.Fatalf("failed to write initial Corefile: %v", err) - } - - kubeDNSCM := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-dns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "stubDomains": `{"other-domain.com":["8.8.8.8"]}`, - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(routerSvc, dnsSvc, kubeDNSCM). - Build() - - reloader := &mockConfigReloader{} - controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: reloader, - } - - // Run one reconciliation loop - ctx := context.Background() - err := controller.reconcile(ctx) - if err != nil { - t.Fatalf("reconcile failed: %v", err) - } - - if !reloader.reloaded { - t.Errorf("expected ConfigReloader to be invoked, but it was not") - } - - // Verify the Corefile on disk has been updated with the router IP - updatedCorefileBytes, err := os.ReadFile(corefilePath) - if err != nil { - t.Fatalf("failed to read updated Corefile from disk: %v", err) - } - updatedCorefile := string(updatedCorefileBytes) - if !strings.Contains(updatedCorefile, `answer "{{ .Name }} 60 IN A 10.0.0.1"`) { - t.Errorf("expected Corefile on disk to contain updated answer line, but got: %s", updatedCorefile) - } - - // Verify kube-system:kube-dns ConfigMap contains the new stub domain without wiping out other-domain - updatedKubeDNSCM := &corev1.ConfigMap{} - err = client.Get(ctx, types.NamespacedName{Name: "kube-dns", Namespace: "kube-system"}, updatedKubeDNSCM) - if err != nil { - t.Fatalf("failed to get updated kube-dns ConfigMap: %v", err) - } - - stubDomainsStr := updatedKubeDNSCM.Data["stubDomains"] - var stubDomains map[string][]string - if err := json.Unmarshal([]byte(stubDomainsStr), &stubDomains); err != nil { - t.Fatalf("failed to unmarshal updated stubDomains: %v", err) - } - - ips, exists := stubDomains["actors.resources.substrate.ate.dev"] - if !exists || len(ips) != 1 || ips[0] != "10.0.0.2" { - t.Errorf("expected stubDomains to map actors.resources.substrate.ate.dev to [10.0.0.2], but got: %v", stubDomains) - } - - otherIPs, exists := stubDomains["other-domain.com"] - if !exists || len(otherIPs) != 1 || otherIPs[0] != "8.8.8.8" { - t.Errorf("expected stubDomains to preserve other-domain.com mapping, but got: %v", stubDomains) - } -} - -func TestReconcileKubeDNSNotFound(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - - routerSvc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "atenet-router", - Namespace: "ate-system", - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.0.0.1", - }, - } - - dnsSvc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dns", - Namespace: "ate-system", - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.0.0.2", - }, - } - - // Set up local Corefile on disk - tempDir := t.TempDir() - corefilePath := filepath.Join(tempDir, "Corefile") - initialCorefile := `answer "{{ .Name }} 60 IN A "` - if err := os.WriteFile(corefilePath, []byte(initialCorefile), 0644); err != nil { - t.Fatalf("failed to write initial Corefile: %v", err) - } - - // kube-dns ConfigMap is omitted to test gracefulness - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(routerSvc, dnsSvc). - Build() - - controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: &mockConfigReloader{}, - } - - ctx := context.Background() - err := controller.reconcile(ctx) - if err != nil { - t.Fatalf("reconcile should handle missing kube-dns configmap gracefully but failed with: %v", err) - } -} diff --git a/cmd/atenet/internal/root.go b/cmd/atenet/internal/root.go index 1ab52398a2..c992844efc 100644 --- a/cmd/atenet/internal/root.go +++ b/cmd/atenet/internal/root.go @@ -40,6 +40,5 @@ func Execute() { func init() { rootCmd.AddCommand(router.NewRouterCmd()) - rootCmd.AddCommand(NewDnsCmd()) rootCmd.AddCommand(sdsmint.NewSdsmintCmd()) } diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 7719a5ce6c..e25abc8bc7 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -63,7 +63,15 @@ cannot pick the egress path by crafting one. `router` itself does the wiring. ## adding a dataplane attribute The filter-state objects and request attributes a proxy carries alongside -a request are declared once, in `extproc/attributes.go`. +a request are declared once, in `extproc/attributes.go`. + +| key | direction | purpose | +| --- | --- | --- | +| `dev.ate.actor.name` | ingress | carries the actor name across CONNECT re-entry | +| `dev.ate.actor.atespace` | ingress | carries the atespace across CONNECT re-entry | +| `dev.ate.connect.authority` | ingress | carries the outer CONNECT authority across re-entry for target-port selection | +| `dev.ate.actor.identity` | egress | carries the authenticated actor identity for logs and additional ext_proc services | +| `dev.ate.extproc.direction` | egress | selects the egress handler for dataplanes without Envoy filter chains | ### name it @@ -94,13 +102,11 @@ reserved. ### keep it trustworthy -An attribute is only as trustworthy as the thing that set it. Every value here -comes from something the dataplane itself derived — a peer certificate Envoy -verified against the actor-identity CA, an `:authority` captured before the -request entered a tunnel — never from a client header, which an actor controls -end to end. That is what makes filter state a sound carrier across the -CONNECT/MITM boundary in the first place, and a new attribute sourced from a -header gives the property back. +An attribute is only as trustworthy as its source. The actor name and atespace +carry client-selected routing coordinates, while the CONNECT authority carries +only the requested target port across tunnel re-entry. Security-sensitive +attributes, such as actor identity, must come from dataplane-authenticated +state rather than a client header. ## modes diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index 97eab8a5a0..1169b93df9 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -32,9 +32,8 @@ type dataplaneHealthCheck struct { } // Both dataplanes resolve the worker address from ext_proc's dynamic -// metadata (see ingress.OriginalDstMetadataKey) and leave :authority/Host -// untouched, so atunnel always authorizes by the actor's own DNS name -- -// ingress.New needs no per-dataplane routing mode. +// metadata (see ingress.OriginalDstMetadataKey). Actor identity is carried in +// explicit headers, so ingress.New needs no per-dataplane routing mode. func (r atenetRouter) healthCheck() dataplaneHealthCheck { switch r { diff --git a/cmd/atenet/internal/router/extproc/attributes.go b/cmd/atenet/internal/router/extproc/attributes.go index 6cfb929a96..72992469fa 100644 --- a/cmd/atenet/internal/router/extproc/attributes.go +++ b/cmd/atenet/internal/router/extproc/attributes.go @@ -27,13 +27,17 @@ package extproc // substrate's own metric dimensions and stay on dotted "ate.". Neither is the // "ate.dev/" slash form, which is Kubernetes labels only. const ( - // AuthorityFilterStateKey is the filter-state key holding an ingress - // request's :authority, set by xds.go's authorityFilterStateFilter. - // Ingress-only: it names the actor a request is addressed to. - AuthorityFilterStateKey = "dev.ate.authority" - // AuthorityFilterStateAttribute is the CEL expression ext_proc evaluates to - // read AuthorityFilterStateKey back out. - AuthorityFilterStateAttribute = "filter_state['" + AuthorityFilterStateKey + "']" + // ActorNameFilterStateKey and AtespaceFilterStateKey carry the ingress actor + // routing coordinates across Envoy's CONNECT internal-listener hop. + ActorNameFilterStateKey = "dev.ate.actor.name" + AtespaceFilterStateKey = "dev.ate.actor.atespace" + ConnectAuthorityFilterStateKey = "dev.ate.connect.authority" + + // ActorNameFilterStateAttribute and AtespaceFilterStateAttribute are the CEL + // expressions ext_proc evaluates to read the corresponding filter state. + ActorNameFilterStateAttribute = "filter_state['" + ActorNameFilterStateKey + "']" + AtespaceFilterStateAttribute = "filter_state['" + AtespaceFilterStateKey + "']" + ConnectAuthorityFilterStateAttribute = "filter_state['" + ConnectAuthorityFilterStateKey + "']" // ActorIdentityFilterStateKey is the filter-state key holding the actor // identity the egress gateway read from the peer certificate it verified diff --git a/cmd/atenet/internal/router/extproc/attributes_test.go b/cmd/atenet/internal/router/extproc/attributes_test.go new file mode 100644 index 0000000000..acafa90235 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/attributes_test.go @@ -0,0 +1,63 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "strings" + "testing" +) + +func TestAttributeKeys(t *testing.T) { + tests := []struct { + name string + got string + want string + }{ + {"actor name filter state", ActorNameFilterStateKey, "dev.ate.actor.name"}, + {"actor name attribute", ActorNameFilterStateAttribute, "filter_state['dev.ate.actor.name']"}, + {"atespace filter state", AtespaceFilterStateKey, "dev.ate.actor.atespace"}, + {"atespace attribute", AtespaceFilterStateAttribute, "filter_state['dev.ate.actor.atespace']"}, + {"CONNECT authority filter state", ConnectAuthorityFilterStateKey, "dev.ate.connect.authority"}, + {"CONNECT authority attribute", ConnectAuthorityFilterStateAttribute, "filter_state['dev.ate.connect.authority']"}, + {"actor identity filter state", ActorIdentityFilterStateKey, "dev.ate.actor.identity"}, + {"direction attribute", directionAttribute, "dev.ate.extproc.direction"}, + {"filter chain name attribute", FilterChainNameAttribute, "xds.filter_chain_name"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.got != tt.want { + t.Errorf("key = %q, want %q; update dataplane configuration to match", tt.got, tt.want) + } + }) + } +} + +func TestSubstrateKeysShareOnePrefix(t *testing.T) { + const prefix = "dev.ate." + for _, key := range []string{ + ActorNameFilterStateKey, + AtespaceFilterStateKey, + ConnectAuthorityFilterStateKey, + ActorIdentityFilterStateKey, + directionAttribute, + } { + if !strings.HasPrefix(key, prefix) { + t.Errorf("key %q is not rooted at %q", key, prefix) + } + if strings.Contains(key, "/") { + t.Errorf("key %q uses the ate.dev/ Kubernetes label form", key) + } + } +} diff --git a/cmd/atenet/internal/router/extproc/dispatch.go b/cmd/atenet/internal/router/extproc/dispatch.go index 3f0f4e7766..cae5b9471e 100644 --- a/cmd/atenet/internal/router/extproc/dispatch.go +++ b/cmd/atenet/internal/router/extproc/dispatch.go @@ -47,8 +47,8 @@ const EgressFilterChainName = "egress" // filter chain name; the request cannot influence it. // // An unrecognized or absent attribute means ingress, the fail-safe direction: -// an egress request misrouted to the ingress handler fails to parse as an actor -// DNS name and 404s, whereas the reverse leaks control-plane state. +// an egress request misrouted to the ingress handler lacks the required actor +// routing headers and fails, whereas the reverse leaks control-plane state. func directionOf(req *extprocv3.ProcessingRequest) Direction { if requestAttribute(req, directionAttribute) == string(DirectionEgress) { return DirectionEgress diff --git a/cmd/atenet/internal/router/extproc/handler.go b/cmd/atenet/internal/router/extproc/handler.go index a5622252e8..04844d1d21 100644 --- a/cmd/atenet/internal/router/extproc/handler.go +++ b/cmd/atenet/internal/router/extproc/handler.go @@ -37,7 +37,7 @@ type Handler interface { // A returned error denies the request: a *ReqError carries the status code // and client-safe body to answer with, anything else becomes a 500. The // Result is read even when an error is returned, so a handler that got far - // enough to learn the metric attributes (template identity, resume outcome) + // enough to learn the metric attributes (template coordinates, resume outcome) // should still fill them in. HandleRequestHeaders(ctx context.Context, md *RequestMetadata) (Result, error) } diff --git a/cmd/atenet/internal/router/extproc/metadata_test.go b/cmd/atenet/internal/router/extproc/metadata_test.go index 302ff5cefe..9cc2086b2e 100644 --- a/cmd/atenet/internal/router/extproc/metadata_test.go +++ b/cmd/atenet/internal/router/extproc/metadata_test.go @@ -124,7 +124,7 @@ func TestRequestMetadataAttribute(t *testing.T) { attrs := map[string]*structpb.Struct{ "envoy.filters.http.ext_proc": { Fields: map[string]*structpb.Value{ - "filter_state['dev.ate.authority']": structpb.NewStringValue("actor-1.team-a.actors.resources.substrate.ate.dev"), + ActorNameFilterStateAttribute: structpb.NewStringValue("actor-1"), }, }, } @@ -134,10 +134,28 @@ func TestRequestMetadataAttribute(t *testing.T) { // The lookup scans every filter's attributes rather than hardcoding which // filter reported the value (see filterChainName in dispatch.go for why), // so it must not matter which filter name the value arrived under. - if got, want := md.Attribute("filter_state['dev.ate.authority']"), "actor-1.team-a.actors.resources.substrate.ate.dev"; got != want { + if got, want := md.Attribute(ActorNameFilterStateAttribute), "actor-1"; got != want { t.Errorf("Attribute() = %q, want %q", got, want) } if got := md.Attribute("filter_state['does.not.exist']"); got != "" { t.Errorf("Attribute() for a missing key = %q, want \"\"", got) } } + +func TestRequestMetadataHeaderIsCaseInsensitive(t *testing.T) { + md := NewRequestMetadata([]*corev3.HeaderValue{ + {Key: "X-ATE-Actor-Name", Value: "actor-1"}, + {Key: "x-ate-ATESPACE", Value: "team-a"}, + }, nil) + + for name, want := range map[string]string{ + "x-ate-actor-name": "actor-1", + "X-Ate-Actor-Name": "actor-1", + "x-ate-atespace": "team-a", + "X-ATE-ATESPACE": "team-a", + } { + if got := md.Header(name); got != want { + t.Errorf("Header(%q) = %q, want %q", name, got, want) + } + } +} diff --git a/cmd/atenet/internal/router/ingress/actorref_test.go b/cmd/atenet/internal/router/ingress/actorref_test.go deleted file mode 100644 index d7aab2ac85..0000000000 --- a/cmd/atenet/internal/router/ingress/actorref_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ingress - -import ( - "testing" - - "github.com/agent-substrate/substrate/internal/resources" -) - -func TestParseActorRef(t *testing.T) { - tests := []struct { - name string - host string - want resources.ActorRef - wantErr bool - }{ - { - name: "valid host without port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev:8443", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with trailing dot", - host: "my-actor.team-a.actors.resources.substrate.ate.dev.", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with trailing dot and port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev.:8080", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "mixed-case host", - host: "My-Actor.Team-A.actors.resources.substrate.ate.dev", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "missing atespace label", - host: "my-actor.actors.resources.substrate.ate.dev", - wantErr: true, - }, - { - name: "invalid suffix", - host: "my-actor.team-a.example.com", - wantErr: true, - }, - { - name: "invalid host port format", - host: "my-actor.team-a.actors.resources.substrate.ate.dev:invalid:port", - wantErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got, err := parseActorRef(tc.host) - if (err != nil) != tc.wantErr { - t.Errorf("parseActorRef(%q) error = %v, wantErr %v", tc.host, err, tc.wantErr) - return - } - if got != tc.want { - t.Errorf("parseActorRef(%q) = %+v, want %+v", tc.host, got, tc.want) - } - }) - } -} diff --git a/cmd/atenet/internal/router/ingress/errors.go b/cmd/atenet/internal/router/ingress/errors.go index 1c3bc89562..ffbd626a2a 100644 --- a/cmd/atenet/internal/router/ingress/errors.go +++ b/cmd/atenet/internal/router/ingress/errors.go @@ -32,12 +32,6 @@ func actorNotFoundErr(actorRef resources.ActorRef) error { return extproc.NewReqError(envoy_type.StatusCode_NotFound, "actor %s not found", actorRef) } -// invalidHostErr returns a 404 denial explaining why the request host was -// rejected. The cause is preserved for log inspection via Unwrap. -func invalidHostErr(host string, cause error) error { - return extproc.WrapReqError(envoy_type.StatusCode_NotFound, cause, "invalid host %q: %v", host, cause) -} - // statusDescription returns the gRPC status description of err, unwrapping // any wrapper (e.g. budgetExhaustedError) first. status.Convert on a wrapping // error replaces the description with the wrapper's full "rpc error: ..." diff --git a/cmd/atenet/internal/router/ingress/errors_test.go b/cmd/atenet/internal/router/ingress/errors_test.go index 2893523671..105c736c1f 100644 --- a/cmd/atenet/internal/router/ingress/errors_test.go +++ b/cmd/atenet/internal/router/ingress/errors_test.go @@ -43,27 +43,6 @@ func TestActorNotFoundErr(t *testing.T) { } } -func TestInvalidHostErr(t *testing.T) { - t.Parallel() - - cause := errors.New("missing suffix") - err := invalidHostErr("foo.example.com", cause) - - var reqErr *extproc.ReqError - if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", err) - } - if reqErr.StatusCode != int(envoy_type.StatusCode_NotFound) { - t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_NotFound) - } - if got, want := err.Error(), `invalid host "foo.example.com": missing suffix`; got != want { - t.Errorf("Error() = %q, want %q", got, want) - } - if !errors.Is(err, cause) { - t.Errorf("errors.Is(err, cause) = false, want true (cause should be wrapped for logging)") - } -} - func TestMapResumeError(t *testing.T) { t.Parallel() diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 407adc282a..847f9776f9 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -18,18 +18,16 @@ // saturated), and points the dataplane at the worker that ends up hosting it. // // Everything reaching this handler is unauthenticated client input. The -// opposite trust model — an actor identity carried by a CA-signed client -// certificate — belongs to the sibling egress package, and the two are kept -// apart deliberately. +// certificate authentication used for egress belongs to the sibling egress +// package, and the two are kept apart deliberately. package ingress import ( "context" - "fmt" "log/slog" "net" + "net/http" "strconv" - "strings" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" @@ -39,6 +37,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -87,23 +86,23 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ExtProc.RequestHeaders") defer span.End() - // Resolved from filter state rather than Host/:authority directly: a - // reinjected CONNECT tunnel's own :authority has nothing to do with the - // actor, so xds.go captures the real one at connect_terminate instead. - authority := md.Attribute(extproc.AuthorityFilterStateAttribute) - if authority == "" { - return extproc.Result{}, invalidHostErr(md.Host, fmt.Errorf("missing %s request attribute", extproc.AuthorityFilterStateAttribute)) + actorRef := resources.ActorRef{ + Name: routingValue(md, atenet.ActorNameHeader, extproc.ActorNameFilterStateAttribute), + Atespace: routingValue(md, atenet.AtespaceHeader, extproc.AtespaceFilterStateAttribute), } - actorRef, err := parseActorRef(authority) - if err != nil { - // Authority is invalid, respond with 404. - return extproc.Result{}, invalidHostErr(authority, err) + if !resources.IsValidResourceName(actorRef.Name) || !resources.IsValidResourceName(actorRef.Atespace) { + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_NotFound, "invalid actor reference") } - // CONNECT traffic can name a port other than defaultActorPort in the - // authority. + // CONNECT traffic can name a port other than defaultActorPort. After Envoy + // terminates CONNECT, filter state retains the outer authority while md.Host + // belongs to the inner request. targetPort := defaultActorPort - if _, portStr, err := net.SplitHostPort(authority); err == nil { + targetAuthority := md.Attribute(extproc.ConnectAuthorityFilterStateAttribute) + if targetAuthority == "" && md.Method == http.MethodConnect { + targetAuthority = md.Host + } + if _, portStr, err := net.SplitHostPort(targetAuthority); err == nil { if p, ok := atunnel.ParsePort(portStr); ok { targetPort = p } @@ -126,7 +125,7 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM return extproc.Result{Resume: string(resumeOutcome)}, mapResumeError(actorRef, err) } - // Actor template identity, used as low-cardinality route-latency metric + // Actor template coordinates, used as low-cardinality route-latency metric // attributes. res := extproc.Result{ TemplateAtespace: actor.GetActorTemplate().GetAtespace(), @@ -164,17 +163,21 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM "actor %s routing failed", actorRef) } - // :authority/Host stays untouched, so atunnel authorizes by the actor's - // own DNS name. The target port still goes as a header: atunnel can't - // read dynamic metadata. + // Overwrite the routing headers so client-provided values cannot select a + // different actor after this request has been resolved. mutation := &extprocv3.HeaderMutation{} - mutation.SetHeaders = append(mutation.SetHeaders, &corev3.HeaderValueOption{ - Header: &corev3.HeaderValue{ - Key: atunnel.TargetPortHeader, - RawValue: []byte(strconv.Itoa(targetPort)), - }, - AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, - }) + for _, header := range []struct{ name, value string }{ + {atenet.ActorNameHeader, actorRef.Name}, + {atenet.AtespaceHeader, actorRef.Atespace}, + } { + mutation.SetHeaders = append(mutation.SetHeaders, &corev3.HeaderValueOption{ + Header: &corev3.HeaderValue{ + Key: header.name, + RawValue: []byte(header.value), + }, + AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, + }) + } res.Target = targetAddr res.Response = &extprocv3.HeadersResponse{ @@ -186,18 +189,9 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM return res, nil } -// parseActorRef extracts the actor an incoming request is addressed to from its -// Host/:authority, which has the form -// "..actors.resources.substrate.ate.dev" (optionally with a -// port). The atespace is part of the name because an actor name is only unique -// within its atespace. -func parseActorRef(host string) (resources.ActorRef, error) { - if strings.Contains(host, ":") { - h, _, err := net.SplitHostPort(host) - if err != nil { - return resources.ActorRef{}, err - } - host = h +func routingValue(md *extproc.RequestMetadata, header, attribute string) string { + if value := md.Header(header); value != "" { + return value } - return resources.ParseActorDNSName(host) + return md.Attribute(attribute) } diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index 76fed77c39..cbe58fbd3e 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -32,6 +32,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -45,29 +46,37 @@ func (m *mockClient) ResumeActor(ctx context.Context, in *ateapipb.ResumeActorRe return m.resumeFn(ctx, in, opts...) } -// authorityAttributes builds the ProcessingRequest.Attributes map the mux -// hands the handler -- the forwarded filter_state['dev.ate.authority'] -// CEL attribute that xds.go's buildHcm (backed by authorityFilterStateFilter, -// or for CONNECT, connect_terminate's own capture) requests from Envoy. It -// replaces the :authority header as the source of routing truth; tests still -// set the header too, since RequestMetadata still logs it. -func authorityAttributes(t *testing.T, authority string) map[string]*structpb.Struct { - t.Helper() - s, err := structpb.NewStruct(map[string]any{extproc.AuthorityFilterStateAttribute: authority}) - if err != nil { - t.Fatalf("build authority attributes: %v", err) - } - return map[string]*structpb.Struct{ - "envoy.filters.http.ext_proc": s, - } +func requestMetadata(actorName, atespace string, headers ...*corev3.HeaderValue) *extproc.RequestMetadata { + headers = append(headers, + &corev3.HeaderValue{Key: atenet.ActorNameHeader, Value: actorName}, + &corev3.HeaderValue{Key: atenet.AtespaceHeader, Value: atespace}, + ) + return extproc.NewRequestMetadata(headers, nil) } -// requestMetadata builds the metadata the ext_proc mux would hand the handler -// for a request with these headers, with authority forwarded via filter-state -// attribute the way Envoy actually delivers it (see authorityAttributes). -func requestMetadata(t *testing.T, authority string, headers ...*corev3.HeaderValue) *extproc.RequestMetadata { - t.Helper() - return extproc.NewRequestMetadata(headers, authorityAttributes(t, authority)) +func TestHandleRequestHeadersAcceptsMixedCaseRoutingHeaders(t *testing.T) { + clientMock := &mockClient{ + resumeFn: func(_ context.Context, in *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + if got, want := in.GetActor().GetName(), "actor-1"; got != want { + t.Errorf("actor name = %q, want %q", got, want) + } + if got, want := in.GetActor().GetAtespace(), "team-a"; got != want { + t.Errorf("atespace = %q, want %q", got, want) + } + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}, + }}, nil + }, + } + h := New(clientMock, ParkedRequestConfig{}, nil) + md := extproc.NewRequestMetadata([]*corev3.HeaderValue{ + {Key: "X-ATE-Actor-Name", Value: "actor-1"}, + {Key: "x-ate-ATESPACE", Value: "team-a"}, + }, nil) + + if _, err := h.HandleRequestHeaders(context.Background(), md); err != nil { + t.Fatalf("HandleRequestHeaders() error = %v", err) + } } // dynamicMetadataTarget extracts the resolved worker address @@ -77,10 +86,7 @@ func dynamicMetadataTarget(dynamicMetadata *structpb.Struct) string { } // dynamicMetadataPort extracts the target port HandleRequestHeaders reports -// via OriginalDstMetadataKey/OriginalDstPortKey. xds.go's buildRoutes derives -// a real atunnel.TargetPortHeader from this at the route level via a -// %DYNAMIC_METADATA(...)% format string; HandleRequestHeaders also sets that -// same header directly (see its own doc comment for why). +// via OriginalDstMetadataKey/OriginalDstPortKey. func dynamicMetadataPort(dynamicMetadata *structpb.Struct) string { return dynamicMetadata.GetFields()[OriginalDstMetadataKey].GetStructValue().GetFields()[OriginalDstPortKey].GetStringValue() } @@ -101,7 +107,7 @@ func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { }, }, ParkedRequestConfig{}, nil) - md := requestMetadata(t, authority, + md := requestMetadata(testUUID, "team-a", &corev3.HeaderValue{Key: ":path", Value: "/api/v1/reset?token=" + secret}, &corev3.HeaderValue{Key: ":authority", Value: authority}, &corev3.HeaderValue{Key: ":method", Value: "POST"}, @@ -138,6 +144,8 @@ func TestHandleRequestHeaders(t *testing.T) { tests := []struct { name string + actorName string + atespace string authority string resumeResp *ateapipb.ResumeActorResponse resumeErr error @@ -148,10 +156,12 @@ func TestHandleRequestHeaders(t *testing.T) { expectedTargetPort string }{ { - name: "invalid host returns 404 identifying the host", + name: "invalid actor header returns 404", + actorName: "INVALID", + atespace: "team-a", authority: "invalid-host.com", expectErr: true, - expectedErrStr: `invalid host "invalid-host.com": invalid actor DNS name: must end with actors.resources.substrate.ate.dev, got "invalid-host.com"`, + expectedErrStr: `invalid actor reference`, expectedStatus: envoy_type.StatusCode_NotFound, }, { @@ -207,8 +217,8 @@ func TestHandleRequestHeaders(t *testing.T) { expectedStatus: envoy_type.StatusCode_InternalServerError, }, { - name: "Successful resume", - authority: testUUID + ".team-a.actors.resources.substrate.ate.dev", + name: "successful resume ignores host and port for actor routing", + authority: "127.0.0.1:44681", resumeResp: &ateapipb.ResumeActorResponse{ Actor: &ateapipb.Actor{ Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}, @@ -222,6 +232,14 @@ func TestHandleRequestHeaders(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + actorName := tc.actorName + if actorName == "" { + actorName = testUUID + } + atespace := tc.atespace + if atespace == "" { + atespace = "team-a" + } clientMock := &mockClient{ resumeFn: func(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { if in.GetActor().GetName() != testUUID { @@ -240,7 +258,7 @@ func TestHandleRequestHeaders(t *testing.T) { // resumer_test.go. h := New(clientMock, ParkedRequestConfig{}, nil) - md := requestMetadata(t, tc.authority, + md := requestMetadata(actorName, atespace, &corev3.HeaderValue{Key: ":path", Value: "/v1/actors/invoke"}, &corev3.HeaderValue{Key: ":authority", Value: tc.authority}, &corev3.HeaderValue{Key: ":method", Value: "POST"}, @@ -275,16 +293,22 @@ func TestHandleRequestHeaders(t *testing.T) { } mutation := res.Response.GetResponse().GetHeaderMutation() - if len(mutation.GetSetHeaders()) != 1 { - t.Fatalf("expected exactly one header option (TargetPortHeader), found: %v", mutation.GetSetHeaders()) + if len(mutation.GetSetHeaders()) != 2 { + t.Fatalf("expected actor routing headers, found: %v", mutation.GetSetHeaders()) } gotMutations := map[string]string{} for _, headerOption := range mutation.GetSetHeaders() { gotMutations[strings.ToLower(headerOption.Header.Key)] = string(headerOption.Header.RawValue) } - if got := gotMutations[strings.ToLower(atunnel.TargetPortHeader)]; got != tc.expectedTargetPort { - t.Errorf("target port mutation = %q, want %q", got, tc.expectedTargetPort) + if _, ok := gotMutations[strings.ToLower(atunnel.TargetPortHeader)]; ok { + t.Errorf("target port must be emitted only as dynamic metadata") + } + if got := gotMutations[strings.ToLower(atenet.ActorNameHeader)]; got != testUUID { + t.Errorf("actor name mutation = %q, want %q", got, testUUID) + } + if got := gotMutations[strings.ToLower(atenet.AtespaceHeader)]; got != "team-a" { + t.Errorf("atespace mutation = %q, want %q", got, "team-a") } if got := dynamicMetadataTarget(res.DynamicMetadata); got != tc.expectedTarget { t.Errorf("invalid destination mapping found: %s, expected: %s", got, tc.expectedTarget) @@ -301,9 +325,8 @@ func TestHandleRequestHeaders(t *testing.T) { // travels in :authority, e.g. ":9090") resolves the actor, // produces the same ":443" original-dst mutation as an ordinary // request (the router only ever dials the worker's atunnel server), and -// reports the arbitrary port itself via -// OriginalDstMetadataKey/OriginalDstPortKey, which xds.go's buildRoutes turns -// into atunnel.TargetPortHeader for atunnel. +// reports the arbitrary port itself in dynamic metadata for Envoy to write to +// atunnel.TargetPortHeader. func TestHandleRequestHeadersHandlesConnectMethod(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" authority := testUUID + ".team-a.actors.resources.substrate.ate.dev:9090" @@ -316,7 +339,7 @@ func TestHandleRequestHeadersHandlesConnectMethod(t *testing.T) { h := New(clientMock, ParkedRequestConfig{}, nil) // CONNECT requests carry no :path; the request-target lives in :authority. - md := requestMetadata(t, authority, + md := requestMetadata(testUUID, "team-a", &corev3.HeaderValue{Key: ":authority", Value: authority}, &corev3.HeaderValue{Key: ":method", Value: "CONNECT"}, ) @@ -338,6 +361,37 @@ func TestHandleRequestHeadersHandlesConnectMethod(t *testing.T) { } } +func TestHandleRequestHeadersUsesRetainedConnectAuthorityForPort(t *testing.T) { + const testUUID = "123e4567-e89b-12d3-a456-426614174000" + clientMock := &mockClient{ + resumeFn: func(context.Context, *ateapipb.ResumeActorRequest, ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}, + }}, nil + }, + } + attrs := map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + }, + }, + } + md := extproc.NewRequestMetadata([]*corev3.HeaderValue{ + {Key: atenet.ActorNameHeader, Value: testUUID}, + {Key: atenet.AtespaceHeader, Value: "team-a"}, + {Key: ":authority", Value: "inner.example"}, + }, attrs) + + res, err := New(clientMock, ParkedRequestConfig{}, nil).HandleRequestHeaders(context.Background(), md) + if err != nil { + t.Fatalf("HandleRequestHeaders() error = %v", err) + } + if got, want := dynamicMetadataPort(res.DynamicMetadata), "9090"; got != want { + t.Errorf("target port = %q, want %q", got, want) + } +} + // TestHandleRequestHeaders_ParkingLotFull verifies that when the parking lot is at capacity // the request is shed with a 503 before any resume is attempted. func TestHandleRequestHeaders_ParkingLotFull(t *testing.T) { @@ -361,7 +415,7 @@ func TestHandleRequestHeaders_ParkingLotFull(t *testing.T) { } defer release(parkOutcomeServed) - md := requestMetadata(t, authority, + md := requestMetadata(testUUID, "team-a", &corev3.HeaderValue{Key: ":authority", Value: authority}, ) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index dca8a5d101..4956cfd267 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -34,7 +34,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" - "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/atenet" accesslogv3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" @@ -71,6 +71,7 @@ import ( "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "github.com/agent-substrate/substrate/internal/atunnel" ) const ( @@ -99,19 +100,14 @@ const ( // OriginalDstClusterName routes actor traffic to the worker's atunnel // ingress by the IP:port ext_proc reports in dynamic metadata (see - // ingress.OriginalDstMetadataKey), while the request :authority stays the - // actor DNS name so atunnel can identify the active actor. + // ingress.OriginalDstMetadataKey). Actor identity remains in explicit + // request headers for atunnel to authorize. OriginalDstClusterName = "actor_original_dst" WildcardIP = "0.0.0.0" ConnectUpgradeType = "CONNECT" MainInternalName = "main_internal" - // dynamicMetadataPortFormat is the %DYNAMIC_METADATA(...)% header-value - // command operator (see buildRoutes) that derives atunnel.TargetPortHeader - // from ingress.OriginalDstMetadataKey/ingress.OriginalDstPortKey. - dynamicMetadataPortFormat = "%DYNAMIC_METADATA(" + ingress.OriginalDstMetadataKey + ":" + ingress.OriginalDstPortKey + ")%" - // httpExtProcFilterName is envoy.filters.http.ext_proc's own well-known // name, used as the HttpFilter.Name in buildHcm. httpExtProcFilterName = "envoy.filters.http.ext_proc" @@ -766,9 +762,8 @@ func (x *XdsServer) buildMainInternalCluster() *clusterv3.Cluster { // buildOriginalDstCluster dials the exact worker atunnel address supplied by // ext_proc in dynamic metadata (see ingress.OriginalDstMetadataKey). It does -// not derive the destination from :authority, so the request keeps the actor -// DNS name as its Host for atunnel to authorize. mTLS to atunnel is applied -// via the shared upstream transport socket (SPIFFE URI validation). +// not derive the destination from :authority. mTLS to atunnel is applied via +// the shared upstream transport socket (SPIFFE URI validation). func (x *XdsServer) buildOriginalDstCluster() *clusterv3.Cluster { cluster := &clusterv3.Cluster{ Name: OriginalDstClusterName, @@ -840,15 +835,12 @@ func (x *XdsServer) buildRoutes() *routev3.RouteConfiguration { IdleTimeout: durationpb.New(x.routeIdleTimeout()), }, }, - // atunnel reads the actor's target port from a header, since - // it can't see Envoy's dynamic metadata; this derives it - // declaratively from the same metadata ext_proc wrote (see - // ingress.OriginalDstPortKey). RequestHeadersToAdd: []*corev3.HeaderValueOption{ { Header: &corev3.HeaderValue{ - Key: atunnel.TargetPortHeader, - Value: dynamicMetadataPortFormat, + Key: atunnel.TargetPortHeader, + Value: fmt.Sprintf("%%DYNAMIC_METADATA(%s:%s)%%", + ingress.OriginalDstMetadataKey, ingress.OriginalDstPortKey), }, AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, }, @@ -894,39 +886,44 @@ func (x *XdsServer) buildMainInternalListener() *listenerv3.Listener { } } -// authorityFilterStateFilter captures :authority into -// extproc.AuthorityFilterStateKey filter state, so main_internal's HTTP leg -// can read it back across the internal-listener hop (see buildHcm, -// ingress.HandleRequestHeaders). buildConnectTerminateHCM and buildHcm's -// ingress listeners use it; main_internal itself must not, since that would -// capture the tunneled protocol's own, unrelated :authority instead. -func authorityFilterStateFilter() *hcmv3.HttpFilter { - return &hcmv3.HttpFilter{ - Name: "envoy.filters.http.set_filter_state", - ConfigType: &hcmv3.HttpFilter_TypedConfig{ - TypedConfig: newAny(&setfilterstatev3.Config{ - OnRequestHeaders: []*setfilterstatecommonv3.FilterStateValue{ - { - Key: &setfilterstatecommonv3.FilterStateValue_ObjectKey{ - ObjectKey: extproc.AuthorityFilterStateKey, - }, - // extproc.AuthorityFilterStateKey is a custom (non-well-known) - // key, so the generic string factory is required. - FactoryKey: "envoy.string", - Value: &setfilterstatecommonv3.FilterStateValue_FormatString{ - FormatString: &corev3.SubstitutionFormatString{ - Format: &corev3.SubstitutionFormatString_TextFormatSource{ - TextFormatSource: &corev3.DataSource{ - Specifier: &corev3.DataSource_InlineString{ - InlineString: "%REQ(:AUTHORITY)%", - }, - }, - }, +// actorRoutingFilterStateFilter captures actor routing headers so +// main_internal can read them across the CONNECT internal-listener hop. +func actorRoutingFilterStateFilter(captureAuthority bool) *hcmv3.HttpFilter { + values := make([]*setfilterstatecommonv3.FilterStateValue, 0, 3) + routingFields := []struct{ key, header string }{ + {extproc.ActorNameFilterStateKey, atenet.ActorNameHeader}, + {extproc.AtespaceFilterStateKey, atenet.AtespaceHeader}, + } + if captureAuthority { + routingFields = append(routingFields, struct{ key, header string }{ + extproc.ConnectAuthorityFilterStateKey, extproc.AuthorityHeader, + }) + } + for _, routingField := range routingFields { + values = append(values, &setfilterstatecommonv3.FilterStateValue{ + Key: &setfilterstatecommonv3.FilterStateValue_ObjectKey{ + ObjectKey: routingField.key, + }, + FactoryKey: "envoy.string", + Value: &setfilterstatecommonv3.FilterStateValue_FormatString{ + FormatString: &corev3.SubstitutionFormatString{ + Format: &corev3.SubstitutionFormatString_TextFormatSource{ + TextFormatSource: &corev3.DataSource{ + Specifier: &corev3.DataSource_InlineString{ + InlineString: "%REQ(" + routingField.header + ")%", }, }, - SharedWithUpstream: setfilterstatecommonv3.FilterStateValue_ONCE, }, }, + }, + SharedWithUpstream: setfilterstatecommonv3.FilterStateValue_ONCE, + }) + } + return &hcmv3.HttpFilter{ + Name: "envoy.filters.http.set_filter_state", + ConfigType: &hcmv3.HttpFilter_TypedConfig{ + TypedConfig: newAny(&setfilterstatev3.Config{ + OnRequestHeaders: values, }), }, } @@ -961,7 +958,7 @@ func (x *XdsServer) buildConnectTerminateHCM(statPrefix string) *anypb.Any { }, CodecType: hcmv3.HttpConnectionManager_AUTO, HttpFilters: []*hcmv3.HttpFilter{ - authorityFilterStateFilter(), + actorRoutingFilterStateFilter(true), { Name: "envoy.filters.http.router", ConfigType: &hcmv3.HttpFilter_TypedConfig{ @@ -1016,12 +1013,9 @@ func buildConnectRoutes() *routev3.RouteConfiguration { } // buildHcm builds the HTTP ext_proc-fronted HCM shared by the ingress_http, -// ingress_https, and main_internal listeners. captureAuthority runs -// authorityFilterStateFilter to populate the actor's :authority for ingress -// listeners; main_internal passes false, since connect_terminate already -// shared the correct value and re-deriving it here would clobber it with -// the tunneled protocol's own, unrelated :authority. -func (x *XdsServer) buildHcm(statPrefix string, captureAuthority bool) *anypb.Any { +// ingress_https, and main_internal listeners. captureActorRouting preserves +// the two actor routing headers across CONNECT re-entry. +func (x *XdsServer) buildHcm(statPrefix string, captureActorRouting bool) *anypb.Any { extProcConfig := newAny(&extprocv3filter.ExternalProcessor{ GrpcService: &corev3.GrpcService{ TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{ @@ -1046,11 +1040,11 @@ func (x *XdsServer) buildHcm(statPrefix string, captureAuthority bool) *anypb.An RequestTrailerMode: extprocv3filter.ProcessingMode_SKIP, ResponseTrailerMode: extprocv3filter.ProcessingMode_SKIP, }, - // Passes the resolved actor's authority as a request attribute (see - // extproc.AuthorityFilterStateAttribute, ingress.HandleRequestHeaders) - // and lets the response write the resolved worker address into - // ingress.OriginalDstMetadataKey. - RequestAttributes: []string{extproc.AuthorityFilterStateAttribute}, + RequestAttributes: []string{ + extproc.ActorNameFilterStateAttribute, + extproc.AtespaceFilterStateAttribute, + extproc.ConnectAuthorityFilterStateAttribute, + }, MetadataOptions: &extprocv3filter.MetadataOptions{ ForwardingNamespaces: &extprocv3filter.MetadataOptions_MetadataNamespaces{ Untyped: []string{ingress.OriginalDstMetadataKey}, @@ -1066,8 +1060,8 @@ func (x *XdsServer) buildHcm(statPrefix string, captureAuthority bool) *anypb.An accessLogConfig := newAny(&streamaccesslogv3.StdoutAccessLog{}) httpFilters := []*hcmv3.HttpFilter{} - if captureAuthority { - httpFilters = append(httpFilters, authorityFilterStateFilter()) + if captureActorRouting { + httpFilters = append(httpFilters, actorRoutingFilterStateFilter(false)) } httpFilters = append(httpFilters, &hcmv3.HttpFilter{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index ea5d71e92c..bb2f4d36b7 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -38,6 +38,7 @@ import ( corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + setfilterstatev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/set_filter_state/v3" hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" @@ -46,10 +47,58 @@ import ( cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" "github.com/agent-substrate/substrate/internal/atunnel" ) +func TestActorRoutingFilterStateFilter(t *testing.T) { + for _, tc := range []struct { + name string + captureAuthority bool + want map[string]string + }{ + { + name: "ordinary ingress", + want: map[string]string{ + extproc.ActorNameFilterStateKey: "%REQ(x-ate-actor-name)%", + extproc.AtespaceFilterStateKey: "%REQ(x-ate-atespace)%", + }, + }, + { + name: "CONNECT termination", + captureAuthority: true, + want: map[string]string{ + extproc.ActorNameFilterStateKey: "%REQ(x-ate-actor-name)%", + extproc.AtespaceFilterStateKey: "%REQ(x-ate-atespace)%", + extproc.ConnectAuthorityFilterStateKey: "%REQ(:authority)%", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + filter := actorRoutingFilterStateFilter(tc.captureAuthority) + config := &setfilterstatev3.Config{} + if err := filter.GetTypedConfig().UnmarshalTo(config); err != nil { + t.Fatalf("unmarshal set_filter_state config: %v", err) + } + + if len(config.GetOnRequestHeaders()) != len(tc.want) { + t.Fatalf("captured values = %d, want %d", len(config.GetOnRequestHeaders()), len(tc.want)) + } + for _, value := range config.GetOnRequestHeaders() { + key := value.GetObjectKey() + format := value.GetFormatString().GetTextFormatSource().GetInlineString() + if format != tc.want[key] { + t.Errorf("capture %q = %q, want %q", key, format, tc.want[key]) + } + if key != extproc.ConnectAuthorityFilterStateKey && strings.Contains(strings.ToLower(format), ":authority") { + t.Errorf("capture %q derives actor routing from authority", key) + } + } + }) + } +} + // assertDualStackIngress checks an ingress listener keeps its 0.0.0.0 primary // and gains exactly one "::" socket on the same port. func assertDualStackIngress(t *testing.T, l *listenerv3.Listener, wantPort uint32) { @@ -1093,31 +1142,25 @@ func TestXdsServer_ActorClusterProtocolOptions(t *testing.T) { } } -// TestXdsServer_BuildRoutes_DerivesTargetPortHeader covers the fix for atunnel -// needing the target port as a real header (it can't read Envoy's dynamic -// metadata directly): rather than ext_proc building that header mutation -// itself, the route derives it declaratively from the same -// ingress.OriginalDstMetadataKey/ingress.OriginalDstPortKey metadata ext_proc -// already writes for the cluster's own MetadataKey, via a -// %DYNAMIC_METADATA(...)% command operator. -func TestXdsServer_BuildRoutes_DerivesTargetPortHeader(t *testing.T) { +// TestXdsServer_BuildRoutesWritesTargetPortHeader ensures the route overwrites +// the target-port header with the value from ext_proc's trusted metadata. +func TestXdsServer_BuildRoutesWritesTargetPortHeader(t *testing.T) { x := NewXdsServer(18000) route := x.buildRoutes().GetVirtualHosts()[0].GetRoutes()[0] headers := route.GetRequestHeadersToAdd() if len(headers) != 1 { - t.Fatalf("Expected exactly 1 request header to add, got %d: %v", len(headers), headers) + t.Fatalf("route adds request headers %v, want exactly one", headers) } - h := headers[0] - if got, want := h.GetHeader().GetKey(), atunnel.TargetPortHeader; got != want { - t.Errorf("Expected header key %q, got %q", want, got) + header := headers[0] + if got, want := header.GetHeader().GetKey(), atunnel.TargetPortHeader; got != want { + t.Errorf("header key = %q, want %q", got, want) } - wantValue := "%DYNAMIC_METADATA(" + ingress.OriginalDstMetadataKey + ":" + ingress.OriginalDstPortKey + ")%" - if got := h.GetHeader().GetValue(); got != wantValue { - t.Errorf("Expected header value %q, got %q", wantValue, got) + if got, want := header.GetHeader().GetValue(), "%DYNAMIC_METADATA(envoy.filters.listener.original_dst:port)%"; got != want { + t.Errorf("header value = %q, want %q", got, want) } - if got, want := h.GetAppendAction(), corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD; got != want { - t.Errorf("Expected append action %s, got %s", want, got) + if got, want := header.GetAppendAction(), corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD; got != want { + t.Errorf("append action = %v, want %v", got, want) } } diff --git a/demos/autoscaled-workerpool/README.md b/demos/autoscaled-workerpool/README.md index 5028c46a8f..04793ce269 100644 --- a/demos/autoscaled-workerpool/README.md +++ b/demos/autoscaled-workerpool/README.md @@ -132,7 +132,7 @@ In a separate terminal, send requests in a retry loop across all hosts to activa ```sh for attempt in {1..10}; do for i in {001..015}; do - curl -s -H "Host: c$i.ate-demo-autoscaled-workerpool.actors.resources.substrate.ate.dev" http://localhost:8000 >/dev/null + curl -s -H "X-Ate-Actor-Name: c$i" -H "X-Ate-Atespace: ate-demo-autoscaled-workerpool" http://localhost:8000 >/dev/null done sleep 2 done diff --git a/demos/counter/README.md b/demos/counter/README.md index 1fd7643c70..ec331d38b6 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -80,7 +80,10 @@ When you send an HTTP request through the router, Substrate automatically detect 1. Send an HTTP POST request to increment the counter: ```bash -curl -X POST -H "Host: my-counter-1.ate-demo-counter.actors.resources.substrate.ate.dev" http://localhost:8000 +curl -X POST \ + -H "X-Ate-Actor-Name: my-counter-1" \ + -H "X-Ate-Atespace: ate-demo-counter" \ + http://localhost:8000 ``` 2. Verify that the actor is now in a `RUNNING` state and assigned to a worker pod: @@ -117,7 +120,10 @@ through it to the named port. proxy behavior wouldn't do: ```bash -curl -p -x http://localhost:8001 http://my-counter-1.ate-demo-counter.actors.resources.substrate.ate.dev:9090/ +curl -p -x http://localhost:8001 \ + -H "X-Ate-Actor-Name: my-counter-1" \ + -H "X-Ate-Atespace: ate-demo-counter" \ + http://my-counter-1:9090/ ``` This reaches the same actor's second listener and resumes it exactly like any diff --git a/demos/egress/README.md b/demos/egress/README.md index b2ece9a4b8..cd4171e2d8 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -27,7 +27,7 @@ intercepted and carried over mTLS to a gateway that verifies who is making the r │ (forwards the peer chain │ verify chain + ActorIdentity extension │ as x-forwarded-client-cert)│ GetActor → UID must match, must be RUNNING │ • dynamic_forward_proxy │ allow / deny 403 - │ │ + │ │ └───────────┼───────────────────────────────────────┘ ▼ real destination (the CONNECT authority, an IP:port) @@ -40,7 +40,7 @@ intercepted and carried over mTLS to a gateway that verifies who is making the r 3. **Guide 3 — HTTP-only actors, identity carried by the certificate.** The Actor only dials plain HTTP. atunnel presents the actor's own certificate — minted per actor by ateapi off the actor-identity CA, carrying an `ActorIdentity` X.509 extension — and sends a bare `CONNECT` - with no identity headers at all. + with no actor routing headers at all. 4. **Identity authentication.** Envoy requires a client certificate signed by the actor-identity CA, so a non-actor client is refused at the handshake. It then forwards the verified chain to `ext_proc` as `x-forwarded-client-cert`, and the **atenet router** (co-located in the gateway @@ -119,7 +119,8 @@ kubectl ate resume actor egress-demo -a ate-demo-egress # wait for ACTOR_STATE # 3. Drive the Actor's egress through the ingress gateway. kubectl -n ate-system port-forward service/atenet-router 8000:80 & curl -s -X POST http://localhost:8000/ \ - -H 'Host: egress-demo.ate-demo-egress.actors.resources.substrate.ate.dev' \ + -H 'X-Ate-Actor-Name: egress-demo' \ + -H 'X-Ate-Atespace: ate-demo-egress' \ -H 'Content-Type: application/json' \ -d "{\"url\":\"http://${TARGET_IP}:80/\"}" ``` diff --git a/demos/egress/test-egress.sh b/demos/egress/test-egress.sh index 71bcf2a292..5faff2ff4c 100755 --- a/demos/egress/test-egress.sh +++ b/demos/egress/test-egress.sh @@ -101,7 +101,8 @@ BEFORE=$(egress_log_count) ${K} -n ate-system port-forward service/atenet-router 18099:80 >/tmp/egress-pf.log 2>&1 & PF=$!; sleep 4 CODE=$(curl -s -o /tmp/egress-body.txt -w '%{http_code}' -X POST http://localhost:18099/ \ - -H "Host: ${ACTOR}.${ATESPACE}.actors.resources.substrate.ate.dev" \ + -H "X-Ate-Actor-Name: ${ACTOR}" \ + -H "X-Ate-Atespace: ${ATESPACE}" \ -H 'Content-Type: application/json' \ -d "{\"url\":\"http://${TARGET_IP}:80/\"}" || true) kill "${PF}" >/dev/null 2>&1 || true diff --git a/demos/jupyter/README.md b/demos/jupyter/README.md index 2f57755611..a50fa42f85 100644 --- a/demos/jupyter/README.md +++ b/demos/jupyter/README.md @@ -53,7 +53,7 @@ kubectl ate create actor jupyter-notebook -a ate-demo-jupyter --template-ref jup ### 2. Access Jupyter via the Proxy! -Substrate routes HTTP traffic using the `Host` header. To make this easy without modifying local `/etc/hosts` files, this demo includes a lightweight NGINX reverse proxy (`jupyter-proxy`) that automatically injects the proper `Host` header (`jupyter-notebook.ate-demo-jupyter.actors.resources.substrate.ate.dev`) and forwards traffic internally to the Substrate router. +Substrate routes HTTP traffic using `X-Ate-Actor-Name` and `X-Ate-Atespace`. This demo includes a lightweight NGINX reverse proxy (`jupyter-proxy`) that injects those routing headers and forwards traffic internally to the Substrate router. 1. **Port-forward the lightweight proxy to your local machine:** @@ -75,7 +75,7 @@ print("hello world") ### 4. Suspending and Resuming the Notebook -When you're not using the notebook, instead of leaving the container running, Substrate can checkpoint and suspend it to disk. +When you're not using the notebook, instead of leaving the container running, Substrate can checkpoint and suspend it to disk. ```bash kubectl ate suspend actor jupyter-notebook -a ate-demo-jupyter @@ -93,7 +93,7 @@ To **resume** the notebook, you can either explicitly resume it via CLI: kubectl ate resume actor jupyter-notebook -a ate-demo-jupyter ``` -Or, even easier, you can rely on "transparent resume" — just refresh the page in your browser or make another request to the URL while it's suspended. Substrate will automatically restore its state and serve your request without any downtime. +Or, even easier, you can rely on "transparent resume" — just refresh the page in your browser or make another request to the URL while it's suspended. Substrate will automatically restore its state and serve your request without any downtime. ### Clean up diff --git a/demos/jupyter/jupyter.yaml.tmpl b/demos/jupyter/jupyter.yaml.tmpl index 3651a9d8ab..203e87cb83 100644 --- a/demos/jupyter/jupyter.yaml.tmpl +++ b/demos/jupyter/jupyter.yaml.tmpl @@ -48,7 +48,8 @@ data: listen 80; location / { proxy_pass http://atenet-router.ate-system.svc.cluster.local; - proxy_set_header Host jupyter-notebook.ate-demo-jupyter.actors.resources.substrate.ate.dev; + proxy_set_header X-Ate-Actor-Name jupyter-notebook; + proxy_set_header X-Ate-Atespace ate-demo-jupyter; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; diff --git a/demos/multi-template/README.md b/demos/multi-template/README.md index 52e1820f60..bb3bf25aa3 100644 --- a/demos/multi-template/README.md +++ b/demos/multi-template/README.md @@ -65,11 +65,11 @@ When you send an HTTP request through the router, Substrate automatically detect ```bash # counter binary -curl -s -H "Host: c1.ate-demo-multi-template-counter.actors.resources.substrate.ate.dev" http://localhost:8000 +curl -s -H "X-Ate-Actor-Name: c1" -H "X-Ate-Atespace: ate-demo-multi-template-counter" http://localhost:8000 # -> hello from: | preserved memory count: 1 # fspersist binary -curl -s -H "Host: f1.ate-demo-multi-template-fspersist.actors.resources.substrate.ate.dev" http://localhost:8000 +curl -s -H "X-Ate-Actor-Name: f1" -H "X-Ate-Atespace: ate-demo-multi-template-fspersist" http://localhost:8000 # -> pod: # --- history --- # pod= | count=0 | time= @@ -87,7 +87,7 @@ preserves that state across the snapshot/restore cycle: ```bash kubectl ate suspend actor f1 -a ate-demo-multi-template-fspersist -curl -s -H "Host: f1.ate-demo-multi-template-fspersist.actors.resources.substrate.ate.dev" http://localhost:8000 # history persists; count keeps climbing +curl -s -H "X-Ate-Actor-Name: f1" -H "X-Ate-Atespace: ate-demo-multi-template-fspersist" http://localhost:8000 # history persists; count keeps climbing ``` ## How to Uninstall diff --git a/demos/parking/README.md b/demos/parking/README.md index 00bb10be43..b6033a9af2 100644 --- a/demos/parking/README.md +++ b/demos/parking/README.md @@ -40,8 +40,7 @@ This command will: ### 2. Create more actors than workers -Actors live in the demo's **atespace** (`ate-demo-parking`), and their DNS names -embed it (`..actors.resources.substrate.ate.dev`). `--template-ref` +Actors live in the demo's **atespace** (`ate-demo-parking`). `--template-ref` names the template, resolved in the actor's atespace: ```bash @@ -70,8 +69,8 @@ Parking is **on by default** (`--parked-request-budget=5s`, Fill both workers by requesting two actors, leaving them `RUNNING`: ```bash -curl -s -H "Host: p1.ate-demo-parking.actors.resources.substrate.ate.dev" http://localhost:8000 -curl -s -H "Host: p2.ate-demo-parking.actors.resources.substrate.ate.dev" http://localhost:8000 +curl -s -H "X-Ate-Actor-Name: p1" -H "X-Ate-Atespace: ate-demo-parking" http://localhost:8000 +curl -s -H "X-Ate-Actor-Name: p2" -H "X-Ate-Atespace: ate-demo-parking" http://localhost:8000 kubectl ate get workers # both workers are now bound to p1 and p2 kubectl ate get actors # p1,p2 RUNNING; p3,p4 SUSPENDED @@ -82,7 +81,7 @@ the `curl` hangs while the router retries the resume: ```bash curl -s -w '\n-> HTTP %{http_code} in %{time_total}s\n' \ - -H "Host: p3.ate-demo-parking.actors.resources.substrate.ate.dev" http://localhost:8000 + -H "X-Ate-Actor-Name: p3" -H "X-Ate-Atespace: ate-demo-parking" http://localhost:8000 ``` While that is hanging, in a **second terminal** free a worker by suspending p1 diff --git a/demos/parking/load.sh b/demos/parking/load.sh index b76a79958c..2b79be3eda 100755 --- a/demos/parking/load.sh +++ b/demos/parking/load.sh @@ -46,7 +46,6 @@ ROUTER="http://localhost:8000" # The template name, resolved in the actors' atespace (--template-ref). TEMPLATE="parking" ATESPACE="ate-demo-parking" -SUFFIX="actors.resources.substrate.ate.dev" usage() { cat <<'EOF' @@ -102,12 +101,13 @@ done # One worker per actor: hammer it with request->suspend until the deadline. worker() { - local actor="$1" host="$1.${ATESPACE}.${SUFFIX}" log="${TMP}/$1.log" + local actor="$1" log="${TMP}/$1.log" local deadline=$(( $(date +%s) + DURATION )) while [[ $(date +%s) -lt ${deadline} ]]; do # %{http_code} lets us tally outcomes; %{time_total} reveals parking waits. curl -s -o /dev/null -w '%{http_code} %{time_total}\n' \ - -H "Host: ${host}" "${ROUTER}" >>"${log}" 2>/dev/null + -H "X-Ate-Actor-Name: ${actor}" \ + -H "X-Ate-Atespace: ${ATESPACE}" "${ROUTER}" >>"${log}" 2>/dev/null # Free the worker so a parked competitor can proceed (simulate going idle). kubectl ate suspend actor "${actor}" --atespace "${ATESPACE}" >/dev/null 2>&1 || true done diff --git a/demos/sandbox/client/main.go b/demos/sandbox/client/main.go index 11269e0246..7a2e3d0b90 100644 --- a/demos/sandbox/client/main.go +++ b/demos/sandbox/client/main.go @@ -28,6 +28,7 @@ import ( "strings" "syscall" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/pflag" @@ -185,7 +186,8 @@ func runCommand(ctx context.Context, atenetAddr string, actorRef resources.Actor return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Content-Type", "application/json") - req.Host = resources.ActorDNSName(actorRef) + req.Header.Set(atenet.ActorNameHeader, actorRef.Name) + req.Header.Set(atenet.AtespaceHeader, actorRef.Atespace) resp, err := http.DefaultClient.Do(req) if err != nil { diff --git a/docs/api-guide.md b/docs/api-guide.md index 5a5302ca8c..72c40562c8 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -142,10 +142,11 @@ Unlike a Pod, an actor is sized by its **`limits`** (CPU and Memory): the size i Container environment variables support literal `value` entries only. Values are not interpolated (`$(VAR)` references are not expanded), and Kubernetes `envFrom`/`valueFrom` sources are not supported. -### Workload Connectivity (Uniform DNS) -Substrate uses a **Uniform DNS Mesh**: every actor created from a template is automatically reachable through the **Substrate Router** via its atespace and name: +### Workload Connectivity -**Format:** `..actors.resources.substrate.ate.dev` +A higher-order system reaches an actor through the **Substrate Router** by +setting `X-Ate-Actor-Name` to the actor name and `X-Ate-Atespace` to its +atespace. Substrate does not provide DNS discovery for actors. ### SystemInfo Volumes @@ -154,7 +155,7 @@ To deliver identity information, including credentials, to a running actor, you Available information sources: #### actorMetadata -The actorMetadata data source projects the actor's identity fields to files, one per item, analogous to the [Kubernetes downwardAPI volume](https://kubernetes.io/docs/concepts/storage/downward-api/). Each item selects a `field` — `name` (unique within an atespace), `atespace` (together with the name, the actor's full identity and DNS name), or `uid` (server-generated, distinguishes incarnations of the same name) — and the relative `path` the value is written to, raw with no trailing newline. +The actorMetadata data source projects the actor's identity fields to files, one per item, analogous to the [Kubernetes downwardAPI volume](https://kubernetes.io/docs/concepts/storage/downward-api/). Each item selects a `field` — `name` (unique within an atespace), `atespace` (together with the name, the actor's full identity), or `uid` (server-generated, distinguishes incarnations of the same name) — and the relative `path` the value is written to, raw with no trailing newline. ```yaml spec: diff --git a/docs/architecture.md b/docs/architecture.md index a6dcbbde73..0d52209602 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -330,16 +330,16 @@ A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging, with container rootfs writes captured in guest RAM via a `tmpfs` overlay. `DurableDir` volumes are host-backed instead, served over a second (writable) virtio-fs share and shipped in snapshots as a tar, so a `Data`-scope snapshot can capture them without any guest memory. Each volume is a subdirectory of that one share, so an actor can have several at no extra cost in devices — which is why the micro-VM class lifts the single-`DurableDir` limit that still applies to gVisor. -### Networking Stack (`atenet` DNS + `atunnel`) +### Networking Stack (`atenet` + `atunnel`) Handles actor-aware routing and automatic re-animation. - * **Uniform DNS Mesh**: Substrate provides a location-transparent actor discovery scheme via a global DNS suffix (`..actors.resources.substrate.ate.dev`). - * **Ingress Routing**: `atenet-router` runs Envoy with an `ext_proc` external - processor and accepts HTTP traffic for the Actor DNS suffix. The ext_proc - extracts the Actor name and Atespace from the `Host` header and calls the - Control Plane to resume the Actor and resolve its current worker assignment. + processor. A higher-order system connects to the router and supplies the + Actor name in `X-Ate-Actor-Name` and the Atespace in `X-Ate-Atespace`. + The ext_proc calls the Control Plane to resume the Actor and resolve its + current worker assignment. `Host` remains application authority and does + not select the Actor. * **Worker Tunnel**: After resolving the assignment, `atenet-router` opens an authenticated TLS tunnel to the worker's `atunnel` listener on port 443. @@ -368,7 +368,6 @@ suspended (UML sequence diagram): ```mermaid sequenceDiagram actor Client - participant DNS as atenet DNS participant Gateway as atenet-router participant API as ate-api-server participant Atelet as atelet @@ -376,9 +375,7 @@ sequenceDiagram participant A as Actor participant Store as snapshot storage - Client->>DNS: resolve actor DNS name - DNS-->>Client: ingress gateway address - Client->>Gateway: HTTP request (Host = actor) + Client->>Gateway: HTTP request (X-Ate-Actor-Name, X-Ate-Atespace) Gateway->>API: ResumeActor(atespace, actor name) API->>Atelet: Restore Store-->>Atelet: download snapshot @@ -500,10 +497,9 @@ Agent Substrate is built on a **Defense-in-Depth** model: versions. * **Request Authorization**: The system currently performs **Identity-Aware - Routing** by utilizing a uniform DNS routing scheme - (`..actors.resources.substrate.ate.dev`) - at the gateway to extract and validate actor identifiers from incoming traffic. This - ensures requests are only routed to recognized, registered actors. + Routing** by extracting and validating the `X-Ate-Actor-Name` and + `X-Ate-Atespace` headers at the gateway. This ensures requests are only + routed to recognized, registered actors. Pluggable, granular authorization policies are planned for future milestones. diff --git a/docs/glossary.md b/docs/glossary.md index 202f4f2ab7..6c7010c2ff 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -62,9 +62,8 @@ because they change too frequently for etcd. sandbox runtime on behalf of atelet. This decouples the physical pod lifecycle from the sandboxed agent process. -- **atenet**: the networking stack. It provides a DNS server for actor - resolution and a router that resumes suspended Actors on demand and routes - traffic to the right worker pod. +- **atenet**: the networking stack. Its router resumes suspended Actors on + demand and routes traffic to the right worker pod. - **podcertcontroller**: issues short-lived pod certificates that components use as their TLS identity to authenticate connections to one another @@ -154,6 +153,6 @@ because they change too frequently for etcd. ## Networking -- **Uniform DNS Mesh**: every Actor is reachable at a uniform address, - `..actors.resources.substrate.ate.dev`, resolved by atenet. Traffic to - that name is routed (and the Actor resumed if needed) automatically. +- **Actor routing headers**: a higher-order system sends traffic to the + Substrate router with `X-Ate-Actor-Name` and `X-Ate-Atespace`. The router + uses these headers to locate and resume the Actor. diff --git a/docs/metrics/substrate.yaml b/docs/metrics/substrate.yaml index cde6df7e5c..786643b1ce 100644 --- a/docs/metrics/substrate.yaml +++ b/docs/metrics/substrate.yaml @@ -221,11 +221,6 @@ blind_spots: The hit and miss counter exists. But the garbage collector (the 85% and 80% limits) and the time to prepare an image send nothing. Thus you cannot see the disk pressure from the layer pool. - - area: atenet-dns - code: cmd/atenet/internal/dns.go - brief: > - There are no instruments, and the Corefile has no prometheus plugin. If a - reload fails, the answers stay old and no signal shows it. - area: podcertcontroller code: cmd/podcertcontroller brief: This component has no metrics. diff --git a/docs/roadmap.md b/docs/roadmap.md index a2730f2339..b042209019 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -44,7 +44,7 @@ Below is a collection of finer-grained efforts which we believe align with the a * Actor security boundary implementation, default deny with explicit ACLs at scale with low latency. This overlaps with some of the security items (see below). * Policy definition: between framework (outside) and Actors, between Actors, Actor Egress. -* Standardized DNS Mesh: Moving to a production-grade routing format (\.actors.resources.substrate.ate.dev) for location-transparent actor-to-actor communication. +* Actor-to-actor routing through explicit identity headers. ### Storage diff --git a/docs/threat-model.md b/docs/threat-model.md index 41fc2bc2f7..80187692d5 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -39,8 +39,8 @@ Substrate is an early, fast moving product. It is full of debate and subject to * **Worker:** Preprovisioned Pods that actors get scheduled to. * **Actor:** The core compute primitive, gets scheduled to/from worker via Run for cold start and Resume for snapshot resume. * **Actor Network:** `ateom` creates a private point-to-point veth network for the active Actor inside a worker Pod. The Actor is not served directly from the worker Pod's port 80; ingress enters through `atunnel`'s authenticated listener on port 443. -* **Actor DNS:** Each Actor gets a DNS name like `..actors.resources.substrate.ate.dev`. Substrate runs a custom CoreDNS instance that returns the `atenet-router` Service IP for A record queries matching the Actor DNS pattern. A controller keeps this target current and configures kube-dns with a stub domain for `actors.resources.substrate.ate.dev`, enabling traditional Kubernetes Pods to resolve Actor names. -* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. The ext_proc extracts the Actor name and Atespace from the HTTP `Host` header, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. The router then connects with mTLS to `atunnel` on worker port 443; `atunnel` validates the router identity and forwards traffic only to the Actor currently assigned to that worker. +* **Actor Routing:** Requests sent to `atenet-router` identify the target Actor with `X-Ate-Actor-Name` and `X-Ate-Atespace`. Host and authority values are application metadata and do not select the Actor. +* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. The ext_proc reads the Actor name and Atespace from `X-Ate-Actor-Name` and `X-Ate-Atespace`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites both routing headers before forwarding. The router then connects with mTLS to `atunnel`; `atunnel` validates the router identity and authorizes the header pair against the Actor currently assigned to that worker. * **Object Storage:** Used to store actor snapshots. * **Filesystem support:** Container local filesystem is saved in snapshots, future integrations likely to include networked storage. * **Substrate Database:** PostgreSQL. diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 2e9fd0c03c..fb78dd8d3c 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -818,10 +818,8 @@ deploy_atenet() { ensure_egress_mitm_ca_pool_secret apply_atenet_egress - run_ko apply -f manifests/ate-install/atenet-dns.yaml run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/dns -n ate-system --timeout="$(rollout_timeout)" } # get_actor_state echoes the actor's state enum (e.g. ACTOR_STATE_SUSPENDED). @@ -1089,7 +1087,6 @@ delete_atenet() { run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-egress.yaml run_kubectl delete --ignore-not-found \ -f manifests/ate-install/atenet-egress-with-sdsmint.yaml - run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-dns.yaml } deploy_benchmarks() { diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 19cd97daa1..7e4c380053 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -120,7 +120,8 @@ cat < "${CURL_OUT}" 2>&1 ) & CURL_PID=$! sleep 1 # inside the 5s park budget; the request is parked on the router diff --git a/hack/verify-egress-demo.sh b/hack/verify-egress-demo.sh index 51c622ac72..01405dc282 100755 --- a/hack/verify-egress-demo.sh +++ b/hack/verify-egress-demo.sh @@ -54,7 +54,8 @@ ${K} -n ate-system port-forward service/atenet-router 18000:80 >/tmp/pf.log 2>&1 PF=$!; trap 'kill ${PF} 2>/dev/null || true' EXIT sleep 3 RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://localhost:18000/ \ - -H "Host: ${ACTOR}.${ATESPACE}.actors.resources.substrate.ate.dev" \ + -H "X-Ate-Actor-Name: ${ACTOR}" \ + -H "X-Ate-Atespace: ${ATESPACE}" \ -H 'Content-Type: application/json' \ -d "{\"url\":\"${TARGET_URL}\"}") || true echo "actor round-trip HTTP ${RESP} (200 = the actor fetched ${TARGET_URL} through egress)" diff --git a/internal/atenet/headers.go b/internal/atenet/headers.go new file mode 100644 index 0000000000..2e067719bb --- /dev/null +++ b/internal/atenet/headers.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package atenet defines the shared contract for Substrate actor networking. +package atenet + +const ( + // ActorNameHeader and AtespaceHeader identify the actor selected for ingress + // routing. HTTP field names are case-insensitive; these use their HTTP/2 wire + // form so dataplane configuration and header mutations are native. + ActorNameHeader = "x-ate-actor-name" + AtespaceHeader = "x-ate-atespace" +) diff --git a/internal/atunnel/client_test.go b/internal/atunnel/client_test.go index a7ddb1724f..d24b3070ef 100644 --- a/internal/atunnel/client_test.go +++ b/internal/atunnel/client_test.go @@ -63,7 +63,7 @@ func TestClientDialContext(t *testing.T) { } defer conn.Close() - gotRequest := <-request + gotRequest := receiveWithin(t, request, "CONNECT request") if gotRequest.Method != http.MethodConnect { t.Errorf("method = %q, want CONNECT", gotRequest.Method) } @@ -302,6 +302,10 @@ func serveTestConnectGateway(t *testing.T, ca *testCA, handle func(net.Conn, *ht return } defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Errorf("setting gateway connection deadline: %v", err) + return + } req, err := http.ReadRequest(bufio.NewReader(conn)) if err != nil { t.Errorf("reading CONNECT request: %v", err) diff --git a/internal/atunnel/credential_test.go b/internal/atunnel/credential_test.go index 057b6fe9e8..abef065d6c 100644 --- a/internal/atunnel/credential_test.go +++ b/internal/atunnel/credential_test.go @@ -47,7 +47,8 @@ func TestBrokerCertificateSourceMintsAndReusesKey(t *testing.T) { t.Fatal(err) } } - first, second := <-broker.publicKeys, <-broker.publicKeys + first := receiveWithin(t, broker.publicKeys, "first certificate public key") + second := receiveWithin(t, broker.publicKeys, "second certificate public key") if string(first) != string(second) { t.Fatal("renewal replaced the actor private key") } diff --git a/internal/atunnel/egress_test.go b/internal/atunnel/egress_test.go index 18440ec5f2..d3b49e4637 100644 --- a/internal/atunnel/egress_test.go +++ b/internal/atunnel/egress_test.go @@ -224,9 +224,9 @@ func TestEgressDeactivationDropsConcurrentRenewal(t *testing.T) { } done := make(chan error, 1) go func() { done <- egress.Deactivate(context.Background()) }() - <-active.ctx.Done() + receiveWithin(t, active.ctx.Done(), "egress cancellation") close(release) - if err := <-done; err != nil { + if err := receiveWithin(t, done, "egress deactivation"); err != nil { t.Fatal(err) } if !active.expiresAt.IsZero() { @@ -287,7 +287,7 @@ func TestEgressEndToEnd(t *testing.T) { }) egress.handle(downstreamProxy) - req := <-requests + req := receiveWithin(t, requests, "gateway CONNECT request") if req.Method != http.MethodConnect || req.Host != "192.0.2.10:443" { t.Errorf("request = %s %s, want CONNECT 192.0.2.10:443", req.Method, req.Host) } @@ -313,7 +313,7 @@ func TestEgressEndToEnd(t *testing.T) { if string(gotAtActor) != "from gateway" { t.Errorf("actor payload = %q, want %q", gotAtActor, "from gateway") } - <-gatewayDone + receiveWithin(t, gatewayDone, "gateway completion") if err := egress.Deactivate(context.Background()); err != nil { t.Fatal(err) diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress.go index c083895f65..02109256f0 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress.go @@ -33,6 +33,7 @@ import ( "sync" "time" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/resources" ) @@ -44,12 +45,6 @@ const ( // StaleAssignmentHeader distinguishes an atunnel routing rejection from a // 421 returned by the actor application itself. StaleAssignmentHeader = "X-Ate-Assignment-Stale" - // OriginalHostHeader carries the actor authority across router dataplanes - // that must use :authority to select the worker as their dynamic backend. - // atunnel only accepts mTLS-authenticated router clients, and the router's - // ext_proc server overwrites this header before every request. - OriginalHostHeader = "X-Ate-Original-Host" - // TargetPortHeader carries the port to reach on the actor: the CONNECT // :authority's port for arbitrary-port ingress, or the default 80 // otherwise (see atenet-router's HandleRequestHeaders). cfg.Upstream is @@ -105,7 +100,7 @@ func NewServer(cfg Config) (*Server, error) { return nil, fmt.Errorf("atunnel: trust bundle path is required") } if cfg.AllowedClientID == "" { - return nil, fmt.Errorf("atunnel: allowed client identity is required") + return nil, fmt.Errorf("atunnel: allowed client ID is required") } if cfg.Upstream == nil || cfg.Upstream.Scheme == "" || cfg.Upstream.Host == "" { return nil, fmt.Errorf("atunnel: upstream URL is required") @@ -130,13 +125,14 @@ func NewServer(cfg Config) (*Server, error) { proxy := &httputil.ReverseProxy{ Rewrite: func(pr *httputil.ProxyRequest) { pr.SetURL(cfg.Upstream) - // Retain the actor's stable mesh hostname rather than the - // upstream's, matching NewSingleHostReverseProxy's default - // behavior. + // Retain the client's Host rather than the upstream's, matching + // NewSingleHostReverseProxy's default behavior. pr.Out.Host = pr.In.Host port := pr.In.Header.Get(TargetPortHeader) pr.Out.Header.Del(TargetPortHeader) + pr.Out.Header.Del(atenet.ActorNameHeader) + pr.Out.Header.Del(atenet.AtespaceHeader) if p, ok := ParsePort(port); ok { pr.Out.URL.Host = net.JoinHostPort(cfg.Upstream.Hostname(), strconv.Itoa(p)) } @@ -414,7 +410,7 @@ func (w flushingWriter) Write(p []byte) (int, error) { // active actor per worker. func (s *Server) Activate(atespace, actorName string) error { if !resources.IsValidResourceName(atespace) || !resources.IsValidResourceName(actorName) { - return fmt.Errorf("atunnel: invalid actor identity %q/%q", atespace, actorName) + return fmt.Errorf("atunnel: invalid actor reference %q/%q", atespace, actorName) } s.mu.Lock() defer s.mu.Unlock() @@ -465,7 +461,7 @@ func (s *Server) closeIdleUpstreamConnections() { } } -// ServeHTTP validates the actor hostname on every request before proxying it. +// ServeHTTP validates the actor routing headers on every request before proxying it. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, requestCtx, release, ok := s.authorize(r) if !ok { @@ -474,32 +470,16 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } defer release() - // Do not expose the router-only routing header to actor code. Restore Host - // so dataplanes that route dynamically on worker IP still give the actor its - // stable actor DNS name. - actorHost := r.Header.Get(OriginalHostHeader) - if actorHost == "" { - actorHost = r.Host - } - r.Header.Del(OriginalHostHeader) - r.Host = actorHost - - // ReverseProxy changes the URL destination but intentionally retains Host, - // allowing the actor application to observe its stable actor DNS name. + // ReverseProxy changes the URL destination but intentionally retains Host. s.proxy.ServeHTTP(w, r.WithContext(requestCtx)) } func (s *Server) authorize(r *http.Request) (resources.ActorRef, context.Context, func(), bool) { - actorHost := r.Header.Get(OriginalHostHeader) - if actorHost == "" { - actorHost = r.Host - } - host, err := requestHostname(actorHost) - if err != nil { - return resources.ActorRef{}, nil, nil, false + ref := resources.ActorRef{ + Name: r.Header.Get(atenet.ActorNameHeader), + Atespace: r.Header.Get(atenet.AtespaceHeader), } - ref, err := resources.ParseActorDNSName(host) - if err != nil { + if !resources.IsValidResourceName(ref.Name) || !resources.IsValidResourceName(ref.Atespace) { return resources.ActorRef{}, nil, nil, false } @@ -525,22 +505,3 @@ func (s *Server) reject(w http.ResponseWriter) { w.Header().Set(StaleAssignmentHeader, "true") http.Error(w, "misdirected request", http.StatusMisdirectedRequest) } - -func requestHostname(hostport string) (string, error) { - if hostport == "" { - return "", fmt.Errorf("empty host") - } - host := hostport - if strings.Contains(hostport, ":") { - var port string - var err error - host, port, err = net.SplitHostPort(hostport) - if err != nil { - return "", fmt.Errorf("invalid host %q: %w", hostport, err) - } - if _, ok := ParsePort(port); !ok { - return "", fmt.Errorf("invalid port in host %q", hostport) - } - } - return strings.ToLower(host), nil -} diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index a93fd7cc6d..1e22f1149d 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -34,6 +34,8 @@ import ( "path/filepath" "testing" "time" + + "github.com/agent-substrate/substrate/internal/atenet" ) func TestRelayIngressWithHalfClose(t *testing.T) { @@ -55,7 +57,7 @@ func TestRelayIngressWithHalfClose(t *testing.T) { t.Fatal(err) } defer actor.Close() - upstream := <-accepted + upstream := receiveWithin(t, accepted, "accepted connection") defer upstream.Close() clientReader, clientInput := io.Pipe() @@ -129,7 +131,7 @@ func TestRelayIngressCancellationClosesBothSides(t *testing.T) { } func TestServeHTTP(t *testing.T) { - upstreamHost := make(chan string, 4) + var upstreamHosts []string upstreamURL, err := url.Parse("http://actor.internal:80") if err != nil { t.Fatal(err) @@ -137,7 +139,7 @@ func TestServeHTTP(t *testing.T) { s := newTestServer(t, upstreamURL) s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { - upstreamHost <- r.Host + upstreamHosts = append(upstreamHosts, r.Host) return &http.Response{ StatusCode: http.StatusNoContent, Header: make(http.Header), @@ -149,27 +151,33 @@ func TestServeHTTP(t *testing.T) { } tests := []struct { - name string - host string - originalHost string - wantStatus int + name string + host string + actorName string + atespace string + mixedCase bool + wantStatus int }{ - {name: "active actor", host: "actor-1.team-a.actors.resources.substrate.ate.dev", wantStatus: http.StatusNoContent}, - {name: "active actor with port", host: "actor-1.team-a.actors.resources.substrate.ate.dev:443", wantStatus: http.StatusNoContent}, - {name: "DNS case insensitive", host: "ACTOR-1.TEAM-A.ACTORS.RESOURCES.SUBSTRATE.ATE.DEV", wantStatus: http.StatusNoContent}, - {name: "router original host", host: "10.0.0.52:443", originalHost: "actor-1.team-a.actors.resources.substrate.ate.dev", wantStatus: http.StatusNoContent}, - {name: "wrong actor", host: "actor-2.team-a.actors.resources.substrate.ate.dev", wantStatus: http.StatusMisdirectedRequest}, - {name: "wrong atespace", host: "actor-1.team-b.actors.resources.substrate.ate.dev", wantStatus: http.StatusMisdirectedRequest}, - {name: "suffix confusion", host: "actor-1.team-a.actors.resources.substrate.ate.dev.example.com", wantStatus: http.StatusMisdirectedRequest}, - {name: "malformed port", host: "actor-1.team-a.actors.resources.substrate.ate.dev:nope", wantStatus: http.StatusMisdirectedRequest}, - {name: "empty", wantStatus: http.StatusMisdirectedRequest}, + {name: "active actor", host: "client.example", actorName: "actor-1", atespace: "team-a", wantStatus: http.StatusNoContent}, + {name: "mixed-case routing headers", actorName: "actor-1", atespace: "team-a", mixedCase: true, wantStatus: http.StatusNoContent}, + {name: "host does not identify actor", host: "actor-2.team-b.example", actorName: "actor-1", atespace: "team-a", wantStatus: http.StatusNoContent}, + {name: "empty host", actorName: "actor-1", atespace: "team-a", wantStatus: http.StatusNoContent}, + {name: "wrong actor", actorName: "actor-2", atespace: "team-a", wantStatus: http.StatusMisdirectedRequest}, + {name: "wrong atespace", actorName: "actor-1", atespace: "team-b", wantStatus: http.StatusMisdirectedRequest}, + {name: "missing actor name", atespace: "team-a", wantStatus: http.StatusMisdirectedRequest}, + {name: "missing atespace", actorName: "actor-1", wantStatus: http.StatusMisdirectedRequest}, + {name: "invalid actor name", actorName: "INVALID", atespace: "team-a", wantStatus: http.StatusMisdirectedRequest}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "https://worker/hello", nil) req.Host = tt.host - if tt.originalHost != "" { - req.Header.Set(OriginalHostHeader, tt.originalHost) + if tt.mixedCase { + req.Header.Set("X-ATE-Actor-Name", tt.actorName) + req.Header.Set("x-ate-ATESPACE", tt.atespace) + } else { + req.Header.Set(atenet.ActorNameHeader, tt.actorName) + req.Header.Set(atenet.AtespaceHeader, tt.atespace) } rec := httptest.NewRecorder() s.ServeHTTP(rec, req) @@ -182,9 +190,13 @@ func TestServeHTTP(t *testing.T) { }) } - for range 4 { - if got := <-upstreamHost; got != "actor-1.team-a.actors.resources.substrate.ate.dev" && got != "actor-1.team-a.actors.resources.substrate.ate.dev:443" && got != "ACTOR-1.TEAM-A.ACTORS.RESOURCES.SUBSTRATE.ATE.DEV" { - t.Errorf("upstream Host = %q", got) + wantHosts := []string{"client.example", "", "actor-2.team-b.example", ""} + if len(upstreamHosts) != len(wantHosts) { + t.Fatalf("upstream requests = %d, want %d", len(upstreamHosts), len(wantHosts)) + } + for i, want := range wantHosts { + if got := upstreamHosts[i]; got != want { + t.Errorf("upstream Host = %q, want %q", got, want) } } } @@ -196,11 +208,11 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { } s := newTestServer(t, upstreamURL) - var gotURLHost, gotHost, gotHeader string + var gotURLHost, gotHost http.Header s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { - gotURLHost = r.URL.Host - gotHost = r.Host - gotHeader = r.Header.Get(TargetPortHeader) + gotURLHost = http.Header{"Host": []string{r.URL.Host}} + gotHost = r.Header.Clone() + gotHost.Set("Host", r.Host) return &http.Response{ StatusCode: http.StatusNoContent, Header: make(http.Header), @@ -224,7 +236,9 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "https://worker/hello", nil) - req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Host = "client.example" + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") if tt.targetPort != "" { req.Header.Set(TargetPortHeader, tt.targetPort) } @@ -233,14 +247,16 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { if rec.Code != http.StatusNoContent { t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent) } - if gotURLHost != tt.wantDialedHost { - t.Errorf("dialed host = %q, want %q", gotURLHost, tt.wantDialedHost) + if gotURLHost.Get("Host") != tt.wantDialedHost { + t.Errorf("dialed host = %q, want %q", gotURLHost.Get("Host"), tt.wantDialedHost) } - if gotHost != "actor-1.team-a.actors.resources.substrate.ate.dev" { - t.Errorf("Host header changed to %q; the actor should see its stable mesh hostname", gotHost) + if gotHost.Get("Host") != "client.example" { + t.Errorf("Host header changed to %q", gotHost.Get("Host")) } - if gotHeader != "" { - t.Errorf("%s leaked to the actor upstream: %q", TargetPortHeader, gotHeader) + for _, header := range []string{TargetPortHeader, atenet.ActorNameHeader, atenet.AtespaceHeader} { + if got := gotHost.Get(header); got != "" { + t.Errorf("%s leaked to the actor upstream: %q", header, got) + } } }) } @@ -264,13 +280,18 @@ func TestServeConnectHTTPValidatesMethodAndAuthority(t *testing.T) { }{ {name: "rejects non CONNECT", method: http.MethodGet, host: "actor-1.team-a.actors.resources.substrate.ate.dev:9090", want: http.StatusMethodNotAllowed}, {name: "requires authority port", method: http.MethodConnect, host: "actor-1.team-a.actors.resources.substrate.ate.dev", want: http.StatusBadRequest}, - {name: "rejects invalid authority port", method: http.MethodConnect, host: "actor-1.team-a.actors.resources.substrate.ate.dev:70000", want: http.StatusMisdirectedRequest}, + {name: "rejects invalid authority port", method: http.MethodConnect, host: "actor-1.team-a.actors.resources.substrate.ate.dev:70000", want: http.StatusBadRequest}, {name: "rejects inactive actor", method: http.MethodConnect, host: "actor-2.team-a.actors.resources.substrate.ate.dev:9090", want: http.StatusMisdirectedRequest}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(tt.method, "https://worker/", nil) req.Host = tt.host + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") + if tt.name == "rejects inactive actor" { + req.Header.Set(atenet.ActorNameHeader, "actor-2") + } rec := httptest.NewRecorder() s.ServeConnectHTTP(rec, req) if rec.Code != tt.want { @@ -316,6 +337,8 @@ func TestDeactivateClosesIdleUpstreamConnections(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "https://worker/", nil) req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") rec := httptest.NewRecorder() s.ServeHTTP(rec, req) if rec.Code != http.StatusNoContent { @@ -337,6 +360,8 @@ func TestInactive(t *testing.T) { s := newTestServer(t, upstream) req := httptest.NewRequest(http.MethodGet, "https://worker/", nil) req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") for _, phase := range []string{"before activation", "after deactivation"} { t.Run(phase, func(t *testing.T) { @@ -357,7 +382,7 @@ func TestInactive(t *testing.T) { } } -func TestMutualTLSClientIdentity(t *testing.T) { +func TestMutualTLSClientAuthentication(t *testing.T) { dir := t.TempDir() ca := newTestCA(t) serverCert := ca.issue(t, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}) @@ -391,16 +416,16 @@ func TestMutualTLSClientIdentity(t *testing.T) { wantErr bool }{ { - name: "allowed identity", + name: "allowed client", cert: ca.issue(t, "spiffe://cluster.local/ns/ate-system/sa/atenet-router", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}), }, { - name: "wrong identity", + name: "wrong client ID", cert: ca.issue(t, "spiffe://cluster.local/ns/ate-system/sa/not-the-gateway", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}), wantErr: true, }, { - name: "untrusted identity", + name: "untrusted client", cert: untrustedCA.issue(t, "spiffe://cluster.local/ns/ate-system/sa/atenet-router", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}), wantErr: true, }, @@ -461,13 +486,16 @@ func TestServeNegotiatesH2(t *testing.T) { if err != nil { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) served := make(chan error, 1) - go func() { served <- s.Serve(ctx, lis) }() + go func() { served <- s.Serve(t.Context(), lis) }() t.Cleanup(func() { - cancel() - if err := <-served; err != nil { - t.Errorf("Serve: %v", err) + select { + case err := <-served: + if err != nil { + t.Errorf("Serve: %v", err) + } + case <-time.After(time.Second): + t.Error("Serve did not stop after cancellation") } }) @@ -475,7 +503,7 @@ func TestServeNegotiatesH2(t *testing.T) { transport := &http.Transport{ TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, - InsecureSkipVerify: true, // The handshake identity checks live in TestMutualTLSClientIdentity. + InsecureSkipVerify: true, // The handshake checks live in TestMutualTLSClientAuthentication. Certificates: []tls.Certificate{clientCert}, }, // With h2, the transport offers "h2" via ALPN like Envoy's @@ -483,6 +511,7 @@ func TestServeNegotiatesH2(t *testing.T) { // HTTP/1.1 pool. ForceAttemptHTTP2: h2, } + t.Cleanup(transport.CloseIdleConnections) return &http.Client{Transport: transport, Timeout: 10 * time.Second} } request := func(t *testing.T, c *http.Client, method, contentType string) *http.Response { @@ -491,7 +520,9 @@ func TestServeNegotiatesH2(t *testing.T) { if err != nil { t.Fatal(err) } - req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Host = "team-a-agent.example.com" // Use a custom Host header to prove that we no longer rely on Host to route to the actor + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") if contentType != "" { req.Header.Set("Content-Type", contentType) } @@ -508,13 +539,13 @@ func TestServeNegotiatesH2(t *testing.T) { if res.Proto != "HTTP/2.0" { t.Fatalf("h2-only client negotiated %s, want HTTP/2.0 — Envoy's mirrored HTTP/2 pool cannot connect", res.Proto) } - if got := <-protoSeen; got != "HTTP/2.0" { + if got := receiveWithin(t, protoSeen, "gRPC backend protocol"); got != "HTTP/2.0" { t.Errorf("gRPC-shaped request reached the actor as %s, want HTTP/2.0", got) } // A non-gRPC request on the same negotiated h2 connection is downgraded // before the actor. request(t, h2Client, http.MethodGet, "") - if got := <-protoSeen; got != "HTTP/1.1" { + if got := receiveWithin(t, protoSeen, "plain HTTP/2 backend protocol"); got != "HTTP/1.1" { t.Errorf("plain GET over h2 reached the actor as %s, want HTTP/1.1", got) } @@ -524,7 +555,7 @@ func TestServeNegotiatesH2(t *testing.T) { if res.Proto != "HTTP/1.1" { t.Errorf("http/1.1-only client negotiated %s, want HTTP/1.1", res.Proto) } - if got := <-protoSeen; got != "HTTP/1.1" { + if got := receiveWithin(t, protoSeen, "HTTP/1.1 backend protocol"); got != "HTTP/1.1" { t.Errorf("HTTP/1.1 request reached the actor as %s, want HTTP/1.1", got) } } @@ -550,13 +581,15 @@ func TestDeactivateCancelsInflightRequest(t *testing.T) { defer close(done) req := httptest.NewRequest(http.MethodGet, "https://worker/", nil) req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") s.ServeHTTP(httptest.NewRecorder(), req) }() - <-started + receiveWithin(t, started, "in-flight request") if err := s.Deactivate(context.Background()); err != nil { t.Fatal(err) } - <-done + receiveWithin(t, done, "canceled in-flight request") } func newTestServer(t *testing.T, upstream *url.URL) *Server { @@ -711,6 +744,9 @@ func writeCredentialBundle(t *testing.T, path string, cert tls.Certificate) { func tlsHandshake(serverConfig, clientConfig *tls.Config) (serverErr, clientErr error) { serverConn, clientConn := net.Pipe() + deadline := time.Now().Add(5 * time.Second) + _ = serverConn.SetDeadline(deadline) + _ = clientConn.SetDeadline(deadline) serverTLS := tls.Server(serverConn, serverConfig) clientTLS := tls.Client(clientConn, clientConfig) done := make(chan error, 1) @@ -840,9 +876,21 @@ func TestProtocolMirrorTransport(t *testing.T) { t.Fatalf("RoundTrip: %v", err) } res.Body.Close() - if got := <-protoSeen; got != tt.want { + if got := receiveWithin(t, protoSeen, "backend protocol"); got != tt.want { t.Errorf("upstream saw %s, want %s", got, tt.want) } }) } } + +func receiveWithin[T any](t *testing.T, channel <-chan T, description string) T { + t.Helper() + select { + case value := <-channel: + return value + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for %s", description) + var zero T + return zero + } +} diff --git a/internal/benchmarking/boomer/glutton/durdir.go b/internal/benchmarking/boomer/glutton/durdir.go index f526be6a71..87fd7a6777 100644 --- a/internal/benchmarking/boomer/glutton/durdir.go +++ b/internal/benchmarking/boomer/glutton/durdir.go @@ -29,6 +29,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" @@ -127,7 +128,6 @@ func (r *durDirRuntime) startUser(ctx context.Context, dynCfg dynconfig.Config) templateName: tmpl, userClass: durDirUserClass, } - u.hostHeader = u.actorName + "." + u.cfg.Atespace + "." + actorDomain bmetrics.UpdateUsers(durDirUserClass, 1) if err := u.ensureAtespace(ctx); err != nil { bmetrics.UpdateUsers(durDirUserClass, -1) @@ -157,7 +157,6 @@ func (r *durDirRuntime) shutdown(ctx context.Context) { type durDirUser struct { cfg *userclass.Config actorName string - hostHeader string templateName string userClass string expectedDigest string @@ -413,8 +412,9 @@ func (u *durDirUser) httpProtoCall(ctx context.Context, metricName, route string bmetrics.RecordFailure("http", metricName, u.userClass, 0, err.Error()) return nil, err } - httpReq.Host = u.hostHeader httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set(atenet.ActorNameHeader, u.actorName) + httpReq.Header.Set(atenet.AtespaceHeader, u.cfg.Atespace) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) start := time.Now() diff --git a/internal/benchmarking/boomer/glutton/fixture_test.go b/internal/benchmarking/boomer/glutton/fixture_test.go index 89bb595669..f3f7097246 100644 --- a/internal/benchmarking/boomer/glutton/fixture_test.go +++ b/internal/benchmarking/boomer/glutton/fixture_test.go @@ -110,7 +110,6 @@ func newTestDurDirUser(t *testing.T, srv *fake.Server, cfg *userclass.Config) *d return &durDirUser{ cfg: c, actorName: "duractor", - hostHeader: "duractor.benchmark." + actorDomain, templateName: defaultDurTemplate, userClass: durDirUserClass, expectedSize: int64(len(srv.Data)), diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index 5ade96de2a..6d75e68511 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -30,6 +30,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" @@ -50,7 +51,6 @@ const ( userClass = "GluttonUser" templateName = "glutton" templateAtespace = "benchmark-workloads" - actorDomain = "actors.resources.substrate.ate.dev" pingPath = "/ping" writeRAMPath = "/writeram" readRAMPath = "/readram" @@ -131,7 +131,6 @@ func (r *taskRuntime) startUser(ctx context.Context) (*gluttonUser, error) { actorName: "sb-" + uuid.NewString(), firstResume: true, } - u.hostHeader = u.actorName + "." + u.cfg.Atespace + "." + actorDomain bmetrics.UpdateUsers(userClass, 1) if err := u.ensureAtespace(ctx); err != nil { bmetrics.UpdateUsers(userClass, -1) @@ -172,7 +171,6 @@ func (r *taskRuntime) dynamicWait() time.Duration { type gluttonUser struct { cfg *userclass.Config actorName string - hostHeader string firstResume bool actorRunning bool ramFilled bool @@ -298,8 +296,9 @@ func (u *gluttonUser) ping(ctx context.Context) { bmetrics.RecordFailure("http", "GluttonPing", userClass, 0, err.Error()) return } - httpReq.Host = u.hostHeader httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set(atenet.ActorNameHeader, u.actorName) + httpReq.Header.Set(atenet.AtespaceHeader, u.cfg.Atespace) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) start := time.Now() @@ -458,8 +457,9 @@ func (u *gluttonUser) postProto(ctx context.Context, path string, req, resp prot if err != nil { return err } - httpReq.Host = u.hostHeader httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set(atenet.ActorNameHeader, u.actorName) + httpReq.Header.Set(atenet.AtespaceHeader, u.cfg.Atespace) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) httpResp, err := u.cfg.HTTPClient.Do(httpReq) diff --git a/internal/benchmarking/boomer/glutton/memfill_test.go b/internal/benchmarking/boomer/glutton/memfill_test.go index d979bfa48d..d63a39eee8 100644 --- a/internal/benchmarking/boomer/glutton/memfill_test.go +++ b/internal/benchmarking/boomer/glutton/memfill_test.go @@ -28,9 +28,8 @@ func newTestGluttonUser(t *testing.T, srv *fake.Server, dyn dynconfig.Config) *g t.Helper() cfg := newTestConfig(t, srv, &userclass.Config{Dyn: dynconfig.NewHolder(dyn)}) return &gluttonUser{ - cfg: cfg, - actorName: "memactor", - hostHeader: "memactor.benchmark." + actorDomain, + cfg: cfg, + actorName: "memactor", } } diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 7a9e536079..88ffb38c58 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -29,6 +29,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/internal/resources" "k8s.io/client-go/kubernetes" @@ -106,8 +107,7 @@ func (c *RouterClient) BaseURL() string { return c.baseURL } -// Get issues GET path to actor through the router, setting the actor's DNS Host -// so the router routes (and resumes) it. The caller must close the body. +// Get issues GET path to actor through the router. The caller must close the body. func (c *RouterClient) Get(ctx context.Context, actorRef resources.ActorRef, path string) (*http.Response, error) { return c.request(ctx, http.MethodGet, actorRef, path, nil) } @@ -126,8 +126,8 @@ func (c *RouterClient) request(ctx context.Context, method string, actorRef reso if method == http.MethodPost { req.Header.Set("Content-Type", "application/json") } - // The router routes on the Host/:authority, not a header. - req.Host = resources.ActorDNSName(actorRef) + req.Header.Set(atenet.ActorNameHeader, actorRef.Name) + req.Header.Set(atenet.AtespaceHeader, actorRef.Atespace) return c.http.Do(req) } @@ -149,11 +149,15 @@ func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, return nil, fmt.Errorf("connecting to router's CONNECT listener: %w", err) } - destination := net.JoinHostPort(resources.ActorDNSName(actorRef), strconv.Itoa(port)) + destination := net.JoinHostPort(actorRef.Name, strconv.Itoa(port)) req := &http.Request{ Method: http.MethodConnect, URL: &url.URL{Host: destination}, Host: destination, + Header: http.Header{ + atenet.ActorNameHeader: []string{actorRef.Name}, + atenet.AtespaceHeader: []string{actorRef.Atespace}, + }, } if err := req.Write(rawConn); err != nil { _ = rawConn.Close() diff --git a/internal/e2e/router_client_test.go b/internal/e2e/router_client_test.go index e963510847..d50fc8849e 100644 --- a/internal/e2e/router_client_test.go +++ b/internal/e2e/router_client_test.go @@ -31,8 +31,8 @@ func TestRouterClientPostJSON(t *testing.T) { if request.Method != http.MethodPost { t.Errorf("method = %q, want POST", request.Method) } - if request.Host != "fetcher.demo.actors.resources.substrate.ate.dev" { - t.Errorf("host = %q", request.Host) + if request.Host != "router.test" { + t.Errorf("host = %q, want router.test", request.Host) } if request.URL.Path != "/fetch" { t.Errorf("path = %q, want /fetch", request.URL.Path) diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 38e6bea07b..0fc48578ba 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -1287,7 +1288,8 @@ func callActorPathOnce(t *testing.T, actorRef resources.ActorRef, method, path s if err != nil { return "", fmt.Errorf("failed to create request: %w", err) } - reqHttp.Host = resources.ActorDNSName(actorRef) + reqHttp.Header.Set(atenet.ActorNameHeader, actorRef.Name) + reqHttp.Header.Set(atenet.AtespaceHeader, actorRef.Atespace) httpClient := &http.Client{Timeout: 15 * time.Second} resp, err := httpClient.Do(reqHttp) diff --git a/internal/e2e/suites/networking/arbitraryport_test.go b/internal/e2e/suites/networking/arbitraryport_test.go index 9217b8fb51..af0231656f 100644 --- a/internal/e2e/suites/networking/arbitraryport_test.go +++ b/internal/e2e/suites/networking/arbitraryport_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" ) @@ -106,7 +107,8 @@ func TestActorArbitraryPortAccess(t *testing.T) { defer conn.Close() conn.SetDeadline(time.Now().Add(10 * time.Second)) - if _, err := conn.Write([]byte("GET / HTTP/1.1\r\nHost: " + resources.ActorDNSName(actorRef) + "\r\nConnection: close\r\n\r\n")); err != nil { + if _, err := fmt.Fprintf(conn, "GET / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: %s\r\nConnection: close\r\n\r\n", + actorRef.Name, atenet.ActorNameHeader, actorRef.Name, atenet.AtespaceHeader, actorRef.Atespace); err != nil { t.Fatalf("writing tunneled request: %v", err) } resp, err := http.ReadResponse(bufio.NewReader(conn), nil) @@ -140,7 +142,7 @@ func waitForTunneledRouteReady(t *testing.T, ctx context.Context, router *e2e.Ro for { conn, err := router.Connect(ctx, actorRef, port) if err == nil { - resp, body, requestErr := requestTunneled(conn, resources.ActorDNSName(actorRef)) + resp, body, requestErr := requestTunneled(conn, actorRef.Name) _ = conn.Close() if requestErr == nil && resp.StatusCode == http.StatusOK { return body diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 3829e388ee..5ff12d937c 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -26,8 +26,10 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/internal/proto/grpcechopb" @@ -69,7 +71,8 @@ func TestIngressProtocolDowngrade(t *testing.T) { if err != nil { return nil, err } - req.Host = resources.ActorDNSName(actorRef) + req.Header.Set(atenet.ActorNameHeader, actorRef.Name) + req.Header.Set(atenet.AtespaceHeader, actorRef.Atespace) if contentType != "" { req.Header.Set("Content-Type", contentType) } @@ -141,17 +144,19 @@ func TestIngressGRPC(t *testing.T) { fixture := deployGRPCEchoTemplate(t, ctx, env["BUCKET_NAME"]) actorName, _ := createAndResumeSubstrateActor(t, ctx, "grpcingress", fixture) actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + ctx = metadata.AppendToOutgoingContext(ctx, + atenet.ActorNameHeader, actorRef.Name, + atenet.AtespaceHeader, actorRef.Atespace, + ) - // Cleartext h2c to the router's HTTP port, with the Actor's DNS name as the - // :authority — the same routing key every other ingress test in this suite - // uses, just carried by a gRPC client instead of an HTTP one. The h2 ALPN - // offer is about the *TLS* listener; nothing here needs it. + // Cleartext h2c to the router's HTTP port. Explicit metadata identifies the + // Actor; the conventional actor authority remains application metadata. The + // h2 ALPN offer is about the *TLS* listener; nothing here needs it. conn, err := grpc.NewClient(routerAddress(t, ctx), grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithAuthority(resources.ActorDNSName(actorRef)), ) if err != nil { - t.Fatalf("creating the gRPC client for %s: %v", resources.ActorDNSName(actorRef), err) + t.Fatalf("creating the gRPC client for %s: %v", actorRef, err) } defer conn.Close() client := grpcechopb.NewEchoClient(conn) diff --git a/internal/e2e/suites/networking/websocketingress_test.go b/internal/e2e/suites/networking/websocketingress_test.go index d0afb3b54a..3618f2f05c 100644 --- a/internal/e2e/suites/networking/websocketingress_test.go +++ b/internal/e2e/suites/networking/websocketingress_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" "github.com/gorilla/websocket" @@ -67,7 +68,8 @@ func TestWebsocketIngressPing(t *testing.T) { actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} header := http.Header{} - header.Set("Host", resources.ActorDNSName(actorRef)) + header.Set(atenet.ActorNameHeader, actorRef.Name) + header.Set(atenet.AtespaceHeader, actorRef.Atespace) var c *websocket.Conn diff --git a/internal/resources/actor.go b/internal/resources/actor.go index cf9479835f..13682a2d40 100644 --- a/internal/resources/actor.go +++ b/internal/resources/actor.go @@ -18,9 +18,6 @@ const ( // ResourceNameRegexPattern is the regular expression pattern for a valid // Substrate resource name. ResourceNameRegexPattern = `[a-z0-9]([-a-z0-9]*[a-z0-9])?` - // ActorDNSSuffix is suffix to the DNS name for direct access to Actor - // "..actors.resources.substrate.ate.dev" - ActorDNSSuffix = "actors.resources.substrate.ate.dev" // GoldenActorAtespace is the reserved system atespace that per-template golden // actors live in. GoldenActorAtespace = "ate-golden" diff --git a/internal/resources/resourceref.go b/internal/resources/resourceref.go index c7fb91245d..9c8e9543e8 100644 --- a/internal/resources/resourceref.go +++ b/internal/resources/resourceref.go @@ -15,10 +15,8 @@ package resources import ( - "fmt" "log/slog" "reflect" - "strings" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -77,53 +75,6 @@ func ActorRefFromActor(a *ateapipb.Actor) ActorRef { } } -// ActorDNSName returns the uniform DNS name the actor is reachable at. -// This is: "..actors.resources.substrate.ate.dev". -func ActorDNSName(r ActorRef) string { - return r.Name + "." + r.Atespace + "." + ActorDNSSuffix -} - -// lowerASCII folds A-Z and leaves every other byte alone. Resource names are -// DNS-1123 labels, so ASCII is the whole alphabet here, and strings.ToLower -// would additionally fold characters outside it onto ASCII letters (the Kelvin -// sign U+212A onto "k", U+017F onto "s"), letting a non-ASCII host reach an -// actor under a spelling that is not its name. -func lowerASCII(s string) string { - b := []byte(s) - for i, c := range b { - if c >= 'A' && c <= 'Z' { - b[i] = c + ('a' - 'A') - } - } - return string(b) -} - -// ParseActorDNSName parses a DNS name for a given actor. -// -// The name is folded to lower case first. DNS lookups are case-insensitive -// (RFC 4343), so a client that resolved "MyActor.MySpace." reaches us -// with that spelling preserved in the Host header, while actor and atespace -// names are always lower case. Folding keeps the request addressed to the same -// actor its DNS lookup resolved to instead of failing to parse. -func ParseActorDNSName(name string) (ActorRef, error) { - normalized := lowerASCII(strings.TrimSuffix(name, ".")) - rest, found := strings.CutSuffix(normalized, "."+ActorDNSSuffix) - if !found { - return ActorRef{}, fmt.Errorf("invalid actor DNS name: must end with %s, got %q", ActorDNSSuffix, name) - } - actorName, atespace, found := strings.Cut(rest, ".") - if !found { - return ActorRef{}, fmt.Errorf("invalid actor DNS name: expected ..%s, got %q", ActorDNSSuffix, name) - } - if !IsValidResourceName(actorName) { - return ActorRef{}, fmt.Errorf("invalid actor DNS name %q: %q is not a valid actor name", name, actorName) - } - if !IsValidResourceName(atespace) { - return ActorRef{}, fmt.Errorf("invalid actor DNS name %q: %q is not a valid atespace", name, atespace) - } - return ActorRef{Atespace: atespace, Name: actorName}, nil -} - // ActorTemplateRef identifies an ActorTemplate by the (atespace, name). type ActorTemplateRef = ResourceRef[*ateapipb.ActorTemplate] diff --git a/internal/resources/resourceref_test.go b/internal/resources/resourceref_test.go index a7bf12671c..e28ef67bbe 100644 --- a/internal/resources/resourceref_test.go +++ b/internal/resources/resourceref_test.go @@ -89,59 +89,6 @@ func TestActorRefString(t *testing.T) { } } -func TestActorRefDNSName(t *testing.T) { - actorRef := ActorRef{Atespace: "team-a", Name: "act-1"} - - got := ActorDNSName(actorRef) - want := "act-1.team-a.actors.resources.substrate.ate.dev" - if got != want { - t.Errorf("ActorDNSName() = %q, want %q", got, want) - } - - parsed, err := ParseActorDNSName(got) - if err != nil { - t.Fatalf("ParseActorDNSName(%q) error = %v", got, err) - } - if parsed != actorRef { - t.Errorf("round-trip = %+v, want %+v", parsed, actorRef) - } -} - -func TestParseActorDNSName(t *testing.T) { - tests := []struct { - name string - input string - want ActorRef - wantErr bool - }{ - {"valid", "act-1.team-a.actors.resources.substrate.ate.dev", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - {"valid trailing dot", "act-1.team-a.actors.resources.substrate.ate.dev.", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - {"wrong suffix", "act-1.team-a.example.com", ActorRef{}, true}, - {"missing atespace", "act-1.actors.resources.substrate.ate.dev", ActorRef{}, true}, - {"mixed-case actor name", "ACT-1.team-a.actors.resources.substrate.ate.dev", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - {"mixed-case atespace", "act-1.TEAM-A.actors.resources.substrate.ate.dev", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - {"mixed-case suffix", "act-1.team-a.Actors.Resources.Substrate.Ate.Dev", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - // strings.ToLower would fold the Kelvin sign onto "k" and hand "act-1k" a - // request addressed to a name no actor can have. - {"non-ASCII actor name", "act-1K.team-a.actors.resources.substrate.ate.dev", ActorRef{}, true}, - {"invalid actor name", "act_1.team-a.actors.resources.substrate.ate.dev", ActorRef{}, true}, - {"invalid atespace", "act-1.team_a.actors.resources.substrate.ate.dev", ActorRef{}, true}, - {"host:port not accepted", "act-1.team-a.actors.resources.substrate.ate.dev:8080", ActorRef{}, true}, - {"empty", "", ActorRef{}, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ParseActorDNSName(tt.input) - if (err != nil) != tt.wantErr { - t.Fatalf("ParseActorDNSName(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) - } - if got != tt.want { - t.Errorf("ParseActorDNSName(%q) = %+v, want %+v", tt.input, got, tt.want) - } - }) - } -} - func TestActorRefObjectRefRoundTrip(t *testing.T) { actorRef := ActorRef{Atespace: "team-a", Name: "act-1"} diff --git a/manifests/ate-install/atenet-dns.yaml b/manifests/ate-install/atenet-dns.yaml deleted file mode 100644 index 77d71bf594..0000000000 --- a/manifests/ate-install/atenet-dns.yaml +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: atenet-dns - namespace: ate-system - labels: - app: dns ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: atenet-dns - namespace: ate-system -rules: -- apiGroups: [""] - resources: ["services"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: atenet-dns - namespace: ate-system -subjects: -- kind: ServiceAccount - name: atenet-dns - namespace: ate-system -roleRef: - kind: Role - name: atenet-dns - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: atenet-dns - namespace: kube-system -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: atenet-dns - namespace: kube-system -subjects: -- kind: ServiceAccount - name: atenet-dns - namespace: ate-system -roleRef: - kind: Role - name: atenet-dns - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dns - namespace: ate-system - labels: - app: dns -spec: - replicas: 1 - selector: - matchLabels: - app: dns - template: - metadata: - labels: - app: dns - spec: - serviceAccountName: atenet-dns - shareProcessNamespace: true - initContainers: - - name: init-dns - image: busybox:1.36 - command: ["sh", "-c"] - # Initial core file is sufficient to start CoreDNS but does not contain - # any additional configuration. The controller will update the Corefile. - args: - - | - cat <<'EOF' > /etc/coredns/Corefile - .:53 { - errors - health :8080 - ready :8181 - reload - } - EOF - volumeMounts: - - name: dns-config-volume - mountPath: /etc/coredns - containers: - - name: coredns - image: coredns/coredns:1.11.1 - imagePullPolicy: IfNotPresent - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: dns-config-volume - mountPath: /etc/coredns - ports: - - name: dns - containerPort: 53 - protocol: UDP - - name: dns-tcp - containerPort: 53 - protocol: TCP - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - initialDelaySeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - - name: dns-controller - image: ko://github.com/agent-substrate/substrate/cmd/atenet - args: - - "dns" - - "--log-level=debug" - - "--interval=10s" - - "--corefile-path=/etc/coredns/Corefile" - volumeMounts: - - name: dns-config-volume - mountPath: /etc/coredns - volumes: - - name: dns-config-volume - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: dns - namespace: ate-system - labels: - app: dns -spec: - selector: - app: dns - type: ClusterIP - # Prefer, not Require: Require fails Service creation on a single-stack cluster. - ipFamilyPolicy: PreferDualStack - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP \ No newline at end of file diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index 2a9b83a969..6a2031ef70 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -22,7 +22,6 @@ resources: - ../ate-api-server.yaml - ../ate-controller.yaml - ../atelet.yaml - - ../atenet-dns.yaml - ../atenet-router.yaml - ../pod-certificate-controller.yaml - ../ate-otel-config.yaml diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 1fabcdec4d..5e13815263 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -25,7 +25,6 @@ resources: - ../ate-api-server.yaml - ../ate-controller.yaml - ./atelet - - ../atenet-dns.yaml - ../atenet-router.yaml - ../pod-certificate-controller.yaml - ate-otel-config.yaml