From af09bfa8a5991e580d253d695ca0ef06577c169a Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Mon, 31 Aug 2026 08:01:16 -0500 Subject: [PATCH 01/10] Move from Host header to explicit headers for actor and atespace Signed-off-by: Keith Mattix II --- README.md | 5 +- .../automation/testtypes/nighthawk_ingress.py | 2 +- benchmarking/locust/tests/counter_demo.py | 15 ++-- benchmarking/nighthawk-ingress/README.md | 12 +-- benchmarking/nighthawk-ingress/actors.py | 8 +- benchmarking/nighthawk-ingress/runner.py | 6 +- benchmarking/nighthawk-ingress/spec.py | 22 +++-- .../nighthawk-ingress/tests/test_spec.py | 25 ++++-- cmd/atenet/internal/router/README.md | 20 +++-- .../internal/router/extproc/attributes.go | 16 ++-- .../router/extproc/attributes_test.go | 60 +++++++++++++ .../internal/router/extproc/metadata_test.go | 4 +- .../internal/router/ingress/actorref_test.go | 89 ------------------- cmd/atenet/internal/router/ingress/errors.go | 6 -- .../internal/router/ingress/errors_test.go | 21 ----- cmd/atenet/internal/router/ingress/ingress.go | 61 ++++++------- .../internal/router/ingress/ingress_test.go | 67 +++++++------- cmd/atenet/internal/router/xds.go | 81 ++++++++--------- cmd/atenet/internal/router/xds_test.go | 28 ++++++ demos/autoscaled-workerpool/README.md | 2 +- demos/counter/README.md | 10 ++- demos/egress/README.md | 2 + demos/egress/test-egress.sh | 2 + demos/jupyter/README.md | 2 +- demos/jupyter/jupyter.yaml.tmpl | 3 +- demos/multi-template/README.md | 6 +- demos/parking/README.md | 6 +- demos/parking/load.sh | 4 +- demos/sandbox/client/main.go | 3 + docs/architecture.md | 8 +- docs/threat-model.md | 2 +- internal/atunnel/ingress.go | 63 +++---------- internal/atunnel/ingress_test.go | 78 +++++++++------- .../benchmarking/boomer/glutton/durdir.go | 3 + .../benchmarking/boomer/glutton/lifecycle.go | 5 ++ internal/e2e/router_client.go | 11 ++- internal/e2e/suites/demo/demo_test.go | 3 + .../suites/networking/arbitraryport_test.go | 4 +- 38 files changed, 385 insertions(+), 380 deletions(-) create mode 100644 cmd/atenet/internal/router/extproc/attributes_test.go delete mode 100644 cmd/atenet/internal/router/ingress/actorref_test.go diff --git a/README.md b/README.md index 01d22a4a25..3780ab9240 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,10 @@ kubectl port-forward -n ate-system svc/atenet-router 8000:80 3. In a **separate terminal**, send an HTTP request to increment the counter: ```shell -curl -X POST -H "Host: my-counter-1.ate-demo-counter.actors.resources.substrate.ate.dev" -i http://localhost:8000/ +curl -X POST \ + -H "X-Ate-Actor-Name: my-counter-1" \ + -H "X-Ate-Atespace: ate-demo-counter" \ + -i http://localhost:8000/ ``` Worker capacity is versioned: the dataplane (the atelet DaemonSet and the diff --git a/benchmarking/automation/testtypes/nighthawk_ingress.py b/benchmarking/automation/testtypes/nighthawk_ingress.py index 1245cee476..624904e821 100644 --- a/benchmarking/automation/testtypes/nighthawk_ingress.py +++ b/benchmarking/automation/testtypes/nighthawk_ingress.py @@ -102,7 +102,7 @@ def validate(test: dict[str, Any]) -> None: if not re.fullmatch(r"[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?", nh["atespace"]): raise ValueError( f"nighthawk-ingress test {name!r} atespace {nh['atespace']!r} " - f"must be a DNS label (it is routed as a Host-header subdomain)" + f"must be a DNS label" ) diff --git a/benchmarking/locust/tests/counter_demo.py b/benchmarking/locust/tests/counter_demo.py index c99343ace4..f01a36aec9 100644 --- a/benchmarking/locust/tests/counter_demo.py +++ b/benchmarking/locust/tests/counter_demo.py @@ -48,9 +48,8 @@ tracer = get_tracer(__name__) -# Atenet router fronts all actor traffic. Actors are addressed by setting -# the HTTP Host header to ..actors.resources.substrate.ate.dev; -# the router resolves that to the actor's current worker pod. +# Atenet router fronts all actor traffic. The actor identity headers select +# the actor whose current worker pod the router resolves. ROUTER_URL = "http://atenet-router.ate-system.svc.cluster.local" ACTOR_DOMAIN = "actors.resources.substrate.ate.dev" @@ -99,9 +98,7 @@ def on_start(self) -> None: except Exception as e: logger.error(f"Failed to create actor {self.actor_name}: {e}") - # One HTTP session per user, talking to the router. The Host header - # pins each request to this actor regardless of which worker pod - # hosts it after a resume. + # One HTTP session per user, talking to the router. self.http_session = requests.Session() self.run_url = f"{ROUTER_URL}/" self.host_header = f"{self.actor_name}.{ATESPACE}.{ACTOR_DOMAIN}" @@ -139,7 +136,11 @@ def run_and_suspend(self) -> None: # 2. Run/Increment (HTTP via atenet-router) start_time = time.time() with tracer.start_as_current_span("RunCounter") as span: - headers = {"Host": self.host_header} + headers = { + "Host": self.host_header, + "X-Ate-Actor-Name": self.actor_name, + "X-Ate-Atespace": ATESPACE, + } inject(headers) try: response = self.http_session.post(self.run_url, headers=headers) diff --git a/benchmarking/nighthawk-ingress/README.md b/benchmarking/nighthawk-ingress/README.md index 6b19eaa1eb..75d1aa0301 100644 --- a/benchmarking/nighthawk-ingress/README.md +++ b/benchmarking/nighthawk-ingress/README.md @@ -30,14 +30,14 @@ just don't steer the search.* ## What it measures Every request exercises the **full production routing path** — created and -warmed sandboxed actors, Host-header routing, the ext_proc routing +warmed sandboxed actors, identity-header routing, the ext_proc routing decision, and the mTLS hop to the worker: ```mermaid flowchart LR subgraph job["nighthawk runner Job"] alc["nighthawk_adaptive_load_client
exponential ramp + binary search"] - svc["nighthawk_service
16 event loops, Host header
rotated across all actors"] + svc["nighthawk_service
16 event loops, actor headers
rotated across all actors"] alc -->|gRPC| svc end @@ -54,7 +54,7 @@ flowchart LR atunnel["atunnel :443"] --> glutton["glutton actor
POST /ping :80"] end - svc -->|"HTTP :80, Host:
actor-N.benchmark.actors..."| envoy + svc -->|"HTTP :80
X-Ate-Actor-Name + X-Ate-Atespace"| envoy extproc -->|ResumeActor| ateapi envoy -->|"mTLS :443"| atunnel ``` @@ -84,9 +84,9 @@ One Kubernetes Job per `type: nighthawk-ingress` tests.yaml entry, driven by down afterwards, so nothing leaks. 2. **Create + warm actors.** The runner creates one glutton actor per WorkerPool worker (the entry's `workerCount`) via ateapi and POSTs - `/ping` through the router with each actor's Host header until it + `/ping` through the router with each actor's identity headers until it answers 200. -3. **Adaptive search.** Open-loop traffic with the Host header rotated +3. **Adaptive search.** Open-loop traffic with the actor identity headers rotated across all actors; `clientConcurrency` event loops (default 16, decoupled from `envoyCpu`) and large per-loop pools so the harness is never the bottleneck. Exponential ramp → binary search → a 60s @@ -174,7 +174,7 @@ Start with `capacity.json`, drill into `stats.jsonl`: The fleet size is the entry's top-level `workerCount` (required): the benchmark warms one actor per worker, so it is also the number of glutton -actors receiving rotated-Host traffic. Everything else lives in the +actors receiving rotated identity-header traffic. Everything else lives in the `nighthawk-ingress:` block: | Knob | Default | Meaning | diff --git a/benchmarking/nighthawk-ingress/actors.py b/benchmarking/nighthawk-ingress/actors.py index 90dbc8bc23..07ad997da2 100644 --- a/benchmarking/nighthawk-ingress/actors.py +++ b/benchmarking/nighthawk-ingress/actors.py @@ -110,7 +110,7 @@ def _warm_actor( ) -> None: """Bring one created actor to serving, so the load ladder never measures a cold start: resume the actor, then poll POST /ping through - the router (addressed by the actor's Host header) until it answers + the router until it answers 200 or the deadline expires. Resume errors are retried: ateapi returns FailedPrecondition/Unavailable until a worker frees up.""" ref = ateapi_pb2.ObjectRef(atespace=atespace, name=name) @@ -125,7 +125,11 @@ def _warm_actor( try: resp = session.post( f"{router_url.rstrip('/')}/ping", - headers={"Host": host}, + headers={ + "Host": host, + "X-Ate-Actor-Name": name, + "X-Ate-Atespace": atespace, + }, data=b"", timeout=10, ) diff --git a/benchmarking/nighthawk-ingress/runner.py b/benchmarking/nighthawk-ingress/runner.py index f5abf09bf8..921673f8fc 100644 --- a/benchmarking/nighthawk-ingress/runner.py +++ b/benchmarking/nighthawk-ingress/runner.py @@ -268,12 +268,10 @@ def main() -> None: log=lambda m: tee(logs, m), ) - hosts = [ - spec_mod.actor_host(n, args.atespace) for n in actor_names - ] spec_dict = spec_mod.build_spec_dict( uri=f"{args.router_url.rstrip('/')}/ping", - hosts=hosts, + actor_names=actor_names, + atespace=args.atespace, client_concurrency=args.client_concurrency, connections=args.connections, max_pending_requests=args.max_pending, diff --git a/benchmarking/nighthawk-ingress/spec.py b/benchmarking/nighthawk-ingress/spec.py index 32ebd742ce..0ef83af3ac 100644 --- a/benchmarking/nighthawk-ingress/spec.py +++ b/benchmarking/nighthawk-ingress/spec.py @@ -59,7 +59,7 @@ def actor_host(actor: str, atespace: str) -> str: - """Host header the router routes on (internal/resources/actor.go).""" + """Return an actor's stable mesh DNS name.""" return f"{actor}.{atespace}.actors.resources.substrate.ate.dev" @@ -91,7 +91,8 @@ def _binary_threshold( def build_spec_dict( *, uri: str, - hosts: list[str], + actor_names: list[str], + atespace: str, client_concurrency: int, connections: int, max_pending_requests: int, @@ -117,12 +118,23 @@ def build_spec_dict( "request_method": "POST", "request_headers": [ { - "header": {"key": "host", "value": host}, + "header": { + "key": "host", + "value": actor_host(actor_name, atespace), + }, "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", - } + }, + { + "header": {"key": "x-ate-actor-name", "value": actor_name}, + "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", + }, + { + "header": {"key": "x-ate-atespace", "value": atespace}, + "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", + }, ], } - for host in hosts + for actor_name in actor_names ] # Threshold roles: tail latency (mean+2stdev, ~p95 proxy — no true # percentiles in the builtin adaptive metrics) is the SLO bound; diff --git a/benchmarking/nighthawk-ingress/tests/test_spec.py b/benchmarking/nighthawk-ingress/tests/test_spec.py index e2b565cb8c..3652a76a6e 100644 --- a/benchmarking/nighthawk-ingress/tests/test_spec.py +++ b/benchmarking/nighthawk-ingress/tests/test_spec.py @@ -33,9 +33,8 @@ def build(**overrides): kwargs = dict( uri="http://atenet-router.ate-system.svc.cluster.local:80/ping", - hosts=[ - spec_mod.actor_host(f"sb-{i}", "benchmark") for i in range(3) - ], + actor_names=[f"sb-{i}" for i in range(3)], + atespace="benchmark", client_concurrency=4, connections=1000, max_pending_requests=10000, @@ -67,14 +66,24 @@ def test_traffic_template_shape(): } -def test_host_rotation_covers_all_actors(): - hosts = [spec_mod.actor_host(f"sb-{i}", "benchmark") for i in range(5)] - spec = build(hosts=hosts) +def test_identity_rotation_covers_all_actors(): + actor_names = [f"sb-{i}" for i in range(5)] + spec = build(actor_names=actor_names) plugin = spec["nighthawk_traffic_template"]["request_source_plugin_config"] assert plugin["name"] == spec_mod.REQUEST_SOURCE_PLUGIN options = plugin["typed_config"]["options_list"]["options"] - got = [o["request_headers"][0]["header"]["value"] for o in options] - assert got == hosts + got = [ + {header["header"]["key"]: header["header"]["value"] for header in o["request_headers"]} + for o in options + ] + assert got == [ + { + "host": spec_mod.actor_host(actor_name, "benchmark"), + "x-ate-actor-name": actor_name, + "x-ate-atespace": "benchmark", + } + for actor_name in actor_names + ] assert all(o["request_method"] == "POST" for o in options) # 0 = loop the list indefinitely. assert plugin["typed_config"]["num_requests"] == 0 diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index 7719a5ce6c..e366e248d0 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -63,7 +63,14 @@ cannot pick the egress path by crafting one. `router` itself does the wiring. ## adding a dataplane attribute The filter-state objects and request attributes a proxy carries alongside -a request are declared once, in `extproc/attributes.go`. +a request are declared once, in `extproc/attributes.go`. + +| key | direction | purpose | +| --- | --- | --- | +| `dev.ate.actor.name` | ingress | carries the actor name across CONNECT re-entry | +| `dev.ate.actor.atespace` | ingress | carries the atespace across CONNECT re-entry | +| `dev.ate.actor.identity` | egress | carries the authenticated actor identity for logs and additional ext_proc services | +| `dev.ate.extproc.direction` | egress | selects the egress handler for dataplanes without Envoy filter chains | ### name it @@ -94,13 +101,10 @@ reserved. ### keep it trustworthy -An attribute is only as trustworthy as the thing that set it. Every value here -comes from something the dataplane itself derived — a peer certificate Envoy -verified against the actor-identity CA, an `:authority` captured before the -request entered a tunnel — never from a client header, which an actor controls -end to end. That is what makes filter state a sound carrier across the -CONNECT/MITM boundary in the first place, and a new attribute sourced from a -header gives the property back. +An attribute is only as trustworthy as its source. The actor name and atespace +carry client-selected routing coordinates. Security-sensitive attributes, such +as actor identity, must come from dataplane-authenticated state rather than a +client header. ## modes diff --git a/cmd/atenet/internal/router/extproc/attributes.go b/cmd/atenet/internal/router/extproc/attributes.go index 6cfb929a96..451063870f 100644 --- a/cmd/atenet/internal/router/extproc/attributes.go +++ b/cmd/atenet/internal/router/extproc/attributes.go @@ -27,13 +27,15 @@ package extproc // substrate's own metric dimensions and stay on dotted "ate.". Neither is the // "ate.dev/" slash form, which is Kubernetes labels only. const ( - // AuthorityFilterStateKey is the filter-state key holding an ingress - // request's :authority, set by xds.go's authorityFilterStateFilter. - // Ingress-only: it names the actor a request is addressed to. - AuthorityFilterStateKey = "dev.ate.authority" - // AuthorityFilterStateAttribute is the CEL expression ext_proc evaluates to - // read AuthorityFilterStateKey back out. - AuthorityFilterStateAttribute = "filter_state['" + AuthorityFilterStateKey + "']" + // ActorNameFilterStateKey and AtespaceFilterStateKey carry the ingress actor + // routing coordinates across Envoy's CONNECT internal-listener hop. + ActorNameFilterStateKey = "dev.ate.actor.name" + AtespaceFilterStateKey = "dev.ate.actor.atespace" + + // ActorNameFilterStateAttribute and AtespaceFilterStateAttribute are the CEL + // expressions ext_proc evaluates to read the corresponding filter state. + ActorNameFilterStateAttribute = "filter_state['" + ActorNameFilterStateKey + "']" + AtespaceFilterStateAttribute = "filter_state['" + AtespaceFilterStateKey + "']" // ActorIdentityFilterStateKey is the filter-state key holding the actor // identity the egress gateway read from the peer certificate it verified diff --git a/cmd/atenet/internal/router/extproc/attributes_test.go b/cmd/atenet/internal/router/extproc/attributes_test.go new file mode 100644 index 0000000000..65f73f30d7 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/attributes_test.go @@ -0,0 +1,60 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "strings" + "testing" +) + +func TestAttributeKeys(t *testing.T) { + tests := []struct { + name string + got string + want string + }{ + {"actor name filter state", ActorNameFilterStateKey, "dev.ate.actor.name"}, + {"actor name attribute", ActorNameFilterStateAttribute, "filter_state['dev.ate.actor.name']"}, + {"atespace filter state", AtespaceFilterStateKey, "dev.ate.actor.atespace"}, + {"atespace attribute", AtespaceFilterStateAttribute, "filter_state['dev.ate.actor.atespace']"}, + {"actor identity filter state", ActorIdentityFilterStateKey, "dev.ate.actor.identity"}, + {"direction attribute", directionAttribute, "dev.ate.extproc.direction"}, + {"filter chain name attribute", FilterChainNameAttribute, "xds.filter_chain_name"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.got != tt.want { + t.Errorf("key = %q, want %q; update dataplane configuration to match", tt.got, tt.want) + } + }) + } +} + +func TestSubstrateKeysShareOnePrefix(t *testing.T) { + const prefix = "dev.ate." + for _, key := range []string{ + ActorNameFilterStateKey, + AtespaceFilterStateKey, + ActorIdentityFilterStateKey, + directionAttribute, + } { + if !strings.HasPrefix(key, prefix) { + t.Errorf("key %q is not rooted at %q", key, prefix) + } + if strings.Contains(key, "/") { + t.Errorf("key %q uses the ate.dev/ Kubernetes label form", key) + } + } +} diff --git a/cmd/atenet/internal/router/extproc/metadata_test.go b/cmd/atenet/internal/router/extproc/metadata_test.go index 302ff5cefe..03648f6a44 100644 --- a/cmd/atenet/internal/router/extproc/metadata_test.go +++ b/cmd/atenet/internal/router/extproc/metadata_test.go @@ -124,7 +124,7 @@ func TestRequestMetadataAttribute(t *testing.T) { attrs := map[string]*structpb.Struct{ "envoy.filters.http.ext_proc": { Fields: map[string]*structpb.Value{ - "filter_state['dev.ate.authority']": structpb.NewStringValue("actor-1.team-a.actors.resources.substrate.ate.dev"), + ActorNameFilterStateAttribute: structpb.NewStringValue("actor-1"), }, }, } @@ -134,7 +134,7 @@ func TestRequestMetadataAttribute(t *testing.T) { // The lookup scans every filter's attributes rather than hardcoding which // filter reported the value (see filterChainName in dispatch.go for why), // so it must not matter which filter name the value arrived under. - if got, want := md.Attribute("filter_state['dev.ate.authority']"), "actor-1.team-a.actors.resources.substrate.ate.dev"; got != want { + if got, want := md.Attribute(ActorNameFilterStateAttribute), "actor-1"; got != want { t.Errorf("Attribute() = %q, want %q", got, want) } if got := md.Attribute("filter_state['does.not.exist']"); got != "" { diff --git a/cmd/atenet/internal/router/ingress/actorref_test.go b/cmd/atenet/internal/router/ingress/actorref_test.go deleted file mode 100644 index d7aab2ac85..0000000000 --- a/cmd/atenet/internal/router/ingress/actorref_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ingress - -import ( - "testing" - - "github.com/agent-substrate/substrate/internal/resources" -) - -func TestParseActorRef(t *testing.T) { - tests := []struct { - name string - host string - want resources.ActorRef - wantErr bool - }{ - { - name: "valid host without port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev:8443", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with trailing dot", - host: "my-actor.team-a.actors.resources.substrate.ate.dev.", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "valid host with trailing dot and port", - host: "my-actor.team-a.actors.resources.substrate.ate.dev.:8080", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "mixed-case host", - host: "My-Actor.Team-A.actors.resources.substrate.ate.dev", - want: resources.ActorRef{Atespace: "team-a", Name: "my-actor"}, - wantErr: false, - }, - { - name: "missing atespace label", - host: "my-actor.actors.resources.substrate.ate.dev", - wantErr: true, - }, - { - name: "invalid suffix", - host: "my-actor.team-a.example.com", - wantErr: true, - }, - { - name: "invalid host port format", - host: "my-actor.team-a.actors.resources.substrate.ate.dev:invalid:port", - wantErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got, err := parseActorRef(tc.host) - if (err != nil) != tc.wantErr { - t.Errorf("parseActorRef(%q) error = %v, wantErr %v", tc.host, err, tc.wantErr) - return - } - if got != tc.want { - t.Errorf("parseActorRef(%q) = %+v, want %+v", tc.host, got, tc.want) - } - }) - } -} diff --git a/cmd/atenet/internal/router/ingress/errors.go b/cmd/atenet/internal/router/ingress/errors.go index 1c3bc89562..ffbd626a2a 100644 --- a/cmd/atenet/internal/router/ingress/errors.go +++ b/cmd/atenet/internal/router/ingress/errors.go @@ -32,12 +32,6 @@ func actorNotFoundErr(actorRef resources.ActorRef) error { return extproc.NewReqError(envoy_type.StatusCode_NotFound, "actor %s not found", actorRef) } -// invalidHostErr returns a 404 denial explaining why the request host was -// rejected. The cause is preserved for log inspection via Unwrap. -func invalidHostErr(host string, cause error) error { - return extproc.WrapReqError(envoy_type.StatusCode_NotFound, cause, "invalid host %q: %v", host, cause) -} - // statusDescription returns the gRPC status description of err, unwrapping // any wrapper (e.g. budgetExhaustedError) first. status.Convert on a wrapping // error replaces the description with the wrapper's full "rpc error: ..." diff --git a/cmd/atenet/internal/router/ingress/errors_test.go b/cmd/atenet/internal/router/ingress/errors_test.go index 2893523671..105c736c1f 100644 --- a/cmd/atenet/internal/router/ingress/errors_test.go +++ b/cmd/atenet/internal/router/ingress/errors_test.go @@ -43,27 +43,6 @@ func TestActorNotFoundErr(t *testing.T) { } } -func TestInvalidHostErr(t *testing.T) { - t.Parallel() - - cause := errors.New("missing suffix") - err := invalidHostErr("foo.example.com", cause) - - var reqErr *extproc.ReqError - if !errors.As(err, &reqErr) { - t.Fatalf("errors.As(*extproc.ReqError) = false, want true; err type = %T", err) - } - if reqErr.StatusCode != int(envoy_type.StatusCode_NotFound) { - t.Errorf("StatusCode = %d, want %d", reqErr.StatusCode, envoy_type.StatusCode_NotFound) - } - if got, want := err.Error(), `invalid host "foo.example.com": missing suffix`; got != want { - t.Errorf("Error() = %q, want %q", got, want) - } - if !errors.Is(err, cause) { - t.Errorf("errors.Is(err, cause) = false, want true (cause should be wrapped for logging)") - } -} - func TestMapResumeError(t *testing.T) { t.Parallel() diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 407adc282a..8a0ba908d6 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -25,11 +25,9 @@ package ingress import ( "context" - "fmt" "log/slog" "net" "strconv" - "strings" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" @@ -87,23 +85,18 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM ctx, span := otel.Tracer(extproc.ServiceName).Start(ctx, "ExtProc.RequestHeaders") defer span.End() - // Resolved from filter state rather than Host/:authority directly: a - // reinjected CONNECT tunnel's own :authority has nothing to do with the - // actor, so xds.go captures the real one at connect_terminate instead. - authority := md.Attribute(extproc.AuthorityFilterStateAttribute) - if authority == "" { - return extproc.Result{}, invalidHostErr(md.Host, fmt.Errorf("missing %s request attribute", extproc.AuthorityFilterStateAttribute)) + actorRef := resources.ActorRef{ + Name: identityValue(md, atunnel.ActorNameHeader, extproc.ActorNameFilterStateAttribute), + Atespace: identityValue(md, atunnel.AtespaceHeader, extproc.AtespaceFilterStateAttribute), } - actorRef, err := parseActorRef(authority) - if err != nil { - // Authority is invalid, respond with 404. - return extproc.Result{}, invalidHostErr(authority, err) + if !resources.IsValidResourceName(actorRef.Name) || !resources.IsValidResourceName(actorRef.Atespace) { + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_NotFound, "invalid actor identity") } // CONNECT traffic can name a port other than defaultActorPort in the // authority. targetPort := defaultActorPort - if _, portStr, err := net.SplitHostPort(authority); err == nil { + if _, portStr, err := net.SplitHostPort(md.Host); err == nil { if p, ok := atunnel.ParsePort(portStr); ok { targetPort = p } @@ -164,17 +157,22 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM "actor %s routing failed", actorRef) } - // :authority/Host stays untouched, so atunnel authorizes by the actor's - // own DNS name. The target port still goes as a header: atunnel can't - // read dynamic metadata. + // Overwrite the routing headers so client-provided values cannot select a + // different actor after this request has been resolved. mutation := &extprocv3.HeaderMutation{} - mutation.SetHeaders = append(mutation.SetHeaders, &corev3.HeaderValueOption{ - Header: &corev3.HeaderValue{ - Key: atunnel.TargetPortHeader, - RawValue: []byte(strconv.Itoa(targetPort)), - }, - AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, - }) + for _, header := range []struct{ name, value string }{ + {atunnel.ActorNameHeader, actorRef.Name}, + {atunnel.AtespaceHeader, actorRef.Atespace}, + {atunnel.TargetPortHeader, strconv.Itoa(targetPort)}, + } { + mutation.SetHeaders = append(mutation.SetHeaders, &corev3.HeaderValueOption{ + Header: &corev3.HeaderValue{ + Key: header.name, + RawValue: []byte(header.value), + }, + AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, + }) + } res.Target = targetAddr res.Response = &extprocv3.HeadersResponse{ @@ -186,18 +184,9 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM return res, nil } -// parseActorRef extracts the actor an incoming request is addressed to from its -// Host/:authority, which has the form -// "..actors.resources.substrate.ate.dev" (optionally with a -// port). The atespace is part of the name because an actor name is only unique -// within its atespace. -func parseActorRef(host string) (resources.ActorRef, error) { - if strings.Contains(host, ":") { - h, _, err := net.SplitHostPort(host) - if err != nil { - return resources.ActorRef{}, err - } - host = h +func identityValue(md *extproc.RequestMetadata, header, attribute string) string { + if value := md.Header(header); value != "" { + return value } - return resources.ParseActorDNSName(host) + return md.Attribute(attribute) } diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index 76fed77c39..038ad34911 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -45,29 +45,12 @@ func (m *mockClient) ResumeActor(ctx context.Context, in *ateapipb.ResumeActorRe return m.resumeFn(ctx, in, opts...) } -// authorityAttributes builds the ProcessingRequest.Attributes map the mux -// hands the handler -- the forwarded filter_state['dev.ate.authority'] -// CEL attribute that xds.go's buildHcm (backed by authorityFilterStateFilter, -// or for CONNECT, connect_terminate's own capture) requests from Envoy. It -// replaces the :authority header as the source of routing truth; tests still -// set the header too, since RequestMetadata still logs it. -func authorityAttributes(t *testing.T, authority string) map[string]*structpb.Struct { - t.Helper() - s, err := structpb.NewStruct(map[string]any{extproc.AuthorityFilterStateAttribute: authority}) - if err != nil { - t.Fatalf("build authority attributes: %v", err) - } - return map[string]*structpb.Struct{ - "envoy.filters.http.ext_proc": s, - } -} - -// requestMetadata builds the metadata the ext_proc mux would hand the handler -// for a request with these headers, with authority forwarded via filter-state -// attribute the way Envoy actually delivers it (see authorityAttributes). -func requestMetadata(t *testing.T, authority string, headers ...*corev3.HeaderValue) *extproc.RequestMetadata { - t.Helper() - return extproc.NewRequestMetadata(headers, authorityAttributes(t, authority)) +func requestMetadata(actorName, atespace string, headers ...*corev3.HeaderValue) *extproc.RequestMetadata { + headers = append(headers, + &corev3.HeaderValue{Key: atunnel.ActorNameHeader, Value: actorName}, + &corev3.HeaderValue{Key: atunnel.AtespaceHeader, Value: atespace}, + ) + return extproc.NewRequestMetadata(headers, nil) } // dynamicMetadataTarget extracts the resolved worker address @@ -101,7 +84,7 @@ func TestHandleRequestHeadersDoesNotLogSensitiveData(t *testing.T) { }, }, ParkedRequestConfig{}, nil) - md := requestMetadata(t, authority, + md := requestMetadata(testUUID, "team-a", &corev3.HeaderValue{Key: ":path", Value: "/api/v1/reset?token=" + secret}, &corev3.HeaderValue{Key: ":authority", Value: authority}, &corev3.HeaderValue{Key: ":method", Value: "POST"}, @@ -138,6 +121,8 @@ func TestHandleRequestHeaders(t *testing.T) { tests := []struct { name string + actorName string + atespace string authority string resumeResp *ateapipb.ResumeActorResponse resumeErr error @@ -148,10 +133,12 @@ func TestHandleRequestHeaders(t *testing.T) { expectedTargetPort string }{ { - name: "invalid host returns 404 identifying the host", + name: "invalid actor header returns 404", + actorName: "INVALID", + atespace: "team-a", authority: "invalid-host.com", expectErr: true, - expectedErrStr: `invalid host "invalid-host.com": invalid actor DNS name: must end with actors.resources.substrate.ate.dev, got "invalid-host.com"`, + expectedErrStr: `invalid actor identity`, expectedStatus: envoy_type.StatusCode_NotFound, }, { @@ -207,8 +194,8 @@ func TestHandleRequestHeaders(t *testing.T) { expectedStatus: envoy_type.StatusCode_InternalServerError, }, { - name: "Successful resume", - authority: testUUID + ".team-a.actors.resources.substrate.ate.dev", + name: "successful resume ignores host for actor identity", + authority: "application.example", resumeResp: &ateapipb.ResumeActorResponse{ Actor: &ateapipb.Actor{ Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}, @@ -222,6 +209,14 @@ func TestHandleRequestHeaders(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + actorName := tc.actorName + if actorName == "" { + actorName = testUUID + } + atespace := tc.atespace + if atespace == "" { + atespace = "team-a" + } clientMock := &mockClient{ resumeFn: func(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { if in.GetActor().GetName() != testUUID { @@ -240,7 +235,7 @@ func TestHandleRequestHeaders(t *testing.T) { // resumer_test.go. h := New(clientMock, ParkedRequestConfig{}, nil) - md := requestMetadata(t, tc.authority, + md := requestMetadata(actorName, atespace, &corev3.HeaderValue{Key: ":path", Value: "/v1/actors/invoke"}, &corev3.HeaderValue{Key: ":authority", Value: tc.authority}, &corev3.HeaderValue{Key: ":method", Value: "POST"}, @@ -275,8 +270,8 @@ func TestHandleRequestHeaders(t *testing.T) { } mutation := res.Response.GetResponse().GetHeaderMutation() - if len(mutation.GetSetHeaders()) != 1 { - t.Fatalf("expected exactly one header option (TargetPortHeader), found: %v", mutation.GetSetHeaders()) + if len(mutation.GetSetHeaders()) != 3 { + t.Fatalf("expected actor identity and target port headers, found: %v", mutation.GetSetHeaders()) } gotMutations := map[string]string{} @@ -286,6 +281,12 @@ func TestHandleRequestHeaders(t *testing.T) { if got := gotMutations[strings.ToLower(atunnel.TargetPortHeader)]; got != tc.expectedTargetPort { t.Errorf("target port mutation = %q, want %q", got, tc.expectedTargetPort) } + if got := gotMutations[strings.ToLower(atunnel.ActorNameHeader)]; got != testUUID { + t.Errorf("actor name mutation = %q, want %q", got, testUUID) + } + if got := gotMutations[strings.ToLower(atunnel.AtespaceHeader)]; got != "team-a" { + t.Errorf("atespace mutation = %q, want %q", got, "team-a") + } if got := dynamicMetadataTarget(res.DynamicMetadata); got != tc.expectedTarget { t.Errorf("invalid destination mapping found: %s, expected: %s", got, tc.expectedTarget) } @@ -316,7 +317,7 @@ func TestHandleRequestHeadersHandlesConnectMethod(t *testing.T) { h := New(clientMock, ParkedRequestConfig{}, nil) // CONNECT requests carry no :path; the request-target lives in :authority. - md := requestMetadata(t, authority, + md := requestMetadata(testUUID, "team-a", &corev3.HeaderValue{Key: ":authority", Value: authority}, &corev3.HeaderValue{Key: ":method", Value: "CONNECT"}, ) @@ -361,7 +362,7 @@ func TestHandleRequestHeaders_ParkingLotFull(t *testing.T) { } defer release(parkOutcomeServed) - md := requestMetadata(t, authority, + md := requestMetadata(testUUID, "team-a", &corev3.HeaderValue{Key: ":authority", Value: authority}, ) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index dca8a5d101..21d4e487be 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -894,39 +894,38 @@ func (x *XdsServer) buildMainInternalListener() *listenerv3.Listener { } } -// authorityFilterStateFilter captures :authority into -// extproc.AuthorityFilterStateKey filter state, so main_internal's HTTP leg -// can read it back across the internal-listener hop (see buildHcm, -// ingress.HandleRequestHeaders). buildConnectTerminateHCM and buildHcm's -// ingress listeners use it; main_internal itself must not, since that would -// capture the tunneled protocol's own, unrelated :authority instead. -func authorityFilterStateFilter() *hcmv3.HttpFilter { - return &hcmv3.HttpFilter{ - Name: "envoy.filters.http.set_filter_state", - ConfigType: &hcmv3.HttpFilter_TypedConfig{ - TypedConfig: newAny(&setfilterstatev3.Config{ - OnRequestHeaders: []*setfilterstatecommonv3.FilterStateValue{ - { - Key: &setfilterstatecommonv3.FilterStateValue_ObjectKey{ - ObjectKey: extproc.AuthorityFilterStateKey, - }, - // extproc.AuthorityFilterStateKey is a custom (non-well-known) - // key, so the generic string factory is required. - FactoryKey: "envoy.string", - Value: &setfilterstatecommonv3.FilterStateValue_FormatString{ - FormatString: &corev3.SubstitutionFormatString{ - Format: &corev3.SubstitutionFormatString_TextFormatSource{ - TextFormatSource: &corev3.DataSource{ - Specifier: &corev3.DataSource_InlineString{ - InlineString: "%REQ(:AUTHORITY)%", - }, - }, - }, +// actorIdentityFilterStateFilter captures the actor identity headers so +// main_internal can read them across the CONNECT internal-listener hop. +func actorIdentityFilterStateFilter() *hcmv3.HttpFilter { + values := make([]*setfilterstatecommonv3.FilterStateValue, 0, 2) + for _, identity := range []struct{ key, header string }{ + {extproc.ActorNameFilterStateKey, atunnel.ActorNameHeader}, + {extproc.AtespaceFilterStateKey, atunnel.AtespaceHeader}, + } { + values = append(values, &setfilterstatecommonv3.FilterStateValue{ + Key: &setfilterstatecommonv3.FilterStateValue_ObjectKey{ + ObjectKey: identity.key, + }, + FactoryKey: "envoy.string", + Value: &setfilterstatecommonv3.FilterStateValue_FormatString{ + FormatString: &corev3.SubstitutionFormatString{ + Format: &corev3.SubstitutionFormatString_TextFormatSource{ + TextFormatSource: &corev3.DataSource{ + Specifier: &corev3.DataSource_InlineString{ + InlineString: "%REQ(" + identity.header + ")%", }, }, - SharedWithUpstream: setfilterstatecommonv3.FilterStateValue_ONCE, }, }, + }, + SharedWithUpstream: setfilterstatecommonv3.FilterStateValue_ONCE, + }) + } + return &hcmv3.HttpFilter{ + Name: "envoy.filters.http.set_filter_state", + ConfigType: &hcmv3.HttpFilter_TypedConfig{ + TypedConfig: newAny(&setfilterstatev3.Config{ + OnRequestHeaders: values, }), }, } @@ -961,7 +960,7 @@ func (x *XdsServer) buildConnectTerminateHCM(statPrefix string) *anypb.Any { }, CodecType: hcmv3.HttpConnectionManager_AUTO, HttpFilters: []*hcmv3.HttpFilter{ - authorityFilterStateFilter(), + actorIdentityFilterStateFilter(), { Name: "envoy.filters.http.router", ConfigType: &hcmv3.HttpFilter_TypedConfig{ @@ -1016,12 +1015,9 @@ func buildConnectRoutes() *routev3.RouteConfiguration { } // buildHcm builds the HTTP ext_proc-fronted HCM shared by the ingress_http, -// ingress_https, and main_internal listeners. captureAuthority runs -// authorityFilterStateFilter to populate the actor's :authority for ingress -// listeners; main_internal passes false, since connect_terminate already -// shared the correct value and re-deriving it here would clobber it with -// the tunneled protocol's own, unrelated :authority. -func (x *XdsServer) buildHcm(statPrefix string, captureAuthority bool) *anypb.Any { +// ingress_https, and main_internal listeners. captureActorIdentity preserves +// the two actor identity headers across CONNECT re-entry. +func (x *XdsServer) buildHcm(statPrefix string, captureActorIdentity bool) *anypb.Any { extProcConfig := newAny(&extprocv3filter.ExternalProcessor{ GrpcService: &corev3.GrpcService{ TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{ @@ -1046,11 +1042,10 @@ func (x *XdsServer) buildHcm(statPrefix string, captureAuthority bool) *anypb.An RequestTrailerMode: extprocv3filter.ProcessingMode_SKIP, ResponseTrailerMode: extprocv3filter.ProcessingMode_SKIP, }, - // Passes the resolved actor's authority as a request attribute (see - // extproc.AuthorityFilterStateAttribute, ingress.HandleRequestHeaders) - // and lets the response write the resolved worker address into - // ingress.OriginalDstMetadataKey. - RequestAttributes: []string{extproc.AuthorityFilterStateAttribute}, + RequestAttributes: []string{ + extproc.ActorNameFilterStateAttribute, + extproc.AtespaceFilterStateAttribute, + }, MetadataOptions: &extprocv3filter.MetadataOptions{ ForwardingNamespaces: &extprocv3filter.MetadataOptions_MetadataNamespaces{ Untyped: []string{ingress.OriginalDstMetadataKey}, @@ -1066,8 +1061,8 @@ func (x *XdsServer) buildHcm(statPrefix string, captureAuthority bool) *anypb.An accessLogConfig := newAny(&streamaccesslogv3.StdoutAccessLog{}) httpFilters := []*hcmv3.HttpFilter{} - if captureAuthority { - httpFilters = append(httpFilters, authorityFilterStateFilter()) + if captureActorIdentity { + httpFilters = append(httpFilters, actorIdentityFilterStateFilter()) } httpFilters = append(httpFilters, &hcmv3.HttpFilter{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index ea5d71e92c..4cd4c909dd 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -38,6 +38,7 @@ import ( corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + setfilterstatev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/set_filter_state/v3" hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" @@ -46,10 +47,37 @@ import ( cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" "github.com/agent-substrate/substrate/internal/atunnel" ) +func TestActorIdentityFilterStateFilter(t *testing.T) { + filter := actorIdentityFilterStateFilter() + config := &setfilterstatev3.Config{} + if err := filter.GetTypedConfig().UnmarshalTo(config); err != nil { + t.Fatalf("unmarshal set_filter_state config: %v", err) + } + + want := map[string]string{ + extproc.ActorNameFilterStateKey: "%REQ(" + atunnel.ActorNameHeader + ")%", + extproc.AtespaceFilterStateKey: "%REQ(" + atunnel.AtespaceHeader + ")%", + } + if len(config.GetOnRequestHeaders()) != len(want) { + t.Fatalf("captured values = %d, want %d", len(config.GetOnRequestHeaders()), len(want)) + } + for _, value := range config.GetOnRequestHeaders() { + key := value.GetObjectKey() + format := value.GetFormatString().GetTextFormatSource().GetInlineString() + if format != want[key] { + t.Errorf("capture %q = %q, want %q", key, format, want[key]) + } + if strings.Contains(format, ":AUTHORITY") { + t.Errorf("capture %q still derives actor identity from authority", key) + } + } +} + // assertDualStackIngress checks an ingress listener keeps its 0.0.0.0 primary // and gains exactly one "::" socket on the same port. func assertDualStackIngress(t *testing.T, l *listenerv3.Listener, wantPort uint32) { diff --git a/demos/autoscaled-workerpool/README.md b/demos/autoscaled-workerpool/README.md index 5028c46a8f..04793ce269 100644 --- a/demos/autoscaled-workerpool/README.md +++ b/demos/autoscaled-workerpool/README.md @@ -132,7 +132,7 @@ In a separate terminal, send requests in a retry loop across all hosts to activa ```sh for attempt in {1..10}; do for i in {001..015}; do - curl -s -H "Host: c$i.ate-demo-autoscaled-workerpool.actors.resources.substrate.ate.dev" http://localhost:8000 >/dev/null + curl -s -H "X-Ate-Actor-Name: c$i" -H "X-Ate-Atespace: ate-demo-autoscaled-workerpool" http://localhost:8000 >/dev/null done sleep 2 done diff --git a/demos/counter/README.md b/demos/counter/README.md index 1fd7643c70..ebe02f9f7c 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -80,7 +80,10 @@ When you send an HTTP request through the router, Substrate automatically detect 1. Send an HTTP POST request to increment the counter: ```bash -curl -X POST -H "Host: my-counter-1.ate-demo-counter.actors.resources.substrate.ate.dev" http://localhost:8000 +curl -X POST \ + -H "X-Ate-Actor-Name: my-counter-1" \ + -H "X-Ate-Atespace: ate-demo-counter" \ + http://localhost:8000 ``` 2. Verify that the actor is now in a `RUNNING` state and assigned to a worker pod: @@ -117,7 +120,10 @@ through it to the named port. proxy behavior wouldn't do: ```bash -curl -p -x http://localhost:8001 http://my-counter-1.ate-demo-counter.actors.resources.substrate.ate.dev:9090/ +curl -p -x http://localhost:8001 \ + -H "X-Ate-Actor-Name: my-counter-1" \ + -H "X-Ate-Atespace: ate-demo-counter" \ + http://my-counter-1.ate-demo-counter.actors.resources.substrate.ate.dev:9090/ ``` This reaches the same actor's second listener and resumes it exactly like any diff --git a/demos/egress/README.md b/demos/egress/README.md index b2ece9a4b8..063bccaeba 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -120,6 +120,8 @@ kubectl ate resume actor egress-demo -a ate-demo-egress # wait for ACTOR_STATE kubectl -n ate-system port-forward service/atenet-router 8000:80 & curl -s -X POST http://localhost:8000/ \ -H 'Host: egress-demo.ate-demo-egress.actors.resources.substrate.ate.dev' \ + -H 'X-Ate-Actor-Name: egress-demo' \ + -H 'X-Ate-Atespace: demo' \ -H 'Content-Type: application/json' \ -d "{\"url\":\"http://${TARGET_IP}:80/\"}" ``` diff --git a/demos/egress/test-egress.sh b/demos/egress/test-egress.sh index 71bcf2a292..7ea28b83d0 100755 --- a/demos/egress/test-egress.sh +++ b/demos/egress/test-egress.sh @@ -102,6 +102,8 @@ ${K} -n ate-system port-forward service/atenet-router 18099:80 >/tmp/egress-pf.l PF=$!; sleep 4 CODE=$(curl -s -o /tmp/egress-body.txt -w '%{http_code}' -X POST http://localhost:18099/ \ -H "Host: ${ACTOR}.${ATESPACE}.actors.resources.substrate.ate.dev" \ + -H "X-Ate-Actor-Name: ${ACTOR}" \ + -H "X-Ate-Atespace: ${ATESPACE}" \ -H 'Content-Type: application/json' \ -d "{\"url\":\"http://${TARGET_IP}:80/\"}" || true) kill "${PF}" >/dev/null 2>&1 || true diff --git a/demos/jupyter/README.md b/demos/jupyter/README.md index 2f57755611..ceac907431 100644 --- a/demos/jupyter/README.md +++ b/demos/jupyter/README.md @@ -53,7 +53,7 @@ kubectl ate create actor jupyter-notebook -a ate-demo-jupyter --template-ref jup ### 2. Access Jupyter via the Proxy! -Substrate routes HTTP traffic using the `Host` header. To make this easy without modifying local `/etc/hosts` files, this demo includes a lightweight NGINX reverse proxy (`jupyter-proxy`) that automatically injects the proper `Host` header (`jupyter-notebook.ate-demo-jupyter.actors.resources.substrate.ate.dev`) and forwards traffic internally to the Substrate router. +Substrate routes HTTP traffic using `X-Ate-Actor-Name` and `X-Ate-Atespace`. This demo includes a lightweight NGINX reverse proxy (`jupyter-proxy`) that injects those routing headers and forwards traffic internally to the Substrate router. 1. **Port-forward the lightweight proxy to your local machine:** diff --git a/demos/jupyter/jupyter.yaml.tmpl b/demos/jupyter/jupyter.yaml.tmpl index 3651a9d8ab..203e87cb83 100644 --- a/demos/jupyter/jupyter.yaml.tmpl +++ b/demos/jupyter/jupyter.yaml.tmpl @@ -48,7 +48,8 @@ data: listen 80; location / { proxy_pass http://atenet-router.ate-system.svc.cluster.local; - proxy_set_header Host jupyter-notebook.ate-demo-jupyter.actors.resources.substrate.ate.dev; + proxy_set_header X-Ate-Actor-Name jupyter-notebook; + proxy_set_header X-Ate-Atespace ate-demo-jupyter; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; diff --git a/demos/multi-template/README.md b/demos/multi-template/README.md index 52e1820f60..bb3bf25aa3 100644 --- a/demos/multi-template/README.md +++ b/demos/multi-template/README.md @@ -65,11 +65,11 @@ When you send an HTTP request through the router, Substrate automatically detect ```bash # counter binary -curl -s -H "Host: c1.ate-demo-multi-template-counter.actors.resources.substrate.ate.dev" http://localhost:8000 +curl -s -H "X-Ate-Actor-Name: c1" -H "X-Ate-Atespace: ate-demo-multi-template-counter" http://localhost:8000 # -> hello from: | preserved memory count: 1 # fspersist binary -curl -s -H "Host: f1.ate-demo-multi-template-fspersist.actors.resources.substrate.ate.dev" http://localhost:8000 +curl -s -H "X-Ate-Actor-Name: f1" -H "X-Ate-Atespace: ate-demo-multi-template-fspersist" http://localhost:8000 # -> pod: # --- history --- # pod= | count=0 | time= @@ -87,7 +87,7 @@ preserves that state across the snapshot/restore cycle: ```bash kubectl ate suspend actor f1 -a ate-demo-multi-template-fspersist -curl -s -H "Host: f1.ate-demo-multi-template-fspersist.actors.resources.substrate.ate.dev" http://localhost:8000 # history persists; count keeps climbing +curl -s -H "X-Ate-Actor-Name: f1" -H "X-Ate-Atespace: ate-demo-multi-template-fspersist" http://localhost:8000 # history persists; count keeps climbing ``` ## How to Uninstall diff --git a/demos/parking/README.md b/demos/parking/README.md index 00bb10be43..f9d32bd896 100644 --- a/demos/parking/README.md +++ b/demos/parking/README.md @@ -70,8 +70,8 @@ Parking is **on by default** (`--parked-request-budget=5s`, Fill both workers by requesting two actors, leaving them `RUNNING`: ```bash -curl -s -H "Host: p1.ate-demo-parking.actors.resources.substrate.ate.dev" http://localhost:8000 -curl -s -H "Host: p2.ate-demo-parking.actors.resources.substrate.ate.dev" http://localhost:8000 +curl -s -H "X-Ate-Actor-Name: p1" -H "X-Ate-Atespace: ate-demo-parking" http://localhost:8000 +curl -s -H "X-Ate-Actor-Name: p2" -H "X-Ate-Atespace: ate-demo-parking" http://localhost:8000 kubectl ate get workers # both workers are now bound to p1 and p2 kubectl ate get actors # p1,p2 RUNNING; p3,p4 SUSPENDED @@ -82,7 +82,7 @@ the `curl` hangs while the router retries the resume: ```bash curl -s -w '\n-> HTTP %{http_code} in %{time_total}s\n' \ - -H "Host: p3.ate-demo-parking.actors.resources.substrate.ate.dev" http://localhost:8000 + -H "X-Ate-Actor-Name: p3" -H "X-Ate-Atespace: ate-demo-parking" http://localhost:8000 ``` While that is hanging, in a **second terminal** free a worker by suspending p1 diff --git a/demos/parking/load.sh b/demos/parking/load.sh index b76a79958c..25147241f8 100755 --- a/demos/parking/load.sh +++ b/demos/parking/load.sh @@ -107,7 +107,9 @@ worker() { while [[ $(date +%s) -lt ${deadline} ]]; do # %{http_code} lets us tally outcomes; %{time_total} reveals parking waits. curl -s -o /dev/null -w '%{http_code} %{time_total}\n' \ - -H "Host: ${host}" "${ROUTER}" >>"${log}" 2>/dev/null + -H "Host: ${host}" \ + -H "X-Ate-Actor-Name: ${actor}" \ + -H "X-Ate-Atespace: ${ATESPACE}" "${ROUTER}" >>"${log}" 2>/dev/null # Free the worker so a parked competitor can proceed (simulate going idle). kubectl ate suspend actor "${actor}" --atespace "${ATESPACE}" >/dev/null 2>&1 || true done diff --git a/demos/sandbox/client/main.go b/demos/sandbox/client/main.go index 11269e0246..b1d76d48ff 100644 --- a/demos/sandbox/client/main.go +++ b/demos/sandbox/client/main.go @@ -28,6 +28,7 @@ import ( "strings" "syscall" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/pflag" @@ -185,6 +186,8 @@ func runCommand(ctx context.Context, atenetAddr string, actorRef resources.Actor return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Content-Type", "application/json") + req.Header.Set(atunnel.ActorNameHeader, actorRef.Name) + req.Header.Set(atunnel.AtespaceHeader, actorRef.Atespace) req.Host = resources.ActorDNSName(actorRef) resp, err := http.DefaultClient.Do(req) diff --git a/docs/architecture.md b/docs/architecture.md index a6dcbbde73..2f4dd09508 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -338,8 +338,10 @@ Handles actor-aware routing and automatic re-animation. * **Ingress Routing**: `atenet-router` runs Envoy with an `ext_proc` external processor and accepts HTTP traffic for the Actor DNS suffix. The ext_proc - extracts the Actor name and Atespace from the `Host` header and calls the - Control Plane to resume the Actor and resolve its current worker assignment. + reads the Actor name from `X-Ate-Actor-Name` and the Atespace from + `X-Ate-Atespace`, then calls the Control Plane to resume the Actor and + resolve its current worker assignment. `Host` remains application authority + and is not used as actor identity. * **Worker Tunnel**: After resolving the assignment, `atenet-router` opens an authenticated TLS tunnel to the worker's `atunnel` listener on port 443. @@ -378,7 +380,7 @@ sequenceDiagram Client->>DNS: resolve actor DNS name DNS-->>Client: ingress gateway address - Client->>Gateway: HTTP request (Host = actor) + Client->>Gateway: HTTP request (X-Ate-Actor-Name, X-Ate-Atespace) Gateway->>API: ResumeActor(atespace, actor name) API->>Atelet: Restore Store-->>Atelet: download snapshot diff --git a/docs/threat-model.md b/docs/threat-model.md index 41fc2bc2f7..754215e6a9 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -40,7 +40,7 @@ Substrate is an early, fast moving product. It is full of debate and subject to * **Actor:** The core compute primitive, gets scheduled to/from worker via Run for cold start and Resume for snapshot resume. * **Actor Network:** `ateom` creates a private point-to-point veth network for the active Actor inside a worker Pod. The Actor is not served directly from the worker Pod's port 80; ingress enters through `atunnel`'s authenticated listener on port 443. * **Actor DNS:** Each Actor gets a DNS name like `..actors.resources.substrate.ate.dev`. Substrate runs a custom CoreDNS instance that returns the `atenet-router` Service IP for A record queries matching the Actor DNS pattern. A controller keeps this target current and configures kube-dns with a stub domain for `actors.resources.substrate.ate.dev`, enabling traditional Kubernetes Pods to resolve Actor names. -* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. The ext_proc extracts the Actor name and Atespace from the HTTP `Host` header, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. The router then connects with mTLS to `atunnel` on worker port 443; `atunnel` validates the router identity and forwards traffic only to the Actor currently assigned to that worker. +* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. The ext_proc reads the Actor name and Atespace from `X-Ate-Actor-Name` and `X-Ate-Atespace`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites both identity headers before forwarding. The router then connects with mTLS to `atunnel`; `atunnel` validates the router identity and authorizes the header pair against the Actor currently assigned to that worker. * **Object Storage:** Used to store actor snapshots. * **Filesystem support:** Container local filesystem is saved in snapshots, future integrations likely to include networked storage. * **Substrate Database:** PostgreSQL. diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress.go index c083895f65..d580c81481 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress.go @@ -44,11 +44,10 @@ const ( // StaleAssignmentHeader distinguishes an atunnel routing rejection from a // 421 returned by the actor application itself. StaleAssignmentHeader = "X-Ate-Assignment-Stale" - // OriginalHostHeader carries the actor authority across router dataplanes - // that must use :authority to select the worker as their dynamic backend. - // atunnel only accepts mTLS-authenticated router clients, and the router's - // ext_proc server overwrites this header before every request. - OriginalHostHeader = "X-Ate-Original-Host" + // ActorNameHeader and AtespaceHeader identify the actor selected by the + // trusted ingress router. + ActorNameHeader = "X-Ate-Actor-Name" + AtespaceHeader = "X-Ate-Atespace" // TargetPortHeader carries the port to reach on the actor: the CONNECT // :authority's port for arbitrary-port ingress, or the default 80 @@ -130,13 +129,14 @@ func NewServer(cfg Config) (*Server, error) { proxy := &httputil.ReverseProxy{ Rewrite: func(pr *httputil.ProxyRequest) { pr.SetURL(cfg.Upstream) - // Retain the actor's stable mesh hostname rather than the - // upstream's, matching NewSingleHostReverseProxy's default - // behavior. + // Retain the client's Host rather than the upstream's, matching + // NewSingleHostReverseProxy's default behavior. pr.Out.Host = pr.In.Host port := pr.In.Header.Get(TargetPortHeader) pr.Out.Header.Del(TargetPortHeader) + pr.Out.Header.Del(ActorNameHeader) + pr.Out.Header.Del(AtespaceHeader) if p, ok := ParsePort(port); ok { pr.Out.URL.Host = net.JoinHostPort(cfg.Upstream.Hostname(), strconv.Itoa(p)) } @@ -465,7 +465,7 @@ func (s *Server) closeIdleUpstreamConnections() { } } -// ServeHTTP validates the actor hostname on every request before proxying it. +// ServeHTTP validates the actor identity headers on every request before proxying it. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, requestCtx, release, ok := s.authorize(r) if !ok { @@ -474,32 +474,16 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } defer release() - // Do not expose the router-only routing header to actor code. Restore Host - // so dataplanes that route dynamically on worker IP still give the actor its - // stable actor DNS name. - actorHost := r.Header.Get(OriginalHostHeader) - if actorHost == "" { - actorHost = r.Host - } - r.Header.Del(OriginalHostHeader) - r.Host = actorHost - - // ReverseProxy changes the URL destination but intentionally retains Host, - // allowing the actor application to observe its stable actor DNS name. + // ReverseProxy changes the URL destination but intentionally retains Host. s.proxy.ServeHTTP(w, r.WithContext(requestCtx)) } func (s *Server) authorize(r *http.Request) (resources.ActorRef, context.Context, func(), bool) { - actorHost := r.Header.Get(OriginalHostHeader) - if actorHost == "" { - actorHost = r.Host - } - host, err := requestHostname(actorHost) - if err != nil { - return resources.ActorRef{}, nil, nil, false + ref := resources.ActorRef{ + Name: r.Header.Get(ActorNameHeader), + Atespace: r.Header.Get(AtespaceHeader), } - ref, err := resources.ParseActorDNSName(host) - if err != nil { + if !resources.IsValidResourceName(ref.Name) || !resources.IsValidResourceName(ref.Atespace) { return resources.ActorRef{}, nil, nil, false } @@ -525,22 +509,3 @@ func (s *Server) reject(w http.ResponseWriter) { w.Header().Set(StaleAssignmentHeader, "true") http.Error(w, "misdirected request", http.StatusMisdirectedRequest) } - -func requestHostname(hostport string) (string, error) { - if hostport == "" { - return "", fmt.Errorf("empty host") - } - host := hostport - if strings.Contains(hostport, ":") { - var port string - var err error - host, port, err = net.SplitHostPort(hostport) - if err != nil { - return "", fmt.Errorf("invalid host %q: %w", hostport, err) - } - if _, ok := ParsePort(port); !ok { - return "", fmt.Errorf("invalid port in host %q", hostport) - } - } - return strings.ToLower(host), nil -} diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index a93fd7cc6d..6872c5d1f6 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -129,7 +129,7 @@ func TestRelayIngressCancellationClosesBothSides(t *testing.T) { } func TestServeHTTP(t *testing.T) { - upstreamHost := make(chan string, 4) + upstreamHost := make(chan string, 3) upstreamURL, err := url.Parse("http://actor.internal:80") if err != nil { t.Fatal(err) @@ -149,28 +149,27 @@ func TestServeHTTP(t *testing.T) { } tests := []struct { - name string - host string - originalHost string - wantStatus int + name string + host string + actorName string + atespace string + wantStatus int }{ - {name: "active actor", host: "actor-1.team-a.actors.resources.substrate.ate.dev", wantStatus: http.StatusNoContent}, - {name: "active actor with port", host: "actor-1.team-a.actors.resources.substrate.ate.dev:443", wantStatus: http.StatusNoContent}, - {name: "DNS case insensitive", host: "ACTOR-1.TEAM-A.ACTORS.RESOURCES.SUBSTRATE.ATE.DEV", wantStatus: http.StatusNoContent}, - {name: "router original host", host: "10.0.0.52:443", originalHost: "actor-1.team-a.actors.resources.substrate.ate.dev", wantStatus: http.StatusNoContent}, - {name: "wrong actor", host: "actor-2.team-a.actors.resources.substrate.ate.dev", wantStatus: http.StatusMisdirectedRequest}, - {name: "wrong atespace", host: "actor-1.team-b.actors.resources.substrate.ate.dev", wantStatus: http.StatusMisdirectedRequest}, - {name: "suffix confusion", host: "actor-1.team-a.actors.resources.substrate.ate.dev.example.com", wantStatus: http.StatusMisdirectedRequest}, - {name: "malformed port", host: "actor-1.team-a.actors.resources.substrate.ate.dev:nope", wantStatus: http.StatusMisdirectedRequest}, - {name: "empty", wantStatus: http.StatusMisdirectedRequest}, + {name: "active actor", host: "client.example", actorName: "actor-1", atespace: "team-a", wantStatus: http.StatusNoContent}, + {name: "host does not identify actor", host: "actor-2.team-b.example", actorName: "actor-1", atespace: "team-a", wantStatus: http.StatusNoContent}, + {name: "empty host", actorName: "actor-1", atespace: "team-a", wantStatus: http.StatusNoContent}, + {name: "wrong actor", actorName: "actor-2", atespace: "team-a", wantStatus: http.StatusMisdirectedRequest}, + {name: "wrong atespace", actorName: "actor-1", atespace: "team-b", wantStatus: http.StatusMisdirectedRequest}, + {name: "missing actor name", atespace: "team-a", wantStatus: http.StatusMisdirectedRequest}, + {name: "missing atespace", actorName: "actor-1", wantStatus: http.StatusMisdirectedRequest}, + {name: "invalid actor name", actorName: "INVALID", atespace: "team-a", wantStatus: http.StatusMisdirectedRequest}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "https://worker/hello", nil) req.Host = tt.host - if tt.originalHost != "" { - req.Header.Set(OriginalHostHeader, tt.originalHost) - } + req.Header.Set(ActorNameHeader, tt.actorName) + req.Header.Set(AtespaceHeader, tt.atespace) rec := httptest.NewRecorder() s.ServeHTTP(rec, req) if rec.Code != tt.wantStatus { @@ -182,9 +181,9 @@ func TestServeHTTP(t *testing.T) { }) } - for range 4 { - if got := <-upstreamHost; got != "actor-1.team-a.actors.resources.substrate.ate.dev" && got != "actor-1.team-a.actors.resources.substrate.ate.dev:443" && got != "ACTOR-1.TEAM-A.ACTORS.RESOURCES.SUBSTRATE.ATE.DEV" { - t.Errorf("upstream Host = %q", got) + for _, want := range []string{"client.example", "actor-2.team-b.example", ""} { + if got := <-upstreamHost; got != want { + t.Errorf("upstream Host = %q, want %q", got, want) } } } @@ -196,11 +195,11 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { } s := newTestServer(t, upstreamURL) - var gotURLHost, gotHost, gotHeader string + var gotURLHost, gotHost http.Header s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { - gotURLHost = r.URL.Host - gotHost = r.Host - gotHeader = r.Header.Get(TargetPortHeader) + gotURLHost = http.Header{"Host": []string{r.URL.Host}} + gotHost = r.Header.Clone() + gotHost.Set("Host", r.Host) return &http.Response{ StatusCode: http.StatusNoContent, Header: make(http.Header), @@ -224,7 +223,9 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "https://worker/hello", nil) - req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Host = "client.example" + req.Header.Set(ActorNameHeader, "actor-1") + req.Header.Set(AtespaceHeader, "team-a") if tt.targetPort != "" { req.Header.Set(TargetPortHeader, tt.targetPort) } @@ -233,14 +234,16 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { if rec.Code != http.StatusNoContent { t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent) } - if gotURLHost != tt.wantDialedHost { - t.Errorf("dialed host = %q, want %q", gotURLHost, tt.wantDialedHost) + if gotURLHost.Get("Host") != tt.wantDialedHost { + t.Errorf("dialed host = %q, want %q", gotURLHost.Get("Host"), tt.wantDialedHost) } - if gotHost != "actor-1.team-a.actors.resources.substrate.ate.dev" { - t.Errorf("Host header changed to %q; the actor should see its stable mesh hostname", gotHost) + if gotHost.Get("Host") != "client.example" { + t.Errorf("Host header changed to %q", gotHost.Get("Host")) } - if gotHeader != "" { - t.Errorf("%s leaked to the actor upstream: %q", TargetPortHeader, gotHeader) + for _, header := range []string{TargetPortHeader, ActorNameHeader, AtespaceHeader} { + if got := gotHost.Get(header); got != "" { + t.Errorf("%s leaked to the actor upstream: %q", header, got) + } } }) } @@ -264,13 +267,18 @@ func TestServeConnectHTTPValidatesMethodAndAuthority(t *testing.T) { }{ {name: "rejects non CONNECT", method: http.MethodGet, host: "actor-1.team-a.actors.resources.substrate.ate.dev:9090", want: http.StatusMethodNotAllowed}, {name: "requires authority port", method: http.MethodConnect, host: "actor-1.team-a.actors.resources.substrate.ate.dev", want: http.StatusBadRequest}, - {name: "rejects invalid authority port", method: http.MethodConnect, host: "actor-1.team-a.actors.resources.substrate.ate.dev:70000", want: http.StatusMisdirectedRequest}, + {name: "rejects invalid authority port", method: http.MethodConnect, host: "actor-1.team-a.actors.resources.substrate.ate.dev:70000", want: http.StatusBadRequest}, {name: "rejects inactive actor", method: http.MethodConnect, host: "actor-2.team-a.actors.resources.substrate.ate.dev:9090", want: http.StatusMisdirectedRequest}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(tt.method, "https://worker/", nil) req.Host = tt.host + req.Header.Set(ActorNameHeader, "actor-1") + req.Header.Set(AtespaceHeader, "team-a") + if tt.name == "rejects inactive actor" { + req.Header.Set(ActorNameHeader, "actor-2") + } rec := httptest.NewRecorder() s.ServeConnectHTTP(rec, req) if rec.Code != tt.want { @@ -316,6 +324,8 @@ func TestDeactivateClosesIdleUpstreamConnections(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "https://worker/", nil) req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Header.Set(ActorNameHeader, "actor-1") + req.Header.Set(AtespaceHeader, "team-a") rec := httptest.NewRecorder() s.ServeHTTP(rec, req) if rec.Code != http.StatusNoContent { @@ -337,6 +347,8 @@ func TestInactive(t *testing.T) { s := newTestServer(t, upstream) req := httptest.NewRequest(http.MethodGet, "https://worker/", nil) req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Header.Set(ActorNameHeader, "actor-1") + req.Header.Set(AtespaceHeader, "team-a") for _, phase := range []string{"before activation", "after deactivation"} { t.Run(phase, func(t *testing.T) { @@ -550,6 +562,8 @@ func TestDeactivateCancelsInflightRequest(t *testing.T) { defer close(done) req := httptest.NewRequest(http.MethodGet, "https://worker/", nil) req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Header.Set(ActorNameHeader, "actor-1") + req.Header.Set(AtespaceHeader, "team-a") s.ServeHTTP(httptest.NewRecorder(), req) }() <-started diff --git a/internal/benchmarking/boomer/glutton/durdir.go b/internal/benchmarking/boomer/glutton/durdir.go index f526be6a71..26f6a5ccc6 100644 --- a/internal/benchmarking/boomer/glutton/durdir.go +++ b/internal/benchmarking/boomer/glutton/durdir.go @@ -28,6 +28,7 @@ import ( "sync" "time" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" @@ -415,6 +416,8 @@ func (u *durDirUser) httpProtoCall(ctx context.Context, metricName, route string } httpReq.Host = u.hostHeader httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set(atunnel.ActorNameHeader, u.actorName) + httpReq.Header.Set(atunnel.AtespaceHeader, u.cfg.Atespace) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) start := time.Now() diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index 5ade96de2a..0cae4996f7 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -29,6 +29,7 @@ import ( "sync" "time" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" @@ -300,6 +301,8 @@ func (u *gluttonUser) ping(ctx context.Context) { } httpReq.Host = u.hostHeader httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set(atunnel.ActorNameHeader, u.actorName) + httpReq.Header.Set(atunnel.AtespaceHeader, u.cfg.Atespace) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) start := time.Now() @@ -460,6 +463,8 @@ func (u *gluttonUser) postProto(ctx context.Context, path string, req, resp prot } httpReq.Host = u.hostHeader httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set(atunnel.ActorNameHeader, u.actorName) + httpReq.Header.Set(atunnel.AtespaceHeader, u.cfg.Atespace) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) httpResp, err := u.cfg.HTTPClient.Do(httpReq) diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 7a9e536079..23edbf1996 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -28,6 +28,7 @@ import ( "sync" "time" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/ateclient" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/internal/resources" @@ -106,8 +107,7 @@ func (c *RouterClient) BaseURL() string { return c.baseURL } -// Get issues GET path to actor through the router, setting the actor's DNS Host -// so the router routes (and resumes) it. The caller must close the body. +// Get issues GET path to actor through the router. The caller must close the body. func (c *RouterClient) Get(ctx context.Context, actorRef resources.ActorRef, path string) (*http.Response, error) { return c.request(ctx, http.MethodGet, actorRef, path, nil) } @@ -126,7 +126,8 @@ func (c *RouterClient) request(ctx context.Context, method string, actorRef reso if method == http.MethodPost { req.Header.Set("Content-Type", "application/json") } - // The router routes on the Host/:authority, not a header. + req.Header.Set(atunnel.ActorNameHeader, actorRef.Name) + req.Header.Set(atunnel.AtespaceHeader, actorRef.Atespace) req.Host = resources.ActorDNSName(actorRef) return c.http.Do(req) } @@ -154,6 +155,10 @@ func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, Method: http.MethodConnect, URL: &url.URL{Host: destination}, Host: destination, + Header: http.Header{ + atunnel.ActorNameHeader: []string{actorRef.Name}, + atunnel.AtespaceHeader: []string{actorRef.Atespace}, + }, } if err := req.Write(rawConn); err != nil { _ = rawConn.Close() diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 38e6bea07b..4a578234ba 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/ateclient" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" @@ -1287,6 +1288,8 @@ func callActorPathOnce(t *testing.T, actorRef resources.ActorRef, method, path s if err != nil { return "", fmt.Errorf("failed to create request: %w", err) } + reqHttp.Header.Set(atunnel.ActorNameHeader, actorRef.Name) + reqHttp.Header.Set(atunnel.AtespaceHeader, actorRef.Atespace) reqHttp.Host = resources.ActorDNSName(actorRef) httpClient := &http.Client{Timeout: 15 * time.Second} diff --git a/internal/e2e/suites/networking/arbitraryport_test.go b/internal/e2e/suites/networking/arbitraryport_test.go index 9217b8fb51..56e2f84cbd 100644 --- a/internal/e2e/suites/networking/arbitraryport_test.go +++ b/internal/e2e/suites/networking/arbitraryport_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" ) @@ -106,7 +107,8 @@ func TestActorArbitraryPortAccess(t *testing.T) { defer conn.Close() conn.SetDeadline(time.Now().Add(10 * time.Second)) - if _, err := conn.Write([]byte("GET / HTTP/1.1\r\nHost: " + resources.ActorDNSName(actorRef) + "\r\nConnection: close\r\n\r\n")); err != nil { + if _, err := fmt.Fprintf(conn, "GET / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: %s\r\nConnection: close\r\n\r\n", + resources.ActorDNSName(actorRef), atunnel.ActorNameHeader, actorRef.Name, atunnel.AtespaceHeader, actorRef.Atespace); err != nil { t.Fatalf("writing tunneled request: %v", err) } resp, err := http.ReadResponse(bufio.NewReader(conn), nil) From e6b1ae4f899b24927de2a2ea86ddc47091b69d58 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Mon, 31 Aug 2026 08:48:39 -0500 Subject: [PATCH 02/10] Clarify routing vs. identity terminology and add tests for case sensitivity in extproc Signed-off-by: Keith Mattix II --- benchmarking/locust/tests/counter_demo.py | 2 +- benchmarking/nighthawk-ingress/README.md | 4 +- cmd/atenet/internal/router/extproc/handler.go | 2 +- .../internal/router/extproc/metadata_test.go | 18 +++++++++ cmd/atenet/internal/router/ingress/ingress.go | 15 ++++---- .../internal/router/ingress/ingress_test.go | 31 +++++++++++++-- cmd/atenet/internal/router/xds.go | 22 +++++------ cmd/atenet/internal/router/xds_test.go | 10 ++--- demos/egress/README.md | 4 +- demos/jupyter/README.md | 4 +- docs/architecture.md | 2 +- docs/threat-model.md | 38 +++++++++---------- internal/atunnel/ingress.go | 13 ++++--- internal/atunnel/ingress_test.go | 23 +++++++---- internal/e2e/router_client.go | 2 +- 15 files changed, 120 insertions(+), 70 deletions(-) diff --git a/benchmarking/locust/tests/counter_demo.py b/benchmarking/locust/tests/counter_demo.py index f01a36aec9..079d7424c5 100644 --- a/benchmarking/locust/tests/counter_demo.py +++ b/benchmarking/locust/tests/counter_demo.py @@ -48,7 +48,7 @@ tracer = get_tracer(__name__) -# Atenet router fronts all actor traffic. The actor identity headers select +# Atenet router fronts all actor traffic. The actor routing headers select # the actor whose current worker pod the router resolves. ROUTER_URL = "http://atenet-router.ate-system.svc.cluster.local" ACTOR_DOMAIN = "actors.resources.substrate.ate.dev" diff --git a/benchmarking/nighthawk-ingress/README.md b/benchmarking/nighthawk-ingress/README.md index 75d1aa0301..923d7c75d7 100644 --- a/benchmarking/nighthawk-ingress/README.md +++ b/benchmarking/nighthawk-ingress/README.md @@ -84,9 +84,9 @@ One Kubernetes Job per `type: nighthawk-ingress` tests.yaml entry, driven by down afterwards, so nothing leaks. 2. **Create + warm actors.** The runner creates one glutton actor per WorkerPool worker (the entry's `workerCount`) via ateapi and POSTs - `/ping` through the router with each actor's identity headers until it + `/ping` through the router with each actor's routing headers until it answers 200. -3. **Adaptive search.** Open-loop traffic with the actor identity headers rotated +3. **Adaptive search.** Open-loop traffic with the actor routing headers rotated across all actors; `clientConcurrency` event loops (default 16, decoupled from `envoyCpu`) and large per-loop pools so the harness is never the bottleneck. Exponential ramp → binary search → a 60s diff --git a/cmd/atenet/internal/router/extproc/handler.go b/cmd/atenet/internal/router/extproc/handler.go index a5622252e8..04844d1d21 100644 --- a/cmd/atenet/internal/router/extproc/handler.go +++ b/cmd/atenet/internal/router/extproc/handler.go @@ -37,7 +37,7 @@ type Handler interface { // A returned error denies the request: a *ReqError carries the status code // and client-safe body to answer with, anything else becomes a 500. The // Result is read even when an error is returned, so a handler that got far - // enough to learn the metric attributes (template identity, resume outcome) + // enough to learn the metric attributes (template coordinates, resume outcome) // should still fill them in. HandleRequestHeaders(ctx context.Context, md *RequestMetadata) (Result, error) } diff --git a/cmd/atenet/internal/router/extproc/metadata_test.go b/cmd/atenet/internal/router/extproc/metadata_test.go index 03648f6a44..9cc2086b2e 100644 --- a/cmd/atenet/internal/router/extproc/metadata_test.go +++ b/cmd/atenet/internal/router/extproc/metadata_test.go @@ -141,3 +141,21 @@ func TestRequestMetadataAttribute(t *testing.T) { t.Errorf("Attribute() for a missing key = %q, want \"\"", got) } } + +func TestRequestMetadataHeaderIsCaseInsensitive(t *testing.T) { + md := NewRequestMetadata([]*corev3.HeaderValue{ + {Key: "X-ATE-Actor-Name", Value: "actor-1"}, + {Key: "x-ate-ATESPACE", Value: "team-a"}, + }, nil) + + for name, want := range map[string]string{ + "x-ate-actor-name": "actor-1", + "X-Ate-Actor-Name": "actor-1", + "x-ate-atespace": "team-a", + "X-ATE-ATESPACE": "team-a", + } { + if got := md.Header(name); got != want { + t.Errorf("Header(%q) = %q, want %q", name, got, want) + } + } +} diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 8a0ba908d6..064ff1f6db 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -18,9 +18,8 @@ // saturated), and points the dataplane at the worker that ends up hosting it. // // Everything reaching this handler is unauthenticated client input. The -// opposite trust model — an actor identity carried by a CA-signed client -// certificate — belongs to the sibling egress package, and the two are kept -// apart deliberately. +// certificate authentication used for egress belongs to the sibling egress +// package, and the two are kept apart deliberately. package ingress import ( @@ -86,11 +85,11 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM defer span.End() actorRef := resources.ActorRef{ - Name: identityValue(md, atunnel.ActorNameHeader, extproc.ActorNameFilterStateAttribute), - Atespace: identityValue(md, atunnel.AtespaceHeader, extproc.AtespaceFilterStateAttribute), + Name: routingValue(md, atunnel.ActorNameHeader, extproc.ActorNameFilterStateAttribute), + Atespace: routingValue(md, atunnel.AtespaceHeader, extproc.AtespaceFilterStateAttribute), } if !resources.IsValidResourceName(actorRef.Name) || !resources.IsValidResourceName(actorRef.Atespace) { - return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_NotFound, "invalid actor identity") + return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_NotFound, "invalid actor reference") } // CONNECT traffic can name a port other than defaultActorPort in the @@ -119,7 +118,7 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM return extproc.Result{Resume: string(resumeOutcome)}, mapResumeError(actorRef, err) } - // Actor template identity, used as low-cardinality route-latency metric + // Actor template coordinates, used as low-cardinality route-latency metric // attributes. res := extproc.Result{ TemplateAtespace: actor.GetActorTemplate().GetAtespace(), @@ -184,7 +183,7 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM return res, nil } -func identityValue(md *extproc.RequestMetadata, header, attribute string) string { +func routingValue(md *extproc.RequestMetadata, header, attribute string) string { if value := md.Header(header); value != "" { return value } diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index 038ad34911..ee1f311686 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -53,6 +53,31 @@ func requestMetadata(actorName, atespace string, headers ...*corev3.HeaderValue) return extproc.NewRequestMetadata(headers, nil) } +func TestHandleRequestHeadersAcceptsMixedCaseRoutingHeaders(t *testing.T) { + clientMock := &mockClient{ + resumeFn: func(_ context.Context, in *ateapipb.ResumeActorRequest, _ ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + if got, want := in.GetActor().GetName(), "actor-1"; got != want { + t.Errorf("actor name = %q, want %q", got, want) + } + if got, want := in.GetActor().GetAtespace(), "team-a"; got != want { + t.Errorf("atespace = %q, want %q", got, want) + } + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}, + }}, nil + }, + } + h := New(clientMock, ParkedRequestConfig{}, nil) + md := extproc.NewRequestMetadata([]*corev3.HeaderValue{ + {Key: "X-ATE-Actor-Name", Value: "actor-1"}, + {Key: "x-ate-ATESPACE", Value: "team-a"}, + }, nil) + + if _, err := h.HandleRequestHeaders(context.Background(), md); err != nil { + t.Fatalf("HandleRequestHeaders() error = %v", err) + } +} + // dynamicMetadataTarget extracts the resolved worker address // HandleRequestHeaders reports via OriginalDstMetadataKey/OriginalDstAddressKey. func dynamicMetadataTarget(dynamicMetadata *structpb.Struct) string { @@ -138,7 +163,7 @@ func TestHandleRequestHeaders(t *testing.T) { atespace: "team-a", authority: "invalid-host.com", expectErr: true, - expectedErrStr: `invalid actor identity`, + expectedErrStr: `invalid actor reference`, expectedStatus: envoy_type.StatusCode_NotFound, }, { @@ -194,7 +219,7 @@ func TestHandleRequestHeaders(t *testing.T) { expectedStatus: envoy_type.StatusCode_InternalServerError, }, { - name: "successful resume ignores host for actor identity", + name: "successful resume ignores host for actor routing", authority: "application.example", resumeResp: &ateapipb.ResumeActorResponse{ Actor: &ateapipb.Actor{ @@ -271,7 +296,7 @@ func TestHandleRequestHeaders(t *testing.T) { mutation := res.Response.GetResponse().GetHeaderMutation() if len(mutation.GetSetHeaders()) != 3 { - t.Fatalf("expected actor identity and target port headers, found: %v", mutation.GetSetHeaders()) + t.Fatalf("expected actor routing and target port headers, found: %v", mutation.GetSetHeaders()) } gotMutations := map[string]string{} diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 21d4e487be..465212ef74 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -894,17 +894,17 @@ func (x *XdsServer) buildMainInternalListener() *listenerv3.Listener { } } -// actorIdentityFilterStateFilter captures the actor identity headers so +// actorRoutingFilterStateFilter captures the actor routing headers so // main_internal can read them across the CONNECT internal-listener hop. -func actorIdentityFilterStateFilter() *hcmv3.HttpFilter { +func actorRoutingFilterStateFilter() *hcmv3.HttpFilter { values := make([]*setfilterstatecommonv3.FilterStateValue, 0, 2) - for _, identity := range []struct{ key, header string }{ + for _, routingField := range []struct{ key, header string }{ {extproc.ActorNameFilterStateKey, atunnel.ActorNameHeader}, {extproc.AtespaceFilterStateKey, atunnel.AtespaceHeader}, } { values = append(values, &setfilterstatecommonv3.FilterStateValue{ Key: &setfilterstatecommonv3.FilterStateValue_ObjectKey{ - ObjectKey: identity.key, + ObjectKey: routingField.key, }, FactoryKey: "envoy.string", Value: &setfilterstatecommonv3.FilterStateValue_FormatString{ @@ -912,7 +912,7 @@ func actorIdentityFilterStateFilter() *hcmv3.HttpFilter { Format: &corev3.SubstitutionFormatString_TextFormatSource{ TextFormatSource: &corev3.DataSource{ Specifier: &corev3.DataSource_InlineString{ - InlineString: "%REQ(" + identity.header + ")%", + InlineString: "%REQ(" + routingField.header + ")%", }, }, }, @@ -960,7 +960,7 @@ func (x *XdsServer) buildConnectTerminateHCM(statPrefix string) *anypb.Any { }, CodecType: hcmv3.HttpConnectionManager_AUTO, HttpFilters: []*hcmv3.HttpFilter{ - actorIdentityFilterStateFilter(), + actorRoutingFilterStateFilter(), { Name: "envoy.filters.http.router", ConfigType: &hcmv3.HttpFilter_TypedConfig{ @@ -1015,9 +1015,9 @@ func buildConnectRoutes() *routev3.RouteConfiguration { } // buildHcm builds the HTTP ext_proc-fronted HCM shared by the ingress_http, -// ingress_https, and main_internal listeners. captureActorIdentity preserves -// the two actor identity headers across CONNECT re-entry. -func (x *XdsServer) buildHcm(statPrefix string, captureActorIdentity bool) *anypb.Any { +// ingress_https, and main_internal listeners. captureActorRouting preserves +// the two actor routing headers across CONNECT re-entry. +func (x *XdsServer) buildHcm(statPrefix string, captureActorRouting bool) *anypb.Any { extProcConfig := newAny(&extprocv3filter.ExternalProcessor{ GrpcService: &corev3.GrpcService{ TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{ @@ -1061,8 +1061,8 @@ func (x *XdsServer) buildHcm(statPrefix string, captureActorIdentity bool) *anyp accessLogConfig := newAny(&streamaccesslogv3.StdoutAccessLog{}) httpFilters := []*hcmv3.HttpFilter{} - if captureActorIdentity { - httpFilters = append(httpFilters, actorIdentityFilterStateFilter()) + if captureActorRouting { + httpFilters = append(httpFilters, actorRoutingFilterStateFilter()) } httpFilters = append(httpFilters, &hcmv3.HttpFilter{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 4cd4c909dd..11ce116c34 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -52,16 +52,16 @@ import ( "github.com/agent-substrate/substrate/internal/atunnel" ) -func TestActorIdentityFilterStateFilter(t *testing.T) { - filter := actorIdentityFilterStateFilter() +func TestActorRoutingFilterStateFilter(t *testing.T) { + filter := actorRoutingFilterStateFilter() config := &setfilterstatev3.Config{} if err := filter.GetTypedConfig().UnmarshalTo(config); err != nil { t.Fatalf("unmarshal set_filter_state config: %v", err) } want := map[string]string{ - extproc.ActorNameFilterStateKey: "%REQ(" + atunnel.ActorNameHeader + ")%", - extproc.AtespaceFilterStateKey: "%REQ(" + atunnel.AtespaceHeader + ")%", + extproc.ActorNameFilterStateKey: "%REQ(x-ate-actor-name)%", + extproc.AtespaceFilterStateKey: "%REQ(x-ate-atespace)%", } if len(config.GetOnRequestHeaders()) != len(want) { t.Fatalf("captured values = %d, want %d", len(config.GetOnRequestHeaders()), len(want)) @@ -73,7 +73,7 @@ func TestActorIdentityFilterStateFilter(t *testing.T) { t.Errorf("capture %q = %q, want %q", key, format, want[key]) } if strings.Contains(format, ":AUTHORITY") { - t.Errorf("capture %q still derives actor identity from authority", key) + t.Errorf("capture %q still derives actor routing from authority", key) } } } diff --git a/demos/egress/README.md b/demos/egress/README.md index 063bccaeba..91bd428028 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -27,7 +27,7 @@ intercepted and carried over mTLS to a gateway that verifies who is making the r │ (forwards the peer chain │ verify chain + ActorIdentity extension │ as x-forwarded-client-cert)│ GetActor → UID must match, must be RUNNING │ • dynamic_forward_proxy │ allow / deny 403 - │ │ + │ │ └───────────┼───────────────────────────────────────┘ ▼ real destination (the CONNECT authority, an IP:port) @@ -40,7 +40,7 @@ intercepted and carried over mTLS to a gateway that verifies who is making the r 3. **Guide 3 — HTTP-only actors, identity carried by the certificate.** The Actor only dials plain HTTP. atunnel presents the actor's own certificate — minted per actor by ateapi off the actor-identity CA, carrying an `ActorIdentity` X.509 extension — and sends a bare `CONNECT` - with no identity headers at all. + with no actor routing headers at all. 4. **Identity authentication.** Envoy requires a client certificate signed by the actor-identity CA, so a non-actor client is refused at the handshake. It then forwards the verified chain to `ext_proc` as `x-forwarded-client-cert`, and the **atenet router** (co-located in the gateway diff --git a/demos/jupyter/README.md b/demos/jupyter/README.md index ceac907431..a50fa42f85 100644 --- a/demos/jupyter/README.md +++ b/demos/jupyter/README.md @@ -75,7 +75,7 @@ print("hello world") ### 4. Suspending and Resuming the Notebook -When you're not using the notebook, instead of leaving the container running, Substrate can checkpoint and suspend it to disk. +When you're not using the notebook, instead of leaving the container running, Substrate can checkpoint and suspend it to disk. ```bash kubectl ate suspend actor jupyter-notebook -a ate-demo-jupyter @@ -93,7 +93,7 @@ To **resume** the notebook, you can either explicitly resume it via CLI: kubectl ate resume actor jupyter-notebook -a ate-demo-jupyter ``` -Or, even easier, you can rely on "transparent resume" — just refresh the page in your browser or make another request to the URL while it's suspended. Substrate will automatically restore its state and serve your request without any downtime. +Or, even easier, you can rely on "transparent resume" — just refresh the page in your browser or make another request to the URL while it's suspended. Substrate will automatically restore its state and serve your request without any downtime. ### Clean up diff --git a/docs/architecture.md b/docs/architecture.md index 2f4dd09508..4011c9f36f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -341,7 +341,7 @@ Handles actor-aware routing and automatic re-animation. reads the Actor name from `X-Ate-Actor-Name` and the Atespace from `X-Ate-Atespace`, then calls the Control Plane to resume the Actor and resolve its current worker assignment. `Host` remains application authority - and is not used as actor identity. + and does not select the Actor. * **Worker Tunnel**: After resolving the assignment, `atenet-router` opens an authenticated TLS tunnel to the worker's `atunnel` listener on port 443. diff --git a/docs/threat-model.md b/docs/threat-model.md index 754215e6a9..545e4883eb 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -11,21 +11,21 @@ Substrate is an early, fast moving product. It is full of debate and subject to # Intended Outcome -* The suggestions from this threat model should be added to Substrate's official roadmap after review and agreement with the community. +* The suggestions from this threat model should be added to Substrate's official roadmap after review and agreement with the community. * Security review SKILLs for AI-assisted security review should be extracted from this threat model and used for continuous review on the upstream [agent-substrate/substrate](https://github.com/agent-substrate/substrate) repository. # Goals -* Identify threats relevant to any system trying to achieve the same goal, with roughly the same basic building blocks as Substrate (Kubernetes, containers, sandboxes, snapshots, and lightweight "actors" dynamically scheduled to reusable "worker" Pods). -* Identify general "mitigating invariants" which, if implemented, eliminate or significantly decrease the severity of the threat. -* Suggest *possible* implementation approaches, which may change if Substrate changes. +* Identify threats relevant to any system trying to achieve the same goal, with roughly the same basic building blocks as Substrate (Kubernetes, containers, sandboxes, snapshots, and lightweight "actors" dynamically scheduled to reusable "worker" Pods). +* Identify general "mitigating invariants" which, if implemented, eliminate or significantly decrease the severity of the threat. +* Suggest *possible* implementation approaches, which may change if Substrate changes. * Prioritize the threats, so that resources can be spent of reducing the biggest risks first. # Non-Goals -* This threat model does not attempt to predict Substrate's future architectural decisions. -* This threat model does not demand any particular approach to mitigation. -* While shown in diagrams for reference, this threat model does not focus on AI frameworks that may be used on top of Substrate. +* This threat model does not attempt to predict Substrate's future architectural decisions. +* This threat model does not demand any particular approach to mitigation. +* While shown in diagrams for reference, this threat model does not focus on AI frameworks that may be used on top of Substrate. # Architecture of Substrate @@ -33,16 +33,16 @@ Substrate is an early, fast moving product. It is full of debate and subject to # Key Components -* **ate-api-server:** Actor creation and scheduling and credential issuance. -* **atelet:** Per-node daemon, performs snapshot/resume. -* **ateom:** Per-worker Pod sidecar, running inside the worker Pod. Ateom sets up "interior" sandboxes in the worker Pod and manages sandbox lifecycle, including image pulls. It currently uses gvisor but Substrate will support multiple microvm solutions. -* **Worker:** Preprovisioned Pods that actors get scheduled to. -* **Actor:** The core compute primitive, gets scheduled to/from worker via Run for cold start and Resume for snapshot resume. +* **ate-api-server:** Actor creation and scheduling and credential issuance. +* **atelet:** Per-node daemon, performs snapshot/resume. +* **ateom:** Per-worker Pod sidecar, running inside the worker Pod. Ateom sets up "interior" sandboxes in the worker Pod and manages sandbox lifecycle, including image pulls. It currently uses gvisor but Substrate will support multiple microvm solutions. +* **Worker:** Preprovisioned Pods that actors get scheduled to. +* **Actor:** The core compute primitive, gets scheduled to/from worker via Run for cold start and Resume for snapshot resume. * **Actor Network:** `ateom` creates a private point-to-point veth network for the active Actor inside a worker Pod. The Actor is not served directly from the worker Pod's port 80; ingress enters through `atunnel`'s authenticated listener on port 443. * **Actor DNS:** Each Actor gets a DNS name like `..actors.resources.substrate.ate.dev`. Substrate runs a custom CoreDNS instance that returns the `atenet-router` Service IP for A record queries matching the Actor DNS pattern. A controller keeps this target current and configures kube-dns with a stub domain for `actors.resources.substrate.ate.dev`, enabling traditional Kubernetes Pods to resolve Actor names. -* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. The ext_proc reads the Actor name and Atespace from `X-Ate-Actor-Name` and `X-Ate-Atespace`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites both identity headers before forwarding. The router then connects with mTLS to `atunnel`; `atunnel` validates the router identity and authorizes the header pair against the Actor currently assigned to that worker. -* **Object Storage:** Used to store actor snapshots. -* **Filesystem support:** Container local filesystem is saved in snapshots, future integrations likely to include networked storage. +* **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. The ext_proc reads the Actor name and Atespace from `X-Ate-Actor-Name` and `X-Ate-Atespace`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites both routing headers before forwarding. The router then connects with mTLS to `atunnel`; `atunnel` validates the router identity and authorizes the header pair against the Actor currently assigned to that worker. +* **Object Storage:** Used to store actor snapshots. +* **Filesystem support:** Container local filesystem is saved in snapshots, future integrations likely to include networked storage. * **Substrate Database:** PostgreSQL. * **Kubernetes:** The underlying infrastructure that Substrate runs on is expected to be Kubernetes. @@ -50,10 +50,10 @@ Substrate is an early, fast moving product. It is full of debate and subject to **Table Schema:** -* **Priority:** Critical, High, Medium, Low -* **Threats:** Expected risks in a naïve implementation of Substrate. -* **Mitigating Invariants:** High level properties which, if true, mitigate the threat. -* **Suggested Concrete Mitigations:** Specific options for implementing the mitigating invariants, based on current understanding of Substrate. +* **Priority:** Critical, High, Medium, Low +* **Threats:** Expected risks in a naïve implementation of Substrate. +* **Mitigating Invariants:** High level properties which, if true, mitigate the threat. +* **Suggested Concrete Mitigations:** Specific options for implementing the mitigating invariants, based on current understanding of Substrate. * **Notes:** Additional relevant information. ## Attacks from External Networks diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress.go index d580c81481..60f44b76a7 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress.go @@ -45,9 +45,10 @@ const ( // 421 returned by the actor application itself. StaleAssignmentHeader = "X-Ate-Assignment-Stale" // ActorNameHeader and AtespaceHeader identify the actor selected by the - // trusted ingress router. - ActorNameHeader = "X-Ate-Actor-Name" - AtespaceHeader = "X-Ate-Atespace" + // trusted ingress router for routing. HTTP field names are case-insensitive; use their + // HTTP/2 wire form so Envoy configuration and header mutations are native. + ActorNameHeader = "x-ate-actor-name" + AtespaceHeader = "x-ate-atespace" // TargetPortHeader carries the port to reach on the actor: the CONNECT // :authority's port for arbitrary-port ingress, or the default 80 @@ -104,7 +105,7 @@ func NewServer(cfg Config) (*Server, error) { return nil, fmt.Errorf("atunnel: trust bundle path is required") } if cfg.AllowedClientID == "" { - return nil, fmt.Errorf("atunnel: allowed client identity is required") + return nil, fmt.Errorf("atunnel: allowed client ID is required") } if cfg.Upstream == nil || cfg.Upstream.Scheme == "" || cfg.Upstream.Host == "" { return nil, fmt.Errorf("atunnel: upstream URL is required") @@ -414,7 +415,7 @@ func (w flushingWriter) Write(p []byte) (int, error) { // active actor per worker. func (s *Server) Activate(atespace, actorName string) error { if !resources.IsValidResourceName(atespace) || !resources.IsValidResourceName(actorName) { - return fmt.Errorf("atunnel: invalid actor identity %q/%q", atespace, actorName) + return fmt.Errorf("atunnel: invalid actor reference %q/%q", atespace, actorName) } s.mu.Lock() defer s.mu.Unlock() @@ -465,7 +466,7 @@ func (s *Server) closeIdleUpstreamConnections() { } } -// ServeHTTP validates the actor identity headers on every request before proxying it. +// ServeHTTP validates the actor routing headers on every request before proxying it. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, requestCtx, release, ok := s.authorize(r) if !ok { diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index 6872c5d1f6..7ae9d696eb 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -153,9 +153,11 @@ func TestServeHTTP(t *testing.T) { host string actorName string atespace string + mixedCase bool wantStatus int }{ {name: "active actor", host: "client.example", actorName: "actor-1", atespace: "team-a", wantStatus: http.StatusNoContent}, + {name: "mixed-case routing headers", actorName: "actor-1", atespace: "team-a", mixedCase: true, wantStatus: http.StatusNoContent}, {name: "host does not identify actor", host: "actor-2.team-b.example", actorName: "actor-1", atespace: "team-a", wantStatus: http.StatusNoContent}, {name: "empty host", actorName: "actor-1", atespace: "team-a", wantStatus: http.StatusNoContent}, {name: "wrong actor", actorName: "actor-2", atespace: "team-a", wantStatus: http.StatusMisdirectedRequest}, @@ -168,8 +170,13 @@ func TestServeHTTP(t *testing.T) { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "https://worker/hello", nil) req.Host = tt.host - req.Header.Set(ActorNameHeader, tt.actorName) - req.Header.Set(AtespaceHeader, tt.atespace) + if tt.mixedCase { + req.Header.Set("X-ATE-Actor-Name", tt.actorName) + req.Header.Set("x-ate-ATESPACE", tt.atespace) + } else { + req.Header.Set(ActorNameHeader, tt.actorName) + req.Header.Set(AtespaceHeader, tt.atespace) + } rec := httptest.NewRecorder() s.ServeHTTP(rec, req) if rec.Code != tt.wantStatus { @@ -181,7 +188,7 @@ func TestServeHTTP(t *testing.T) { }) } - for _, want := range []string{"client.example", "actor-2.team-b.example", ""} { + for _, want := range []string{"client.example", "", "actor-2.team-b.example", ""} { if got := <-upstreamHost; got != want { t.Errorf("upstream Host = %q, want %q", got, want) } @@ -369,7 +376,7 @@ func TestInactive(t *testing.T) { } } -func TestMutualTLSClientIdentity(t *testing.T) { +func TestMutualTLSClientAuthentication(t *testing.T) { dir := t.TempDir() ca := newTestCA(t) serverCert := ca.issue(t, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}) @@ -403,16 +410,16 @@ func TestMutualTLSClientIdentity(t *testing.T) { wantErr bool }{ { - name: "allowed identity", + name: "allowed client", cert: ca.issue(t, "spiffe://cluster.local/ns/ate-system/sa/atenet-router", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}), }, { - name: "wrong identity", + name: "wrong client ID", cert: ca.issue(t, "spiffe://cluster.local/ns/ate-system/sa/not-the-gateway", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}), wantErr: true, }, { - name: "untrusted identity", + name: "untrusted client", cert: untrustedCA.issue(t, "spiffe://cluster.local/ns/ate-system/sa/atenet-router", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}), wantErr: true, }, @@ -487,7 +494,7 @@ func TestServeNegotiatesH2(t *testing.T) { transport := &http.Transport{ TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, - InsecureSkipVerify: true, // The handshake identity checks live in TestMutualTLSClientIdentity. + InsecureSkipVerify: true, // The handshake checks live in TestMutualTLSClientAuthentication. Certificates: []tls.Certificate{clientCert}, }, // With h2, the transport offers "h2" via ALPN like Envoy's diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 23edbf1996..0fe3f915fe 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -28,8 +28,8 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/internal/resources" "k8s.io/client-go/kubernetes" From be1105dcfcba10e685c342f72b46399447a8f6ad Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Mon, 31 Aug 2026 09:57:55 -0500 Subject: [PATCH 03/10] Add back authority propagation because we need the port Signed-off-by: Keith Mattix II --- cmd/atenet/internal/router/README.md | 8 +++-- .../internal/router/extproc/attributes.go | 10 +++--- .../router/extproc/attributes_test.go | 3 ++ cmd/atenet/internal/router/ingress/ingress.go | 11 +++++-- .../internal/router/ingress/ingress_test.go | 31 +++++++++++++++++++ cmd/atenet/internal/router/xds.go | 9 ++++-- cmd/atenet/internal/router/xds_test.go | 9 +++--- .../e2e/suites/networking/grpcingress_test.go | 8 +++++ .../networking/websocketingress_test.go | 3 ++ 9 files changed, 75 insertions(+), 17 deletions(-) diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index e366e248d0..e25abc8bc7 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -69,6 +69,7 @@ a request are declared once, in `extproc/attributes.go`. | --- | --- | --- | | `dev.ate.actor.name` | ingress | carries the actor name across CONNECT re-entry | | `dev.ate.actor.atespace` | ingress | carries the atespace across CONNECT re-entry | +| `dev.ate.connect.authority` | ingress | carries the outer CONNECT authority across re-entry for target-port selection | | `dev.ate.actor.identity` | egress | carries the authenticated actor identity for logs and additional ext_proc services | | `dev.ate.extproc.direction` | egress | selects the egress handler for dataplanes without Envoy filter chains | @@ -102,9 +103,10 @@ reserved. ### keep it trustworthy An attribute is only as trustworthy as its source. The actor name and atespace -carry client-selected routing coordinates. Security-sensitive attributes, such -as actor identity, must come from dataplane-authenticated state rather than a -client header. +carry client-selected routing coordinates, while the CONNECT authority carries +only the requested target port across tunnel re-entry. Security-sensitive +attributes, such as actor identity, must come from dataplane-authenticated +state rather than a client header. ## modes diff --git a/cmd/atenet/internal/router/extproc/attributes.go b/cmd/atenet/internal/router/extproc/attributes.go index 451063870f..72992469fa 100644 --- a/cmd/atenet/internal/router/extproc/attributes.go +++ b/cmd/atenet/internal/router/extproc/attributes.go @@ -29,13 +29,15 @@ package extproc const ( // ActorNameFilterStateKey and AtespaceFilterStateKey carry the ingress actor // routing coordinates across Envoy's CONNECT internal-listener hop. - ActorNameFilterStateKey = "dev.ate.actor.name" - AtespaceFilterStateKey = "dev.ate.actor.atespace" + ActorNameFilterStateKey = "dev.ate.actor.name" + AtespaceFilterStateKey = "dev.ate.actor.atespace" + ConnectAuthorityFilterStateKey = "dev.ate.connect.authority" // ActorNameFilterStateAttribute and AtespaceFilterStateAttribute are the CEL // expressions ext_proc evaluates to read the corresponding filter state. - ActorNameFilterStateAttribute = "filter_state['" + ActorNameFilterStateKey + "']" - AtespaceFilterStateAttribute = "filter_state['" + AtespaceFilterStateKey + "']" + ActorNameFilterStateAttribute = "filter_state['" + ActorNameFilterStateKey + "']" + AtespaceFilterStateAttribute = "filter_state['" + AtespaceFilterStateKey + "']" + ConnectAuthorityFilterStateAttribute = "filter_state['" + ConnectAuthorityFilterStateKey + "']" // ActorIdentityFilterStateKey is the filter-state key holding the actor // identity the egress gateway read from the peer certificate it verified diff --git a/cmd/atenet/internal/router/extproc/attributes_test.go b/cmd/atenet/internal/router/extproc/attributes_test.go index 65f73f30d7..acafa90235 100644 --- a/cmd/atenet/internal/router/extproc/attributes_test.go +++ b/cmd/atenet/internal/router/extproc/attributes_test.go @@ -29,6 +29,8 @@ func TestAttributeKeys(t *testing.T) { {"actor name attribute", ActorNameFilterStateAttribute, "filter_state['dev.ate.actor.name']"}, {"atespace filter state", AtespaceFilterStateKey, "dev.ate.actor.atespace"}, {"atespace attribute", AtespaceFilterStateAttribute, "filter_state['dev.ate.actor.atespace']"}, + {"CONNECT authority filter state", ConnectAuthorityFilterStateKey, "dev.ate.connect.authority"}, + {"CONNECT authority attribute", ConnectAuthorityFilterStateAttribute, "filter_state['dev.ate.connect.authority']"}, {"actor identity filter state", ActorIdentityFilterStateKey, "dev.ate.actor.identity"}, {"direction attribute", directionAttribute, "dev.ate.extproc.direction"}, {"filter chain name attribute", FilterChainNameAttribute, "xds.filter_chain_name"}, @@ -47,6 +49,7 @@ func TestSubstrateKeysShareOnePrefix(t *testing.T) { for _, key := range []string{ ActorNameFilterStateKey, AtespaceFilterStateKey, + ConnectAuthorityFilterStateKey, ActorIdentityFilterStateKey, directionAttribute, } { diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 064ff1f6db..2d5c35caef 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -92,10 +92,15 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_NotFound, "invalid actor reference") } - // CONNECT traffic can name a port other than defaultActorPort in the - // authority. + // CONNECT traffic can name a port other than defaultActorPort. After Envoy + // terminates CONNECT, filter state retains the outer authority while md.Host + // belongs to the inner request. targetPort := defaultActorPort - if _, portStr, err := net.SplitHostPort(md.Host); err == nil { + targetAuthority := md.Attribute(extproc.ConnectAuthorityFilterStateAttribute) + if targetAuthority == "" { + targetAuthority = md.Host + } + if _, portStr, err := net.SplitHostPort(targetAuthority); err == nil { if p, ok := atunnel.ParsePort(portStr); ok { targetPort = p } diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index ee1f311686..5146f24a96 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -364,6 +364,37 @@ func TestHandleRequestHeadersHandlesConnectMethod(t *testing.T) { } } +func TestHandleRequestHeadersUsesRetainedConnectAuthorityForPort(t *testing.T) { + const testUUID = "123e4567-e89b-12d3-a456-426614174000" + clientMock := &mockClient{ + resumeFn: func(context.Context, *ateapipb.ResumeActorRequest, ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { + return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{ + Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}, + }}, nil + }, + } + attrs := map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + extproc.ConnectAuthorityFilterStateAttribute: structpb.NewStringValue("unrelated.example:9090"), + }, + }, + } + md := extproc.NewRequestMetadata([]*corev3.HeaderValue{ + {Key: atunnel.ActorNameHeader, Value: testUUID}, + {Key: atunnel.AtespaceHeader, Value: "team-a"}, + {Key: ":authority", Value: "inner.example"}, + }, attrs) + + res, err := New(clientMock, ParkedRequestConfig{}, nil).HandleRequestHeaders(context.Background(), md) + if err != nil { + t.Fatalf("HandleRequestHeaders() error = %v", err) + } + if got, want := dynamicMetadataPort(res.DynamicMetadata), "9090"; got != want { + t.Errorf("target port = %q, want %q", got, want) + } +} + // TestHandleRequestHeaders_ParkingLotFull verifies that when the parking lot is at capacity // the request is shed with a 503 before any resume is attempted. func TestHandleRequestHeaders_ParkingLotFull(t *testing.T) { diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 465212ef74..c4aefb35b6 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -894,13 +894,15 @@ func (x *XdsServer) buildMainInternalListener() *listenerv3.Listener { } } -// actorRoutingFilterStateFilter captures the actor routing headers so -// main_internal can read them across the CONNECT internal-listener hop. +// actorRoutingFilterStateFilter captures actor routing headers and the target +// authority so main_internal can read them across the CONNECT internal-listener +// hop. The authority supplies only the target port, never the actor reference. func actorRoutingFilterStateFilter() *hcmv3.HttpFilter { - values := make([]*setfilterstatecommonv3.FilterStateValue, 0, 2) + values := make([]*setfilterstatecommonv3.FilterStateValue, 0, 3) for _, routingField := range []struct{ key, header string }{ {extproc.ActorNameFilterStateKey, atunnel.ActorNameHeader}, {extproc.AtespaceFilterStateKey, atunnel.AtespaceHeader}, + {extproc.ConnectAuthorityFilterStateKey, extproc.AuthorityHeader}, } { values = append(values, &setfilterstatecommonv3.FilterStateValue{ Key: &setfilterstatecommonv3.FilterStateValue_ObjectKey{ @@ -1045,6 +1047,7 @@ func (x *XdsServer) buildHcm(statPrefix string, captureActorRouting bool) *anypb RequestAttributes: []string{ extproc.ActorNameFilterStateAttribute, extproc.AtespaceFilterStateAttribute, + extproc.ConnectAuthorityFilterStateAttribute, }, MetadataOptions: &extprocv3filter.MetadataOptions{ ForwardingNamespaces: &extprocv3filter.MetadataOptions_MetadataNamespaces{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 11ce116c34..ebe847ca59 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -60,8 +60,9 @@ func TestActorRoutingFilterStateFilter(t *testing.T) { } want := map[string]string{ - extproc.ActorNameFilterStateKey: "%REQ(x-ate-actor-name)%", - extproc.AtespaceFilterStateKey: "%REQ(x-ate-atespace)%", + extproc.ActorNameFilterStateKey: "%REQ(x-ate-actor-name)%", + extproc.AtespaceFilterStateKey: "%REQ(x-ate-atespace)%", + extproc.ConnectAuthorityFilterStateKey: "%REQ(:authority)%", } if len(config.GetOnRequestHeaders()) != len(want) { t.Fatalf("captured values = %d, want %d", len(config.GetOnRequestHeaders()), len(want)) @@ -72,8 +73,8 @@ func TestActorRoutingFilterStateFilter(t *testing.T) { if format != want[key] { t.Errorf("capture %q = %q, want %q", key, format, want[key]) } - if strings.Contains(format, ":AUTHORITY") { - t.Errorf("capture %q still derives actor routing from authority", key) + if key != extproc.ConnectAuthorityFilterStateKey && strings.Contains(strings.ToLower(format), ":authority") { + t.Errorf("capture %q derives actor routing from authority", key) } } } diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 3829e388ee..0bacc5b895 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -26,8 +26,10 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/internal/proto/grpcechopb" @@ -70,6 +72,8 @@ func TestIngressProtocolDowngrade(t *testing.T) { return nil, err } req.Host = resources.ActorDNSName(actorRef) + req.Header.Set(atunnel.ActorNameHeader, actorRef.Name) + req.Header.Set(atunnel.AtespaceHeader, actorRef.Atespace) if contentType != "" { req.Header.Set("Content-Type", contentType) } @@ -141,6 +145,10 @@ func TestIngressGRPC(t *testing.T) { fixture := deployGRPCEchoTemplate(t, ctx, env["BUCKET_NAME"]) actorName, _ := createAndResumeSubstrateActor(t, ctx, "grpcingress", fixture) actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + ctx = metadata.AppendToOutgoingContext(ctx, + atunnel.ActorNameHeader, actorRef.Name, + atunnel.AtespaceHeader, actorRef.Atespace, + ) // Cleartext h2c to the router's HTTP port, with the Actor's DNS name as the // :authority — the same routing key every other ingress test in this suite diff --git a/internal/e2e/suites/networking/websocketingress_test.go b/internal/e2e/suites/networking/websocketingress_test.go index d0afb3b54a..4e8ec4d962 100644 --- a/internal/e2e/suites/networking/websocketingress_test.go +++ b/internal/e2e/suites/networking/websocketingress_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" "github.com/gorilla/websocket" @@ -68,6 +69,8 @@ func TestWebsocketIngressPing(t *testing.T) { actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} header := http.Header{} header.Set("Host", resources.ActorDNSName(actorRef)) + header.Set(atunnel.ActorNameHeader, actorRef.Name) + header.Set(atunnel.AtespaceHeader, actorRef.Atespace) var c *websocket.Conn From c607b24b8a1322e07c58eb47c46acd5e0a199c91 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Mon, 31 Aug 2026 10:33:28 -0500 Subject: [PATCH 04/10] Fix test deadlock Signed-off-by: Keith Mattix II --- internal/atunnel/ingress_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index 7ae9d696eb..10ada81a67 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -129,7 +129,7 @@ func TestRelayIngressCancellationClosesBothSides(t *testing.T) { } func TestServeHTTP(t *testing.T) { - upstreamHost := make(chan string, 3) + upstreamHost := make(chan string, 4) upstreamURL, err := url.Parse("http://actor.internal:80") if err != nil { t.Fatal(err) From 74f8eb556c0af6aff67bbd0237b12dd412fd37a7 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Mon, 31 Aug 2026 11:04:03 -0500 Subject: [PATCH 05/10] Fix one more stale expectation - confirmed with race this time Signed-off-by: Keith Mattix II --- internal/atunnel/client_test.go | 6 +++- internal/atunnel/credential_test.go | 3 +- internal/atunnel/egress_test.go | 8 ++--- internal/atunnel/ingress_test.go | 55 +++++++++++++++++++++-------- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/internal/atunnel/client_test.go b/internal/atunnel/client_test.go index a7ddb1724f..d24b3070ef 100644 --- a/internal/atunnel/client_test.go +++ b/internal/atunnel/client_test.go @@ -63,7 +63,7 @@ func TestClientDialContext(t *testing.T) { } defer conn.Close() - gotRequest := <-request + gotRequest := receiveWithin(t, request, "CONNECT request") if gotRequest.Method != http.MethodConnect { t.Errorf("method = %q, want CONNECT", gotRequest.Method) } @@ -302,6 +302,10 @@ func serveTestConnectGateway(t *testing.T, ca *testCA, handle func(net.Conn, *ht return } defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Errorf("setting gateway connection deadline: %v", err) + return + } req, err := http.ReadRequest(bufio.NewReader(conn)) if err != nil { t.Errorf("reading CONNECT request: %v", err) diff --git a/internal/atunnel/credential_test.go b/internal/atunnel/credential_test.go index 057b6fe9e8..abef065d6c 100644 --- a/internal/atunnel/credential_test.go +++ b/internal/atunnel/credential_test.go @@ -47,7 +47,8 @@ func TestBrokerCertificateSourceMintsAndReusesKey(t *testing.T) { t.Fatal(err) } } - first, second := <-broker.publicKeys, <-broker.publicKeys + first := receiveWithin(t, broker.publicKeys, "first certificate public key") + second := receiveWithin(t, broker.publicKeys, "second certificate public key") if string(first) != string(second) { t.Fatal("renewal replaced the actor private key") } diff --git a/internal/atunnel/egress_test.go b/internal/atunnel/egress_test.go index 18440ec5f2..d3b49e4637 100644 --- a/internal/atunnel/egress_test.go +++ b/internal/atunnel/egress_test.go @@ -224,9 +224,9 @@ func TestEgressDeactivationDropsConcurrentRenewal(t *testing.T) { } done := make(chan error, 1) go func() { done <- egress.Deactivate(context.Background()) }() - <-active.ctx.Done() + receiveWithin(t, active.ctx.Done(), "egress cancellation") close(release) - if err := <-done; err != nil { + if err := receiveWithin(t, done, "egress deactivation"); err != nil { t.Fatal(err) } if !active.expiresAt.IsZero() { @@ -287,7 +287,7 @@ func TestEgressEndToEnd(t *testing.T) { }) egress.handle(downstreamProxy) - req := <-requests + req := receiveWithin(t, requests, "gateway CONNECT request") if req.Method != http.MethodConnect || req.Host != "192.0.2.10:443" { t.Errorf("request = %s %s, want CONNECT 192.0.2.10:443", req.Method, req.Host) } @@ -313,7 +313,7 @@ func TestEgressEndToEnd(t *testing.T) { if string(gotAtActor) != "from gateway" { t.Errorf("actor payload = %q, want %q", gotAtActor, "from gateway") } - <-gatewayDone + receiveWithin(t, gatewayDone, "gateway completion") if err := egress.Deactivate(context.Background()); err != nil { t.Fatal(err) diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index 10ada81a67..aef525ac04 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -55,7 +55,7 @@ func TestRelayIngressWithHalfClose(t *testing.T) { t.Fatal(err) } defer actor.Close() - upstream := <-accepted + upstream := receiveWithin(t, accepted, "accepted connection") defer upstream.Close() clientReader, clientInput := io.Pipe() @@ -129,7 +129,7 @@ func TestRelayIngressCancellationClosesBothSides(t *testing.T) { } func TestServeHTTP(t *testing.T) { - upstreamHost := make(chan string, 4) + var upstreamHosts []string upstreamURL, err := url.Parse("http://actor.internal:80") if err != nil { t.Fatal(err) @@ -137,7 +137,7 @@ func TestServeHTTP(t *testing.T) { s := newTestServer(t, upstreamURL) s.proxy.Transport = roundTripFunc(func(r *http.Request) (*http.Response, error) { - upstreamHost <- r.Host + upstreamHosts = append(upstreamHosts, r.Host) return &http.Response{ StatusCode: http.StatusNoContent, Header: make(http.Header), @@ -188,8 +188,12 @@ func TestServeHTTP(t *testing.T) { }) } - for _, want := range []string{"client.example", "", "actor-2.team-b.example", ""} { - if got := <-upstreamHost; got != want { + wantHosts := []string{"client.example", "", "actor-2.team-b.example", ""} + if len(upstreamHosts) != len(wantHosts) { + t.Fatalf("upstream requests = %d, want %d", len(upstreamHosts), len(wantHosts)) + } + for i, want := range wantHosts { + if got := upstreamHosts[i]; got != want { t.Errorf("upstream Host = %q, want %q", got, want) } } @@ -485,8 +489,13 @@ func TestServeNegotiatesH2(t *testing.T) { go func() { served <- s.Serve(ctx, lis) }() t.Cleanup(func() { cancel() - if err := <-served; err != nil { - t.Errorf("Serve: %v", err) + select { + case err := <-served: + if err != nil { + t.Errorf("Serve: %v", err) + } + case <-time.After(time.Second): + t.Error("Serve did not stop after cancellation") } }) @@ -502,6 +511,7 @@ func TestServeNegotiatesH2(t *testing.T) { // HTTP/1.1 pool. ForceAttemptHTTP2: h2, } + t.Cleanup(transport.CloseIdleConnections) return &http.Client{Transport: transport, Timeout: 10 * time.Second} } request := func(t *testing.T, c *http.Client, method, contentType string) *http.Response { @@ -510,7 +520,9 @@ func TestServeNegotiatesH2(t *testing.T) { if err != nil { t.Fatal(err) } - req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" + req.Host = "team-a-agent.example.com" // Use a custom Host header to prove that we no longer rely on Host to route to the actor + req.Header.Set(ActorNameHeader, "actor-1") + req.Header.Set(AtespaceHeader, "team-a") if contentType != "" { req.Header.Set("Content-Type", contentType) } @@ -527,13 +539,13 @@ func TestServeNegotiatesH2(t *testing.T) { if res.Proto != "HTTP/2.0" { t.Fatalf("h2-only client negotiated %s, want HTTP/2.0 — Envoy's mirrored HTTP/2 pool cannot connect", res.Proto) } - if got := <-protoSeen; got != "HTTP/2.0" { + if got := receiveWithin(t, protoSeen, "gRPC backend protocol"); got != "HTTP/2.0" { t.Errorf("gRPC-shaped request reached the actor as %s, want HTTP/2.0", got) } // A non-gRPC request on the same negotiated h2 connection is downgraded // before the actor. request(t, h2Client, http.MethodGet, "") - if got := <-protoSeen; got != "HTTP/1.1" { + if got := receiveWithin(t, protoSeen, "plain HTTP/2 backend protocol"); got != "HTTP/1.1" { t.Errorf("plain GET over h2 reached the actor as %s, want HTTP/1.1", got) } @@ -543,7 +555,7 @@ func TestServeNegotiatesH2(t *testing.T) { if res.Proto != "HTTP/1.1" { t.Errorf("http/1.1-only client negotiated %s, want HTTP/1.1", res.Proto) } - if got := <-protoSeen; got != "HTTP/1.1" { + if got := receiveWithin(t, protoSeen, "HTTP/1.1 backend protocol"); got != "HTTP/1.1" { t.Errorf("HTTP/1.1 request reached the actor as %s, want HTTP/1.1", got) } } @@ -573,11 +585,11 @@ func TestDeactivateCancelsInflightRequest(t *testing.T) { req.Header.Set(AtespaceHeader, "team-a") s.ServeHTTP(httptest.NewRecorder(), req) }() - <-started + receiveWithin(t, started, "in-flight request") if err := s.Deactivate(context.Background()); err != nil { t.Fatal(err) } - <-done + receiveWithin(t, done, "canceled in-flight request") } func newTestServer(t *testing.T, upstream *url.URL) *Server { @@ -732,6 +744,9 @@ func writeCredentialBundle(t *testing.T, path string, cert tls.Certificate) { func tlsHandshake(serverConfig, clientConfig *tls.Config) (serverErr, clientErr error) { serverConn, clientConn := net.Pipe() + deadline := time.Now().Add(5 * time.Second) + _ = serverConn.SetDeadline(deadline) + _ = clientConn.SetDeadline(deadline) serverTLS := tls.Server(serverConn, serverConfig) clientTLS := tls.Client(clientConn, clientConfig) done := make(chan error, 1) @@ -861,9 +876,21 @@ func TestProtocolMirrorTransport(t *testing.T) { t.Fatalf("RoundTrip: %v", err) } res.Body.Close() - if got := <-protoSeen; got != tt.want { + if got := receiveWithin(t, protoSeen, "backend protocol"); got != tt.want { t.Errorf("upstream saw %s, want %s", got, tt.want) } }) } } + +func receiveWithin[T any](t *testing.T, channel <-chan T, description string) T { + t.Helper() + select { + case value := <-channel: + return value + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for %s", description) + var zero T + return zero + } +} From 58285d2727bd5e56516c6f4e1ddc1546e1a1ce74 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Mon, 31 Aug 2026 11:36:10 -0500 Subject: [PATCH 06/10] Remove CoreDNS plugin Signed-off-by: Keith Mattix II --- README.md | 2 +- cmd/ate-setup/internal/cmd/deploy.go | 2 +- cmd/ate-setup/internal/steps/delete.go | 1 - cmd/ate-setup/internal/steps/deploy.go | 7 +- cmd/atenet/README.md | 14 +- cmd/atenet/internal/dns.go | 99 ------- cmd/atenet/internal/dns/README.md | 36 --- cmd/atenet/internal/dns/corefile.go | 64 ----- cmd/atenet/internal/dns/corefile_test.go | 65 ----- cmd/atenet/internal/dns/dns.go | 252 ------------------ cmd/atenet/internal/dns/dns_test.go | 199 -------------- cmd/atenet/internal/root.go | 1 - cmd/atenet/internal/router/dataplane.go | 5 +- .../internal/router/extproc/dispatch.go | 4 +- cmd/atenet/internal/router/xds.go | 9 +- demos/parking/README.md | 3 +- docs/api-guide.md | 9 +- docs/architecture.md | 24 +- docs/glossary.md | 11 +- docs/metrics/substrate.yaml | 5 - docs/roadmap.md | 2 +- docs/threat-model.md | 2 +- hack/install-ate.sh | 2 - .../benchmarking/boomer/glutton/durdir.go | 2 +- .../benchmarking/boomer/glutton/lifecycle.go | 2 +- internal/e2e/suites/demo/demo_test.go | 2 +- .../e2e/suites/networking/grpcingress_test.go | 7 +- internal/resources/actor.go | 3 +- internal/resources/resourceref.go | 47 +--- internal/resources/resourceref_test.go | 43 --- manifests/ate-install/atenet-dns.yaml | 178 ------------- manifests/ate-install/base/kustomization.yaml | 1 - manifests/ate-install/kind/kustomization.yaml | 1 - 33 files changed, 45 insertions(+), 1059 deletions(-) delete mode 100644 cmd/atenet/internal/dns.go delete mode 100644 cmd/atenet/internal/dns/README.md delete mode 100644 cmd/atenet/internal/dns/corefile.go delete mode 100644 cmd/atenet/internal/dns/corefile_test.go delete mode 100644 cmd/atenet/internal/dns/dns.go delete mode 100644 cmd/atenet/internal/dns/dns_test.go delete mode 100644 manifests/ate-install/atenet-dns.yaml diff --git a/README.md b/README.md index 3780ab9240..5bd7df07e2 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit * `cmd/ateapi`: The core control plane API server exposing gRPC endpoints to manage actor and worker lifecycles. * `cmd/atelet`: A node-level DaemonSet that supervises physical worker pods, coordinates snapshotting, and manages state transfers. * `cmd/atecontroller`: A Kubernetes controller that reconciles WorkerPool custom resources. -* `cmd/atenet`: A combined networking controller providing DNS, Envoy routing, and proxy sidecars. +* `cmd/atenet`: A combined networking controller providing Envoy routing and proxy sidecars. * `cmd/ateom-gvisor`: An interior-pod helper running inside sandboxed worker pods to execute `runsc` checkpoint and restore commands. * `cmd/ateom-microvm`: The micro-VM peer of `ateom-gvisor`, running actors as cloud-hypervisor VMs. * `cmd/podcertcontroller`: A "polyfill" that provides Pod Certificate signers that diff --git a/cmd/ate-setup/internal/cmd/deploy.go b/cmd/ate-setup/internal/cmd/deploy.go index 0d67133b6c..3f8d95f151 100644 --- a/cmd/ate-setup/internal/cmd/deploy.go +++ b/cmd/ate-setup/internal/cmd/deploy.go @@ -75,7 +75,7 @@ var deployControllerCmd = &cobra.Command{ var deployAtenetCmd = &cobra.Command{ Use: "atenet", - Short: "Deploy the atenet dataplane only: router, egress, and DNS", + Short: "Deploy the atenet dataplane only: router and egress", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return env.DeployAtenet(cmd.Context()) diff --git a/cmd/ate-setup/internal/steps/delete.go b/cmd/ate-setup/internal/steps/delete.go index 73317c0e96..8d0f6d2554 100644 --- a/cmd/ate-setup/internal/steps/delete.go +++ b/cmd/ate-setup/internal/steps/delete.go @@ -75,7 +75,6 @@ func (e *Env) DeleteAtenet(ctx context.Context) error { // other does not. {"atenet-egress.yaml"}, {"atenet-egress-with-sdsmint.yaml"}, - {"atenet-dns.yaml"}, } { if err := e.Kube.DeletePath(ctx, e.Cfg.Manifest(path...)); err != nil { return err diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 9c450adef5..d3e8a720c8 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -289,7 +289,7 @@ func (e *Env) DeployAtelet(ctx context.Context) error { return e.Kube.RolloutStatus(ctx, kube.KindDaemonSet, NamespaceAteSystem, ateletName, e.Cfg.RolloutTimeout) } -// DeployAtenet redeploys the atenet dataplane: router, egress, and DNS. +// DeployAtenet redeploys the atenet dataplane: router and egress. func (e *Env) DeployAtenet(ctx context.Context) error { log.Step("deploy_atenet") @@ -316,11 +316,8 @@ func (e *Env) DeployAtenet(ctx context.Context) error { if err := e.applyAtenetEgress(ctx); err != nil { return err } - if err := e.ResolveAndApply(ctx, e.Cfg.Manifest("atenet-dns.yaml")); err != nil { - return err - } - for _, name := range []string{"atenet-router", "atenet-egress", "dns"} { + for _, name := range []string{"atenet-router", "atenet-egress"} { if err := e.Kube.RolloutStatus(ctx, kube.KindDeployment, NamespaceAteSystem, name, e.Cfg.RolloutTimeout); err != nil { return err } diff --git a/cmd/atenet/README.md b/cmd/atenet/README.md index f2ab3f1a2b..bf3917e015 100644 --- a/cmd/atenet/README.md +++ b/cmd/atenet/README.md @@ -2,7 +2,6 @@ atenet is a combined daemon for all networking functionality. -* DNS server for ATE Actor resolution: `atenet dns` * Envoy control plane for programming ATE resolution. `atenet router` This is built as a single binary for convenience in the prototyping. @@ -31,8 +30,8 @@ likely be split in the future for better scalability.) a separate process on the worker pod, not part of Envoy -- so the port to reach on the actor itself (its default port, or an arbitrary one for CONNECT) still travels as a real header, `atunnel.TargetPortHeader`. - `:authority`/`Host` reaches atunnel unmodified either way, so it authorizes - the actor by its own DNS name. + `X-Ate-Actor-Name` and `X-Ate-Atespace` identify the Actor independently of + `:authority`/`Host`. * Termination: the router drains gracefully on SIGTERM (readiness flip → endpoint propagation → Envoy admin-API drain → ext_proc drain), and the Envoy container's `preStop` hook waits for the router's drain-complete @@ -45,15 +44,6 @@ likely be split in the future for better scalability.) RBAC permissions: * get, list, watch on ate-system EndpointSlices -### dns - -* `atenet dns` will be deployed as: - * Deployment - * Service exposing tcp and udp 53 - -* read, list on kube-system services -* read, list on ate-system services - ## testing Run the package tests with `go test ./cmd/atenet/...`. Cluster e2e diff --git a/cmd/atenet/internal/dns.go b/cmd/atenet/internal/dns.go deleted file mode 100644 index 85798fa73f..0000000000 --- a/cmd/atenet/internal/dns.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "context" - "fmt" - "log/slog" - "os" - "os/signal" - "syscall" - "time" - - "github.com/agent-substrate/substrate/internal/serverboot" - "github.com/spf13/cobra" - "k8s.io/client-go/tools/clientcmd" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" - - "github.com/agent-substrate/substrate/cmd/atenet/internal/dns" -) - -type DnsConfig struct { - LogLevel string - Kubeconfig string - ReconcileInterval time.Duration - CorefilePath string -} - -func NewDnsCmd() *cobra.Command { - var cfg DnsConfig - - cmd := &cobra.Command{ - Use: "dns", - Short: "Orchestrates CoreDNS and GKE stub resolver configuration", - RunE: func(cmd *cobra.Command, args []string) error { - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() - - serverboot.InitLogger() - if err := serverboot.SetLogLevel(cfg.LogLevel); err != nil { - return err - } - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-sigChan - cancel() - }() - - k8sCfg, err := config.GetConfig() - if err != nil { - if cfg.Kubeconfig != "" { - k8sCfg, err = clientcmd.BuildConfigFromFlags("", cfg.Kubeconfig) - if err != nil { - return fmt.Errorf("failed to read config from path %s: %w", cfg.Kubeconfig, err) - } - } else { - return fmt.Errorf("unable to establish Kubernetes configuration parameters: %w", err) - } - } - - k8sClient, err := client.New(k8sCfg, client.Options{}) - if err != nil { - return fmt.Errorf("failed to initialize cluster client: %w", err) - } - - dnsController := &dns.Controller{ - Client: k8sClient, - Interval: cfg.ReconcileInterval, - CorefilePath: cfg.CorefilePath, - Reloader: dns.NewConfigReloader(), - } - - slog.InfoContext(ctx, "Starting DNS Controller subsystem") - return dnsController.Run(ctx) - }, - } - - cmd.Flags().StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error") - cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file") - cmd.Flags().DurationVar(&cfg.ReconcileInterval, "interval", 10*time.Second, "Interval for reconciling DNS configurations") - cmd.Flags().StringVar(&cfg.CorefilePath, "corefile-path", "/etc/coredns/Corefile", "Path to the local Corefile configuration on shared volume") - - return cmd -} diff --git a/cmd/atenet/internal/dns/README.md b/cmd/atenet/internal/dns/README.md deleted file mode 100644 index a134e829ab..0000000000 --- a/cmd/atenet/internal/dns/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# DNS Controller - -The DNS Controller orchestrates the configuration needed to setup the ATE routing. - -We want to resolve requests for ..actors.resources.substrate.ate.dev to the router service address. - -* Stub resolver mode: orchestrate running a CoreDNS instance with the actor name mapped to the atenet-router service address. - -Cluster resources: - -* Deployment `ate-system:dns`. Label: app=dns -* Service `ate-system:dns`. -* ConfigMap `ate-system:dns`. - -These are defined in manifests/ate-install/atenet-dns.yaml. - -## Stub resolver mode - -* Ensure stub resolver CoreDNS is running as: - * Deployment `ate-system:dns`. - * Service `ate-system:dns` pointing to the Deployment. - -ConfigMap `ate-system:dns`: - -``` -# Match any 'A' query for an actor name + atespace pattern under actors.resources.substrate.ate.dev - template IN A actors.resources.substrate.ate.dev { - match "^[a-z0-9]([-a-z0-9]*[a-z0-9])?\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?\\.actors\\.resources\\.substrate\\.ate\\.dev\\.$" - answer "{{ .Name }} 60 IN A " - } -``` - -## Integration - -* CoreDNS: Update CoreDNS ConfigMap to add the stub resolver. -* GKE DNS: Update the GKE DNS ConfigMap to add the stub resolver. diff --git a/cmd/atenet/internal/dns/corefile.go b/cmd/atenet/internal/dns/corefile.go deleted file mode 100644 index 0b301e7e29..0000000000 --- a/cmd/atenet/internal/dns/corefile.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "fmt" - "strings" - "time" - - "github.com/agent-substrate/substrate/internal/resources" -) - -// corefileTemplate is a Sprintf template for the CoreDNS configuration. -var corefileTemplate string - -func init() { - corefileTemplate = buildTemplate() -} - -func buildTemplate() string { - // Build up the corefileTemplate programmatically to make it easier to understand. - var directives []string - // Plugins to enable. - directives = append(directives, "log") - directives = append(directives, "errors") - directives = append(directives, "health :8080") - directives = append(directives, "ready :8181") - directives = append(directives, "reload") - - // Construct match pattern for ... Both the - // actor name and the atespace are DNS-1123 labels (same regex). - directives = append(directives, fmt.Sprintf("template IN A %s {", resources.ActorDNSSuffix)) - // Escape the suffix's dots so they match literally; the final \. matches the FQDN's trailing dot. - escapedSuffix := strings.ReplaceAll(resources.ActorDNSSuffix, ".", `\.`) - directives = append(directives, fmt.Sprintf(` match "^%s\.%s\.%s\.$"`, resources.ResourceNameRegexPattern, resources.ResourceNameRegexPattern, escapedSuffix)) - // Note the %s -- this will be filled with the router IP. - directives = append(directives, ` answer "{{ .Name }} 60 IN A %s"`) - directives = append(directives, "}") - - // Generate the template. - b := strings.Builder{} - fmt.Fprintf(&b, "# Generated at %s\n", time.Now()) - fmt.Fprintf(&b, "%s:53 {\n ", resources.ActorDNSSuffix) - fmt.Fprint(&b, strings.Join(directives, "\n ")) - fmt.Fprint(&b, "\n}\n") - - return b.String() -} - -func makeCoreFile(routerIP string) string { - return fmt.Sprintf(corefileTemplate, routerIP) -} diff --git a/cmd/atenet/internal/dns/corefile_test.go b/cmd/atenet/internal/dns/corefile_test.go deleted file mode 100644 index f13429e475..0000000000 --- a/cmd/atenet/internal/dns/corefile_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "strings" - "testing" - - "github.com/agent-substrate/substrate/internal/resources" -) - -func TestMakeCoreFile(t *testing.T) { - tests := []struct { - name string - routerIP string - expected []string - }{ - { - name: "standard local IP", - routerIP: "10.240.0.10", - expected: []string{ - "actors.resources.substrate.ate.dev:53 {", - "log", - "errors", - "health :8080", - "ready :8181", - "reload", - "template IN A actors.resources.substrate.ate.dev {", - `match "^` + resources.ResourceNameRegexPattern + `\.` + resources.ResourceNameRegexPattern + `\.actors\.resources\.substrate\.ate\.dev\.$"`, - `answer "{{ .Name }} 60 IN A 10.240.0.10"`, - }, - }, - { - name: "different IP", - routerIP: "192.168.1.1", - expected: []string{ - "actors.resources.substrate.ate.dev:53 {", - `answer "{{ .Name }} 60 IN A 192.168.1.1"`, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := makeCoreFile(tc.routerIP) - for _, exp := range tc.expected { - if !strings.Contains(got, exp) { - t.Errorf("makeCoreFile(%q) missing expected substring %q\nGot:\n%s", tc.routerIP, exp, got) - } - } - }) - } -} diff --git a/cmd/atenet/internal/dns/dns.go b/cmd/atenet/internal/dns/dns.go deleted file mode 100644 index cf2db99b69..0000000000 --- a/cmd/atenet/internal/dns/dns.go +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "os" - "path/filepath" - "strconv" - "strings" - "syscall" - "time" - - "github.com/agent-substrate/substrate/internal/resources" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - // serviceName is the name of the CoreDNS service. - serviceName = "dns" - systemNamespace = "ate-system" -) - -// Controller manages the DNS configuration for the ATE. -type Controller struct { - Client client.Client - Interval time.Duration - CorefilePath string - Reloader ConfigReloader -} - -// Run the DNS orchestration loop until ctx is canceled. -func (c *Controller) Run(ctx context.Context) error { - slog.InfoContext(ctx, "DNS Controller started", slog.Duration("interval", c.Interval), slog.String("corefile", c.CorefilePath)) - slog.InfoContext(ctx, "Using template", "template", corefileTemplate) - - ticker := time.NewTicker(c.Interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - slog.InfoContext(ctx, "DNS Controller stopped") - return nil - case <-ticker.C: - if err := c.reconcile(ctx); err != nil { - slog.ErrorContext(ctx, "Error during DNS reconciliation", slog.Any("error", err)) - } - } - } -} - -func (c *Controller) reconcile(ctx context.Context) error { - slog.DebugContext(ctx, "Reconciling DNS orchestration configuration...") - - // 1. Get the ClusterIP of atenet-router in ate-system namespace - routerSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: "atenet-router", Namespace: systemNamespace}, routerSvc); err != nil { - if errors.IsNotFound(err) { - slog.WarnContext(ctx, "atenet-router service not found, skipping until it is available") - return nil - } - return fmt.Errorf("failed to get atenet-router service: %w", err) - } - - routerIP := routerSvc.Spec.ClusterIP - if routerIP == "" || routerIP == "None" { - slog.WarnContext(ctx, "atenet-router service has no ClusterIP yet, waiting...") - return nil - } - - // 2. Get the ClusterIP of dns service in ate-system namespace - dnsSvc := &corev1.Service{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: systemNamespace}, dnsSvc); err != nil { - if errors.IsNotFound(err) { - slog.WarnContext(ctx, "dns service not found, skipping until it is available") - return nil - } - return fmt.Errorf("failed to get dns service: %w", err) - } - - dnsIP := dnsSvc.Spec.ClusterIP - if dnsIP == "" || dnsIP == "None" { - slog.WarnContext(ctx, "dns service has no ClusterIP yet, waiting...") - return nil - } - - // 3. Reconcile CoreDNS Corefile on shared volume - if err := c.reconcileCoreDNSConfig(ctx, routerIP); err != nil { - return fmt.Errorf("failed to reconcile CoreDNS config file: %w", err) - } - - // 4. Reconcile GKE kube-dns ConfigMap with dns service IP - if err := c.reconcileKubeDNSConfig(ctx, dnsIP); err != nil { - return fmt.Errorf("failed to reconcile kube-dns configmap: %w", err) - } - - return nil -} - -func (c *Controller) reconcileCoreDNSConfig(ctx context.Context, routerIP string) error { - expectedCorefile := makeCoreFile(routerIP) - - // Read Corefile from local shared volume path to see if it needs updating - corefileBytes, err := os.ReadFile(c.CorefilePath) - if err == nil && string(corefileBytes) == expectedCorefile { - slog.DebugContext(ctx, "CoreDNS Corefile is up-to-date", slog.String("routerIP", routerIP)) - return nil - } - - // Write updated Corefile back to shared volume in its entirety - if err := os.WriteFile(c.CorefilePath, []byte(expectedCorefile), 0644); err != nil { - return fmt.Errorf("failed to write updated Corefile to %s: %w", c.CorefilePath, err) - } - slog.InfoContext(ctx, "CoreDNS Corefile updated", slog.String("routerIP", routerIP)) - - // Signal CoreDNS process to reload - if err := c.Reloader.Reload(ctx); err != nil { - return fmt.Errorf("failed to reload CoreDNS: %w", err) - } - - return nil -} - -// reconcileKubeDNSConfig ensures that the kube-dns ConfigMap has a stub domain for ate-system. -func (c *Controller) reconcileKubeDNSConfig(ctx context.Context, dnsIP string) error { - cm := &corev1.ConfigMap{} - if err := c.Client.Get(ctx, types.NamespacedName{Name: "kube-dns", Namespace: "kube-system"}, cm); err != nil { - if errors.IsNotFound(err) { - slog.WarnContext(ctx, "kube-dns ConfigMap not found in kube-system namespace, skipping stub resolver configuration") - return nil - } - return fmt.Errorf("failed to retrieve kube-dns ConfigMap: %w", err) - } - - if cm.Data == nil { - cm.Data = make(map[string]string) - } - - stubDomainsStr := cm.Data["stubDomains"] - var stubDomains map[string][]string - - if stubDomainsStr != "" { - if err := json.Unmarshal([]byte(stubDomainsStr), &stubDomains); err != nil { - return fmt.Errorf("failed to parse stubDomains JSON: %w", err) - } - } else { - stubDomains = make(map[string][]string) - } - - ips, exists := stubDomains[resources.ActorDNSSuffix] - if exists && len(ips) == 1 && ips[0] == dnsIP { - slog.DebugContext(ctx, "kube-dns stubDomains are already up-to-date", slog.String("dnsIP", dnsIP)) - return nil - } - - stubDomains[resources.ActorDNSSuffix] = []string{dnsIP} - - newStubDomainsBytes, err := json.Marshal(stubDomains) - if err != nil { - return fmt.Errorf("failed to marshal stubDomains JSON: %w", err) - } - - cm.Data["stubDomains"] = string(newStubDomainsBytes) - if err := c.Client.Update(ctx, cm); err != nil { - return fmt.Errorf("failed to update kube-dns ConfigMap: %w", err) - } - - slog.InfoContext(ctx, "kube-dns stubDomains successfully updated with custom DNS IP", slog.String("dnsIP", dnsIP)) - return nil -} - -// ConfigReloader defines an interface for dynamically signaling CoreDNS to reload its configuration. -type ConfigReloader interface { - Reload(ctx context.Context) error -} - -type procConfigReloader struct{} - -func NewConfigReloader() ConfigReloader { - return &procConfigReloader{} -} - -func (r *procConfigReloader) Reload(ctx context.Context) error { - pid, err := findPID("coredns") - if err != nil { - slog.ErrorContext(ctx, "findPID error", slog.Any("error", err)) - return nil - } - - process, err := os.FindProcess(pid) - if err != nil { - return fmt.Errorf("FindProcess %d: %w", pid, err) - } - - // CoreDNS catches SIGUSR1 or SIGHUP to trigger dynamic reload of Corefile - if err := process.Signal(syscall.SIGUSR1); err != nil { - return fmt.Errorf("SendProcess SIGUSR1 %d: %w", pid, err) - } - - slog.InfoContext(ctx, "Successfully signaled reload", slog.Int("pid", pid)) - return nil -} - -func findPID(cmdName string) (int, error) { - files, err := os.ReadDir("/proc") - if err != nil { - return 0, fmt.Errorf("ReadDir /proc: %w", err) - } - - for _, file := range files { - if !file.IsDir() { - continue - } - - pid, err := strconv.Atoi(file.Name()) - if err != nil { - continue - } - - commBytes, err := os.ReadFile(filepath.Join("/proc", file.Name(), "comm")) - if err != nil { - // Process might have terminated - continue - } - - commGot := strings.TrimSpace(string(commBytes)) - if commGot == cmdName { - return pid, nil - } - } - - return 0, fmt.Errorf("%q process not found", cmdName) -} diff --git a/cmd/atenet/internal/dns/dns_test.go b/cmd/atenet/internal/dns/dns_test.go deleted file mode 100644 index 34116db284..0000000000 --- a/cmd/atenet/internal/dns/dns_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dns - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -type mockConfigReloader struct { - reloaded bool -} - -func (m *mockConfigReloader) Reload(ctx context.Context) error { - m.reloaded = true - return nil -} - -func TestReconcile(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - - // 1. Create mock services - routerSvc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "atenet-router", - Namespace: "ate-system", - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.0.0.1", - }, - } - - dnsSvc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dns", - Namespace: "ate-system", - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.0.0.2", - }, - } - - initialCorefile := ` -.:53 { - errors -} -` - - // 2. Set up a temporary local Corefile on disk - tempDir := t.TempDir() - corefilePath := filepath.Join(tempDir, "Corefile") - if err := os.WriteFile(corefilePath, []byte(initialCorefile), 0644); err != nil { - t.Fatalf("failed to write initial Corefile: %v", err) - } - - kubeDNSCM := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-dns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "stubDomains": `{"other-domain.com":["8.8.8.8"]}`, - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(routerSvc, dnsSvc, kubeDNSCM). - Build() - - reloader := &mockConfigReloader{} - controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: reloader, - } - - // Run one reconciliation loop - ctx := context.Background() - err := controller.reconcile(ctx) - if err != nil { - t.Fatalf("reconcile failed: %v", err) - } - - if !reloader.reloaded { - t.Errorf("expected ConfigReloader to be invoked, but it was not") - } - - // Verify the Corefile on disk has been updated with the router IP - updatedCorefileBytes, err := os.ReadFile(corefilePath) - if err != nil { - t.Fatalf("failed to read updated Corefile from disk: %v", err) - } - updatedCorefile := string(updatedCorefileBytes) - if !strings.Contains(updatedCorefile, `answer "{{ .Name }} 60 IN A 10.0.0.1"`) { - t.Errorf("expected Corefile on disk to contain updated answer line, but got: %s", updatedCorefile) - } - - // Verify kube-system:kube-dns ConfigMap contains the new stub domain without wiping out other-domain - updatedKubeDNSCM := &corev1.ConfigMap{} - err = client.Get(ctx, types.NamespacedName{Name: "kube-dns", Namespace: "kube-system"}, updatedKubeDNSCM) - if err != nil { - t.Fatalf("failed to get updated kube-dns ConfigMap: %v", err) - } - - stubDomainsStr := updatedKubeDNSCM.Data["stubDomains"] - var stubDomains map[string][]string - if err := json.Unmarshal([]byte(stubDomainsStr), &stubDomains); err != nil { - t.Fatalf("failed to unmarshal updated stubDomains: %v", err) - } - - ips, exists := stubDomains["actors.resources.substrate.ate.dev"] - if !exists || len(ips) != 1 || ips[0] != "10.0.0.2" { - t.Errorf("expected stubDomains to map actors.resources.substrate.ate.dev to [10.0.0.2], but got: %v", stubDomains) - } - - otherIPs, exists := stubDomains["other-domain.com"] - if !exists || len(otherIPs) != 1 || otherIPs[0] != "8.8.8.8" { - t.Errorf("expected stubDomains to preserve other-domain.com mapping, but got: %v", stubDomains) - } -} - -func TestReconcileKubeDNSNotFound(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - - routerSvc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "atenet-router", - Namespace: "ate-system", - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.0.0.1", - }, - } - - dnsSvc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dns", - Namespace: "ate-system", - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.0.0.2", - }, - } - - // Set up local Corefile on disk - tempDir := t.TempDir() - corefilePath := filepath.Join(tempDir, "Corefile") - initialCorefile := `answer "{{ .Name }} 60 IN A "` - if err := os.WriteFile(corefilePath, []byte(initialCorefile), 0644); err != nil { - t.Fatalf("failed to write initial Corefile: %v", err) - } - - // kube-dns ConfigMap is omitted to test gracefulness - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(routerSvc, dnsSvc). - Build() - - controller := &Controller{ - Client: client, - Interval: 1 * time.Second, - CorefilePath: corefilePath, - Reloader: &mockConfigReloader{}, - } - - ctx := context.Background() - err := controller.reconcile(ctx) - if err != nil { - t.Fatalf("reconcile should handle missing kube-dns configmap gracefully but failed with: %v", err) - } -} diff --git a/cmd/atenet/internal/root.go b/cmd/atenet/internal/root.go index 1ab52398a2..c992844efc 100644 --- a/cmd/atenet/internal/root.go +++ b/cmd/atenet/internal/root.go @@ -40,6 +40,5 @@ func Execute() { func init() { rootCmd.AddCommand(router.NewRouterCmd()) - rootCmd.AddCommand(NewDnsCmd()) rootCmd.AddCommand(sdsmint.NewSdsmintCmd()) } diff --git a/cmd/atenet/internal/router/dataplane.go b/cmd/atenet/internal/router/dataplane.go index 97eab8a5a0..1169b93df9 100644 --- a/cmd/atenet/internal/router/dataplane.go +++ b/cmd/atenet/internal/router/dataplane.go @@ -32,9 +32,8 @@ type dataplaneHealthCheck struct { } // Both dataplanes resolve the worker address from ext_proc's dynamic -// metadata (see ingress.OriginalDstMetadataKey) and leave :authority/Host -// untouched, so atunnel always authorizes by the actor's own DNS name -- -// ingress.New needs no per-dataplane routing mode. +// metadata (see ingress.OriginalDstMetadataKey). Actor identity is carried in +// explicit headers, so ingress.New needs no per-dataplane routing mode. func (r atenetRouter) healthCheck() dataplaneHealthCheck { switch r { diff --git a/cmd/atenet/internal/router/extproc/dispatch.go b/cmd/atenet/internal/router/extproc/dispatch.go index 3f0f4e7766..cae5b9471e 100644 --- a/cmd/atenet/internal/router/extproc/dispatch.go +++ b/cmd/atenet/internal/router/extproc/dispatch.go @@ -47,8 +47,8 @@ const EgressFilterChainName = "egress" // filter chain name; the request cannot influence it. // // An unrecognized or absent attribute means ingress, the fail-safe direction: -// an egress request misrouted to the ingress handler fails to parse as an actor -// DNS name and 404s, whereas the reverse leaks control-plane state. +// an egress request misrouted to the ingress handler lacks the required actor +// routing headers and fails, whereas the reverse leaks control-plane state. func directionOf(req *extprocv3.ProcessingRequest) Direction { if requestAttribute(req, directionAttribute) == string(DirectionEgress) { return DirectionEgress diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index c4aefb35b6..f77a808c4b 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -99,8 +99,8 @@ const ( // OriginalDstClusterName routes actor traffic to the worker's atunnel // ingress by the IP:port ext_proc reports in dynamic metadata (see - // ingress.OriginalDstMetadataKey), while the request :authority stays the - // actor DNS name so atunnel can identify the active actor. + // ingress.OriginalDstMetadataKey). Actor identity remains in explicit + // request headers for atunnel to authorize. OriginalDstClusterName = "actor_original_dst" WildcardIP = "0.0.0.0" @@ -766,9 +766,8 @@ func (x *XdsServer) buildMainInternalCluster() *clusterv3.Cluster { // buildOriginalDstCluster dials the exact worker atunnel address supplied by // ext_proc in dynamic metadata (see ingress.OriginalDstMetadataKey). It does -// not derive the destination from :authority, so the request keeps the actor -// DNS name as its Host for atunnel to authorize. mTLS to atunnel is applied -// via the shared upstream transport socket (SPIFFE URI validation). +// not derive the destination from :authority. mTLS to atunnel is applied via +// the shared upstream transport socket (SPIFFE URI validation). func (x *XdsServer) buildOriginalDstCluster() *clusterv3.Cluster { cluster := &clusterv3.Cluster{ Name: OriginalDstClusterName, diff --git a/demos/parking/README.md b/demos/parking/README.md index f9d32bd896..b6033a9af2 100644 --- a/demos/parking/README.md +++ b/demos/parking/README.md @@ -40,8 +40,7 @@ This command will: ### 2. Create more actors than workers -Actors live in the demo's **atespace** (`ate-demo-parking`), and their DNS names -embed it (`..actors.resources.substrate.ate.dev`). `--template-ref` +Actors live in the demo's **atespace** (`ate-demo-parking`). `--template-ref` names the template, resolved in the actor's atespace: ```bash diff --git a/docs/api-guide.md b/docs/api-guide.md index 5a5302ca8c..72c40562c8 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -142,10 +142,11 @@ Unlike a Pod, an actor is sized by its **`limits`** (CPU and Memory): the size i Container environment variables support literal `value` entries only. Values are not interpolated (`$(VAR)` references are not expanded), and Kubernetes `envFrom`/`valueFrom` sources are not supported. -### Workload Connectivity (Uniform DNS) -Substrate uses a **Uniform DNS Mesh**: every actor created from a template is automatically reachable through the **Substrate Router** via its atespace and name: +### Workload Connectivity -**Format:** `..actors.resources.substrate.ate.dev` +A higher-order system reaches an actor through the **Substrate Router** by +setting `X-Ate-Actor-Name` to the actor name and `X-Ate-Atespace` to its +atespace. Substrate does not provide DNS discovery for actors. ### SystemInfo Volumes @@ -154,7 +155,7 @@ To deliver identity information, including credentials, to a running actor, you Available information sources: #### actorMetadata -The actorMetadata data source projects the actor's identity fields to files, one per item, analogous to the [Kubernetes downwardAPI volume](https://kubernetes.io/docs/concepts/storage/downward-api/). Each item selects a `field` — `name` (unique within an atespace), `atespace` (together with the name, the actor's full identity and DNS name), or `uid` (server-generated, distinguishes incarnations of the same name) — and the relative `path` the value is written to, raw with no trailing newline. +The actorMetadata data source projects the actor's identity fields to files, one per item, analogous to the [Kubernetes downwardAPI volume](https://kubernetes.io/docs/concepts/storage/downward-api/). Each item selects a `field` — `name` (unique within an atespace), `atespace` (together with the name, the actor's full identity), or `uid` (server-generated, distinguishes incarnations of the same name) — and the relative `path` the value is written to, raw with no trailing newline. ```yaml spec: diff --git a/docs/architecture.md b/docs/architecture.md index 4011c9f36f..0d52209602 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -330,18 +330,16 @@ A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class * **micro-VM** (`ateom-microvm`): Runs the workload inside a [Kata Containers](https://katacontainers.io/) guest on the [Cloud Hypervisor](https://www.cloudhypervisor.org/) VMM. Suspend and resume capture a memory-only VM snapshot and restore it on-demand using `userfaultfd` memory demand-paging, with container rootfs writes captured in guest RAM via a `tmpfs` overlay. `DurableDir` volumes are host-backed instead, served over a second (writable) virtio-fs share and shipped in snapshots as a tar, so a `Data`-scope snapshot can capture them without any guest memory. Each volume is a subdirectory of that one share, so an actor can have several at no extra cost in devices — which is why the micro-VM class lifts the single-`DurableDir` limit that still applies to gVisor. -### Networking Stack (`atenet` DNS + `atunnel`) +### Networking Stack (`atenet` + `atunnel`) Handles actor-aware routing and automatic re-animation. - * **Uniform DNS Mesh**: Substrate provides a location-transparent actor discovery scheme via a global DNS suffix (`..actors.resources.substrate.ate.dev`). - * **Ingress Routing**: `atenet-router` runs Envoy with an `ext_proc` external - processor and accepts HTTP traffic for the Actor DNS suffix. The ext_proc - reads the Actor name from `X-Ate-Actor-Name` and the Atespace from - `X-Ate-Atespace`, then calls the Control Plane to resume the Actor and - resolve its current worker assignment. `Host` remains application authority - and does not select the Actor. + processor. A higher-order system connects to the router and supplies the + Actor name in `X-Ate-Actor-Name` and the Atespace in `X-Ate-Atespace`. + The ext_proc calls the Control Plane to resume the Actor and resolve its + current worker assignment. `Host` remains application authority and does + not select the Actor. * **Worker Tunnel**: After resolving the assignment, `atenet-router` opens an authenticated TLS tunnel to the worker's `atunnel` listener on port 443. @@ -370,7 +368,6 @@ suspended (UML sequence diagram): ```mermaid sequenceDiagram actor Client - participant DNS as atenet DNS participant Gateway as atenet-router participant API as ate-api-server participant Atelet as atelet @@ -378,8 +375,6 @@ sequenceDiagram participant A as Actor participant Store as snapshot storage - Client->>DNS: resolve actor DNS name - DNS-->>Client: ingress gateway address Client->>Gateway: HTTP request (X-Ate-Actor-Name, X-Ate-Atespace) Gateway->>API: ResumeActor(atespace, actor name) API->>Atelet: Restore @@ -502,10 +497,9 @@ Agent Substrate is built on a **Defense-in-Depth** model: versions. * **Request Authorization**: The system currently performs **Identity-Aware - Routing** by utilizing a uniform DNS routing scheme - (`..actors.resources.substrate.ate.dev`) - at the gateway to extract and validate actor identifiers from incoming traffic. This - ensures requests are only routed to recognized, registered actors. + Routing** by extracting and validating the `X-Ate-Actor-Name` and + `X-Ate-Atespace` headers at the gateway. This ensures requests are only + routed to recognized, registered actors. Pluggable, granular authorization policies are planned for future milestones. diff --git a/docs/glossary.md b/docs/glossary.md index 202f4f2ab7..6c7010c2ff 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -62,9 +62,8 @@ because they change too frequently for etcd. sandbox runtime on behalf of atelet. This decouples the physical pod lifecycle from the sandboxed agent process. -- **atenet**: the networking stack. It provides a DNS server for actor - resolution and a router that resumes suspended Actors on demand and routes - traffic to the right worker pod. +- **atenet**: the networking stack. Its router resumes suspended Actors on + demand and routes traffic to the right worker pod. - **podcertcontroller**: issues short-lived pod certificates that components use as their TLS identity to authenticate connections to one another @@ -154,6 +153,6 @@ because they change too frequently for etcd. ## Networking -- **Uniform DNS Mesh**: every Actor is reachable at a uniform address, - `..actors.resources.substrate.ate.dev`, resolved by atenet. Traffic to - that name is routed (and the Actor resumed if needed) automatically. +- **Actor routing headers**: a higher-order system sends traffic to the + Substrate router with `X-Ate-Actor-Name` and `X-Ate-Atespace`. The router + uses these headers to locate and resume the Actor. diff --git a/docs/metrics/substrate.yaml b/docs/metrics/substrate.yaml index cde6df7e5c..786643b1ce 100644 --- a/docs/metrics/substrate.yaml +++ b/docs/metrics/substrate.yaml @@ -221,11 +221,6 @@ blind_spots: The hit and miss counter exists. But the garbage collector (the 85% and 80% limits) and the time to prepare an image send nothing. Thus you cannot see the disk pressure from the layer pool. - - area: atenet-dns - code: cmd/atenet/internal/dns.go - brief: > - There are no instruments, and the Corefile has no prometheus plugin. If a - reload fails, the answers stay old and no signal shows it. - area: podcertcontroller code: cmd/podcertcontroller brief: This component has no metrics. diff --git a/docs/roadmap.md b/docs/roadmap.md index a2730f2339..b042209019 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -44,7 +44,7 @@ Below is a collection of finer-grained efforts which we believe align with the a * Actor security boundary implementation, default deny with explicit ACLs at scale with low latency. This overlaps with some of the security items (see below). * Policy definition: between framework (outside) and Actors, between Actors, Actor Egress. -* Standardized DNS Mesh: Moving to a production-grade routing format (\.actors.resources.substrate.ate.dev) for location-transparent actor-to-actor communication. +* Actor-to-actor routing through explicit identity headers. ### Storage diff --git a/docs/threat-model.md b/docs/threat-model.md index 545e4883eb..61f6323333 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -39,7 +39,7 @@ Substrate is an early, fast moving product. It is full of debate and subject to * **Worker:** Preprovisioned Pods that actors get scheduled to. * **Actor:** The core compute primitive, gets scheduled to/from worker via Run for cold start and Resume for snapshot resume. * **Actor Network:** `ateom` creates a private point-to-point veth network for the active Actor inside a worker Pod. The Actor is not served directly from the worker Pod's port 80; ingress enters through `atunnel`'s authenticated listener on port 443. -* **Actor DNS:** Each Actor gets a DNS name like `..actors.resources.substrate.ate.dev`. Substrate runs a custom CoreDNS instance that returns the `atenet-router` Service IP for A record queries matching the Actor DNS pattern. A controller keeps this target current and configures kube-dns with a stub domain for `actors.resources.substrate.ate.dev`, enabling traditional Kubernetes Pods to resolve Actor names. +* **Actor Routing:** A higher-order system connects to `atenet-router` and identifies the target Actor with `X-Ate-Actor-Name` and `X-Ate-Atespace`. Host and authority values are application metadata and do not select the Actor. * **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. The ext_proc reads the Actor name and Atespace from `X-Ate-Actor-Name` and `X-Ate-Atespace`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites both routing headers before forwarding. The router then connects with mTLS to `atunnel`; `atunnel` validates the router identity and authorizes the header pair against the Actor currently assigned to that worker. * **Object Storage:** Used to store actor snapshots. * **Filesystem support:** Container local filesystem is saved in snapshots, future integrations likely to include networked storage. diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 2e9fd0c03c..be023962a2 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -818,7 +818,6 @@ deploy_atenet() { ensure_egress_mitm_ca_pool_secret apply_atenet_egress - run_ko apply -f manifests/ate-install/atenet-dns.yaml run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" run_kubectl rollout status deployment/dns -n ate-system --timeout="$(rollout_timeout)" @@ -1089,7 +1088,6 @@ delete_atenet() { run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-egress.yaml run_kubectl delete --ignore-not-found \ -f manifests/ate-install/atenet-egress-with-sdsmint.yaml - run_kubectl delete --ignore-not-found -f manifests/ate-install/atenet-dns.yaml } deploy_benchmarks() { diff --git a/internal/benchmarking/boomer/glutton/durdir.go b/internal/benchmarking/boomer/glutton/durdir.go index 26f6a5ccc6..835a383a69 100644 --- a/internal/benchmarking/boomer/glutton/durdir.go +++ b/internal/benchmarking/boomer/glutton/durdir.go @@ -28,8 +28,8 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index 0cae4996f7..cc5021ca0b 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -29,8 +29,8 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 4a578234ba..0ff08c471b 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -23,8 +23,8 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 0bacc5b895..02aa497369 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -150,10 +150,9 @@ func TestIngressGRPC(t *testing.T) { atunnel.AtespaceHeader, actorRef.Atespace, ) - // Cleartext h2c to the router's HTTP port, with the Actor's DNS name as the - // :authority — the same routing key every other ingress test in this suite - // uses, just carried by a gRPC client instead of an HTTP one. The h2 ALPN - // offer is about the *TLS* listener; nothing here needs it. + // Cleartext h2c to the router's HTTP port. Explicit metadata identifies the + // Actor; the conventional actor authority remains application metadata. The + // h2 ALPN offer is about the *TLS* listener; nothing here needs it. conn, err := grpc.NewClient(routerAddress(t, ctx), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithAuthority(resources.ActorDNSName(actorRef)), diff --git a/internal/resources/actor.go b/internal/resources/actor.go index cf9479835f..ee93775c89 100644 --- a/internal/resources/actor.go +++ b/internal/resources/actor.go @@ -18,8 +18,7 @@ const ( // ResourceNameRegexPattern is the regular expression pattern for a valid // Substrate resource name. ResourceNameRegexPattern = `[a-z0-9]([-a-z0-9]*[a-z0-9])?` - // ActorDNSSuffix is suffix to the DNS name for direct access to Actor - // "..actors.resources.substrate.ate.dev" + // ActorDNSSuffix is the suffix used by the conventional actor authority. ActorDNSSuffix = "actors.resources.substrate.ate.dev" // GoldenActorAtespace is the reserved system atespace that per-template golden // actors live in. diff --git a/internal/resources/resourceref.go b/internal/resources/resourceref.go index c7fb91245d..6133bb45f9 100644 --- a/internal/resources/resourceref.go +++ b/internal/resources/resourceref.go @@ -15,10 +15,8 @@ package resources import ( - "fmt" "log/slog" "reflect" - "strings" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -77,53 +75,12 @@ func ActorRefFromActor(a *ateapipb.Actor) ActorRef { } } -// ActorDNSName returns the uniform DNS name the actor is reachable at. -// This is: "..actors.resources.substrate.ate.dev". +// ActorDNSName returns the conventional authority name for an actor. +// Substrate does not resolve this name; routing uses explicit actor headers. func ActorDNSName(r ActorRef) string { return r.Name + "." + r.Atespace + "." + ActorDNSSuffix } -// lowerASCII folds A-Z and leaves every other byte alone. Resource names are -// DNS-1123 labels, so ASCII is the whole alphabet here, and strings.ToLower -// would additionally fold characters outside it onto ASCII letters (the Kelvin -// sign U+212A onto "k", U+017F onto "s"), letting a non-ASCII host reach an -// actor under a spelling that is not its name. -func lowerASCII(s string) string { - b := []byte(s) - for i, c := range b { - if c >= 'A' && c <= 'Z' { - b[i] = c + ('a' - 'A') - } - } - return string(b) -} - -// ParseActorDNSName parses a DNS name for a given actor. -// -// The name is folded to lower case first. DNS lookups are case-insensitive -// (RFC 4343), so a client that resolved "MyActor.MySpace." reaches us -// with that spelling preserved in the Host header, while actor and atespace -// names are always lower case. Folding keeps the request addressed to the same -// actor its DNS lookup resolved to instead of failing to parse. -func ParseActorDNSName(name string) (ActorRef, error) { - normalized := lowerASCII(strings.TrimSuffix(name, ".")) - rest, found := strings.CutSuffix(normalized, "."+ActorDNSSuffix) - if !found { - return ActorRef{}, fmt.Errorf("invalid actor DNS name: must end with %s, got %q", ActorDNSSuffix, name) - } - actorName, atespace, found := strings.Cut(rest, ".") - if !found { - return ActorRef{}, fmt.Errorf("invalid actor DNS name: expected ..%s, got %q", ActorDNSSuffix, name) - } - if !IsValidResourceName(actorName) { - return ActorRef{}, fmt.Errorf("invalid actor DNS name %q: %q is not a valid actor name", name, actorName) - } - if !IsValidResourceName(atespace) { - return ActorRef{}, fmt.Errorf("invalid actor DNS name %q: %q is not a valid atespace", name, atespace) - } - return ActorRef{Atespace: atespace, Name: actorName}, nil -} - // ActorTemplateRef identifies an ActorTemplate by the (atespace, name). type ActorTemplateRef = ResourceRef[*ateapipb.ActorTemplate] diff --git a/internal/resources/resourceref_test.go b/internal/resources/resourceref_test.go index a7bf12671c..d0562f07f1 100644 --- a/internal/resources/resourceref_test.go +++ b/internal/resources/resourceref_test.go @@ -97,49 +97,6 @@ func TestActorRefDNSName(t *testing.T) { if got != want { t.Errorf("ActorDNSName() = %q, want %q", got, want) } - - parsed, err := ParseActorDNSName(got) - if err != nil { - t.Fatalf("ParseActorDNSName(%q) error = %v", got, err) - } - if parsed != actorRef { - t.Errorf("round-trip = %+v, want %+v", parsed, actorRef) - } -} - -func TestParseActorDNSName(t *testing.T) { - tests := []struct { - name string - input string - want ActorRef - wantErr bool - }{ - {"valid", "act-1.team-a.actors.resources.substrate.ate.dev", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - {"valid trailing dot", "act-1.team-a.actors.resources.substrate.ate.dev.", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - {"wrong suffix", "act-1.team-a.example.com", ActorRef{}, true}, - {"missing atespace", "act-1.actors.resources.substrate.ate.dev", ActorRef{}, true}, - {"mixed-case actor name", "ACT-1.team-a.actors.resources.substrate.ate.dev", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - {"mixed-case atespace", "act-1.TEAM-A.actors.resources.substrate.ate.dev", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - {"mixed-case suffix", "act-1.team-a.Actors.Resources.Substrate.Ate.Dev", ActorRef{Atespace: "team-a", Name: "act-1"}, false}, - // strings.ToLower would fold the Kelvin sign onto "k" and hand "act-1k" a - // request addressed to a name no actor can have. - {"non-ASCII actor name", "act-1K.team-a.actors.resources.substrate.ate.dev", ActorRef{}, true}, - {"invalid actor name", "act_1.team-a.actors.resources.substrate.ate.dev", ActorRef{}, true}, - {"invalid atespace", "act-1.team_a.actors.resources.substrate.ate.dev", ActorRef{}, true}, - {"host:port not accepted", "act-1.team-a.actors.resources.substrate.ate.dev:8080", ActorRef{}, true}, - {"empty", "", ActorRef{}, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ParseActorDNSName(tt.input) - if (err != nil) != tt.wantErr { - t.Fatalf("ParseActorDNSName(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) - } - if got != tt.want { - t.Errorf("ParseActorDNSName(%q) = %+v, want %+v", tt.input, got, tt.want) - } - }) - } } func TestActorRefObjectRefRoundTrip(t *testing.T) { diff --git a/manifests/ate-install/atenet-dns.yaml b/manifests/ate-install/atenet-dns.yaml deleted file mode 100644 index 77d71bf594..0000000000 --- a/manifests/ate-install/atenet-dns.yaml +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: atenet-dns - namespace: ate-system - labels: - app: dns ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: atenet-dns - namespace: ate-system -rules: -- apiGroups: [""] - resources: ["services"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: atenet-dns - namespace: ate-system -subjects: -- kind: ServiceAccount - name: atenet-dns - namespace: ate-system -roleRef: - kind: Role - name: atenet-dns - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: atenet-dns - namespace: kube-system -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: atenet-dns - namespace: kube-system -subjects: -- kind: ServiceAccount - name: atenet-dns - namespace: ate-system -roleRef: - kind: Role - name: atenet-dns - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dns - namespace: ate-system - labels: - app: dns -spec: - replicas: 1 - selector: - matchLabels: - app: dns - template: - metadata: - labels: - app: dns - spec: - serviceAccountName: atenet-dns - shareProcessNamespace: true - initContainers: - - name: init-dns - image: busybox:1.36 - command: ["sh", "-c"] - # Initial core file is sufficient to start CoreDNS but does not contain - # any additional configuration. The controller will update the Corefile. - args: - - | - cat <<'EOF' > /etc/coredns/Corefile - .:53 { - errors - health :8080 - ready :8181 - reload - } - EOF - volumeMounts: - - name: dns-config-volume - mountPath: /etc/coredns - containers: - - name: coredns - image: coredns/coredns:1.11.1 - imagePullPolicy: IfNotPresent - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: dns-config-volume - mountPath: /etc/coredns - ports: - - name: dns - containerPort: 53 - protocol: UDP - - name: dns-tcp - containerPort: 53 - protocol: TCP - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - initialDelaySeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - - name: dns-controller - image: ko://github.com/agent-substrate/substrate/cmd/atenet - args: - - "dns" - - "--log-level=debug" - - "--interval=10s" - - "--corefile-path=/etc/coredns/Corefile" - volumeMounts: - - name: dns-config-volume - mountPath: /etc/coredns - volumes: - - name: dns-config-volume - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: dns - namespace: ate-system - labels: - app: dns -spec: - selector: - app: dns - type: ClusterIP - # Prefer, not Require: Require fails Service creation on a single-stack cluster. - ipFamilyPolicy: PreferDualStack - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP \ No newline at end of file diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index 2a9b83a969..6a2031ef70 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -22,7 +22,6 @@ resources: - ../ate-api-server.yaml - ../ate-controller.yaml - ../atelet.yaml - - ../atenet-dns.yaml - ../atenet-router.yaml - ../pod-certificate-controller.yaml - ../ate-otel-config.yaml diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 1fabcdec4d..5e13815263 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -25,7 +25,6 @@ resources: - ../ate-api-server.yaml - ../ate-controller.yaml - ./atelet - - ../atenet-dns.yaml - ../atenet-router.yaml - ../pod-certificate-controller.yaml - ate-otel-config.yaml From a1eb26b8d891dd96d4c7f0c2518470ffe37efbb8 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Mon, 31 Aug 2026 12:57:06 -0500 Subject: [PATCH 07/10] Remove DNS wait Signed-off-by: Keith Mattix II --- hack/install-ate.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/hack/install-ate.sh b/hack/install-ate.sh index be023962a2..fb78dd8d3c 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -820,7 +820,6 @@ deploy_atenet() { apply_atenet_egress run_kubectl rollout status deployment/atenet-router -n ate-system --timeout="$(rollout_timeout)" run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout="$(rollout_timeout)" - run_kubectl rollout status deployment/dns -n ate-system --timeout="$(rollout_timeout)" } # get_actor_state echoes the actor's state enum (e.g. ACTOR_STATE_SUSPENDED). From 9c5d850f71f141b6f713c2e0d4e3ef03b643a76d Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Mon, 31 Aug 2026 14:17:21 -0500 Subject: [PATCH 08/10] Remove legacy actor host construction --- benchmarking/locust/tests/counter_demo.py | 3 -- benchmarking/nighthawk-ingress/actors.py | 2 - benchmarking/nighthawk-ingress/spec.py | 12 ------ .../nighthawk-ingress/tests/test_spec.py | 8 ---- cmd/atenet/internal/router/ingress/ingress.go | 9 +++-- .../internal/router/ingress/ingress_test.go | 13 ++++--- cmd/atenet/internal/router/xds.go | 5 ++- demos/counter/README.md | 2 +- demos/egress/README.md | 3 +- demos/egress/test-egress.sh | 1 - demos/parking/load.sh | 4 +- demos/sandbox/client/main.go | 7 ++-- docs/threat-model.md | 38 +++++++++---------- hack/run-microvm-demo.sh | 3 +- hack/verify-atenet-drain.sh | 3 +- hack/verify-egress-demo.sh | 3 +- internal/atenet/headers.go | 24 ++++++++++++ internal/atunnel/ingress.go | 15 +++----- internal/atunnel/ingress_test.go | 38 +++++++++---------- .../benchmarking/boomer/glutton/durdir.go | 9 ++--- .../boomer/glutton/fixture_test.go | 1 - .../benchmarking/boomer/glutton/lifecycle.go | 15 +++----- .../boomer/glutton/memfill_test.go | 5 +-- internal/e2e/router_client.go | 13 +++---- internal/e2e/router_client_test.go | 4 +- internal/e2e/suites/demo/demo_test.go | 7 ++-- .../suites/networking/arbitraryport_test.go | 6 +-- .../e2e/suites/networking/grpcingress_test.go | 14 +++---- .../networking/websocketingress_test.go | 7 ++-- internal/resources/actor.go | 2 - internal/resources/resourceref.go | 6 --- internal/resources/resourceref_test.go | 10 ----- 32 files changed, 127 insertions(+), 165 deletions(-) create mode 100644 internal/atenet/headers.go diff --git a/benchmarking/locust/tests/counter_demo.py b/benchmarking/locust/tests/counter_demo.py index 079d7424c5..81759c3e5d 100644 --- a/benchmarking/locust/tests/counter_demo.py +++ b/benchmarking/locust/tests/counter_demo.py @@ -51,7 +51,6 @@ # Atenet router fronts all actor traffic. The actor routing headers select # the actor whose current worker pod the router resolves. ROUTER_URL = "http://atenet-router.ate-system.svc.cluster.local" -ACTOR_DOMAIN = "actors.resources.substrate.ate.dev" class CounterUser(User): @@ -101,7 +100,6 @@ def on_start(self) -> None: # One HTTP session per user, talking to the router. self.http_session = requests.Session() self.run_url = f"{ROUTER_URL}/" - self.host_header = f"{self.actor_name}.{ATESPACE}.{ACTOR_DOMAIN}" def on_stop(self) -> None: update_user_count(-1, self.__class__.__name__) @@ -137,7 +135,6 @@ def run_and_suspend(self) -> None: start_time = time.time() with tracer.start_as_current_span("RunCounter") as span: headers = { - "Host": self.host_header, "X-Ate-Actor-Name": self.actor_name, "X-Ate-Atespace": ATESPACE, } diff --git a/benchmarking/nighthawk-ingress/actors.py b/benchmarking/nighthawk-ingress/actors.py index 07ad997da2..808fa2da29 100644 --- a/benchmarking/nighthawk-ingress/actors.py +++ b/benchmarking/nighthawk-ingress/actors.py @@ -114,7 +114,6 @@ def _warm_actor( 200 or the deadline expires. Resume errors are retried: ateapi returns FailedPrecondition/Unavailable until a worker frees up.""" ref = ateapi_pb2.ObjectRef(atespace=atespace, name=name) - host = spec_mod.actor_host(name, atespace) session = requests.Session() last_err: str = "not attempted" while time.time() < deadline: @@ -126,7 +125,6 @@ def _warm_actor( resp = session.post( f"{router_url.rstrip('/')}/ping", headers={ - "Host": host, "X-Ate-Actor-Name": name, "X-Ate-Atespace": atespace, }, diff --git a/benchmarking/nighthawk-ingress/spec.py b/benchmarking/nighthawk-ingress/spec.py index 0ef83af3ac..29edc6fc41 100644 --- a/benchmarking/nighthawk-ingress/spec.py +++ b/benchmarking/nighthawk-ingress/spec.py @@ -58,11 +58,6 @@ ) -def actor_host(actor: str, atespace: str) -> str: - """Return an actor's stable mesh DNS name.""" - return f"{actor}.{atespace}.actors.resources.substrate.ate.dev" - - def _metric_spec(name: str) -> dict: return {"metric_name": name, "metrics_plugin_name": BUILTIN_METRICS_PLUGIN} @@ -117,13 +112,6 @@ def build_spec_dict( { "request_method": "POST", "request_headers": [ - { - "header": { - "key": "host", - "value": actor_host(actor_name, atespace), - }, - "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", - }, { "header": {"key": "x-ate-actor-name", "value": actor_name}, "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", diff --git a/benchmarking/nighthawk-ingress/tests/test_spec.py b/benchmarking/nighthawk-ingress/tests/test_spec.py index 3652a76a6e..fb754adf2f 100644 --- a/benchmarking/nighthawk-ingress/tests/test_spec.py +++ b/benchmarking/nighthawk-ingress/tests/test_spec.py @@ -78,7 +78,6 @@ def test_identity_rotation_covers_all_actors(): ] assert got == [ { - "host": spec_mod.actor_host(actor_name, "benchmark"), "x-ate-actor-name": actor_name, "x-ate-atespace": "benchmark", } @@ -125,13 +124,6 @@ def test_tail_latency_slo_threshold(): assert len(build(tail_latency_slo_ms=None)["metric_thresholds"]) == 2 -def test_actor_host_format(): - assert ( - spec_mod.actor_host("sb-1", "benchmark") - == "sb-1.benchmark.actors.resources.substrate.ate.dev" - ) - - def test_spec_round_trips_through_real_protos(): """Round-trip through the real Nighthawk protos; self-skips outside the runner image (no FileDescriptorSet).""" diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 2d5c35caef..021c781db5 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -36,6 +36,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -85,8 +86,8 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM defer span.End() actorRef := resources.ActorRef{ - Name: routingValue(md, atunnel.ActorNameHeader, extproc.ActorNameFilterStateAttribute), - Atespace: routingValue(md, atunnel.AtespaceHeader, extproc.AtespaceFilterStateAttribute), + Name: routingValue(md, atenet.ActorNameHeader, extproc.ActorNameFilterStateAttribute), + Atespace: routingValue(md, atenet.AtespaceHeader, extproc.AtespaceFilterStateAttribute), } if !resources.IsValidResourceName(actorRef.Name) || !resources.IsValidResourceName(actorRef.Atespace) { return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_NotFound, "invalid actor reference") @@ -165,8 +166,8 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM // different actor after this request has been resolved. mutation := &extprocv3.HeaderMutation{} for _, header := range []struct{ name, value string }{ - {atunnel.ActorNameHeader, actorRef.Name}, - {atunnel.AtespaceHeader, actorRef.Atespace}, + {atenet.ActorNameHeader, actorRef.Name}, + {atenet.AtespaceHeader, actorRef.Atespace}, {atunnel.TargetPortHeader, strconv.Itoa(targetPort)}, } { mutation.SetHeaders = append(mutation.SetHeaders, &corev3.HeaderValueOption{ diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index 5146f24a96..cea42e6e3c 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -32,6 +32,7 @@ import ( "google.golang.org/protobuf/types/known/structpb" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/atunnel" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" ) @@ -47,8 +48,8 @@ func (m *mockClient) ResumeActor(ctx context.Context, in *ateapipb.ResumeActorRe func requestMetadata(actorName, atespace string, headers ...*corev3.HeaderValue) *extproc.RequestMetadata { headers = append(headers, - &corev3.HeaderValue{Key: atunnel.ActorNameHeader, Value: actorName}, - &corev3.HeaderValue{Key: atunnel.AtespaceHeader, Value: atespace}, + &corev3.HeaderValue{Key: atenet.ActorNameHeader, Value: actorName}, + &corev3.HeaderValue{Key: atenet.AtespaceHeader, Value: atespace}, ) return extproc.NewRequestMetadata(headers, nil) } @@ -306,10 +307,10 @@ func TestHandleRequestHeaders(t *testing.T) { if got := gotMutations[strings.ToLower(atunnel.TargetPortHeader)]; got != tc.expectedTargetPort { t.Errorf("target port mutation = %q, want %q", got, tc.expectedTargetPort) } - if got := gotMutations[strings.ToLower(atunnel.ActorNameHeader)]; got != testUUID { + if got := gotMutations[strings.ToLower(atenet.ActorNameHeader)]; got != testUUID { t.Errorf("actor name mutation = %q, want %q", got, testUUID) } - if got := gotMutations[strings.ToLower(atunnel.AtespaceHeader)]; got != "team-a" { + if got := gotMutations[strings.ToLower(atenet.AtespaceHeader)]; got != "team-a" { t.Errorf("atespace mutation = %q, want %q", got, "team-a") } if got := dynamicMetadataTarget(res.DynamicMetadata); got != tc.expectedTarget { @@ -381,8 +382,8 @@ func TestHandleRequestHeadersUsesRetainedConnectAuthorityForPort(t *testing.T) { }, } md := extproc.NewRequestMetadata([]*corev3.HeaderValue{ - {Key: atunnel.ActorNameHeader, Value: testUUID}, - {Key: atunnel.AtespaceHeader, Value: "team-a"}, + {Key: atenet.ActorNameHeader, Value: testUUID}, + {Key: atenet.AtespaceHeader, Value: "team-a"}, {Key: ":authority", Value: "inner.example"}, }, attrs) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index f77a808c4b..56dfe0402f 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -34,6 +34,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/atunnel" accesslogv3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" @@ -899,8 +900,8 @@ func (x *XdsServer) buildMainInternalListener() *listenerv3.Listener { func actorRoutingFilterStateFilter() *hcmv3.HttpFilter { values := make([]*setfilterstatecommonv3.FilterStateValue, 0, 3) for _, routingField := range []struct{ key, header string }{ - {extproc.ActorNameFilterStateKey, atunnel.ActorNameHeader}, - {extproc.AtespaceFilterStateKey, atunnel.AtespaceHeader}, + {extproc.ActorNameFilterStateKey, atenet.ActorNameHeader}, + {extproc.AtespaceFilterStateKey, atenet.AtespaceHeader}, {extproc.ConnectAuthorityFilterStateKey, extproc.AuthorityHeader}, } { values = append(values, &setfilterstatecommonv3.FilterStateValue{ diff --git a/demos/counter/README.md b/demos/counter/README.md index ebe02f9f7c..ec331d38b6 100644 --- a/demos/counter/README.md +++ b/demos/counter/README.md @@ -123,7 +123,7 @@ proxy behavior wouldn't do: curl -p -x http://localhost:8001 \ -H "X-Ate-Actor-Name: my-counter-1" \ -H "X-Ate-Atespace: ate-demo-counter" \ - http://my-counter-1.ate-demo-counter.actors.resources.substrate.ate.dev:9090/ + http://my-counter-1:9090/ ``` This reaches the same actor's second listener and resumes it exactly like any diff --git a/demos/egress/README.md b/demos/egress/README.md index 91bd428028..cd4171e2d8 100644 --- a/demos/egress/README.md +++ b/demos/egress/README.md @@ -119,9 +119,8 @@ kubectl ate resume actor egress-demo -a ate-demo-egress # wait for ACTOR_STATE # 3. Drive the Actor's egress through the ingress gateway. kubectl -n ate-system port-forward service/atenet-router 8000:80 & curl -s -X POST http://localhost:8000/ \ - -H 'Host: egress-demo.ate-demo-egress.actors.resources.substrate.ate.dev' \ -H 'X-Ate-Actor-Name: egress-demo' \ - -H 'X-Ate-Atespace: demo' \ + -H 'X-Ate-Atespace: ate-demo-egress' \ -H 'Content-Type: application/json' \ -d "{\"url\":\"http://${TARGET_IP}:80/\"}" ``` diff --git a/demos/egress/test-egress.sh b/demos/egress/test-egress.sh index 7ea28b83d0..5faff2ff4c 100755 --- a/demos/egress/test-egress.sh +++ b/demos/egress/test-egress.sh @@ -101,7 +101,6 @@ BEFORE=$(egress_log_count) ${K} -n ate-system port-forward service/atenet-router 18099:80 >/tmp/egress-pf.log 2>&1 & PF=$!; sleep 4 CODE=$(curl -s -o /tmp/egress-body.txt -w '%{http_code}' -X POST http://localhost:18099/ \ - -H "Host: ${ACTOR}.${ATESPACE}.actors.resources.substrate.ate.dev" \ -H "X-Ate-Actor-Name: ${ACTOR}" \ -H "X-Ate-Atespace: ${ATESPACE}" \ -H 'Content-Type: application/json' \ diff --git a/demos/parking/load.sh b/demos/parking/load.sh index 25147241f8..2b79be3eda 100755 --- a/demos/parking/load.sh +++ b/demos/parking/load.sh @@ -46,7 +46,6 @@ ROUTER="http://localhost:8000" # The template name, resolved in the actors' atespace (--template-ref). TEMPLATE="parking" ATESPACE="ate-demo-parking" -SUFFIX="actors.resources.substrate.ate.dev" usage() { cat <<'EOF' @@ -102,12 +101,11 @@ done # One worker per actor: hammer it with request->suspend until the deadline. worker() { - local actor="$1" host="$1.${ATESPACE}.${SUFFIX}" log="${TMP}/$1.log" + local actor="$1" log="${TMP}/$1.log" local deadline=$(( $(date +%s) + DURATION )) while [[ $(date +%s) -lt ${deadline} ]]; do # %{http_code} lets us tally outcomes; %{time_total} reveals parking waits. curl -s -o /dev/null -w '%{http_code} %{time_total}\n' \ - -H "Host: ${host}" \ -H "X-Ate-Actor-Name: ${actor}" \ -H "X-Ate-Atespace: ${ATESPACE}" "${ROUTER}" >>"${log}" 2>/dev/null # Free the worker so a parked competitor can proceed (simulate going idle). diff --git a/demos/sandbox/client/main.go b/demos/sandbox/client/main.go index b1d76d48ff..7a2e3d0b90 100644 --- a/demos/sandbox/client/main.go +++ b/demos/sandbox/client/main.go @@ -28,7 +28,7 @@ import ( "strings" "syscall" - "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/pflag" @@ -186,9 +186,8 @@ func runCommand(ctx context.Context, atenetAddr string, actorRef resources.Actor return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Content-Type", "application/json") - req.Header.Set(atunnel.ActorNameHeader, actorRef.Name) - req.Header.Set(atunnel.AtespaceHeader, actorRef.Atespace) - req.Host = resources.ActorDNSName(actorRef) + req.Header.Set(atenet.ActorNameHeader, actorRef.Name) + req.Header.Set(atenet.AtespaceHeader, actorRef.Atespace) resp, err := http.DefaultClient.Do(req) if err != nil { diff --git a/docs/threat-model.md b/docs/threat-model.md index 61f6323333..80187692d5 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -11,21 +11,21 @@ Substrate is an early, fast moving product. It is full of debate and subject to # Intended Outcome -* The suggestions from this threat model should be added to Substrate's official roadmap after review and agreement with the community. +* The suggestions from this threat model should be added to Substrate's official roadmap after review and agreement with the community. * Security review SKILLs for AI-assisted security review should be extracted from this threat model and used for continuous review on the upstream [agent-substrate/substrate](https://github.com/agent-substrate/substrate) repository. # Goals -* Identify threats relevant to any system trying to achieve the same goal, with roughly the same basic building blocks as Substrate (Kubernetes, containers, sandboxes, snapshots, and lightweight "actors" dynamically scheduled to reusable "worker" Pods). -* Identify general "mitigating invariants" which, if implemented, eliminate or significantly decrease the severity of the threat. -* Suggest *possible* implementation approaches, which may change if Substrate changes. +* Identify threats relevant to any system trying to achieve the same goal, with roughly the same basic building blocks as Substrate (Kubernetes, containers, sandboxes, snapshots, and lightweight "actors" dynamically scheduled to reusable "worker" Pods). +* Identify general "mitigating invariants" which, if implemented, eliminate or significantly decrease the severity of the threat. +* Suggest *possible* implementation approaches, which may change if Substrate changes. * Prioritize the threats, so that resources can be spent of reducing the biggest risks first. # Non-Goals -* This threat model does not attempt to predict Substrate's future architectural decisions. -* This threat model does not demand any particular approach to mitigation. -* While shown in diagrams for reference, this threat model does not focus on AI frameworks that may be used on top of Substrate. +* This threat model does not attempt to predict Substrate's future architectural decisions. +* This threat model does not demand any particular approach to mitigation. +* While shown in diagrams for reference, this threat model does not focus on AI frameworks that may be used on top of Substrate. # Architecture of Substrate @@ -33,16 +33,16 @@ Substrate is an early, fast moving product. It is full of debate and subject to # Key Components -* **ate-api-server:** Actor creation and scheduling and credential issuance. -* **atelet:** Per-node daemon, performs snapshot/resume. -* **ateom:** Per-worker Pod sidecar, running inside the worker Pod. Ateom sets up "interior" sandboxes in the worker Pod and manages sandbox lifecycle, including image pulls. It currently uses gvisor but Substrate will support multiple microvm solutions. -* **Worker:** Preprovisioned Pods that actors get scheduled to. -* **Actor:** The core compute primitive, gets scheduled to/from worker via Run for cold start and Resume for snapshot resume. +* **ate-api-server:** Actor creation and scheduling and credential issuance. +* **atelet:** Per-node daemon, performs snapshot/resume. +* **ateom:** Per-worker Pod sidecar, running inside the worker Pod. Ateom sets up "interior" sandboxes in the worker Pod and manages sandbox lifecycle, including image pulls. It currently uses gvisor but Substrate will support multiple microvm solutions. +* **Worker:** Preprovisioned Pods that actors get scheduled to. +* **Actor:** The core compute primitive, gets scheduled to/from worker via Run for cold start and Resume for snapshot resume. * **Actor Network:** `ateom` creates a private point-to-point veth network for the active Actor inside a worker Pod. The Actor is not served directly from the worker Pod's port 80; ingress enters through `atunnel`'s authenticated listener on port 443. -* **Actor Routing:** A higher-order system connects to `atenet-router` and identifies the target Actor with `X-Ate-Actor-Name` and `X-Ate-Atespace`. Host and authority values are application metadata and do not select the Actor. +* **Actor Routing:** Requests sent to `atenet-router` identify the target Actor with `X-Ate-Actor-Name` and `X-Ate-Atespace`. Host and authority values are application metadata and do not select the Actor. * **atenet-router:** Substrate runs Envoy with an `ext_proc` external processor to handle Actor ingress. The ext_proc reads the Actor name and Atespace from `X-Ate-Actor-Name` and `X-Ate-Atespace`, calls the Substrate API to resume the Actor and obtain its current worker assignment, and selects that worker as a dynamic backend. It overwrites both routing headers before forwarding. The router then connects with mTLS to `atunnel`; `atunnel` validates the router identity and authorizes the header pair against the Actor currently assigned to that worker. -* **Object Storage:** Used to store actor snapshots. -* **Filesystem support:** Container local filesystem is saved in snapshots, future integrations likely to include networked storage. +* **Object Storage:** Used to store actor snapshots. +* **Filesystem support:** Container local filesystem is saved in snapshots, future integrations likely to include networked storage. * **Substrate Database:** PostgreSQL. * **Kubernetes:** The underlying infrastructure that Substrate runs on is expected to be Kubernetes. @@ -50,10 +50,10 @@ Substrate is an early, fast moving product. It is full of debate and subject to **Table Schema:** -* **Priority:** Critical, High, Medium, Low -* **Threats:** Expected risks in a naïve implementation of Substrate. -* **Mitigating Invariants:** High level properties which, if true, mitigate the threat. -* **Suggested Concrete Mitigations:** Specific options for implementing the mitigating invariants, based on current understanding of Substrate. +* **Priority:** Critical, High, Medium, Low +* **Threats:** Expected risks in a naïve implementation of Substrate. +* **Mitigating Invariants:** High level properties which, if true, mitigate the threat. +* **Suggested Concrete Mitigations:** Specific options for implementing the mitigating invariants, based on current understanding of Substrate. * **Notes:** Additional relevant information. ## Attacks from External Networks diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 19cd97daa1..7e4c380053 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -120,7 +120,8 @@ cat < "${CURL_OUT}" 2>&1 ) & CURL_PID=$! sleep 1 # inside the 5s park budget; the request is parked on the router diff --git a/hack/verify-egress-demo.sh b/hack/verify-egress-demo.sh index 51c622ac72..01405dc282 100755 --- a/hack/verify-egress-demo.sh +++ b/hack/verify-egress-demo.sh @@ -54,7 +54,8 @@ ${K} -n ate-system port-forward service/atenet-router 18000:80 >/tmp/pf.log 2>&1 PF=$!; trap 'kill ${PF} 2>/dev/null || true' EXIT sleep 3 RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://localhost:18000/ \ - -H "Host: ${ACTOR}.${ATESPACE}.actors.resources.substrate.ate.dev" \ + -H "X-Ate-Actor-Name: ${ACTOR}" \ + -H "X-Ate-Atespace: ${ATESPACE}" \ -H 'Content-Type: application/json' \ -d "{\"url\":\"${TARGET_URL}\"}") || true echo "actor round-trip HTTP ${RESP} (200 = the actor fetched ${TARGET_URL} through egress)" diff --git a/internal/atenet/headers.go b/internal/atenet/headers.go new file mode 100644 index 0000000000..2e067719bb --- /dev/null +++ b/internal/atenet/headers.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package atenet defines the shared contract for Substrate actor networking. +package atenet + +const ( + // ActorNameHeader and AtespaceHeader identify the actor selected for ingress + // routing. HTTP field names are case-insensitive; these use their HTTP/2 wire + // form so dataplane configuration and header mutations are native. + ActorNameHeader = "x-ate-actor-name" + AtespaceHeader = "x-ate-atespace" +) diff --git a/internal/atunnel/ingress.go b/internal/atunnel/ingress.go index 60f44b76a7..02109256f0 100644 --- a/internal/atunnel/ingress.go +++ b/internal/atunnel/ingress.go @@ -33,6 +33,7 @@ import ( "sync" "time" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/resources" ) @@ -44,12 +45,6 @@ const ( // StaleAssignmentHeader distinguishes an atunnel routing rejection from a // 421 returned by the actor application itself. StaleAssignmentHeader = "X-Ate-Assignment-Stale" - // ActorNameHeader and AtespaceHeader identify the actor selected by the - // trusted ingress router for routing. HTTP field names are case-insensitive; use their - // HTTP/2 wire form so Envoy configuration and header mutations are native. - ActorNameHeader = "x-ate-actor-name" - AtespaceHeader = "x-ate-atespace" - // TargetPortHeader carries the port to reach on the actor: the CONNECT // :authority's port for arbitrary-port ingress, or the default 80 // otherwise (see atenet-router's HandleRequestHeaders). cfg.Upstream is @@ -136,8 +131,8 @@ func NewServer(cfg Config) (*Server, error) { port := pr.In.Header.Get(TargetPortHeader) pr.Out.Header.Del(TargetPortHeader) - pr.Out.Header.Del(ActorNameHeader) - pr.Out.Header.Del(AtespaceHeader) + pr.Out.Header.Del(atenet.ActorNameHeader) + pr.Out.Header.Del(atenet.AtespaceHeader) if p, ok := ParsePort(port); ok { pr.Out.URL.Host = net.JoinHostPort(cfg.Upstream.Hostname(), strconv.Itoa(p)) } @@ -481,8 +476,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (s *Server) authorize(r *http.Request) (resources.ActorRef, context.Context, func(), bool) { ref := resources.ActorRef{ - Name: r.Header.Get(ActorNameHeader), - Atespace: r.Header.Get(AtespaceHeader), + Name: r.Header.Get(atenet.ActorNameHeader), + Atespace: r.Header.Get(atenet.AtespaceHeader), } if !resources.IsValidResourceName(ref.Name) || !resources.IsValidResourceName(ref.Atespace) { return resources.ActorRef{}, nil, nil, false diff --git a/internal/atunnel/ingress_test.go b/internal/atunnel/ingress_test.go index aef525ac04..1e22f1149d 100644 --- a/internal/atunnel/ingress_test.go +++ b/internal/atunnel/ingress_test.go @@ -34,6 +34,8 @@ import ( "path/filepath" "testing" "time" + + "github.com/agent-substrate/substrate/internal/atenet" ) func TestRelayIngressWithHalfClose(t *testing.T) { @@ -174,8 +176,8 @@ func TestServeHTTP(t *testing.T) { req.Header.Set("X-ATE-Actor-Name", tt.actorName) req.Header.Set("x-ate-ATESPACE", tt.atespace) } else { - req.Header.Set(ActorNameHeader, tt.actorName) - req.Header.Set(AtespaceHeader, tt.atespace) + req.Header.Set(atenet.ActorNameHeader, tt.actorName) + req.Header.Set(atenet.AtespaceHeader, tt.atespace) } rec := httptest.NewRecorder() s.ServeHTTP(rec, req) @@ -235,8 +237,8 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "https://worker/hello", nil) req.Host = "client.example" - req.Header.Set(ActorNameHeader, "actor-1") - req.Header.Set(AtespaceHeader, "team-a") + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") if tt.targetPort != "" { req.Header.Set(TargetPortHeader, tt.targetPort) } @@ -251,7 +253,7 @@ func TestServeHTTPHonorsTargetPortHeader(t *testing.T) { if gotHost.Get("Host") != "client.example" { t.Errorf("Host header changed to %q", gotHost.Get("Host")) } - for _, header := range []string{TargetPortHeader, ActorNameHeader, AtespaceHeader} { + for _, header := range []string{TargetPortHeader, atenet.ActorNameHeader, atenet.AtespaceHeader} { if got := gotHost.Get(header); got != "" { t.Errorf("%s leaked to the actor upstream: %q", header, got) } @@ -285,10 +287,10 @@ func TestServeConnectHTTPValidatesMethodAndAuthority(t *testing.T) { t.Run(tt.name, func(t *testing.T) { req := httptest.NewRequest(tt.method, "https://worker/", nil) req.Host = tt.host - req.Header.Set(ActorNameHeader, "actor-1") - req.Header.Set(AtespaceHeader, "team-a") + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") if tt.name == "rejects inactive actor" { - req.Header.Set(ActorNameHeader, "actor-2") + req.Header.Set(atenet.ActorNameHeader, "actor-2") } rec := httptest.NewRecorder() s.ServeConnectHTTP(rec, req) @@ -335,8 +337,8 @@ func TestDeactivateClosesIdleUpstreamConnections(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "https://worker/", nil) req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" - req.Header.Set(ActorNameHeader, "actor-1") - req.Header.Set(AtespaceHeader, "team-a") + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") rec := httptest.NewRecorder() s.ServeHTTP(rec, req) if rec.Code != http.StatusNoContent { @@ -358,8 +360,8 @@ func TestInactive(t *testing.T) { s := newTestServer(t, upstream) req := httptest.NewRequest(http.MethodGet, "https://worker/", nil) req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" - req.Header.Set(ActorNameHeader, "actor-1") - req.Header.Set(AtespaceHeader, "team-a") + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") for _, phase := range []string{"before activation", "after deactivation"} { t.Run(phase, func(t *testing.T) { @@ -484,11 +486,9 @@ func TestServeNegotiatesH2(t *testing.T) { if err != nil { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) served := make(chan error, 1) - go func() { served <- s.Serve(ctx, lis) }() + go func() { served <- s.Serve(t.Context(), lis) }() t.Cleanup(func() { - cancel() select { case err := <-served: if err != nil { @@ -521,8 +521,8 @@ func TestServeNegotiatesH2(t *testing.T) { t.Fatal(err) } req.Host = "team-a-agent.example.com" // Use a custom Host header to prove that we no longer rely on Host to route to the actor - req.Header.Set(ActorNameHeader, "actor-1") - req.Header.Set(AtespaceHeader, "team-a") + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") if contentType != "" { req.Header.Set("Content-Type", contentType) } @@ -581,8 +581,8 @@ func TestDeactivateCancelsInflightRequest(t *testing.T) { defer close(done) req := httptest.NewRequest(http.MethodGet, "https://worker/", nil) req.Host = "actor-1.team-a.actors.resources.substrate.ate.dev" - req.Header.Set(ActorNameHeader, "actor-1") - req.Header.Set(AtespaceHeader, "team-a") + req.Header.Set(atenet.ActorNameHeader, "actor-1") + req.Header.Set(atenet.AtespaceHeader, "team-a") s.ServeHTTP(httptest.NewRecorder(), req) }() receiveWithin(t, started, "in-flight request") diff --git a/internal/benchmarking/boomer/glutton/durdir.go b/internal/benchmarking/boomer/glutton/durdir.go index 835a383a69..87fd7a6777 100644 --- a/internal/benchmarking/boomer/glutton/durdir.go +++ b/internal/benchmarking/boomer/glutton/durdir.go @@ -29,7 +29,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/dynconfig" bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" @@ -128,7 +128,6 @@ func (r *durDirRuntime) startUser(ctx context.Context, dynCfg dynconfig.Config) templateName: tmpl, userClass: durDirUserClass, } - u.hostHeader = u.actorName + "." + u.cfg.Atespace + "." + actorDomain bmetrics.UpdateUsers(durDirUserClass, 1) if err := u.ensureAtespace(ctx); err != nil { bmetrics.UpdateUsers(durDirUserClass, -1) @@ -158,7 +157,6 @@ func (r *durDirRuntime) shutdown(ctx context.Context) { type durDirUser struct { cfg *userclass.Config actorName string - hostHeader string templateName string userClass string expectedDigest string @@ -414,10 +412,9 @@ func (u *durDirUser) httpProtoCall(ctx context.Context, metricName, route string bmetrics.RecordFailure("http", metricName, u.userClass, 0, err.Error()) return nil, err } - httpReq.Host = u.hostHeader httpReq.Header.Set("Content-Type", "application/x-protobuf") - httpReq.Header.Set(atunnel.ActorNameHeader, u.actorName) - httpReq.Header.Set(atunnel.AtespaceHeader, u.cfg.Atespace) + httpReq.Header.Set(atenet.ActorNameHeader, u.actorName) + httpReq.Header.Set(atenet.AtespaceHeader, u.cfg.Atespace) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) start := time.Now() diff --git a/internal/benchmarking/boomer/glutton/fixture_test.go b/internal/benchmarking/boomer/glutton/fixture_test.go index 89bb595669..f3f7097246 100644 --- a/internal/benchmarking/boomer/glutton/fixture_test.go +++ b/internal/benchmarking/boomer/glutton/fixture_test.go @@ -110,7 +110,6 @@ func newTestDurDirUser(t *testing.T, srv *fake.Server, cfg *userclass.Config) *d return &durDirUser{ cfg: c, actorName: "duractor", - hostHeader: "duractor.benchmark." + actorDomain, templateName: defaultDurTemplate, userClass: durDirUserClass, expectedSize: int64(len(srv.Data)), diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index cc5021ca0b..6d75e68511 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -30,7 +30,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/ateinterceptors" - "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/boomerutil" bmetrics "github.com/agent-substrate/substrate/internal/benchmarking/boomer/metrics" "github.com/agent-substrate/substrate/internal/benchmarking/boomer/userclass" @@ -51,7 +51,6 @@ const ( userClass = "GluttonUser" templateName = "glutton" templateAtespace = "benchmark-workloads" - actorDomain = "actors.resources.substrate.ate.dev" pingPath = "/ping" writeRAMPath = "/writeram" readRAMPath = "/readram" @@ -132,7 +131,6 @@ func (r *taskRuntime) startUser(ctx context.Context) (*gluttonUser, error) { actorName: "sb-" + uuid.NewString(), firstResume: true, } - u.hostHeader = u.actorName + "." + u.cfg.Atespace + "." + actorDomain bmetrics.UpdateUsers(userClass, 1) if err := u.ensureAtespace(ctx); err != nil { bmetrics.UpdateUsers(userClass, -1) @@ -173,7 +171,6 @@ func (r *taskRuntime) dynamicWait() time.Duration { type gluttonUser struct { cfg *userclass.Config actorName string - hostHeader string firstResume bool actorRunning bool ramFilled bool @@ -299,10 +296,9 @@ func (u *gluttonUser) ping(ctx context.Context) { bmetrics.RecordFailure("http", "GluttonPing", userClass, 0, err.Error()) return } - httpReq.Host = u.hostHeader httpReq.Header.Set("Content-Type", "application/x-protobuf") - httpReq.Header.Set(atunnel.ActorNameHeader, u.actorName) - httpReq.Header.Set(atunnel.AtespaceHeader, u.cfg.Atespace) + httpReq.Header.Set(atenet.ActorNameHeader, u.actorName) + httpReq.Header.Set(atenet.AtespaceHeader, u.cfg.Atespace) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) start := time.Now() @@ -461,10 +457,9 @@ func (u *gluttonUser) postProto(ctx context.Context, path string, req, resp prot if err != nil { return err } - httpReq.Host = u.hostHeader httpReq.Header.Set("Content-Type", "application/x-protobuf") - httpReq.Header.Set(atunnel.ActorNameHeader, u.actorName) - httpReq.Header.Set(atunnel.AtespaceHeader, u.cfg.Atespace) + httpReq.Header.Set(atenet.ActorNameHeader, u.actorName) + httpReq.Header.Set(atenet.AtespaceHeader, u.cfg.Atespace) otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(httpReq.Header)) httpResp, err := u.cfg.HTTPClient.Do(httpReq) diff --git a/internal/benchmarking/boomer/glutton/memfill_test.go b/internal/benchmarking/boomer/glutton/memfill_test.go index d979bfa48d..d63a39eee8 100644 --- a/internal/benchmarking/boomer/glutton/memfill_test.go +++ b/internal/benchmarking/boomer/glutton/memfill_test.go @@ -28,9 +28,8 @@ func newTestGluttonUser(t *testing.T, srv *fake.Server, dyn dynconfig.Config) *g t.Helper() cfg := newTestConfig(t, srv, &userclass.Config{Dyn: dynconfig.NewHolder(dyn)}) return &gluttonUser{ - cfg: cfg, - actorName: "memactor", - hostHeader: "memactor.benchmark." + actorDomain, + cfg: cfg, + actorName: "memactor", } } diff --git a/internal/e2e/router_client.go b/internal/e2e/router_client.go index 0fe3f915fe..88ffb38c58 100644 --- a/internal/e2e/router_client.go +++ b/internal/e2e/router_client.go @@ -29,7 +29,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/internal/resources" "k8s.io/client-go/kubernetes" @@ -126,9 +126,8 @@ func (c *RouterClient) request(ctx context.Context, method string, actorRef reso if method == http.MethodPost { req.Header.Set("Content-Type", "application/json") } - req.Header.Set(atunnel.ActorNameHeader, actorRef.Name) - req.Header.Set(atunnel.AtespaceHeader, actorRef.Atespace) - req.Host = resources.ActorDNSName(actorRef) + req.Header.Set(atenet.ActorNameHeader, actorRef.Name) + req.Header.Set(atenet.AtespaceHeader, actorRef.Atespace) return c.http.Do(req) } @@ -150,14 +149,14 @@ func (c *RouterClient) Connect(ctx context.Context, actorRef resources.ActorRef, return nil, fmt.Errorf("connecting to router's CONNECT listener: %w", err) } - destination := net.JoinHostPort(resources.ActorDNSName(actorRef), strconv.Itoa(port)) + destination := net.JoinHostPort(actorRef.Name, strconv.Itoa(port)) req := &http.Request{ Method: http.MethodConnect, URL: &url.URL{Host: destination}, Host: destination, Header: http.Header{ - atunnel.ActorNameHeader: []string{actorRef.Name}, - atunnel.AtespaceHeader: []string{actorRef.Atespace}, + atenet.ActorNameHeader: []string{actorRef.Name}, + atenet.AtespaceHeader: []string{actorRef.Atespace}, }, } if err := req.Write(rawConn); err != nil { diff --git a/internal/e2e/router_client_test.go b/internal/e2e/router_client_test.go index e963510847..d50fc8849e 100644 --- a/internal/e2e/router_client_test.go +++ b/internal/e2e/router_client_test.go @@ -31,8 +31,8 @@ func TestRouterClientPostJSON(t *testing.T) { if request.Method != http.MethodPost { t.Errorf("method = %q, want POST", request.Method) } - if request.Host != "fetcher.demo.actors.resources.substrate.ate.dev" { - t.Errorf("host = %q", request.Host) + if request.Host != "router.test" { + t.Errorf("host = %q, want router.test", request.Host) } if request.URL.Path != "/fetch" { t.Errorf("path = %q, want /fetch", request.URL.Path) diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 0ff08c471b..0fc48578ba 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -24,7 +24,7 @@ import ( "time" "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -1288,9 +1288,8 @@ func callActorPathOnce(t *testing.T, actorRef resources.ActorRef, method, path s if err != nil { return "", fmt.Errorf("failed to create request: %w", err) } - reqHttp.Header.Set(atunnel.ActorNameHeader, actorRef.Name) - reqHttp.Header.Set(atunnel.AtespaceHeader, actorRef.Atespace) - reqHttp.Host = resources.ActorDNSName(actorRef) + reqHttp.Header.Set(atenet.ActorNameHeader, actorRef.Name) + reqHttp.Header.Set(atenet.AtespaceHeader, actorRef.Atespace) httpClient := &http.Client{Timeout: 15 * time.Second} resp, err := httpClient.Do(reqHttp) diff --git a/internal/e2e/suites/networking/arbitraryport_test.go b/internal/e2e/suites/networking/arbitraryport_test.go index 56e2f84cbd..af0231656f 100644 --- a/internal/e2e/suites/networking/arbitraryport_test.go +++ b/internal/e2e/suites/networking/arbitraryport_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" ) @@ -108,7 +108,7 @@ func TestActorArbitraryPortAccess(t *testing.T) { conn.SetDeadline(time.Now().Add(10 * time.Second)) if _, err := fmt.Fprintf(conn, "GET / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: %s\r\nConnection: close\r\n\r\n", - resources.ActorDNSName(actorRef), atunnel.ActorNameHeader, actorRef.Name, atunnel.AtespaceHeader, actorRef.Atespace); err != nil { + actorRef.Name, atenet.ActorNameHeader, actorRef.Name, atenet.AtespaceHeader, actorRef.Atespace); err != nil { t.Fatalf("writing tunneled request: %v", err) } resp, err := http.ReadResponse(bufio.NewReader(conn), nil) @@ -142,7 +142,7 @@ func waitForTunneledRouteReady(t *testing.T, ctx context.Context, router *e2e.Ro for { conn, err := router.Connect(ctx, actorRef, port) if err == nil { - resp, body, requestErr := requestTunneled(conn, resources.ActorDNSName(actorRef)) + resp, body, requestErr := requestTunneled(conn, actorRef.Name) _ = conn.Close() if requestErr == nil && resp.StatusCode == http.StatusOK { return body diff --git a/internal/e2e/suites/networking/grpcingress_test.go b/internal/e2e/suites/networking/grpcingress_test.go index 02aa497369..5ff12d937c 100644 --- a/internal/e2e/suites/networking/grpcingress_test.go +++ b/internal/e2e/suites/networking/grpcingress_test.go @@ -29,7 +29,7 @@ import ( "google.golang.org/grpc/metadata" "github.com/agent-substrate/substrate/internal/ateclient" - "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/portforward" "github.com/agent-substrate/substrate/internal/proto/grpcechopb" @@ -71,9 +71,8 @@ func TestIngressProtocolDowngrade(t *testing.T) { if err != nil { return nil, err } - req.Host = resources.ActorDNSName(actorRef) - req.Header.Set(atunnel.ActorNameHeader, actorRef.Name) - req.Header.Set(atunnel.AtespaceHeader, actorRef.Atespace) + req.Header.Set(atenet.ActorNameHeader, actorRef.Name) + req.Header.Set(atenet.AtespaceHeader, actorRef.Atespace) if contentType != "" { req.Header.Set("Content-Type", contentType) } @@ -146,8 +145,8 @@ func TestIngressGRPC(t *testing.T) { actorName, _ := createAndResumeSubstrateActor(t, ctx, "grpcingress", fixture) actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} ctx = metadata.AppendToOutgoingContext(ctx, - atunnel.ActorNameHeader, actorRef.Name, - atunnel.AtespaceHeader, actorRef.Atespace, + atenet.ActorNameHeader, actorRef.Name, + atenet.AtespaceHeader, actorRef.Atespace, ) // Cleartext h2c to the router's HTTP port. Explicit metadata identifies the @@ -155,10 +154,9 @@ func TestIngressGRPC(t *testing.T) { // h2 ALPN offer is about the *TLS* listener; nothing here needs it. conn, err := grpc.NewClient(routerAddress(t, ctx), grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithAuthority(resources.ActorDNSName(actorRef)), ) if err != nil { - t.Fatalf("creating the gRPC client for %s: %v", resources.ActorDNSName(actorRef), err) + t.Fatalf("creating the gRPC client for %s: %v", actorRef, err) } defer conn.Close() client := grpcechopb.NewEchoClient(conn) diff --git a/internal/e2e/suites/networking/websocketingress_test.go b/internal/e2e/suites/networking/websocketingress_test.go index 4e8ec4d962..3618f2f05c 100644 --- a/internal/e2e/suites/networking/websocketingress_test.go +++ b/internal/e2e/suites/networking/websocketingress_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "github.com/agent-substrate/substrate/internal/atunnel" + "github.com/agent-substrate/substrate/internal/atenet" "github.com/agent-substrate/substrate/internal/e2e" "github.com/agent-substrate/substrate/internal/resources" "github.com/gorilla/websocket" @@ -68,9 +68,8 @@ func TestWebsocketIngressPing(t *testing.T) { actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} header := http.Header{} - header.Set("Host", resources.ActorDNSName(actorRef)) - header.Set(atunnel.ActorNameHeader, actorRef.Name) - header.Set(atunnel.AtespaceHeader, actorRef.Atespace) + header.Set(atenet.ActorNameHeader, actorRef.Name) + header.Set(atenet.AtespaceHeader, actorRef.Atespace) var c *websocket.Conn diff --git a/internal/resources/actor.go b/internal/resources/actor.go index ee93775c89..13682a2d40 100644 --- a/internal/resources/actor.go +++ b/internal/resources/actor.go @@ -18,8 +18,6 @@ const ( // ResourceNameRegexPattern is the regular expression pattern for a valid // Substrate resource name. ResourceNameRegexPattern = `[a-z0-9]([-a-z0-9]*[a-z0-9])?` - // ActorDNSSuffix is the suffix used by the conventional actor authority. - ActorDNSSuffix = "actors.resources.substrate.ate.dev" // GoldenActorAtespace is the reserved system atespace that per-template golden // actors live in. GoldenActorAtespace = "ate-golden" diff --git a/internal/resources/resourceref.go b/internal/resources/resourceref.go index 6133bb45f9..9c8e9543e8 100644 --- a/internal/resources/resourceref.go +++ b/internal/resources/resourceref.go @@ -75,12 +75,6 @@ func ActorRefFromActor(a *ateapipb.Actor) ActorRef { } } -// ActorDNSName returns the conventional authority name for an actor. -// Substrate does not resolve this name; routing uses explicit actor headers. -func ActorDNSName(r ActorRef) string { - return r.Name + "." + r.Atespace + "." + ActorDNSSuffix -} - // ActorTemplateRef identifies an ActorTemplate by the (atespace, name). type ActorTemplateRef = ResourceRef[*ateapipb.ActorTemplate] diff --git a/internal/resources/resourceref_test.go b/internal/resources/resourceref_test.go index d0562f07f1..e28ef67bbe 100644 --- a/internal/resources/resourceref_test.go +++ b/internal/resources/resourceref_test.go @@ -89,16 +89,6 @@ func TestActorRefString(t *testing.T) { } } -func TestActorRefDNSName(t *testing.T) { - actorRef := ActorRef{Atespace: "team-a", Name: "act-1"} - - got := ActorDNSName(actorRef) - want := "act-1.team-a.actors.resources.substrate.ate.dev" - if got != want { - t.Errorf("ActorDNSName() = %q, want %q", got, want) - } -} - func TestActorRefObjectRefRoundTrip(t *testing.T) { actorRef := ActorRef{Atespace: "team-a", Name: "act-1"} From 0847a350149a9b161faf23b162fe31e93f335958 Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Tue, 1 Sep 2026 08:37:15 -0500 Subject: [PATCH 09/10] Keep direct ingress on the default actor port --- cmd/atenet/internal/router/ingress/ingress.go | 3 ++- cmd/atenet/internal/router/ingress/ingress_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 021c781db5..377c2b6ccf 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -26,6 +26,7 @@ import ( "context" "log/slog" "net" + "net/http" "strconv" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -98,7 +99,7 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM // belongs to the inner request. targetPort := defaultActorPort targetAuthority := md.Attribute(extproc.ConnectAuthorityFilterStateAttribute) - if targetAuthority == "" { + if targetAuthority == "" && md.Method == http.MethodConnect { targetAuthority = md.Host } if _, portStr, err := net.SplitHostPort(targetAuthority); err == nil { diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index cea42e6e3c..efaeea6b04 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -220,8 +220,8 @@ func TestHandleRequestHeaders(t *testing.T) { expectedStatus: envoy_type.StatusCode_InternalServerError, }, { - name: "successful resume ignores host for actor routing", - authority: "application.example", + name: "successful resume ignores host and port for actor routing", + authority: "127.0.0.1:44681", resumeResp: &ateapipb.ResumeActorResponse{ Actor: &ateapipb.Actor{ Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.52"}}, From e5b20ecdf41901c67aa18749402f1b1bfb808b1e Mon Sep 17 00:00:00 2001 From: Keith Mattix II Date: Tue, 1 Sep 2026 13:15:33 -0500 Subject: [PATCH 10/10] Make envoy set the X-ate-target-port header from metadata Signed-off-by: Keith Mattix II --- cmd/atenet/internal/router/ingress/ingress.go | 1 - .../internal/router/ingress/ingress_test.go | 18 ++-- cmd/atenet/internal/router/xds.go | 38 ++++---- cmd/atenet/internal/router/xds_test.go | 92 +++++++++++-------- 4 files changed, 77 insertions(+), 72 deletions(-) diff --git a/cmd/atenet/internal/router/ingress/ingress.go b/cmd/atenet/internal/router/ingress/ingress.go index 377c2b6ccf..847f9776f9 100644 --- a/cmd/atenet/internal/router/ingress/ingress.go +++ b/cmd/atenet/internal/router/ingress/ingress.go @@ -169,7 +169,6 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM for _, header := range []struct{ name, value string }{ {atenet.ActorNameHeader, actorRef.Name}, {atenet.AtespaceHeader, actorRef.Atespace}, - {atunnel.TargetPortHeader, strconv.Itoa(targetPort)}, } { mutation.SetHeaders = append(mutation.SetHeaders, &corev3.HeaderValueOption{ Header: &corev3.HeaderValue{ diff --git a/cmd/atenet/internal/router/ingress/ingress_test.go b/cmd/atenet/internal/router/ingress/ingress_test.go index efaeea6b04..cbe58fbd3e 100644 --- a/cmd/atenet/internal/router/ingress/ingress_test.go +++ b/cmd/atenet/internal/router/ingress/ingress_test.go @@ -86,10 +86,7 @@ func dynamicMetadataTarget(dynamicMetadata *structpb.Struct) string { } // dynamicMetadataPort extracts the target port HandleRequestHeaders reports -// via OriginalDstMetadataKey/OriginalDstPortKey. xds.go's buildRoutes derives -// a real atunnel.TargetPortHeader from this at the route level via a -// %DYNAMIC_METADATA(...)% format string; HandleRequestHeaders also sets that -// same header directly (see its own doc comment for why). +// via OriginalDstMetadataKey/OriginalDstPortKey. func dynamicMetadataPort(dynamicMetadata *structpb.Struct) string { return dynamicMetadata.GetFields()[OriginalDstMetadataKey].GetStructValue().GetFields()[OriginalDstPortKey].GetStringValue() } @@ -296,16 +293,16 @@ func TestHandleRequestHeaders(t *testing.T) { } mutation := res.Response.GetResponse().GetHeaderMutation() - if len(mutation.GetSetHeaders()) != 3 { - t.Fatalf("expected actor routing and target port headers, found: %v", mutation.GetSetHeaders()) + if len(mutation.GetSetHeaders()) != 2 { + t.Fatalf("expected actor routing headers, found: %v", mutation.GetSetHeaders()) } gotMutations := map[string]string{} for _, headerOption := range mutation.GetSetHeaders() { gotMutations[strings.ToLower(headerOption.Header.Key)] = string(headerOption.Header.RawValue) } - if got := gotMutations[strings.ToLower(atunnel.TargetPortHeader)]; got != tc.expectedTargetPort { - t.Errorf("target port mutation = %q, want %q", got, tc.expectedTargetPort) + if _, ok := gotMutations[strings.ToLower(atunnel.TargetPortHeader)]; ok { + t.Errorf("target port must be emitted only as dynamic metadata") } if got := gotMutations[strings.ToLower(atenet.ActorNameHeader)]; got != testUUID { t.Errorf("actor name mutation = %q, want %q", got, testUUID) @@ -328,9 +325,8 @@ func TestHandleRequestHeaders(t *testing.T) { // travels in :authority, e.g. ":9090") resolves the actor, // produces the same ":443" original-dst mutation as an ordinary // request (the router only ever dials the worker's atunnel server), and -// reports the arbitrary port itself via -// OriginalDstMetadataKey/OriginalDstPortKey, which xds.go's buildRoutes turns -// into atunnel.TargetPortHeader for atunnel. +// reports the arbitrary port itself in dynamic metadata for Envoy to write to +// atunnel.TargetPortHeader. func TestHandleRequestHeadersHandlesConnectMethod(t *testing.T) { const testUUID = "123e4567-e89b-12d3-a456-426614174000" authority := testUUID + ".team-a.actors.resources.substrate.ate.dev:9090" diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 56dfe0402f..4956cfd267 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -35,7 +35,6 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" "github.com/agent-substrate/substrate/internal/atenet" - "github.com/agent-substrate/substrate/internal/atunnel" accesslogv3 "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v3" clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" @@ -72,6 +71,7 @@ import ( "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/ingress" + "github.com/agent-substrate/substrate/internal/atunnel" ) const ( @@ -108,11 +108,6 @@ const ( ConnectUpgradeType = "CONNECT" MainInternalName = "main_internal" - // dynamicMetadataPortFormat is the %DYNAMIC_METADATA(...)% header-value - // command operator (see buildRoutes) that derives atunnel.TargetPortHeader - // from ingress.OriginalDstMetadataKey/ingress.OriginalDstPortKey. - dynamicMetadataPortFormat = "%DYNAMIC_METADATA(" + ingress.OriginalDstMetadataKey + ":" + ingress.OriginalDstPortKey + ")%" - // httpExtProcFilterName is envoy.filters.http.ext_proc's own well-known // name, used as the HttpFilter.Name in buildHcm. httpExtProcFilterName = "envoy.filters.http.ext_proc" @@ -840,15 +835,12 @@ func (x *XdsServer) buildRoutes() *routev3.RouteConfiguration { IdleTimeout: durationpb.New(x.routeIdleTimeout()), }, }, - // atunnel reads the actor's target port from a header, since - // it can't see Envoy's dynamic metadata; this derives it - // declaratively from the same metadata ext_proc wrote (see - // ingress.OriginalDstPortKey). RequestHeadersToAdd: []*corev3.HeaderValueOption{ { Header: &corev3.HeaderValue{ - Key: atunnel.TargetPortHeader, - Value: dynamicMetadataPortFormat, + Key: atunnel.TargetPortHeader, + Value: fmt.Sprintf("%%DYNAMIC_METADATA(%s:%s)%%", + ingress.OriginalDstMetadataKey, ingress.OriginalDstPortKey), }, AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, }, @@ -894,16 +886,20 @@ func (x *XdsServer) buildMainInternalListener() *listenerv3.Listener { } } -// actorRoutingFilterStateFilter captures actor routing headers and the target -// authority so main_internal can read them across the CONNECT internal-listener -// hop. The authority supplies only the target port, never the actor reference. -func actorRoutingFilterStateFilter() *hcmv3.HttpFilter { +// actorRoutingFilterStateFilter captures actor routing headers so +// main_internal can read them across the CONNECT internal-listener hop. +func actorRoutingFilterStateFilter(captureAuthority bool) *hcmv3.HttpFilter { values := make([]*setfilterstatecommonv3.FilterStateValue, 0, 3) - for _, routingField := range []struct{ key, header string }{ + routingFields := []struct{ key, header string }{ {extproc.ActorNameFilterStateKey, atenet.ActorNameHeader}, {extproc.AtespaceFilterStateKey, atenet.AtespaceHeader}, - {extproc.ConnectAuthorityFilterStateKey, extproc.AuthorityHeader}, - } { + } + if captureAuthority { + routingFields = append(routingFields, struct{ key, header string }{ + extproc.ConnectAuthorityFilterStateKey, extproc.AuthorityHeader, + }) + } + for _, routingField := range routingFields { values = append(values, &setfilterstatecommonv3.FilterStateValue{ Key: &setfilterstatecommonv3.FilterStateValue_ObjectKey{ ObjectKey: routingField.key, @@ -962,7 +958,7 @@ func (x *XdsServer) buildConnectTerminateHCM(statPrefix string) *anypb.Any { }, CodecType: hcmv3.HttpConnectionManager_AUTO, HttpFilters: []*hcmv3.HttpFilter{ - actorRoutingFilterStateFilter(), + actorRoutingFilterStateFilter(true), { Name: "envoy.filters.http.router", ConfigType: &hcmv3.HttpFilter_TypedConfig{ @@ -1065,7 +1061,7 @@ func (x *XdsServer) buildHcm(statPrefix string, captureActorRouting bool) *anypb httpFilters := []*hcmv3.HttpFilter{} if captureActorRouting { - httpFilters = append(httpFilters, actorRoutingFilterStateFilter()) + httpFilters = append(httpFilters, actorRoutingFilterStateFilter(false)) } httpFilters = append(httpFilters, &hcmv3.HttpFilter{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index ebe847ca59..bb2f4d36b7 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -53,29 +53,49 @@ import ( ) func TestActorRoutingFilterStateFilter(t *testing.T) { - filter := actorRoutingFilterStateFilter() - config := &setfilterstatev3.Config{} - if err := filter.GetTypedConfig().UnmarshalTo(config); err != nil { - t.Fatalf("unmarshal set_filter_state config: %v", err) - } + for _, tc := range []struct { + name string + captureAuthority bool + want map[string]string + }{ + { + name: "ordinary ingress", + want: map[string]string{ + extproc.ActorNameFilterStateKey: "%REQ(x-ate-actor-name)%", + extproc.AtespaceFilterStateKey: "%REQ(x-ate-atespace)%", + }, + }, + { + name: "CONNECT termination", + captureAuthority: true, + want: map[string]string{ + extproc.ActorNameFilterStateKey: "%REQ(x-ate-actor-name)%", + extproc.AtespaceFilterStateKey: "%REQ(x-ate-atespace)%", + extproc.ConnectAuthorityFilterStateKey: "%REQ(:authority)%", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + filter := actorRoutingFilterStateFilter(tc.captureAuthority) + config := &setfilterstatev3.Config{} + if err := filter.GetTypedConfig().UnmarshalTo(config); err != nil { + t.Fatalf("unmarshal set_filter_state config: %v", err) + } - want := map[string]string{ - extproc.ActorNameFilterStateKey: "%REQ(x-ate-actor-name)%", - extproc.AtespaceFilterStateKey: "%REQ(x-ate-atespace)%", - extproc.ConnectAuthorityFilterStateKey: "%REQ(:authority)%", - } - if len(config.GetOnRequestHeaders()) != len(want) { - t.Fatalf("captured values = %d, want %d", len(config.GetOnRequestHeaders()), len(want)) - } - for _, value := range config.GetOnRequestHeaders() { - key := value.GetObjectKey() - format := value.GetFormatString().GetTextFormatSource().GetInlineString() - if format != want[key] { - t.Errorf("capture %q = %q, want %q", key, format, want[key]) - } - if key != extproc.ConnectAuthorityFilterStateKey && strings.Contains(strings.ToLower(format), ":authority") { - t.Errorf("capture %q derives actor routing from authority", key) - } + if len(config.GetOnRequestHeaders()) != len(tc.want) { + t.Fatalf("captured values = %d, want %d", len(config.GetOnRequestHeaders()), len(tc.want)) + } + for _, value := range config.GetOnRequestHeaders() { + key := value.GetObjectKey() + format := value.GetFormatString().GetTextFormatSource().GetInlineString() + if format != tc.want[key] { + t.Errorf("capture %q = %q, want %q", key, format, tc.want[key]) + } + if key != extproc.ConnectAuthorityFilterStateKey && strings.Contains(strings.ToLower(format), ":authority") { + t.Errorf("capture %q derives actor routing from authority", key) + } + } + }) } } @@ -1122,31 +1142,25 @@ func TestXdsServer_ActorClusterProtocolOptions(t *testing.T) { } } -// TestXdsServer_BuildRoutes_DerivesTargetPortHeader covers the fix for atunnel -// needing the target port as a real header (it can't read Envoy's dynamic -// metadata directly): rather than ext_proc building that header mutation -// itself, the route derives it declaratively from the same -// ingress.OriginalDstMetadataKey/ingress.OriginalDstPortKey metadata ext_proc -// already writes for the cluster's own MetadataKey, via a -// %DYNAMIC_METADATA(...)% command operator. -func TestXdsServer_BuildRoutes_DerivesTargetPortHeader(t *testing.T) { +// TestXdsServer_BuildRoutesWritesTargetPortHeader ensures the route overwrites +// the target-port header with the value from ext_proc's trusted metadata. +func TestXdsServer_BuildRoutesWritesTargetPortHeader(t *testing.T) { x := NewXdsServer(18000) route := x.buildRoutes().GetVirtualHosts()[0].GetRoutes()[0] headers := route.GetRequestHeadersToAdd() if len(headers) != 1 { - t.Fatalf("Expected exactly 1 request header to add, got %d: %v", len(headers), headers) + t.Fatalf("route adds request headers %v, want exactly one", headers) } - h := headers[0] - if got, want := h.GetHeader().GetKey(), atunnel.TargetPortHeader; got != want { - t.Errorf("Expected header key %q, got %q", want, got) + header := headers[0] + if got, want := header.GetHeader().GetKey(), atunnel.TargetPortHeader; got != want { + t.Errorf("header key = %q, want %q", got, want) } - wantValue := "%DYNAMIC_METADATA(" + ingress.OriginalDstMetadataKey + ":" + ingress.OriginalDstPortKey + ")%" - if got := h.GetHeader().GetValue(); got != wantValue { - t.Errorf("Expected header value %q, got %q", wantValue, got) + if got, want := header.GetHeader().GetValue(), "%DYNAMIC_METADATA(envoy.filters.listener.original_dst:port)%"; got != want { + t.Errorf("header value = %q, want %q", got, want) } - if got, want := h.GetAppendAction(), corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD; got != want { - t.Errorf("Expected append action %s, got %s", want, got) + if got, want := header.GetAppendAction(), corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD; got != want { + t.Errorf("append action = %v, want %v", got, want) } }