feat(gate): warm a revision up through its public hosts once it serves

README.md

@@ -112,8 +112,8 @@ Flagger, `503` a question the gate could not answer. Flagger records a refusal's
   Flagger's edge to the gate is the only one declared for its webhooks.
 - **It serves what it holds, read-only, at `GET /releases`.** For every gated Application,
   ordered by namespace and Application: its members and its migration, from the release-gate
-  inputs, and its record (`serving`, `pinned`, `since`). An Application whose inputs or record do
-  not read says so in `unreadable` and keeps the rest from failing with it. A thousand
+  inputs, and its record (`serving`, `pinned`, `since`, `warmup`). An Application whose inputs or
+  record do not read says so in `unreadable` and keeps the rest from failing with it. A thousand
   Applications at most, with `truncated` set beyond that. This is how the estate dashboard reads
   both without any ConfigMap access of its own (JorisJonkers-dev/estate-dashboard#15): it declares
   an edge to the gate's `http` surface, which its derived policy then admits. Reading it starts
@@ -170,6 +170,22 @@ gated Application itself (`internal/gate/tend.go`): each ConfigMap the render wr
   one it cannot answer from. No rendered object says what
   serves once a new render is applied, so this is the gate's one piece of state, and it survives
   a restart. When it first sees a revision applied and not yet serving, it notes that and when.
+- **It warms up a revision once it serves.** The first round an Application serves a new
+  revision, the gate requests every member's paths once more through each public hostname its
+  Canary hands it (`hostnames`: the hosts of its `anonymous` and `authenticated` routes), so the
+  request crosses Cloudflare and Traefik to the new primary and warms the caches nearest the
+  cluster. Flagger asks nothing after promotion, which is why this loop does it
+  (JorisJonkers-dev/delivery#12).
+  - Each request is `https` on port 443, through no proxy, with no redirect followed. The
+    hostname is resolved as an absolute name, IPv4 only, since the derived internet egress admits
+    no IPv6, and the gate connects only to an address that is public. It decides on the address,
+    not the name: loopback, private, link-local, the tailnet's `100.64.0.0/10` and the reserved
+    blocks are refused.
+  - A `5xx`, or no answer within 10 seconds, stops the warmup. The record keeps what it met in
+    `warmup` (`revision`, and `failed` with the first failing request). A `401`, a `404` or a
+    redirect passes. A revision is warmed once and never again, and nothing is rolled back: the
+    Reporter fails the release's Rollout on it.
+  - An Application no public host reaches is recorded as warmed without a request.
 - **It starts a migration only while its proof holds.** The up Job, `<identity>-<tag>`, is
   unsuspended when the record's serving revision is the inputs' `testedAgainst` and no primary's
   digest has moved since. A release proven against nothing starts only where nothing is recorded

deploy/delivery.project.yml

@@ -93,6 +93,14 @@ applications:
         placement:
           memory: 64Mi
           cpu: 25m
+        # The warmup after promotion crosses the internet to the edge, at a public address only;
+        # its checks during analysis reach each member by the member label instead.
+        egress:
+          to: internet
+          reason: >-
+            after promotion the gate requests a released Application's paths through each of its
+            public hostnames, so the request crosses the edge to the new primary and warms the
+            caches nearest the cluster
         api:
           reason: >-
             the Release Gate answers Flagger from what the cluster holds, so it reads each gated

internal/gate/export_test.go

@@ -0,0 +1,9 @@
+package gate
+
+// What the tests reach that is not the package's API.
+var (
+	PublicDialer       = publicDialer
+	Public             = public
+	ErrNoPublicAddress = errNoPublicAddress
+	ErrNotHTTPS        = errNotHTTPS
+)

internal/gate/gate.go

@@ -98,6 +98,9 @@ type Canary struct {
 	// every Route to the member, and the path its readiness is asked on.
 	Paths     []string
 	Readiness string
+	// Hostnames are the public hosts the Canary's webhooks hand the gate to warm through, once
+	// the Application serves the revision.
+	Hostnames []string
 	// Transitioned is when Flagger last moved the member to another phase.
 	Transitioned time.Time
 }
@@ -165,6 +168,7 @@ type SyncObject struct {
 type Gate struct {
 	cluster Cluster
 	traffic Traffic
+	public  Traffic
 
 	mu sync.Mutex
 	// askedAt is the revision each member last asked a question at, by namespace and Process.
@@ -182,9 +186,12 @@ type Option func(*Gate)
 // WithTraffic has the Gate send its own requests through traffic rather than HTTPTraffic.
 func WithTraffic(traffic Traffic) Option { return func(g *Gate) { g.traffic = traffic } }
 
+// WithPublicTraffic has the Gate warm up through public rather than PublicTraffic.
+func WithPublicTraffic(public Traffic) Option { return func(g *Gate) { g.public = public } }
+
 // New returns a Gate reading cluster.
 func New(cluster Cluster, options ...Option) *Gate {
-	g := &Gate{cluster: cluster, traffic: HTTPTraffic(), askedAt: map[string]string{}, served: served{now: time.Now}}
+	g := &Gate{cluster: cluster, traffic: HTTPTraffic(), public: PublicTraffic(), askedAt: map[string]string{}, served: served{now: time.Now}}
 	for _, option := range options {
 		option(g)
 	}

internal/gate/kube.go

@@ -133,23 +133,27 @@ func (k Kube) Canary(ctx context.Context, namespace, process string) (Canary, er
 	}
 	webhooks, _, _ := unstructured.NestedSlice(object.Object, "spec", "analysis", "webhooks")
 	var read Canary
-	var paths string
+	var paths, hostnames string
 	for i, webhook := range webhooks {
 		fields, _ := webhook.(map[string]any)
 		application, _, _ := unstructured.NestedString(fields, "metadata", "application")
 		revision, _, _ := unstructured.NestedString(fields, "metadata", "revision")
 		routes, _, _ := unstructured.NestedString(fields, "metadata", "paths")
 		readiness, _, _ := unstructured.NestedString(fields, "metadata", "readiness")
+		hosts, _, _ := unstructured.NestedString(fields, "metadata", "hostnames")
 		if i == 0 {
-			read.Application, read.Revision, paths, read.Readiness = application, revision, routes, readiness
+			read.Application, read.Revision, paths, read.Readiness, hostnames = application, revision, routes, readiness, hosts
 		}
-		if application != read.Application || revision != read.Revision || routes != paths || readiness != read.Readiness {
+		if application != read.Application || revision != read.Revision || routes != paths || readiness != read.Readiness || hosts != hostnames {
 			return Canary{}, errNotTheRenders
 		}
 	}
 	if paths != "" {
 		read.Paths = strings.Fields(paths)
 	}
+	if hostnames != "" {
+		read.Hostnames = strings.Fields(hostnames)
+	}
 	port, _, _ := unstructured.NestedInt64(object.Object, "spec", "service", "port")
 	read.Port = int(port)
 	if read.Application == "" || read.Revision == "" {

internal/gate/kube_test.go

@@ -162,13 +162,20 @@ func TestACanaryIsReadWithWhatItsWebhooksCarryAndWhatFlaggerRecorded(t *testing.
 	api := canary("auth-api", handing(rendered(revision), "/api /api/v1/users/register", "/api/actuator/health/readiness"),
 		map[string]any{"phase": "WaitingPromotion", "lastAppliedSpec": "a2", "lastPromotedSpec": "a1", "iterations": int64(4)})
 	api.Object["spec"].(map[string]any)["service"] = map[string]any{"port": int64(8080)}
+	for _, w := range api.Object["spec"].(map[string]any)["analysis"].(map[string]any)["webhooks"].([]any) {
+		w.(map[string]any)["metadata"].(map[string]any)["hostnames"] = "auth.jorisjonkers.dev id.jorisjonkers.dev"
+	}
 	k := kube(nil,
 		api,
 		canary("auth-ui", rendered(revision), nil),
 		canary("two-path-sets", []any{
 			handing([]any{webhook("may-start", "auth", revision)}, "/api", "")[0],
 			handing([]any{webhook("checks", "auth", revision)}, "/", "")[0],
 		}, nil),
+		canary("two-host-sets", []any{
+			map[string]any{"name": "may-start", "metadata": map[string]any{"application": "auth", "revision": revision, "hostnames": "a.example"}},
+			map[string]any{"name": "checks", "metadata": map[string]any{"application": "auth", "revision": revision}},
+		}, nil),
 		canary("two-readiness-paths", []any{
 			handing([]any{webhook("may-start", "auth", revision)}, "/", "/ready")[0],
 			handing([]any{webhook("checks", "auth", revision)}, "/", "/")[0],
@@ -186,14 +193,15 @@ func TestACanaryIsReadWithWhatItsWebhooksCarryAndWhatFlaggerRecorded(t *testing.
 	want := gate.Canary{
 		Application: "auth", Revision: revision, Phase: "WaitingPromotion", Applied: "a2", Promoted: "a1", Iterations: 4,
 		Port: 8080, Paths: []string{"/api", "/api/v1/users/register"}, Readiness: "/api/actuator/health/readiness",
+		Hostnames: []string{"auth.jorisjonkers.dev", "id.jorisjonkers.dev"},
 	}
 	if got, err := k.Canary(t.Context(), ns, "auth-api"); err != nil || !reflect.DeepEqual(got, want) {
 		t.Fatalf("canary = %+v, %v", got, err)
 	}
 	if got, err := k.Canary(t.Context(), ns, "auth-ui"); err != nil || !reflect.DeepEqual(got, gate.Canary{Application: "auth", Revision: revision}) {
 		t.Fatalf("a Canary Flagger has not touched = %+v, %v", got, err)
 	}
-	for _, name := range []string{"two-path-sets", "two-readiness-paths", "two-revisions", "two-applications", "revision-on-one", "no-revision", "no-application", "not-webhooks", "no-webhooks", "absent"} {
+	for _, name := range []string{"two-path-sets", "two-host-sets", "two-readiness-paths", "two-revisions", "two-applications", "revision-on-one", "no-revision", "no-application", "not-webhooks", "no-webhooks", "absent"} {
 		if got, err := k.Canary(t.Context(), ns, name); err == nil {
 			t.Fatalf("%s: read as %+v, want an error", name, got)
 		}

internal/gate/releases.go

@@ -32,6 +32,9 @@ type Release struct {
 	Serving string     `json:"serving,omitempty"`
 	Pinned  string     `json:"pinned,omitempty"`
 	Since   *time.Time `json:"since,omitempty"`
+	// Warmup is what the warmup of the serving revision met, once the gate has warmed it: the
+	// Reporter fails the Rollout on it.
+	Warmup *Warmup `json:"warmup,omitempty"`
 	// Unreadable says why the gate could not read the Application's inputs or record. The
 	// fields that depend on what it could not read are then empty.
 	Unreadable string `json:"unreadable,omitempty"`
@@ -122,7 +125,7 @@ func (g *Gate) release(ctx context.Context, a Gated) (Release, error) {
 	case err != nil:
 		return Release{}, fmt.Errorf("gate: read the release record of %s/%s: %w", a.Namespace, a.Application, err)
 	}
-	release.Serving, release.Pinned = record.Serving, record.Pinned
+	release.Serving, release.Pinned, release.Warmup = record.Serving, record.Pinned, record.Warmup
 	if !record.Since.IsZero() {
 		since := record.Since.UTC()
 		release.Since = &since

internal/gate/tend.go

@@ -47,6 +47,8 @@ type Record struct {
 	// it first saw it: the start of that release as far as the gate can tell.
 	Pinned string    `json:"pinned,omitempty"`
 	Since  time.Time `json:"since,omitzero"`
+	// Warmup is what the warmup of the serving revision met: absent until the gate has warmed it.
+	Warmup *Warmup `json:"warmup,omitempty"`
 }
 
 // ErrNoRecord is what Cluster.Record returns where the gate has recorded nothing yet.
@@ -253,10 +255,7 @@ func (g *Gate) tend(ctx context.Context, a Gated, now time.Time) (*Held, *Refuse
 		return nil, nil, err
 	}
 	if serving {
-		if err := g.record(ctx, r, Record{Serving: r.pinned, Promoted: r.promoted()}); err != nil {
-			return nil, nil, err
-		}
-		return nil, nil, g.sweep(ctx, r)
+		return nil, nil, g.serving(ctx, r)
 	}
 	// A release of the pinned revision is under way, or held.
 	if r, err = g.note(ctx, r, now); err != nil {
@@ -284,6 +283,20 @@ func (g *Gate) tend(ctx context.Context, a Gated, now time.Time) (*Held, *Refuse
 	return held, r.refusal(condition), nil
 }
 
+// serving records that the Application serves the pinned revision, warms that revision up the
+// first round it serves, and sweeps what older revisions synced. A revision is warmed once and
+// never again: a warmup that failed fails the Rollout and rolls nothing back.
+func (g *Gate) serving(ctx context.Context, r release) error {
+	next := Record{Serving: r.pinned, Promoted: r.promoted(), Warmup: r.record.Warmup}
+	if next.Warmup == nil || next.Warmup.Revision != r.pinned {
+		next.Warmup = g.warm(ctx, r)
+	}
+	if err := g.record(ctx, r, next); err != nil {
+		return err
+	}
+	return g.sweep(ctx, r)
+}
+
 // sweep deletes the Application's sync objects of every revision but the one it now serves. Every
 // primary runs that revision, so none reads what they sync any more
 // (deploy-kit spec/v1/55-delivery.md#a-removed-grant-outlives-its-release).
@@ -329,7 +342,9 @@ func (r release) refusal(condition string) *Refused {
 
 // record writes the Application's record where it differs from what the cluster holds.
 func (g *Gate) record(ctx context.Context, r release, next Record) error {
-	if next.Serving == r.record.Serving && next.Pinned == r.record.Pinned && next.Since.Equal(r.record.Since) && maps.Equal(next.Promoted, r.record.Promoted) {
+	sameWarmup := (next.Warmup == nil && r.record.Warmup == nil) ||
+		(next.Warmup != nil && r.record.Warmup != nil && *next.Warmup == *r.record.Warmup)
+	if next.Serving == r.record.Serving && next.Pinned == r.record.Pinned && next.Since.Equal(r.record.Since) && maps.Equal(next.Promoted, r.record.Promoted) && sameWarmup {
 		return nil
 	}
 	if err := g.cluster.SetRecord(ctx, r.Namespace, r.Application, next); err != nil {

internal/gate/warmup.go

@@ -0,0 +1,148 @@
+package gate
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"maps"
+	"net"
+	"net/http"
+	"net/netip"
+	"regexp"
+	"slices"
+	"time"
+)
+
+// The warmup after promotion (deploy-kit spec/v1/55-delivery.md#the-release-gate): once an
+// Application serves a new revision, the gate requests the same paths once more through every
+// public hostname its members' Canaries hand it, so the request crosses the edge to the new
+// primary. A 5xx there fails the release's Rollout and rolls nothing back. The tend loop starts
+// it, since Flagger asks no question after promotion.
+
+// Warmup is what the warmup of a serving revision met.
+type Warmup struct {
+	// Revision is the revision warmed: the one the record says serves.
+	Revision string `json:"revision"`
+	// Failed says what the first failing request met; empty where none failed.
+	Failed string `json:"failed,omitempty"`
+}
+
+// warmupTimeout bounds one warmup request, across the edge and a cache that may be cold.
+const warmupTimeout = 10 * time.Second
+
+// hostname is a host the gate warms through: a lower-case DNS name of two labels or more. The
+// render's schema guarantees it; the gate checks again, and the address it connects to too.
+var hostname = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$`) //nolint:gochecknoglobals // a compiled constant
+
+// warm requests every target of every member through each of its hostnames, in member order,
+// and stops at the first that fails.
+func (g *Gate) warm(ctx context.Context, r release) *Warmup {
+	warmup := &Warmup{Revision: r.pinned}
+	for _, process := range slices.Sorted(maps.Keys(r.members)) {
+		canary := r.members[process]
+		targets := targetsOf(canary)
+		for _, host := range canary.Hostnames {
+			for _, path := range targets {
+				if !hostname.MatchString(host) || !requestable.MatchString(path) {
+					warmup.Failed = fmt.Sprintf("the Canary of %s hands a host or path the gate does not request", process)
+					return warmup
+				}
+				url := "https://" + host + path
+				status, _, err := g.public.Get(ctx, url)
+				switch {
+				case err != nil:
+					warmup.Failed = url + " was not answered"
+					return warmup
+				case status >= http.StatusInternalServerError:
+					warmup.Failed = fmt.Sprintf("%s answered %d", url, status)
+					return warmup
+				}
+			}
+		}
+	}
+	return warmup
+}
+
+// PublicTraffic is Traffic across the internet to the edge: `https` on the default port only, no
+// proxy, no redirect followed, each hostname resolved as an absolute name and connected to only at
+// a public address.
+func PublicTraffic() Traffic {
+	dialer := &net.Dialer{Timeout: 5 * time.Second}
+	lookup := func(ctx context.Context, host string) ([]netip.Addr, error) {
+		// IPv4 only: the derived internet egress admits no IPv6 address.
+		return net.DefaultResolver.LookupNetIP(ctx, "ip4", host)
+	}
+	transport := http.DefaultTransport.(*http.Transport).Clone() //nolint:forcetypeassert // the standard library's own type
+	transport.Proxy = nil
+	transport.DialContext = publicDialer(lookup, dialer.DialContext)
+	return httpTraffic{client: &http.Client{
+		Transport:     transport,
+		Timeout:       warmupTimeout,
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}}
+}
+
+var (
+	errNotHTTPS        = errors.New("gate: a warmup connects to port 443 only")
+	errNoPublicAddress = errors.New("gate: the hostname resolves to no public address")
+)
+
+// publicDialer connects only to a public address on port 443. It resolves the hostname as an
+// absolute name, so the cluster's search path never completes it into a Service, and decides on
+// each address it would connect to, not on the name: a name that resolves somewhere internal is
+// refused whatever it is called.
+func publicDialer(
+	lookup func(ctx context.Context, host string) ([]netip.Addr, error),
+	dial func(ctx context.Context, network, address string) (net.Conn, error),
+) func(ctx context.Context, network, address string) (net.Conn, error) {
+	return func(ctx context.Context, network, address string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(address)
+		if err != nil {
+			return nil, err
+		}
+		if port != "443" {
+			return nil, errNotHTTPS
+		}
+		addresses, err := lookup(ctx, host+".")
+		if err != nil {
+			return nil, err
+		}
+		failed := fmt.Errorf("%w: %s", errNoPublicAddress, host)
+		for _, a := range addresses {
+			if !public(a) {
+				continue
+			}
+			conn, err := dial(ctx, network, netip.AddrPortFrom(a.Unmap(), 443).String())
+			if err == nil {
+				return conn, nil
+			}
+			failed = err
+		}
+		return nil, failed
+	}
+}
+
+// notPublic are the ranges that are global unicast to the standard library and reach no public
+// host: the carrier-grade NAT range the tailnet uses, and the IETF's special-purpose blocks.
+var notPublic = []netip.Prefix{ //nolint:gochecknoglobals // a constant list
+	netip.MustParsePrefix("100.64.0.0/10"),
+	netip.MustParsePrefix("192.0.0.0/24"),
+	netip.MustParsePrefix("198.18.0.0/15"),
+	netip.MustParsePrefix("2001:db8::/32"),
+}
+
+// public reports whether an address is one the warmup may connect to: global unicast, and in no
+// private, shared or special-purpose range. Loopback, link-local, multicast and the cluster's own
+// pod and service ranges, which are private, are all refused.
+func public(a netip.Addr) bool {
+	a = a.Unmap()
+	if !a.IsGlobalUnicast() || a.IsPrivate() {
+		return false
+	}
+	for _, p := range notPublic {
+		if p.Contains(a) {
+			return false
+		}
+	}
+	return true
+}