diff --git a/internal/internaldns/serving/activation.go b/internal/internaldns/serving/activation.go new file mode 100644 index 00000000..74a2c6dd --- /dev/null +++ b/internal/internaldns/serving/activation.go @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +const ( + activationInstalling = "installing" + activationVerifying = "verifying" +) + +type activationState struct { + Phase string `json:"phase"` +} + +func (t *FileTransaction) beginActivation() error { + if t.MarkerPath == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(t.MarkerPath), 0o750); err != nil { + return err + } + if t.activationLock == nil { + lock, err := os.OpenFile(t.MarkerPath+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil { + _ = lock.Close() + return err + } + t.activationLock = lock + } + if err := unix.Flock(int(t.activationLock.Fd()), unix.LOCK_EX); err != nil { + return err + } + return t.writeActivation(activationState{Phase: activationInstalling}) +} + +func (t *FileTransaction) writeActivation(state activationState) error { + if t.MarkerPath == "" { + return nil + } + data, err := json.Marshal(state) + if err != nil { + return err + } + temporary := t.MarkerPath + ".tmp" + if err := writeDurable(temporary, data, 0o644); err != nil { + return err + } + if err := os.Rename(temporary, t.MarkerPath); err != nil { + return err + } + return syncDirs([]string{t.MarkerPath}) +} + +func (t *FileTransaction) filesInstalled() error { + if err := t.writeActivation(activationState{Phase: activationVerifying}); err != nil { + return err + } + if t.activationLock != nil { + return unix.Flock(int(t.activationLock.Fd()), unix.LOCK_SH) + } + return nil +} + +func (t *FileTransaction) commitActivation() error { + if t.MarkerPath != "" { + if err := os.Remove(t.MarkerPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := syncDirs([]string{t.MarkerPath}); err != nil { + return err + } + } + return t.abandonActivation() +} + +// The kernel releases activation ownership when the agent process exits. +func (t *FileTransaction) abandonActivation() error { + if t.activationLock == nil { + return nil + } + err := t.activationLock.Close() + t.activationLock = nil + return err +} + +// daemonPermission holds the shared activation lock through daemon startup so +// installation cannot race with a process loading its configuration files. +func daemonPermission(stateDir string) (start, retain bool, release func()) { + marker := activationMarkerPath(stateDir) + lock, err := os.OpenFile(marker+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return false, false, func() {} + } + release = func() { _ = lock.Close() } + if err := unix.Flock(int(lock.Fd()), unix.LOCK_SH|unix.LOCK_NB); err != nil { + if errors.Is(err, unix.EWOULDBLOCK) { + // Running processes retain their in-memory configuration while the + // installer writes files. New processes must wait. + return false, true, release + } + return false, false, release + } + upgradeErr := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB) + _, readErr := os.ReadFile(marker) + if errors.Is(readErr, os.ErrNotExist) { + return true, true, release + } + if upgradeErr == nil { + // A pending marker without an installer is an abandoned activation. + return false, false, release + } + if !errors.Is(upgradeErr, unix.EWOULDBLOCK) { + return false, false, release + } + // A failed upgrade may release the shared lock. Reacquire it before + // reading the phase or allowing a daemon to load its configuration. + if err := unix.Flock(int(lock.Fd()), unix.LOCK_SH|unix.LOCK_NB); err != nil { + return false, true, release + } + data, readErr := os.ReadFile(marker) + if errors.Is(readErr, os.ErrNotExist) { + return true, true, release + } + var state activationState + if readErr != nil || json.Unmarshal(data, &state) != nil || state.Phase != activationVerifying { + return false, false, release + } + return true, true, release +} diff --git a/internal/internaldns/serving/activation_test.go b/internal/internaldns/serving/activation_test.go new file mode 100644 index 00000000..6ee5421f --- /dev/null +++ b/internal/internaldns/serving/activation_test.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +func TestActivationOwnershipGatesAbandonedUpdatesWithoutHeartbeat(t *testing.T) { + dir := t.TempDir() + tx := FileTransaction{MarkerPath: activationMarkerPath(dir)} + if err := tx.beginActivation(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = tx.abandonActivation() }) + start, retain, release := daemonPermission(dir) + release() + if start || !retain { + t.Fatal("installation allowed a new process or stopped approved in-memory DNS") + } + if err := tx.filesInstalled(); err != nil { + t.Fatal(err) + } + start, retain, release = daemonPermission(dir) + release() + if !start || !retain { + t.Fatal("live installer could not verify its complete configuration") + } + if err := tx.abandonActivation(); err != nil { + t.Fatal(err) + } + start, retain, release = daemonPermission(dir) + release() + if start || retain { + t.Fatal("abandoned activation retained query processes") + } + if err := os.Remove(tx.MarkerPath); err != nil { + t.Fatal(err) + } + start, retain, release = daemonPermission(dir) + release() + if !start || !retain { + t.Fatal("completed configuration required agent ownership") + } +} + +func TestPublicationActivationRemainsOwnedUntilDNSProof(t *testing.T) { + now := time.Now().UTC() + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, snapshot)); err != nil { + t.Fatal(err) + } + approved := agent.state.Active + verified := false + agent.publicationVerifier = publicationVerifierFunc(func(context.Context, model.PublicationPlan, []PublicationView) error { + verified = true + if !activationPending(cfg.StateDir) { + t.Fatal("activation committed before DNS proof") + } + if agent.state.Active != approved { + t.Fatal("unverified publication replaced approved checkpoint") + } + start, retain, release := daemonPermission(cfg.StateDir) + release() + if !start || !retain { + t.Fatal("live verification cannot query installed configuration") + } + if err := agent.tx.abandonActivation(); err != nil { + t.Fatal(err) + } + start, retain, release = daemonPermission(cfg.StateDir) + release() + if start || retain { + t.Fatal("abandoned unverified publication remains queryable") + } + return errors.New("publication proof interrupted") + }) + chunk, manifest := testPublication(t, now) + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err == nil { + t.Fatal("unverified publication accepted") + } + if !verified { + t.Fatal("publication proof was not attempted") + } + if !activationPending(cfg.StateDir) { + t.Fatal("interrupted proof lost recovery marker") + } +} + +func TestRestoreActiveUsesInstalledConfigurationHash(t *testing.T) { + now := time.Now().UTC() + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + actual := hashFiles(agent.state.Active.Files) + agent.state.RenderedHash = "unapproved-candidate" + if err := agent.store.Save(agent.state); err != nil { + t.Fatal(err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if restarted.state.RenderedHash != actual { + t.Fatal("restart retained candidate hash for approved files") + } + saved, err := restarted.store.Load() + if err != nil { + t.Fatal(err) + } + if saved.RenderedHash != actual { + t.Fatal("approved installed hash was not checkpointed") + } +} diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go new file mode 100644 index 00000000..24f8b7bc --- /dev/null +++ b/internal/internaldns/serving/agent.go @@ -0,0 +1,1333 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "path/filepath" + "sort" + "strconv" + "sync" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +type Mode string + +const ( + ModeResolver Mode = "resolver" + ModeCombined Mode = "combined" +) + +type Config struct { + Region string `json:"region"` + Shard string `json:"shard"` + MemberID string `json:"memberID"` + ReplicaID string `json:"replicaID,omitempty"` + StateDir string `json:"stateDir"` + Mode Mode `json:"mode"` + Render RenderConfig `json:"render"` + Transaction TransactionConfig `json:"transaction"` + ReconcileInterval time.Duration `json:"reconcileInterval,omitempty"` + RetryDelay time.Duration `json:"retryDelay,omitempty"` + AckInterval time.Duration `json:"ackInterval,omitempty"` + AckLease time.Duration `json:"ackLease,omitempty"` + AckPublishTimeout time.Duration `json:"ackPublishTimeout,omitempty"` + MaxBindings int `json:"maxBindings,omitempty"` + UnsafeAllowNoFailClosed bool `json:"unsafeAllowNoFailClosed,omitempty"` + PublicationDNSProbe PublicationDNSVerifierConfig `json:"publicationDNSProbe,omitempty"` + ResolverDNSProbe DNSVerifierConfig `json:"resolverDNSProbe,omitempty"` + ResolverVerifyConcurrency int `json:"resolverVerifyConcurrency,omitempty"` + UnsafeAllowNoDNSVerify bool `json:"unsafeAllowNoDNSVerify,omitempty"` +} + +func (c Config) Validate() error { + if c.Region == "" || c.Shard == "" || c.MemberID == "" || !filepath.IsAbs(c.StateDir) { + return errors.New("region, shard, member ID, and absolute state directory are required") + } + if c.Mode != ModeResolver && c.Mode != ModeCombined { + return fmt.Errorf("unsupported serving mode %q", c.Mode) + } + if c.Mode == ModeResolver || c.Mode == ModeCombined { + if err := validateRenderConfig(c.Render); err != nil { + return err + } + if !filepath.IsAbs(c.Transaction.StageDir) { + return errors.New("resolver mode requires an absolute transaction stage directory") + } + } + + if c.AckLease > 2*time.Minute { + return errors.New("ack lease may not exceed two minutes") + } + if c.AckPublishTimeout < 0 || c.AckPublishTimeout > 2*time.Second { + return errors.New("ACK publish timeout must be between zero and two seconds") + } + + if c.MaxBindings < 0 { + return errors.New("maximum bindings may not be negative") + } + if c.ResolverVerifyConcurrency < 0 || c.ResolverVerifyConcurrency > 64 { + return errors.New("resolver verification concurrency must be between zero and 64") + } + if !c.UnsafeAllowNoFailClosed && len(c.Transaction.FailClosed) == 0 { + return errors.New("failClosed command is required; use unsafeAllowNoFailClosed only in tests") + } + if c.Mode == ModeCombined { + if c.Render.Role != RenderCluster && c.Render.Role != RenderBoth && c.Render.Role != RenderRegionalCoreDNS && c.Render.Role != "" { + return errors.New("combined publication serving requires an owned regional CoreDNS process") + } + if c.PublicationDNSProbe.Server != "" && c.PublicationDNSProbe.Server != net.JoinHostPort(c.Render.ClusterCoreDNS.ListenAddress, strconv.Itoa(int(c.Render.ClusterCoreDNS.Port))) { + return errors.New("publication probe must target this member's protected CoreDNS listener") + } + } + return nil +} + +type EventSource interface { + Next(context.Context) (transport.Message, error) +} + +type BootstrapSource interface { + Load(context.Context) ([]model.Envelope, error) +} + +type Verifier interface { + Verify(context.Context, model.Binding) error +} + +type Dependencies struct { + ServingSource EventSource + RecordSource EventSource + AckSource EventSource + Bootstrap BootstrapSource + AckPublisher transport.ExportPublisher + Runner Runner + Verifier Verifier + Clock func() time.Time + PublicationVerifier PublicationVerifier + OnError func(error) +} + +type ackPublishResult struct { + ack transport.PublishAck + err error +} + +// boundedAckPublisher isolates local serving safety from a transport client +// blocked before it observes the publish context (for example, nats.Conn's +// connection-state mutex during a reconnect dial). There is no queue: at most +// one underlying call exists, and later state-derived ACKs expire at their +// caller deadline until that call returns. The next reconciliation publishes +// current state, naturally superseding intermediate heartbeats. +type boundedAckPublisher struct { + delegate transport.ExportPublisher + inflight chan struct{} +} + +func newBoundedAckPublisher(delegate transport.ExportPublisher) transport.ExportPublisher { + if delegate == nil { + return nil + } + if _, ok := delegate.(*boundedAckPublisher); ok { + return delegate + } + return &boundedAckPublisher{delegate: delegate, inflight: make(chan struct{}, 1)} +} + +func (p *boundedAckPublisher) Publish(ctx context.Context, subject string, env model.Envelope) (transport.PublishAck, error) { + if err := ctx.Err(); err != nil { + return transport.PublishAck{}, err + } + select { + case p.inflight <- struct{}{}: + case <-ctx.Done(): + return transport.PublishAck{}, ctx.Err() + } + if err := ctx.Err(); err != nil { + <-p.inflight + return transport.PublishAck{}, err + } + result := make(chan ackPublishResult, 1) + go func() { + ack, err := p.delegate.Publish(ctx, subject, env) + <-p.inflight + result <- ackPublishResult{ack: ack, err: err} + }() + select { + case result := <-result: + return result.ack, result.err + case <-ctx.Done(): + return transport.PublishAck{}, ctx.Err() + } +} + +type Agent struct { + cfg Config + deps Dependencies + store stateStore + tx FileTransaction + publicationVerifier PublicationVerifier + mu sync.Mutex + state checkpoint + needsActivation bool + lastServingAck map[string]servingAckState + lastPublicationAck map[string]time.Time + installedPublications map[string]installedPublication + // render is renderPublications unless a test injects a failure. + render func(RenderConfig, model.ServingSnapshot, map[string]bool, map[string]publicationState) (Rendered, error) +} + +// installedPublication identifies the exact files activated by a validated transaction. +type installedPublication struct { + Identity string + EffectiveHash string + Views []PublicationView +} + +func publicationIdentity(p publicationState) string { + data, _ := json.Marshal(p.Plan) + return fmt.Sprintf("%d/%d/%d/%s", p.Fence.Epoch, p.Fence.Revision, p.LocalRevision, model.Hash(data)) +} + +type servingAckState struct { + phase model.AckPhase + at time.Time +} + +func NewAgent(c Config, d Dependencies) (*Agent, error) { + if err := c.Validate(); err != nil { + return nil, err + } + if d.Clock == nil { + d.Clock = time.Now + } + if d.Runner == nil { + d.Runner = ExecRunner{} + } + d.AckPublisher = newBoundedAckPublisher(d.AckPublisher) + if (c.Mode == ModeResolver || c.Mode == ModeCombined) && c.Render.Role != RenderRegionalCoreDNS && d.Verifier == nil && !c.UnsafeAllowNoDNSVerify { + probeConfig := c.ResolverDNSProbe + if c.Render.Role == RenderCluster || c.Render.Role == RenderRegionalDNSDist { + probeConfig.UseClusterAddress = true + } + d.Verifier = DNSVerifier{Config: probeConfig} + } + publicationVerifier := d.PublicationVerifier + if c.Mode == ModeCombined { + if publicationVerifier == nil && c.PublicationDNSProbe.Server != "" { + publicationVerifier = PublicationDNSVerifier{Config: c.PublicationDNSProbe} + } + if publicationVerifier == nil && !c.UnsafeAllowNoDNSVerify { + return nil, errors.New("regional materialization DNS query verifier is required before readiness ACK") + } + } + store := stateStore{path: filepath.Join(c.StateDir, "checkpoint.json")} + state, err := store.Load() + if err != nil { + gateErr := (FileTransaction{Config: c.Transaction, Runner: d.Runner}).FailClosed(context.Background()) + return nil, errors.Join(err, gateErr) + } + return &Agent{cfg: c, deps: d, store: store, tx: FileTransaction{Config: c.Transaction, Runner: d.Runner, MarkerPath: activationMarkerPath(c.StateDir)}, publicationVerifier: publicationVerifier, state: state, lastServingAck: map[string]servingAckState{}, lastPublicationAck: map[string]time.Time{}, installedPublications: map[string]installedPublication{}}, nil +} + +func (a *Agent) Start(ctx context.Context) error { + defer func() { _ = a.tx.abandonActivation() }() + // Restore locally validated configuration without depending on state delivery. + a.mu.Lock() + if err := a.restoreActive(ctx); err != nil { + a.report(err) + } + a.mu.Unlock() + for { + err := a.bootstrapAndStep(ctx) + if err == nil { + break + } + if ctx.Err() != nil { + return nil + } + a.report(err) + retry := a.cfg.RetryDelay + if retry <= 0 { + retry = time.Second + } + if !waitRetry(ctx, retry) { + return nil + } + } + errCh := make(chan error, 3) + start := func(source EventSource) { + if source == nil { + return + } + go func() { errCh <- a.consume(ctx, source) }() + } + start(a.deps.ServingSource) + start(a.deps.RecordSource) + if a.cfg.Mode == ModeResolver || a.cfg.Mode == ModeCombined { + start(a.deps.AckSource) + } + interval := a.cfg.ReconcileInterval + if interval <= 0 { + interval = time.Second + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case err := <-errCh: + if err != nil && !errors.Is(err, context.Canceled) { + a.report(err) + } + case <-ticker.C: + if err := a.Step(ctx); err != nil { + a.report(err) + } + } + } +} + +func (a *Agent) failClosed(ctx context.Context) error { + a.needsActivation = true + gateCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + return errors.Join(a.tx.beginActivation(), a.tx.FailClosed(gateCtx), a.tx.abandonActivation()) +} + +func (a *Agent) bootstrapAndStep(ctx context.Context) error { + if a.deps.Bootstrap != nil { + if err := a.loadBootstrap(ctx, a.deps.Bootstrap); err != nil { + return err + } + } + if err := a.Step(ctx); err != nil { + return fmt.Errorf("initial serving reconciliation: %w", err) + } + return nil +} + +func (a *Agent) loadBootstrap(ctx context.Context, loader BootstrapSource) error { + envs, err := loader.Load(ctx) + if err != nil { + return fmt.Errorf("load bootstrap snapshot: %w", err) + } + order := func(k model.Kind) int { + switch k { + case model.KindPublicationChunk: + return 0 + case model.KindServingSnapshot: + return 1 + default: + return 2 + } + } + sort.SliceStable(envs, func(i, j int) bool { return order(envs[i].Kind) < order(envs[j].Kind) }) + for _, env := range envs { + if err := a.Handle(ctx, env); err != nil { + return fmt.Errorf("apply bootstrap %s: %w", env.Kind, err) + } + } + return nil +} + +// RunTransport synchronizes desired state independently of local DNS serving. +func (a *Agent) RunTransport(ctx context.Context, loader BootstrapSource, sources ...EventSource) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + if err := a.loadBootstrap(ctx, loader); err != nil { + return err + } + results := make(chan error, len(sources)) + for _, source := range sources { + go func() { results <- a.consume(ctx, source) }() + } + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-results: + return err + } +} + +func (a *Agent) consume(ctx context.Context, source EventSource) error { + retry := a.cfg.RetryDelay + if retry <= 0 { + retry = time.Second + } + for { + msg, err := source.Next(ctx) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + a.report(err) + if !waitRetry(ctx, retry) { + return ctx.Err() + } + continue + } + err = a.Handle(ctx, msg.Envelope()) + if err != nil { + env := msg.Envelope() + a.report(fmt.Errorf("apply %s %s at generation %d revision %d: %w", env.Kind, env.ResourceUID, env.Epoch, env.Revision, err)) + _ = msg.Retry(retry) + continue + } + if err := msg.Ack(ctx); err != nil { + a.report(err) + _ = msg.Retry(retry) + if !waitRetry(ctx, retry) { + return ctx.Err() + } + } + } +} + +func (a *Agent) report(err error) { + if err != nil && a.deps.OnError != nil { + a.deps.OnError(err) + } +} + +func waitRetry(ctx context.Context, d time.Duration) bool { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func (a *Agent) Handle(ctx context.Context, env model.Envelope) error { + if err := env.Validate(); err != nil { + return err + } + if env.Region != a.cfg.Region || env.Shard != a.cfg.Shard { + return fmt.Errorf("event targets %s/%s, agent serves %s/%s", env.Region, env.Shard, a.cfg.Region, a.cfg.Shard) + } + a.mu.Lock() + defer a.mu.Unlock() + if err := ctx.Err(); err != nil { + return err + } + switch env.Kind { + case model.KindServingSnapshot: + return a.handleServing(ctx, env) + case model.KindPublicationChunk: + + return a.handleChunk(env) + case model.KindPublicationManifest: + + if err := a.handleManifest(ctx, env); err != nil { + a.gateRejectedUpdate(ctx, err) + return err + } + return a.finishEvent(ctx) + case model.KindMemberAck: + + return a.handleAck(ctx, env) + default: + return fmt.Errorf("unsupported event kind %q", env.Kind) + } +} + +func (a *Agent) handleServing(ctx context.Context, env model.Envelope) error { + var snapshot model.ServingSnapshot + if err := json.Unmarshal(env.Payload, &snapshot); err != nil { + return err + } + if err := snapshot.Validate(); err != nil { + return err + } + maximum := a.cfg.MaxBindings + if maximum == 0 { + maximum = 4096 + } + if len(snapshot.Bindings) > maximum { + return fmt.Errorf("serving snapshot has %d bindings, exceeding member limit %d", len(snapshot.Bindings), maximum) + } + if snapshot.Region != env.Region || snapshot.Shard != env.Shard || snapshot.ConfigurationEpoch != env.Epoch || snapshot.ConfigurationRevision != env.Revision { + return errors.New("serving envelope identity does not match payload") + } + cmp := model.Compare(env.Epoch, env.Revision, a.state.ServingFence.Epoch, a.state.ServingFence.Revision) + if cmp < 0 { + return nil + } + if cmp == 0 && a.state.Snapshot != nil { + if !equalJSON(snapshot, *a.state.Snapshot) { + return errors.New("conflicting serving snapshot at the same epoch and revision") + } + return a.applyServingSnapshot(ctx, snapshot, false) + } + return a.applyServingSnapshot(ctx, snapshot, true) +} + +func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.ServingSnapshot, advance bool) error { + if advance { + bindingFences, err := advanceBindingFences(a.state.BindingFences, snapshot) + if err != nil { + return err + } + a.state.BindingFences = bindingFences + a.state.ServingFence = fence{snapshot.ConfigurationEpoch, snapshot.ConfigurationRevision} + a.state.Snapshot = &snapshot + if !a.canRetainActive() { + a.state.Active = nil + } + if err := a.store.Save(a.state); err != nil { + _ = a.failClosed(ctx) + return err + } + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.finishEvent(ctx) +} + +func (a *Agent) handleChunk(env model.Envelope) error { + var c model.PublicationChunk + if err := json.Unmarshal(env.Payload, &c); err != nil { + return err + } + if err := c.Validate(); err != nil { + return err + } + if c.ZoneUID != env.ResourceUID || c.WriterEpoch != env.Epoch || c.Revision != env.Revision { + return errors.New("chunk envelope identity does not match payload") + } + current := a.state.Publications[c.ZoneUID].Fence + if model.Compare(c.WriterEpoch, c.Revision, current.Epoch, current.Revision) < 0 { + return nil + } + if a.state.Chunks[c.ManifestUID] == nil { + a.state.Chunks[c.ManifestUID] = map[string]model.PublicationChunk{} + } + key := chunkKey(c.Index) + if old, ok := a.state.Chunks[c.ManifestUID][key]; ok && old.SHA256 != c.SHA256 { + return errors.New("conflicting chunk at the same manifest index") + } + a.state.Chunks[c.ManifestUID][key] = c + return a.store.Save(a.state) +} + +func (a *Agent) handleManifest(ctx context.Context, env model.Envelope) error { + var m model.PublicationManifest + if err := json.Unmarshal(env.Payload, &m); err != nil { + return err + } + if err := m.Validate(); err != nil { + return err + } + if m.ZoneUID != env.ResourceUID || m.WriterEpoch != env.Epoch || m.Revision != env.Revision { + return errors.New("manifest envelope identity does not match payload") + } + old := a.state.Publications[m.ZoneUID] + cmp := model.Compare(m.WriterEpoch, m.Revision, old.Fence.Epoch, old.Fence.Revision) + if cmp < 0 { + return nil + } + if cmp == 0 && old.Fence.Revision > 0 && !equalJSON(m, old.Manifest) { + return errors.New("conflicting publication manifest at the same epoch and revision") + } + if cmp == 0 && old.Verified { + if a.cfg.Mode == ModeResolver { + return a.recordResolverPublication(m.ZoneUID, old) + } + if !equalJSON(m, old.Manifest) { + return errors.New("conflicting publication manifest at the same epoch and revision") + } + if err := a.verifyPublication(ctx, old); err != nil { + _ = a.emitAck(ctx, model.KindPublicationManifest, m.ZoneUID, m.WriterEpoch, m.Revision, model.AckRejected, err.Error()) + return err + } + return a.emitVerifiedPublicationAck(ctx, m.ZoneUID, m.WriterEpoch, m.Revision) + } + // Publications are complete snapshots. The control plane reserves a revision + // before staging immutable artifacts, so a failed or superseded staging pass + // can legitimately leave a gap. Accept a snapshot compiled from a base at or + // ahead of the locally applied revision, while rejecting a newer revision + // compiled from state older than the regional materialization has already committed. + if cmp > 0 && old.Fence.Epoch == m.WriterEpoch && old.Fence.Revision > 0 && m.PreviousRevision < old.Fence.Revision { + return fmt.Errorf("publication revision %d is based on revision %d older than applied revision %d", m.Revision, m.PreviousRevision, old.Fence.Revision) + } + if m.Tombstone { + return a.applyTombstone(ctx, m, old) + } + chunkMap := a.state.Chunks[m.ManifestUID] + chunks := make([]model.PublicationChunk, 0, len(chunkMap)) + for _, c := range chunkMap { + chunks = append(chunks, c) + } + payload, err := model.VerifyManifest(m, chunks) + if err != nil { + return err + } + var plan model.PublicationPlan + if err := json.Unmarshal(payload, &plan); err != nil { + return err + } + if err := plan.Validate(); err != nil { + return err + } + if plan.ZoneUID != m.ZoneUID || model.AbsoluteName(plan.Apex) != model.AbsoluteName(m.Apex) { + return errors.New("publication plan identity does not match manifest") + } + if err := a.validateContributionFences(plan); err != nil { + return err + } + // Persist every accepted observation high-water mark before mutating the + // authoritative server. If the process dies during materialization, restart + // must still reject an older healthy observation that could resurrect data + // withdrawn by this plan. An identical retry remains idempotent. + a.commitContributionFences(plan) + if err := a.store.Save(a.state); err != nil { + return err + } + next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: plan, EffectiveHash: effectivePlanHash(plan)} + if a.cfg.Mode == ModeResolver { + return a.recordResolverPublication(m.ZoneUID, next) + } + return a.activatePublication(ctx, m.ZoneUID, next) +} + +func (a *Agent) applyTombstone(ctx context.Context, m model.PublicationManifest, old publicationState) error { + next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: old.Plan} + if a.cfg.Mode == ModeResolver { + return a.recordResolverPublication(m.ZoneUID, next) + } + return a.activatePublication(ctx, m.ZoneUID, next) +} + +// Commit the immutable fence before installing files. Failed activation stays +// gated and an identical redelivery retries the accepted snapshot. +func (a *Agent) recordResolverPublication(uid string, next publicationState) error { + next.Verified = false + a.state.Publications[uid] = next + if err := a.store.Save(a.state); err != nil { + return err + } + next.Verified = true + a.state.Publications[uid] = next + return a.store.Save(a.state) +} + +func (a *Agent) activatePublication(ctx context.Context, uid string, next publicationState) error { + a.state.Publications[uid] = next + if err := a.store.Save(a.state); err != nil { + return err + } + if err := a.reconcile(ctx); err != nil { + return err + } + if !next.Manifest.Tombstone && len(a.publicationViews(next)) == 0 { + return nil + } + if err := a.verifyPublication(ctx, next); err != nil { + _ = a.failClosed(ctx) + return err + } + next.Verified = true + a.state.Publications[uid] = next + if err := a.store.Save(a.state); err != nil { + return err + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.emitVerifiedPublicationAck(ctx, uid, next.Fence.Epoch, next.Fence.Revision) +} + +func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { + var ack model.MemberAck + if err := json.Unmarshal(env.Payload, &ack); err != nil { + return err + } + if ack.ResourceUID != env.ResourceUID || ack.Epoch != env.Epoch || ack.Revision != env.Revision { + return errors.New("ack envelope identity does not match payload") + } + if ack.ValidUntil.IsZero() || ack.ObservedAt.IsZero() || ack.ValidUntil.After(ack.ObservedAt.Add(2*time.Minute)) { + return errors.New("ack has an invalid serving lease") + } + if a.state.Snapshot == nil { + return nil + } + known := false + for _, m := range a.state.Snapshot.Members { + if m.MemberID == ack.MemberID && m.ReplicaID == ack.ReplicaID { + known = true + break + } + } + if !known { + return fmt.Errorf("ack from unknown member %q replica %q", ack.MemberID, ack.ReplicaID) + } + // A durable ACK consumer can legitimately replay a once-valid lease after a + // backlog. Drain it without changing applied readiness; + // retrying an already expired ACK can otherwise create a permanent NACK loop. + if !a.deps.Clock().Before(ack.ValidUntil) { + return nil + } + if ack.Kind != model.KindPublicationManifest { + return nil + } + if ack.Phase == model.AckExpired { + if err := a.reconcile(ctx); err != nil { + return err + } + return a.finishEvent(ctx) + } + if ack.Phase != model.AckVerified { + if ack.Phase != model.AckRejected && ack.Phase != model.AckApplied { + return errors.New("unsupported publication ACK phase") + } + if a.state.Acks[ack.ResourceUID] == nil { + a.state.Acks[ack.ResourceUID] = map[string]replicaAck{} + } + key := ack.MemberID + "/" + ack.ReplicaID + old := a.state.Acks[ack.ResourceUID][key] + cmp := model.Compare(ack.Epoch, ack.Revision, old.Fence.Epoch, old.Fence.Revision) + if cmp < 0 || (cmp == 0 && !ack.ObservedAt.After(old.ObservedAt)) { + return nil + } + a.state.Acks[ack.ResourceUID][key] = replicaAck{Fence: fence{ack.Epoch, ack.Revision}, ObservedAt: ack.ObservedAt, Rejected: true} + if err := a.store.Save(a.state); err != nil { + return err + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.finishEvent(ctx) + } + if a.state.Acks[ack.ResourceUID] == nil { + a.state.Acks[ack.ResourceUID] = map[string]replicaAck{} + } + key := ack.MemberID + "/" + ack.ReplicaID + old := a.state.Acks[ack.ResourceUID][key] + comparison := model.Compare(ack.Epoch, ack.Revision, old.Fence.Epoch, old.Fence.Revision) + if comparison < 0 || (comparison == 0 && !ack.ObservedAt.After(old.ObservedAt)) { + return nil + } + if comparison > 0 || (ack.Epoch == old.Fence.Epoch && ack.Revision == old.Fence.Revision && ack.ValidUntil.After(old.ValidUntil)) { + a.state.Acks[ack.ResourceUID][key] = replicaAck{Fence: fence{ack.Epoch, ack.Revision}, ValidUntil: ack.ValidUntil, ObservedAt: ack.ObservedAt} + if err := a.store.Save(a.state); err != nil { + return err + } + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.finishEvent(ctx) +} + +func (a *Agent) Step(ctx context.Context) error { + a.mu.Lock() + defer a.mu.Unlock() + if a.state.Snapshot == nil { + return errors.New("no verified serving configuration is available") + } + if a.cfg.Mode == ModeResolver { + if err := a.retryPendingPublications(ctx); err != nil { + a.gateRejectedUpdate(ctx, err) + return err + } + } + if a.cfg.Mode == ModeCombined { + if err := a.retryPendingPublications(ctx); err != nil { + a.gateRejectedUpdate(ctx, err) + return err + } + if a.needsActivation || len(a.installedPublications) == 0 { + if err := a.reconcile(ctx); err != nil { + return err + } + } + if err := a.heartbeatPublications(ctx); err != nil { + return err + } + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.verifyAppliedConfiguration(ctx) +} + +// finishEvent verifies regional materialization before acknowledging an update. +func (a *Agent) finishEvent(ctx context.Context) error { + if a.cfg.Mode == ModeCombined { + if err := a.heartbeatPublications(ctx); err != nil { + return err + } + } + return a.verifyAppliedConfiguration(ctx) +} + +// reconcile installs the desired configuration. When the desired state +// withdraws anything the approved configuration serves (a revoked binding, a +// detached zone or a deleted record), any failure to install it gates serving: +// returning early would leave the daemons answering with withdrawn data while +// every retry hits the same error. +func (a *Agent) reconcile(ctx context.Context) error { + if a.state.Snapshot == nil { + return nil + } + withdrawing := !a.canRetainActive() + err := a.reconcileDesired(ctx) + // Shutdown is not a failed install: restart gates again via restoreActive. + if err != nil && withdrawing && ctx.Err() == nil { + return errors.Join(err, a.failClosed(ctx)) + } + return err +} + +func (a *Agent) renderDesired(c RenderConfig, snapshot model.ServingSnapshot, active map[string]bool) (Rendered, error) { + if a.render != nil { + return a.render(c, snapshot, active, a.state.Publications) + } + return renderPublications(c, snapshot, active, a.state.Publications) +} + +func (a *Agent) reconcileDesired(ctx context.Context) error { + + active := map[string]bool{} + filtered := *a.state.Snapshot + filtered.Bindings = nil + for _, b := range a.state.Snapshot.Bindings { + eligible := !b.Tombstone && !b.ConfigurationPending + if eligible { + active[b.BindingUID] = true + } + if !b.Tombstone { + filtered.Bindings = append(filtered.Bindings, b) + } + } + renderConfig := a.cfg.Render + renderConfig.ReadyMembers = map[string]map[string]bool{} + renderConfig.ReadyZoneMembers = map[string]map[string]map[string]bool{} + for _, binding := range filtered.Bindings { + public := binding + public.Zones = nil + renderConfig.ReadyMembers[binding.BindingUID] = a.eligibleRegionalMembers(public) + renderConfig.ReadyZoneMembers[binding.BindingUID] = map[string]map[string]bool{} + for _, zone := range binding.Zones { + private := binding + private.Zones = []model.ZoneAttachment{zone} + renderConfig.ReadyZoneMembers[binding.BindingUID][zone.ZoneUID] = a.eligibleRegionalMembers(private) + } + } + rendered, err := a.renderDesired(renderConfig, filtered, active) + if err != nil { + return err + } + hash := hashFiles(rendered.Files) + if hash != a.state.RenderedHash || a.needsActivation { + if !a.canRetainActive() { + a.state.Active = nil + if err := a.store.Save(a.state); err != nil { + return err + } + } + var err error + if a.needsActivation { + err = a.tx.ApplyForce(ctx, rendered.Files) + } else { + err = a.tx.Apply(ctx, rendered.Files) + } + if err != nil { + a.gateRejectedUpdate(ctx, err) + return err + } + a.state.RenderedHash = hash + if err := a.store.Save(a.state); err != nil { + return err + } + a.needsActivation = false + } + if a.cfg.Mode == ModeCombined { + installed := map[string]installedPublication{} + for uid, p := range a.state.Publications { + installed[uid] = installedPublication{Identity: publicationIdentity(p), EffectiveHash: effectivePlanHash(p.Plan), Views: a.publicationViewsAt(p)} + } + a.installedPublications = installed + for _, p := range a.state.Publications { + if p.Verified && !p.Manifest.Tombstone && len(a.publicationViews(p)) > 0 { + if err := a.verifyPublication(ctx, p); err != nil { + _ = a.failClosed(ctx) + a.needsActivation = true + return err + } + } + } + } + // End-to-end verification must run after the candidate mapping is live. + // Any binding that fails either UDP or TCP is immediately removed again. + probeFailed := a.verifyResolverBindings(ctx, filtered.Bindings, active) + if probeFailed { + gated, err := a.renderDesired(renderConfig, filtered, active) + if err != nil { + return err + } + gatedHash := hashFiles(gated.Files) + if gatedHash != a.state.RenderedHash { + if err := a.tx.Apply(ctx, gated.Files); err != nil { + _ = a.failClosed(ctx) + return err + } + a.state.RenderedHash = gatedHash + rendered = gated + if err := a.store.Save(a.state); err != nil { + return err + } + } + } + // Keep activation owned until every materialized publication has query proof + // and publication content has passed query verification. + if a.cfg.Mode == ModeCombined { + for _, p := range a.state.Publications { + if !p.Verified && (p.Manifest.Tombstone || len(a.publicationViews(p)) > 0) { + return nil + } + } + } + if err := a.approveConfiguration(rendered.Files); err != nil { + return err + } + // Applied configuration remains independent of broker acknowledgements. + if a.cfg.Mode == ModeResolver { + if err := a.verifyAppliedConfiguration(ctx); err != nil { + return err + } + } + for _, b := range a.state.Snapshot.Bindings { + if b.Tombstone || b.ConfigurationPending { + continue + } + p := model.AckVerified + if !active[b.BindingUID] { + p = model.AckApplied + } + if err := a.emitServingAck(ctx, b, p); err != nil { + a.report(fmt.Errorf("publish serving ACK %s: %w", b.BindingUID, err)) + break + } + } + return nil +} + +type resolverProbeResult struct { + bindingUID string + duration time.Duration + err error +} + +// verifyResolverBindings probes a shard in a bounded worker pool. All probes +// share one deadline so adding resolver contexts cannot make reconciliation grow by one +// complete probe timeout per binding. +func (a *Agent) verifyResolverBindings(ctx context.Context, bindings []model.Binding, active map[string]bool) bool { + if a.deps.Verifier == nil { + return false + } + eligible := make([]model.Binding, 0, len(bindings)) + for _, binding := range bindings { + if active[binding.BindingUID] { + binding.Zones = a.readyZones(binding) + eligible = append(eligible, binding) + } + } + if len(eligible) == 0 { + return false + } + + timeout := a.cfg.ResolverDNSProbe.Timeout + if timeout <= 0 { + timeout = 3 * time.Second + } + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + concurrency := a.cfg.ResolverVerifyConcurrency + if concurrency == 0 { + concurrency = 16 + } + concurrency = min(concurrency, len(eligible)) + jobs := make(chan model.Binding) + results := make(chan resolverProbeResult, len(eligible)) + for range concurrency { + go func() { + for binding := range jobs { + started := time.Now() + err := a.deps.Verifier.Verify(probeCtx, binding) + results <- resolverProbeResult{bindingUID: binding.BindingUID, duration: time.Since(started), err: err} + } + }() + } + go func() { + defer close(jobs) + for _, binding := range eligible { + select { + case jobs <- binding: + case <-probeCtx.Done(): + return + } + } + }() + + pending := make(map[string]bool, len(eligible)) + for _, binding := range eligible { + pending[binding.BindingUID] = true + } + probeFailed := false + for len(pending) > 0 { + select { + case result := <-results: + if !pending[result.bindingUID] { + continue + } + delete(pending, result.bindingUID) + if result.err != nil { + a.report(fmt.Errorf("verify resolver binding %s after %s: %w", result.bindingUID, result.duration.Round(time.Millisecond), result.err)) + delete(active, result.bindingUID) + probeFailed = true + } + case <-probeCtx.Done(): + for bindingUID := range pending { + a.report(fmt.Errorf("verify resolver binding %s within shared %s batch deadline: %w", bindingUID, timeout, probeCtx.Err())) + delete(active, bindingUID) + } + return true + } + } + return probeFailed +} + +func (a *Agent) publicationsReady(b model.Binding) bool { + if b.ConfigurationPending { + return false + } + if len(b.Zones) == 0 { + return true + } + if a.cfg.Mode == ModeCombined { + for _, z := range b.Zones { + p, ok := a.state.Publications[z.ZoneUID] + if z.Pending || !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + return false + } + } + return true + } + return len(a.eligibleRegionalMembers(b)) > 0 +} + +// Every selected member must prove every attached zone at the exact publication +// plan version this resolver holds. One zone per different member is unsafe. +func (a *Agent) eligibleRegionalMembers(b model.Binding) map[string]bool { + out := map[string]bool{} + if a.state.Snapshot == nil || b.ConfigurationPending { + return out + } + if a.cfg.Mode == ModeCombined { + if a.publicationsReady(b) { + out[a.cfg.MemberID] = true + } + return out + } + for _, member := range a.state.Snapshot.Members { + if member.Role != "regional" && member.Role != "cluster" { + continue + } + ready := true + for _, z := range b.Zones { + p, ok := a.state.Publications[z.ZoneUID] + if z.Pending || !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + ready = false + break + } + for _, known := range a.state.Acks[z.ZoneUID] { + if model.Compare(known.Fence.Epoch, known.Fence.Revision, p.Fence.Epoch, p.Fence.Revision) > 0 { + ready = false + } + } + proof := a.state.Acks[z.ZoneUID][member.MemberID+"/"+member.ReplicaID] + if proof.Rejected || proof.ObservedAt.IsZero() || proof.Fence != p.Fence { + ready = false + break + } + } + if ready { + out[member.MemberID] = true + } + } + return out +} + +func (a *Agent) readyZones(b model.Binding) []model.ZoneAttachment { + var ready []model.ZoneAttachment + for _, zone := range b.Zones { + candidate := b + candidate.Zones = []model.ZoneAttachment{zone} + if len(a.eligibleRegionalMembers(candidate)) > 0 { + ready = append(ready, zone) + } + } + return ready +} + +func (a *Agent) verifyPublication(ctx context.Context, p publicationState) error { + if p.Manifest.Tombstone { + return nil + } + if a.publicationVerifier != nil { + if len(a.publicationViews(p)) == 0 { + return nil + } + installed, ok := a.installedPublications[p.Plan.ZoneUID] + if !ok || installed.Identity != publicationIdentity(p) { + return errors.New("publication has no matching activated file identity") + } + return a.publicationVerifier.VerifyPublication(ctx, p.Plan, installed.Views) + } + return nil +} + +func (a *Agent) publicationViews(p publicationState) []PublicationView { + return a.publicationViewsAt(p) +} + +func (a *Agent) publicationViewsAt(p publicationState) []PublicationView { + if a.state.Snapshot == nil { + return nil + } + var out []PublicationView + for _, b := range a.state.Snapshot.Bindings { + if b.Tombstone || b.ConfigurationPending { + continue + } + for _, z := range b.Zones { + if !z.Pending && z.ZoneUID == p.Plan.ZoneUID && model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) >= 0 { + out = append(out, PublicationView{View: b.ViewName(), DestinationAddress: b.ClusterAddress, DestinationPort: b.Port, Serial: publicationSerial(p), FingerprintMailbox: publicationMailbox(p)}) + } + } + } + return out +} + +func (a *Agent) retryPendingPublications(ctx context.Context) error { + for uid, p := range a.state.Publications { + if p.Verified { + continue + } + if a.cfg.Mode == ModeResolver { + if err := a.recordResolverPublication(uid, p); err != nil { + return err + } + continue + } + if !p.Manifest.Tombstone && len(a.publicationViews(p)) == 0 { + continue + } + if err := a.reconcile(ctx); err != nil { + return err + } + if err := a.verifyPublication(ctx, p); err != nil { + return err + } + p.Verified = true + p.EffectiveHash = effectivePlanHash(p.Plan) + a.state.Publications[uid] = p + if err := a.store.Save(a.state); err != nil { + return err + } + } + return nil +} + +func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, revision uint64, phase model.AckPhase, message string) error { + if a.deps.AckPublisher == nil { + return nil + } + now := a.deps.Clock().UTC() + lease := a.cfg.AckLease + if lease <= 0 { + lease = 90 * time.Second + } + leaseUntil := now.Add(lease) + ack := model.MemberAck{MemberID: a.cfg.MemberID, ReplicaID: a.cfg.ReplicaID, ResourceUID: uid, Kind: kind, Epoch: epoch, Revision: revision, Phase: phase, ObservedAt: now, ValidUntil: leaseUntil, Error: message} + eventID := fmt.Sprintf("ack-%s-%s-%d-%d-%s-%d", model.OpaqueToken(a.cfg.MemberID+"/"+a.cfg.ReplicaID), model.OpaqueToken(uid), epoch, revision, phase, now.UnixNano()) + env, err := model.NewEnvelope(model.KindMemberAck, eventID, a.cfg.Region, a.cfg.Shard, uid, epoch, revision, a.deps.Clock(), ack) + if err != nil { + return err + } + publishTimeout := a.cfg.AckPublishTimeout + if publishTimeout <= 0 { + publishTimeout = 500 * time.Millisecond + } + publishCtx, cancel := context.WithTimeout(ctx, publishTimeout) + defer cancel() + _, err = a.deps.AckPublisher.Publish(publishCtx, model.AckSubject(a.cfg.Region, a.cfg.Shard, a.cfg.MemberID), env) + return err +} + +func (a *Agent) emitVerifiedPublicationAck(ctx context.Context, uid string, epoch, revision uint64) error { + // Validate local activation before best-effort acknowledgement delivery. + if err := a.verifyAppliedConfiguration(ctx); err != nil { + return err + } + if err := a.emitAck(ctx, model.KindPublicationManifest, uid, epoch, revision, model.AckVerified, ""); err != nil { + a.report(fmt.Errorf("publish verified publication ACK %s: %w", uid, err)) + return nil + } + // The materialization path has already completed the same staged-file and DNS + // proof required by a periodic heartbeat. Remember it so finishEvent checks + // other due publications without immediately repeating this proof. + a.lastPublicationAck[uid] = a.deps.Clock() + return nil +} + +func (a *Agent) emitServingAck(ctx context.Context, b model.Binding, phase model.AckPhase) error { + now := a.deps.Clock() + interval := a.cfg.AckInterval + if interval <= 0 { + interval = 30 * time.Second + } + last := a.lastServingAck[b.BindingUID] + if last.phase == phase && now.Sub(last.at) < interval { + return nil + } + if err := a.emitAck(ctx, model.KindServingSnapshot, b.BindingUID, a.state.ServingFence.Epoch, a.state.ServingFence.Revision, phase, ""); err != nil { + return err + } + a.lastServingAck[b.BindingUID] = servingAckState{phase: phase, at: now} + return nil +} + +func (a *Agent) heartbeatPublications(ctx context.Context) error { + now := a.deps.Clock() + interval := a.cfg.AckInterval + if interval <= 0 { + interval = 30 * time.Second + } + type duePublication struct { + uid string + p publicationState + } + var due []duePublication + for uid, p := range a.state.Publications { + if !p.Verified || (!p.Manifest.Tombstone && len(a.publicationViews(p)) == 0) { + continue + } + if last := a.lastPublicationAck[uid]; !last.IsZero() && now.Sub(last) < interval { + continue + } + if err := a.verifyPublication(ctx, p); err != nil { + _ = a.emitAck(ctx, model.KindPublicationManifest, uid, p.Fence.Epoch, p.Fence.Revision, model.AckRejected, err.Error()) + return fmt.Errorf("reverify publication %s before ACK renewal: %w", uid, err) + } + due = append(due, duePublication{uid: uid, p: p}) + } + // Publish acknowledgements within a bounded budget after local DNS verification. + for _, publication := range due { + p := publication.p + if err := a.emitAck(ctx, model.KindPublicationManifest, publication.uid, p.Fence.Epoch, p.Fence.Revision, model.AckVerified, ""); err != nil { + a.report(fmt.Errorf("publish verified publication ACK %s: %w", publication.uid, err)) + break + } + a.lastPublicationAck[publication.uid] = now + } + return nil +} + +func hashFiles(files map[string][]byte) string { + keys := make([]string, 0, len(files)) + for k := range files { + keys = append(keys, k) + } + sort.Strings(keys) + size := 0 + for _, key := range keys { + size += len(key) + 1 + len(files[key]) + } + all := make([]byte, 0, size) + for _, k := range keys { + all = append(all, []byte(k)...) + all = append(all, 0) + all = append(all, files[k]...) + } + return model.Hash(all) +} + +func equalJSON(left, right any) bool { + a, errA := json.Marshal(left) + b, errB := json.Marshal(right) + return errA == nil && errB == nil && string(a) == string(b) +} + +func (a *Agent) validateContributionFences(plan model.PublicationPlan) error { + for _, observation := range plan.ObservationFences { + current, ok := a.state.ContributionFences[observation.ContributionUID] + if !ok { + continue + } + if current.Retired && !observation.Retired { + return fmt.Errorf("contribution %q lifetime is retired", observation.ContributionUID) + } + comparison := model.Compare(observation.Generation, observation.Sequence, current.Generation, current.Sequence) + if comparison < 0 { + return fmt.Errorf("contribution %q regressed from epoch %d sequence %d", observation.ContributionUID, current.Generation, current.Sequence) + } + + } + return nil +} + +func (a *Agent) commitContributionFences(plan model.PublicationPlan) { + for _, observation := range plan.ObservationFences { + next := contributionFence{Generation: observation.Generation, Sequence: observation.Sequence, Retired: observation.Retired} + current := a.state.ContributionFences[observation.ContributionUID] + comparison := model.Compare(next.Generation, next.Sequence, current.Generation, current.Sequence) + if comparison > 0 || (comparison == 0 && next.Retired && !current.Retired) { + a.state.ContributionFences[observation.ContributionUID] = next + } + } +} + +func (a *Agent) appliedPublicationError() error { + for uid, p := range a.state.Publications { + if p.Manifest.Tombstone { + continue + } + current := effectivePlanHash(p.Plan) + if a.cfg.Mode == ModeCombined { + if len(a.publicationViewsAt(p)) == 0 { + continue + } + installed, ok := a.installedPublications[uid] + if !ok || installed.Identity != publicationIdentity(p) || installed.EffectiveHash != current { + return fmt.Errorf("publication %s configuration differs from activation; refusing activation", uid) + } + } else if p.EffectiveHash != current { + return fmt.Errorf("publication %s configuration differs from the applied checkpoint; refusing activation", uid) + } + } + return nil +} + +func (a *Agent) verifyAppliedConfiguration(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + if err := a.appliedPublicationError(); err != nil { + return errors.Join(err, a.failClosed(ctx)) + } + return ctx.Err() +} diff --git a/internal/internaldns/serving/agent_test.go b/internal/internaldns/serving/agent_test.go new file mode 100644 index 00000000..332aa506 --- /dev/null +++ b/internal/internaldns/serving/agent_test.go @@ -0,0 +1,681 @@ +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +type fakePublisher struct{ envs []model.Envelope } + +func (f *fakePublisher) Publish(_ context.Context, _ string, e model.Envelope) (transport.PublishAck, error) { + f.envs = append(f.envs, e) + return transport.PublishAck{}, nil +} + +type stalledPublisher struct{} + +func (stalledPublisher) Publish(ctx context.Context, _ string, _ model.Envelope) (transport.PublishAck, error) { + <-ctx.Done() + return transport.PublishAck{}, ctx.Err() +} + +type contextIgnoringPublisher struct { + mu sync.Mutex + calls int + started chan<- struct{} + release <-chan struct{} +} + +func (p *contextIgnoringPublisher) Publish(context.Context, string, model.Envelope) (transport.PublishAck, error) { + p.mu.Lock() + p.calls++ + p.mu.Unlock() + select { + case p.started <- struct{}{}: + default: + } + <-p.release + return transport.PublishAck{}, errors.New("nats: server disconnected") +} + +func (p *contextIgnoringPublisher) callCount() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.calls +} + +func TestBoundedAckPublisherDoesNotStartExpiredRequest(t *testing.T) { + t.Parallel() + started := make(chan struct{}, 1) + release := make(chan struct{}) + close(release) + delegate := &contextIgnoringPublisher{started: started, release: release} + publisher := newBoundedAckPublisher(delegate) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := publisher.Publish(ctx, "ack", model.Envelope{}); !errors.Is(err, context.Canceled) { + t.Fatalf("expired ACK request error = %v, want context cancellation", err) + } + if calls := delegate.callCount(); calls != 0 { + t.Fatalf("expired ACK request invoked delegate %d times", calls) + } +} + +type blockingPublisher struct { + started chan<- struct{} + release <-chan struct{} +} + +type resolverVerifierFunc func(context.Context, model.Binding) error + +func (f resolverVerifierFunc) Verify(ctx context.Context, binding model.Binding) error { + return f(ctx, binding) +} + +func (p blockingPublisher) Publish(ctx context.Context, _ string, _ model.Envelope) (transport.PublishAck, error) { + select { + case p.started <- struct{}{}: + default: + } + select { + case <-p.release: + return transport.PublishAck{}, nil + case <-ctx.Done(): + return transport.PublishAck{}, ctx.Err() + } +} + +type commandChannelRunner struct{ calls chan Command } + +func (r *commandChannelRunner) Run(_ context.Context, command Command) ([]byte, error) { + r.calls <- command + return nil, nil +} + +type boundedResolverVerifier struct { + mu sync.Mutex + active int + maximum int + calls int + release <-chan struct{} + started chan<- struct{} + waitForCtx bool +} + +func (v *boundedResolverVerifier) Verify(ctx context.Context, _ model.Binding) error { + v.mu.Lock() + v.active++ + v.calls++ + if v.active > v.maximum { + v.maximum = v.active + } + v.mu.Unlock() + if v.started != nil { + select { + case v.started <- struct{}{}: + default: + } + } + var err error + if v.waitForCtx { + <-ctx.Done() + err = ctx.Err() + } else { + select { + case <-ctx.Done(): + err = ctx.Err() + case <-v.release: + } + } + v.mu.Lock() + v.active-- + v.mu.Unlock() + return err +} + +func (v *boundedResolverVerifier) snapshot() (calls, maximum int) { + v.mu.Lock() + defer v.mu.Unlock() + return v.calls, v.maximum +} + +func envelope(t *testing.T, kind model.Kind, uid string, epoch, revision uint64, payload any) model.Envelope { + t.Helper() + e, err := model.NewEnvelope(kind, "event-"+string(kind)+uid+time.Now().String(), "east", "s1", uid, epoch, revision, time.Now(), payload) + if err != nil { + t.Fatal(err) + } + return e +} + +func testSnapshot(now time.Time) model.ServingSnapshot { + b := model.Binding{BindingUID: "binding-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 2, RequiredPublicationRevision: 4}}, Version: model.AccessVersion{Epoch: 1, Sequence: 1}} + return model.ServingSnapshot{Region: "east", Shard: "s1", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []model.Binding{b}, Members: []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}}} +} + +func testPublication(t *testing.T, now time.Time) (model.PublicationChunk, model.PublicationManifest) { + t.Helper() + plan := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", Sequence: 1, Generation: 1}}, Owners: []string{"api.prod.internal"}, RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{Content: "192.0.2.10", ContributionUID: "contrib-a", Sequence: 1, Eligible: true, Generation: 1}}}}} + payload, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + chunk := model.PublicationChunk{ManifestUID: "manifest-a", ZoneUID: "zone-a", WriterEpoch: 2, Revision: 4, Index: 0, SHA256: model.Hash(payload), Payload: payload} + manifest := model.PublicationManifest{ManifestUID: "manifest-a", ZoneUID: "zone-a", Apex: "prod.internal", WriterEpoch: 2, Revision: 4, Chunks: []model.ChunkRef{{Index: 0, SHA256: chunk.SHA256, Size: len(payload)}}, ContentHash: model.Hash(payload), GeneratedAt: now} + return chunk, manifest +} + +func TestWithdrawnObservationFencesOlderHealthyRecord(t *testing.T) { + t.Parallel() + agent := &Agent{state: newCheckpoint()} + withdrawn := model.PublicationPlan{ + ZoneUID: "zone-a", Apex: "prod.internal", + ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", Sequence: 33, Generation: 7}}, + RRSets: []model.RRSet{}, + } + if err := withdrawn.Validate(); err != nil { + t.Fatal(err) + } + agent.commitContributionFences(withdrawn) + olderHealthy := model.PublicationPlan{ + ZoneUID: "zone-a", Apex: "prod.internal", + ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", Sequence: 32, Generation: 7}}, + RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{Content: "192.0.2.10", ContributionUID: "contrib-a", Sequence: 32, Eligible: true, Generation: 7}}}}, + } + if err := olderHealthy.Validate(); err != nil { + t.Fatal(err) + } + if err := agent.validateContributionFences(olderHealthy); err == nil { + t.Fatal("older healthy observation was accepted after a higher withdrawn observation") + } +} + +func TestClusterRoleAutomaticallyVerifiesClusterVIP(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, _ := renderFixture(t) + renderConfig.Role = RenderCluster + renderConfig.ClusterDNSDist.Path = filepath.Join(dir, "cluster-dnsdist.conf") + renderConfig.ClusterCoreDNS.Path = filepath.Join(dir, "cluster-coredns.json") + renderConfig.NodeDNSDist = DNSDistRenderConfig{} + renderConfig.NodeCoreDNS = CoreDNSRenderConfig{} + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "cluster-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoFailClosed: true, + }, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + verifier, ok := agent.deps.Verifier.(DNSVerifier) + if !ok || !verifier.Config.UseClusterAddress { + t.Fatalf("cluster agent verifier = %#v, want cluster VIP", agent.deps.Verifier) + } +} + +func TestStartActivatesPersistedDesiredConfiguration(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} + rendered, err := Render(renderConfig, snapshot, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + for path, contents := range rendered.Files { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, contents, 0o644); err != nil { + t.Fatal(err) + } + } + stateDir := filepath.Join(dir, "state") + state := newCheckpoint() + state.Snapshot = &snapshot + state.BindingFences, _ = advanceBindingFences(nil, snapshot) + state.ServingFence = fence{Epoch: snapshot.ConfigurationEpoch, Revision: snapshot.ConfigurationRevision} + state.RenderedHash = hashFiles(rendered.Files) + if err := (stateStore{path: filepath.Join(stateDir, "checkpoint.json")}).Save(state); err != nil { + t.Fatal(err) + } + config := Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: stateDir, Mode: ModeResolver, + Render: renderConfig, UnsafeAllowNoDNSVerify: true, + ReconcileInterval: time.Hour, + Transaction: TransactionConfig{ + StageDir: filepath.Join(dir, "stage"), + FailClosed: []Command{{Path: "gate"}}, + Validate: []Command{ + {Path: "validate-dnsdist", Args: []string{"{stageDir}/node-dnsdist.conf"}, WhenChanged: []string{renderConfig.NodeDNSDist.Path}}, + {Path: "validate-bind", Args: []string{"{stageDir}/node-coredns.json"}, WhenChanged: []string{renderConfig.NodeCoreDNS.Path}}, + }, + Reload: []Command{ + {Path: "reload-dnsdist", WhenChanged: []string{renderConfig.NodeDNSDist.Path}}, + {Path: "reload-bind", WhenChanged: []string{renderConfig.NodeCoreDNS.Path}}, + }, + }, + } + runner := &commandChannelRunner{calls: make(chan Command, 16)} + agent, err := NewAgent(config, Dependencies{Runner: runner}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- agent.Start(ctx) }() + deadline := time.After(2 * time.Second) + for { + if state, err := agent.store.Load(); err == nil && state.Active != nil { + break + } + select { + case <-deadline: + cancel() + t.Fatal("startup did not approve persisted desired configuration") + case <-time.After(5 * time.Millisecond): + } + } + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } + var calls []string + for len(runner.calls) > 0 { + calls = append(calls, (<-runner.calls).Path) + } + for _, required := range []string{"validate-dnsdist", "validate-bind", "reload-dnsdist", "reload-bind"} { + if !slices.Contains(calls, required) { + t.Fatalf("startup omitted %s: %#v", required, calls) + } + } + assertAppliedCheckpoint(t, agent) + +} + +func TestStepBoundsStalledAckPublishAndRetainsAppliedConfiguration(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} + snapshot.Bindings[0].Zones = nil + var reported []error + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoDNSVerify: true, AckPublishTimeout: 30 * time.Millisecond, + }, Dependencies{Runner: &recordingRunner{}, OnError: func(err error) { reported = append(reported, err) }}) + if err != nil { + t.Fatal(err) + } + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + // Complete durable activation before measuring the ACK publication bound. + if err := agent.Step(context.Background()); err != nil { + t.Fatal(err) + } + agent.deps.AckPublisher = stalledPublisher{} + agent.lastServingAck = make(map[string]servingAckState) + started := time.Now() + if err := agent.Step(context.Background()); err != nil { + t.Fatal(err) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("Step blocked %s on an unavailable ACK broker", elapsed) + } + if len(reported) == 0 || !strings.Contains(reported[0].Error(), "publish serving ACK") { + t.Fatalf("stalled ACK publish was not reported: %#v", reported) + } + assertAppliedCheckpoint(t, agent) +} + +func stepUntilBlockedAck(t *testing.T, agent *Agent, started <-chan struct{}, release chan struct{}) func() error { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + finished := make(chan struct{}) + var stepErr error + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(func() { + unblock() + cancel() + select { + case <-finished: + case <-time.After(10 * time.Second): + t.Error("Step did not finish after unconditional ACK release and cancellation") + } + }) + go func() { + stepErr = agent.Step(ctx) + close(finished) + }() + select { + case <-started: + case <-finished: + t.Fatalf("Step finished before blocked ACK verification: %v", stepErr) + case <-time.After(10 * time.Second): + t.Fatal("resolver did not reach broker ACK publication") + } + return func() error { + unblock() + select { + case <-finished: + return stepErr + case <-time.After(10 * time.Second): + return errors.New("Step did not finish after ACK release") + } + } +} + +func TestResolverApprovesConfigurationBeforeBrokerAckCompletes(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} + snapshot.Bindings[0].Zones = nil + started := make(chan struct{}, 1) + release := make(chan struct{}) + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoDNSVerify: true, AckPublishTimeout: time.Second, + }, Dependencies{AckPublisher: blockingPublisher{started: started, release: release}, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + finish := stepUntilBlockedAck(t, agent, started, release) + assertAppliedCheckpoint(t, agent) + if err := finish(); err != nil { + t.Fatal(err) + } +} + +func TestResolverGatesFailedTenantAndRetainsHealthyTenant(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} + snapshot.Bindings[0].Zones = nil + failedUID := snapshot.Bindings[0].BindingUID + healthy := snapshot.Bindings[0] + healthy.BindingUID = "binding-b" + healthy.ProjectUID = "project-b" + healthy.ContextUID = "vpc-b" + healthy.ConsumerAddress = "10.253.0.52" + healthy.ClusterAddress = "10.253.0.42" + snapshot.Bindings = append(snapshot.Bindings, healthy) + started := make(chan struct{}, 1) + release := make(chan struct{}) + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoFailClosed: true, AckPublishTimeout: time.Second, + }, Dependencies{ + AckPublisher: blockingPublisher{started: started, release: release}, Runner: &recordingRunner{}, + Verifier: resolverVerifierFunc(func(_ context.Context, binding model.Binding) error { + if binding.BindingUID == failedUID { + return errors.New("probe failed") + } + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + finish := stepUntilBlockedAck(t, agent, started, release) + config, err := os.ReadFile(renderConfig.NodeDNSDist.Path) + if err != nil { + t.Fatal(err) + } + if bindingReadyInConfig(string(config), snapshot.Bindings[0].ConsumerAddress) { + t.Fatal("failed tenant remained routed") + } + if !strings.Contains(string(config), healthy.ConsumerAddress) { + t.Fatal("healthy tenant was gated with failed tenant") + } + assertAppliedCheckpoint(t, agent) + if err := finish(); err != nil { + t.Fatal(err) + } +} + +func TestStartPreservesAppliedConfigurationWhileBrokerDisconnected(t *testing.T) { + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} + snapshot.Bindings[0].Zones = nil + publishStarted := make(chan struct{}, 1) + publishRelease := make(chan struct{}) + publisher := &contextIgnoringPublisher{started: publishStarted, release: publishRelease} + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, + ReconcileInterval: 20 * time.Millisecond, AckInterval: 50 * time.Millisecond, + AckPublishTimeout: 10 * time.Millisecond, + }, Dependencies{AckPublisher: publisher, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- agent.Start(ctx) }() + var stopOnce sync.Once + var stopErr error + stopAgent := func() { + stopOnce.Do(func() { + cancel() + select { + case stopErr = <-done: + case <-time.After(time.Second): + stopErr = errors.New("resolver did not stop after continuity test") + } + }) + } + defer stopAgent() + var releaseOnce sync.Once + releasePublisher := func() { releaseOnce.Do(func() { close(publishRelease) }) } + defer releasePublisher() + select { + case <-publishStarted: + case <-time.After(10 * time.Second): + t.Fatal("resolver did not enter context-ignoring broker publish") + } + + started := time.Now() + var continuityErr error + for time.Since(started) < 1200*time.Millisecond { + state, err := agent.store.Load() + if err != nil || state.Active == nil || !installedFilesMatch(state.Active.Files) || activationPending(agent.cfg.StateDir) { + continuityErr = fmt.Errorf("applied configuration lost during broker partition: %w", err) + break + } + time.Sleep(20 * time.Millisecond) + } + if calls := publisher.callCount(); calls != 1 { + continuityErr = fmt.Errorf("context-ignoring publisher calls = %d, want one bounded inflight call", calls) + } + releasePublisher() + stopAgent() + if stopErr != nil { + t.Fatal(stopErr) + } + if continuityErr != nil { + t.Fatal(continuityErr) + } +} + +func TestResolverVerificationUsesBoundedConcurrentBatch(t *testing.T) { + t.Parallel() + release := make(chan struct{}) + started := make(chan struct{}, 8) + verifier := &boundedResolverVerifier{release: release, started: started} + agent := &Agent{ + cfg: Config{ResolverVerifyConcurrency: 3, ResolverDNSProbe: DNSVerifierConfig{Timeout: time.Second}}, + deps: Dependencies{Verifier: verifier}, + } + bindings := make([]model.Binding, 8) + active := make(map[string]bool, len(bindings)) + for i := range bindings { + bindings[i].BindingUID = fmt.Sprintf("binding-%d", i) + active[bindings[i].BindingUID] = true + } + done := make(chan bool, 1) + go func() { done <- agent.verifyResolverBindings(context.Background(), bindings, active) }() + for range 3 { + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("verification workers did not start concurrently") + } + } + if calls, maximum := verifier.snapshot(); calls != 3 || maximum != 3 { + t.Fatalf("before release calls=%d max=%d, want exactly three workers", calls, maximum) + } + close(release) + if failed := <-done; failed { + t.Fatal("successful resolver proof batch was marked failed") + } + if calls, maximum := verifier.snapshot(); calls != len(bindings) || maximum > 3 { + t.Fatalf("completed calls=%d max=%d, want calls=%d max<=3", calls, maximum, len(bindings)) + } +} + +func TestResolverVerificationHasOneSharedDeadline(t *testing.T) { + t.Parallel() + verifier := &boundedResolverVerifier{waitForCtx: true} + agent := &Agent{ + cfg: Config{ResolverVerifyConcurrency: 2, ResolverDNSProbe: DNSVerifierConfig{Timeout: 40 * time.Millisecond}}, + deps: Dependencies{Verifier: verifier}, + } + bindings := make([]model.Binding, 10) + active := make(map[string]bool, len(bindings)) + for i := range bindings { + bindings[i].BindingUID = fmt.Sprintf("binding-%d", i) + active[bindings[i].BindingUID] = true + } + started := time.Now() + if failed := agent.verifyResolverBindings(context.Background(), bindings, active); !failed { + t.Fatal("timed out resolver proof batch was accepted") + } + if elapsed := time.Since(started); elapsed > 500*time.Millisecond { + t.Fatalf("shared resolver proof deadline took %s", elapsed) + } + if len(active) != 0 { + t.Fatalf("timed out resolver proof batch left %d bindings active", len(active)) + } +} + +func TestAckEventPreservesAppliedConfigurationAfterReconcile(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} + snapshot.Members = []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}} + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, + }, Dependencies{Clock: func() time.Time { return now }, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", snapshot.ConfigurationEpoch, snapshot.ConfigurationRevision, snapshot)); err != nil { + t.Fatal(err) + } + ack := model.MemberAck{MemberID: "regional-0", ReplicaID: "regional-0", ResourceUID: "zone-a", Kind: model.KindPublicationManifest, Epoch: 2, Revision: 4, Phase: model.AckVerified, ObservedAt: now, ValidUntil: now.Add(time.Minute)} + if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatal(err) + } + assertAppliedCheckpoint(t, agent) +} + +func TestExpiredKnownAckReplayDrainsWithoutStateChange(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + agent := &Agent{ + cfg: Config{Region: "east", Shard: "s1", Mode: ModeResolver}, + deps: Dependencies{Clock: func() time.Time { return now }}, state: newCheckpoint(), + lastServingAck: map[string]servingAckState{}, lastPublicationAck: map[string]time.Time{}, + } + snapshot := testSnapshot(now) + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + ack := model.MemberAck{MemberID: "regional-0", ReplicaID: "regional-0", ResourceUID: "zone-a", Kind: model.KindPublicationManifest, Epoch: 2, Revision: 4, Phase: model.AckVerified, ObservedAt: now.Add(-2 * time.Minute), ValidUntil: now.Add(-time.Minute)} + if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatalf("expired known ACK replay was not drained: %v", err) + } + if len(agent.state.Acks) != 0 { + t.Fatalf("expired ACK changed readiness state: %#v", agent.state.Acks) + } + if agent.state.Active != nil { + t.Fatal("expired ACK changed applied configuration") + } + ack.MemberID = "unknown" + if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err == nil { + t.Fatal("expired ACK from an unknown member was accepted") + } + ack.MemberID = "regional-0" + ack.ValidUntil = ack.ObservedAt.Add(2*time.Minute + time.Nanosecond) + if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err == nil { + t.Fatal("overlong ACK lease was accepted") + } +} + +func assertAppliedCheckpoint(t *testing.T, agent *Agent) { + t.Helper() + state, err := agent.store.Load() + if err != nil { + t.Fatal(err) + } + if state.Active == nil || !installedFilesMatch(state.Active.Files) || activationPending(agent.cfg.StateDir) { + t.Fatal("approved configuration is not durably active") + } +} diff --git a/internal/internaldns/serving/binding_fences.go b/internal/internaldns/serving/binding_fences.go new file mode 100644 index 00000000..2b0bd3dc --- /dev/null +++ b/internal/internaldns/serving/binding_fences.go @@ -0,0 +1,65 @@ +package serving + +import ( + "encoding/json" + "fmt" + "sort" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +// bindingFence retains mapping lifetime and order independently of shard snapshots. +type bindingFence struct { + Identity string `json:"identity"` + ConfigurationGeneration uint64 `json:"configurationGeneration"` + ConfigurationRevision uint64 `json:"configurationRevision"` + Version model.AccessVersion `json:"version"` + Tombstone bool `json:"tombstone"` +} + +func bindingIdentity(b model.Binding) string { + transports := append([]model.Transport(nil), b.Transports...) + sort.Slice(transports, func(i, j int) bool { return transports[i] < transports[j] }) + data, _ := json.Marshal(struct { + Project, Context, Node, Regional string + Port uint16 + Transports []model.Transport + }{b.ProjectUID, b.ContextUID, b.ConsumerAddress, b.ClusterAddress, b.Port, transports}) + return model.Hash(data) +} + +func bindingFenceFor(b model.Binding) bindingFence { + return bindingFence{Identity: bindingIdentity(b), ConfigurationGeneration: b.BindingGeneration, ConfigurationRevision: b.ConfigurationRevision, Version: b.Version, Tombstone: b.Tombstone} +} + +func advanceBindingFences(prior map[string]bindingFence, snapshot model.ServingSnapshot) (map[string]bindingFence, error) { + next := make(map[string]bindingFence, len(prior)+len(snapshot.Bindings)) + for uid, old := range prior { + withdrawn := old + withdrawn.Tombstone = true + next[uid] = withdrawn + } + for _, b := range snapshot.Bindings { + candidate := bindingFenceFor(b) + if old, ok := prior[b.BindingUID]; ok { + if candidate.Identity != old.Identity { + return nil, fmt.Errorf("binding %s changed its UID-pinned source or listeners", b.BindingUID) + } + if candidate.ConfigurationGeneration < old.ConfigurationGeneration || candidate.ConfigurationRevision < old.ConfigurationRevision { + return nil, fmt.Errorf("binding %s configuration regressed", b.BindingUID) + } + cmp := model.Compare(candidate.Version.Epoch, candidate.Version.Sequence, old.Version.Epoch, old.Version.Sequence) + if cmp < 0 { + return nil, fmt.Errorf("binding %s access version regressed", b.BindingUID) + } + if old.Tombstone && !candidate.Tombstone && cmp <= 0 { + return nil, fmt.Errorf("binding %s reactivation requires a newer access version", b.BindingUID) + } + if old.Tombstone != candidate.Tombstone && candidate.ConfigurationRevision <= old.ConfigurationRevision { + return nil, fmt.Errorf("binding %s withdrawal or reactivation requires a newer configuration revision", b.BindingUID) + } + } + next[b.BindingUID] = candidate + } + return next, nil +} diff --git a/internal/internaldns/serving/binding_fences_test.go b/internal/internaldns/serving/binding_fences_test.go new file mode 100644 index 00000000..5f46cd47 --- /dev/null +++ b/internal/internaldns/serving/binding_fences_test.go @@ -0,0 +1,324 @@ +package serving + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +func TestBindingWithdrawalPreservesSourceSequenceAndAllowsNormalRenewal(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + active := testSnapshot(now) + active.Bindings[0].Version.Sequence = 2 + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + chunk, manifest := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } + clock = now.Add(11 * time.Second) + withdrawn := active + withdrawn.Bindings = append([]model.Binding(nil), active.Bindings...) + withdrawn.ConfigurationRevision = 2 + withdrawn.Bindings[0].ConfigurationRevision = 2 + withdrawn.Bindings[0].Tombstone = true + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, withdrawn)); err != nil { + t.Fatal(err) + } + if f := a.state.BindingFences["binding-a"]; !f.Tombstone || f.Version.Sequence != 2 { + t.Fatal("withdrawal fabricated access version") + } + // Withdrawal preserves the source sequence so a newer access version can reactivate. + renewed := withdrawn + renewed.Bindings = append([]model.Binding(nil), withdrawn.Bindings...) + renewed.ConfigurationRevision = 3 + renewed.Bindings[0].ConfigurationRevision = 3 + renewed.Bindings[0].Tombstone = false + renewed.Bindings[0].Version.Sequence = 3 + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, renewed)); err != nil { + t.Fatalf("normal source2 -> source3 renewal rejected: %v", err) + } + config, err := os.ReadFile(cfg.Render.ClusterCoreDNS.Path) + if err != nil || !strings.Contains(string(config), renewed.Bindings[0].ViewName()) { + t.Fatalf("renewal did not restore context: %v", err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, withdrawn)); err != nil { + t.Fatal(err) + } + if a.state.Snapshot.Bindings[0].Tombstone || a.state.ServingFence.Revision != 3 { + t.Fatal("stale withdrawal replay removed current access") + } + // A higher shard revision cannot legitimize an older source revision. + stale := withdrawn + stale.ConfigurationRevision = 4 + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 4, stale)); err == nil { + t.Fatal("new shard revision accepted stale binding authority") + } + if a.state.ServingFence.Revision != 3 { + t.Fatal("rejected binding authority advanced shard fence") + } +} + +func TestBindingOmissionFenceSurvivesRestartAndRequiresNewSourceAuthority(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + active := testSnapshot(now) + active.Bindings[0].Version.Sequence = 2 + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + omitted := active + omitted.ConfigurationRevision = 2 + omitted.Bindings = nil + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, omitted)); err != nil { + t.Fatal(err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if !restarted.state.BindingFences["binding-a"].Tombstone { + t.Fatal("restart forgot omitted UID withdrawal") + } + candidate := active + candidate.Bindings = append([]model.Binding(nil), active.Bindings...) + candidate.ConfigurationRevision = 3 + candidate.Bindings[0].ConfigurationRevision = 2 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err == nil { + t.Fatal("same source sequence resurrected omitted binding") + } + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err == nil { + t.Fatal("same source sequence changed its source revision") + } + candidate.Bindings[0].Version.Sequence = 3 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err != nil { + t.Fatalf("new source sequence could not reactivate omitted binding: %v", err) + } + // A replacement resource has a new UID and independent source sequence. + replacement := candidate + replacement.Bindings = append([]model.Binding(nil), candidate.Bindings...) + replacement.ConfigurationRevision = 4 + replacement.Bindings[0].BindingUID = "replacement-binding" + replacement.Bindings[0].ConfigurationRevision = 1 + replacement.Bindings[0].Version.Sequence = 1 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 4, replacement)); err != nil { + t.Fatal(err) + } + if !restarted.state.BindingFences["binding-a"].Tombstone || restarted.state.BindingFences["replacement-binding"].Version.Sequence != 1 { + t.Fatal("replacement UID discarded old source fences or inherited its sequence") + } + candidate.ConfigurationRevision = 5 + candidate.Bindings[0].ConfigurationRevision = 3 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 5, candidate)); err == nil { + t.Fatal("retired UID replay replaced newly installed resource") + } + mutated := replacement + mutated.Bindings = append([]model.Binding(nil), replacement.Bindings...) + mutated.ConfigurationRevision = 5 + mutated.Bindings[0].ProjectUID = "different-project" + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 5, mutated)); err == nil { + t.Fatal("same binding UID changed pinned project identity") + } +} + +func TestCheckpointRejectsPreBindingFenceFormatWithoutDiscardingIt(t *testing.T) { + t.Parallel() + for _, version := range []int{2, 3, checkpointFormatVersion} { + t.Run(string(rune('0'+version)), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "checkpoint.json") + data, err := json.Marshal(map[string]any{"formatVersion": version, "contributionFences": map[string]any{"original-observation": map[string]any{"epoch": 1, "sequence": 2}}}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + if _, err := (stateStore{path: path}).Load(); err == nil { + t.Fatal("checkpoint without complete binding authority history accepted") + } + retained, err := os.ReadFile(path) + if err != nil || string(retained) != string(data) { + t.Fatal("unsupported checkpoint was modified") + } + }) + } +} + +func TestPendingConfigurationGatesQueriesAndRecoversWithoutAuthorizationRenewal(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + runner := &recordingRunner{} + dependencies := Dependencies{Runner: runner, Clock: func() time.Time { return now }} + agent, err := NewAgent(cfg, dependencies) + if err != nil { + t.Fatal(err) + } + active := testSnapshot(now) + healthy := active.Bindings[0] + healthy.BindingUID, healthy.ProjectUID, healthy.ContextUID = "binding-b", "project-b", "context-b" + healthy.ConsumerAddress, healthy.ClusterAddress = "10.253.0.52", "10.253.0.42" + healthy.Zones = nil + active.Bindings = append(active.Bindings, healthy) + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + chunk, manifest := testPublication(t, now) + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } + original := active.Bindings[0].Version + pending := active + pending.Bindings = append([]model.Binding(nil), active.Bindings...) + pending.ConfigurationRevision = 2 + pending.Bindings[0].ConfigurationRevision = 2 + pending.Bindings[0].ConfigurationPending = true + pending.Bindings[0].Zones = nil + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, pending)); err != nil { + t.Fatal(err) + } + if agent.state.BindingFences["binding-a"].Tombstone || agent.publicationsReady(pending.Bindings[0]) || !agent.publicationsReady(healthy) { + t.Fatal("pending configuration revoked access, activated incomplete data, or gated another context") + } + config, err := os.ReadFile(cfg.Render.ClusterCoreDNS.Path) + if err != nil { + t.Fatal(err) + } + contexts := decodedCoreDNS(t, config).Contexts + found := false + for _, entry := range contexts { + if entry.ID == pending.Bindings[0].ViewName() { + found = true + if !entry.Pending || len(entry.Zones) > 0 { + t.Fatal("pending context can serve private data") + } + } + } + if !found { + t.Fatal("pending context missing") + } + restarted, err := NewAgent(cfg, dependencies) + if err != nil { + t.Fatal(err) + } + ready := active + ready.Bindings = append([]model.Binding(nil), active.Bindings...) + ready.ConfigurationRevision = 3 + ready.Bindings[0].ConfigurationRevision = 2 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, ready)); err != nil { + t.Fatalf("completion required a new authorization: %v", err) + } + if restarted.state.BindingFences["binding-a"].Version != original || !restarted.publicationsReady(ready.Bindings[0]) { + t.Fatal("completion changed authority or remained gated") + } + // The older pending snapshot cannot undo the completed configuration. + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, pending)); err != nil { + t.Fatal(err) + } + if restarted.state.Snapshot.Bindings[0].ConfigurationPending { + t.Fatal("stale pending replay gated the recovered context") + } +} + +func TestWithdrawalRenderFailureStopsServing(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + runner := &recordingRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + active := testSnapshot(now) + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + chunk, manifest := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } + if a.state.Active == nil || a.needsActivation { + t.Fatalf("fixture did not approve serving: active=%t needsActivation=%t", a.state.Active != nil, a.needsActivation) + } + gatesBefore := gateCalls(runner) + + a.render = func(RenderConfig, model.ServingSnapshot, map[string]bool, map[string]publicationState) (Rendered, error) { + return Rendered{}, errors.New("injected render failure") + } + clock = now.Add(11 * time.Second) + withdrawn := active + withdrawn.Bindings = append([]model.Binding(nil), active.Bindings...) + withdrawn.ConfigurationRevision = 2 + withdrawn.Bindings[0].ConfigurationRevision = 2 + withdrawn.Bindings[0].Tombstone = true + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, withdrawn)); err == nil { + t.Fatal("withdrawal with a failed render reported success") + } + if gateCalls(runner) == gatesBefore || !a.needsActivation { + t.Fatalf("withdrawal with a failed render kept serving withdrawn data: gates=%d needsActivation=%t", gateCalls(runner)-gatesBefore, a.needsActivation) + } + + // A render failure that withdraws nothing keeps the approved configuration. + retained := &recordingRunner{} + b, err := NewAgent(regionalTestConfig(t), Dependencies{Runner: retained, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if err := b.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + if err := b.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := b.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } + retainedBefore := gateCalls(retained) + b.render = a.render + if err := b.reconcile(context.Background()); err == nil { + t.Fatal("injected render failure was not returned") + } + if gateCalls(retained) != retainedBefore || b.needsActivation { + t.Fatal("a render failure that withdraws nothing gated serving") + } +} + +func gateCalls(r *recordingRunner) int { + n := 0 + for _, c := range r.calls { + if c.Path == "gate" { + n++ + } + } + return n +} diff --git a/internal/internaldns/serving/bootstrap.go b/internal/internaldns/serving/bootstrap.go new file mode 100644 index 00000000..030f20ad --- /dev/null +++ b/internal/internaldns/serving/bootstrap.go @@ -0,0 +1,146 @@ +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +type SnapshotLister interface { + List(context.Context, string) ([]model.Envelope, error) +} +type SnapshotBootstrap struct { + Store SnapshotLister + Region, Shard string + Attempts int + RetryDelay time.Duration +} + +// Load obtains a self-consistent bootstrap cut. Chunk keys are immutable and +// revision-qualified; stable manifest heads are accepted only when all of their +// referenced chunks are present and hash correctly in the same listing. +func (b SnapshotBootstrap) Load(ctx context.Context) ([]model.Envelope, error) { + if b.Store == nil || b.Region == "" || b.Shard == "" { + return nil, errors.New("snapshot store, region, and shard are required") + } + attempts := b.Attempts + if attempts <= 0 { + attempts = 4 + } + delay := b.RetryDelay + if delay <= 0 { + delay = 100 * time.Millisecond + } + var last error + for attempt := 0; attempt < attempts; attempt++ { + envs, err := b.Store.List(ctx, model.SnapshotPrefix(b.Region, b.Shard)) + if err == nil { + err = verifyBootstrap(envs, b.Region, b.Shard) + } + if err == nil { + envs, err = currentBootstrapEnvelopes(envs) + } + if err == nil { + sort.SliceStable(envs, func(i, j int) bool { return bootstrapOrder(envs[i].Kind) < bootstrapOrder(envs[j].Kind) }) + return envs, nil + } + last = err + if attempt+1 < attempts { + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + } + return nil, fmt.Errorf("consistent internal DNS bootstrap unavailable: %w", last) +} + +// currentBootstrapEnvelopes drops immutable chunk sets that are no longer +// referenced by a stable manifest head. The KV listing may retain them for +// replay/rollback safety, but a cold member must not checkpoint or apply them. +func currentBootstrapEnvelopes(envs []model.Envelope) ([]model.Envelope, error) { + manifests := map[string]bool{} + for _, env := range envs { + if env.Kind != model.KindPublicationManifest { + continue + } + var manifest model.PublicationManifest + if err := json.Unmarshal(env.Payload, &manifest); err != nil { + return nil, err + } + manifests[manifest.ManifestUID] = true + } + out := make([]model.Envelope, 0, len(envs)) + for _, env := range envs { + if env.Kind == model.KindPublicationChunk { + var chunk model.PublicationChunk + if err := json.Unmarshal(env.Payload, &chunk); err != nil { + return nil, err + } + if !manifests[chunk.ManifestUID] { + continue + } + } + out = append(out, env) + } + return out, nil +} + +func verifyBootstrap(envs []model.Envelope, region, shard string) error { + chunks := map[string][]model.PublicationChunk{} + var manifests []model.PublicationManifest + serving := false + for _, env := range envs { + if err := env.Validate(); err != nil { + return err + } + if env.Region != region || env.Shard != shard { + return errors.New("bootstrap contains a foreign region or shard") + } + switch env.Kind { + case model.KindPublicationChunk: + var c model.PublicationChunk + if err := json.Unmarshal(env.Payload, &c); err != nil { + return err + } + chunks[c.ManifestUID] = append(chunks[c.ManifestUID], c) + case model.KindPublicationManifest: + var m model.PublicationManifest + if err := json.Unmarshal(env.Payload, &m); err != nil { + return err + } + manifests = append(manifests, m) + case model.KindServingSnapshot: + serving = true + default: + return fmt.Errorf("bootstrap contains unsupported kind %q", env.Kind) + } + } + if !serving { + return errors.New("bootstrap has no complete serving snapshot") + } + for _, m := range manifests { + if _, err := model.VerifyManifest(m, chunks[m.ManifestUID]); err != nil { + return err + } + } + return nil +} +func bootstrapOrder(k model.Kind) int { + switch k { + case model.KindPublicationChunk: + return 0 + case model.KindServingSnapshot: + return 1 + default: + return 2 + } +} diff --git a/internal/internaldns/serving/bootstrap_test.go b/internal/internaldns/serving/bootstrap_test.go new file mode 100644 index 00000000..3d7e4ac7 --- /dev/null +++ b/internal/internaldns/serving/bootstrap_test.go @@ -0,0 +1,71 @@ +package serving + +import ( + "context" + "errors" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +type bootstrapLister struct { + calls int + results [][]model.Envelope +} + +func (l *bootstrapLister) List(context.Context, string) ([]model.Envelope, error) { + l.calls++ + if len(l.results) == 0 { + return nil, errors.New("empty") + } + i := l.calls - 1 + if i >= len(l.results) { + i = len(l.results) - 1 + } + return l.results[i], nil +} + +func TestSnapshotBootstrapRetriesRacingManifestHead(t *testing.T) { + t.Parallel() + now := testSnapshotNow() + chunk, manifest := testPublication(t, now) + serving := testSnapshot(now) + chunkEnv := envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk) + manifestEnv := envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest) + servingEnv := envelope(t, model.KindServingSnapshot, "shard", 1, 1, serving) + lister := &bootstrapLister{results: [][]model.Envelope{{manifestEnv, servingEnv}, {manifestEnv, chunkEnv, servingEnv}}} + got, err := (SnapshotBootstrap{Store: lister, Region: "east", Shard: "s1", Attempts: 2}).Load(context.Background()) + if err != nil { + t.Fatal(err) + } + if lister.calls != 2 || len(got) != 3 || got[0].Kind != model.KindPublicationChunk { + t.Fatalf("calls=%d events=%v", lister.calls, got) + } +} + +func TestSnapshotBootstrapIgnoresUnreferencedHistoricalChunks(t *testing.T) { + t.Parallel() + now := testSnapshotNow() + chunk, manifest := testPublication(t, now) + historical := chunk + historical.ManifestUID = "old-manifest" + events := []model.Envelope{ + envelope(t, model.KindPublicationChunk, "zone-a", 2, 3, historical), + envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk), + envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest), + envelope(t, model.KindServingSnapshot, "shard", 1, 1, testSnapshot(now)), + } + lister := &bootstrapLister{results: [][]model.Envelope{events}} + got, err := (SnapshotBootstrap{Store: lister, Region: "east", Shard: "s1"}).Load(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("cold bootstrap retained %d events, want current chunk+manifest+serving", len(got)) + } + if got[0].Kind != model.KindPublicationChunk || got[1].Kind != model.KindServingSnapshot || got[2].Kind != model.KindPublicationManifest { + t.Fatalf("unsafe bootstrap order: %#v", got) + } +} +func testSnapshotNow() time.Time { return time.Now().UTC() } diff --git a/internal/internaldns/serving/continuity.go b/internal/internaldns/serving/continuity.go new file mode 100644 index 00000000..74e70f90 --- /dev/null +++ b/internal/internaldns/serving/continuity.go @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "encoding/json" + "errors" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +type activeConfiguration struct { + Snapshot model.ServingSnapshot `json:"snapshot"` + Publications map[string]publicationState `json:"publications"` + Files map[string][]byte `json:"files"` +} + +func (a *Agent) gateRejectedUpdate(ctx context.Context, err error) { + var preserved preservedTransactionError + var uncertain uncertainTransactionError + if !errors.As(err, &uncertain) && a.canRetainActive() && (errors.As(err, &preserved) || !a.needsActivation && a.state.RenderedHash == hashFiles(a.state.Active.Files)) { + return + } + _ = a.failClosed(ctx) +} + +func (a *Agent) canRetainActive() bool { + old := a.state.Active + if old == nil || a.state.Snapshot == nil { + return false + } + next := map[string]model.Binding{} + for _, binding := range a.state.Snapshot.Bindings { + next[binding.BindingUID] = binding + } + for _, binding := range old.Snapshot.Bindings { + candidate, ok := next[binding.BindingUID] + if !ok || candidate.Tombstone || candidate.ConfigurationPending || bindingIdentity(binding) != bindingIdentity(candidate) || binding.Version != candidate.Version { + return false + } + for _, zone := range binding.Zones { + present := false + for _, attached := range candidate.Zones { + if attached.ZoneUID == zone.ZoneUID && attached.Apex == zone.Apex && !attached.Pending { + present = true + } + } + if !present && !zone.Pending { + return false + } + } + } + for uid, publication := range old.Publications { + if publication.Manifest.Tombstone { + continue + } + candidate, ok := a.state.Publications[uid] + if !ok || candidate.Manifest.Tombstone || candidate.Plan.Apex != publication.Plan.Apex { + return false + } + records := map[string]bool{} + for _, rrset := range candidate.Plan.RRSets { + for _, record := range rrset.Records { + if record.Eligible { + records[rrset.Name+"\x00"+rrset.Type+"\x00"+record.Content] = true + } + } + } + for _, rrset := range publication.Plan.RRSets { + for _, record := range rrset.Records { + if record.Eligible && !records[rrset.Name+"\x00"+rrset.Type+"\x00"+record.Content] { + return false + } + } + } + } + return true +} + +func (a *Agent) approveConfiguration(files map[string][]byte) error { + candidate := activeConfiguration{Snapshot: *a.state.Snapshot, Publications: a.state.Publications, Files: files} + data, err := json.Marshal(candidate) + if err != nil { + return err + } + var active activeConfiguration + if err := json.Unmarshal(data, &active); err != nil { + return err + } + if a.state.Active != nil && equalJSON(*a.state.Active, active) { + return a.tx.commitActivation() + } + previous := a.state.Active + a.state.Active = &active + if err := a.store.Save(a.state); err != nil { + a.state.Active = previous + return err + } + return a.tx.commitActivation() +} + +// restoreActive preserves running daemons when their installed configuration +// matches the approved checkpoint. Incomplete installation requires recovery. +func (a *Agent) restoreActive(ctx context.Context) error { + if !a.canRetainActive() { + return a.failClosed(ctx) + } + files := a.state.Active.Files + if !activationPending(a.cfg.StateDir) && installedFilesMatch(files) { + if err := a.probeLocalDaemons(ctx); err == nil { + a.needsActivation = false + if actual := hashFiles(files); a.state.RenderedHash != actual { + a.state.RenderedHash = actual + return a.store.Save(a.state) + } + return nil + } else if ctx.Err() != nil { + return ctx.Err() + } + } + a.needsActivation = true + if err := a.tx.ApplyForce(ctx, files); err != nil { + return err + } + a.state.RenderedHash = hashFiles(files) + if err := a.store.Save(a.state); err != nil { + return err + } + if err := a.tx.commitActivation(); err != nil { + return err + } + a.needsActivation = false + return nil +} diff --git a/internal/internaldns/serving/continuity_test.go b/internal/internaldns/serving/continuity_test.go new file mode 100644 index 00000000..948c2845 --- /dev/null +++ b/internal/internaldns/serving/continuity_test.go @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +func TestRejectedExpansionPreservesServingAndRestarts(t *testing.T) { + now := time.Now().UTC() + runner := &recordingRunner{} + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + activeHash := agent.state.RenderedHash + candidate := *agent.state.Snapshot + candidate.ConfigurationRevision++ + candidate.Bindings[0].ConfigurationRevision++ + candidate.Bindings[0].Zones = append(candidate.Bindings[0].Zones, model.ZoneAttachment{ZoneUID: "pending-zone", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + runner.failAt = len(runner.calls) + 1 + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)); err == nil { + t.Fatal("invalid expansion was accepted") + } + if agent.state.RenderedHash != activeHash || agent.state.Active == nil { + t.Fatal("invalid expansion interrupted the active configuration") + } + now = now.Add(time.Hour) + if !installedFilesMatch(agent.state.Active.Files) { + t.Fatal("rejected update changed the approved files") + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatalf("restart could not recover active configuration: %v", err) + } + withdrawn := candidate + withdrawn.ConfigurationRevision++ + withdrawn.Bindings = nil + runner.failAt = len(runner.calls) + 1 + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", withdrawn.ConfigurationEpoch, withdrawn.ConfigurationRevision, withdrawn)); err == nil { + t.Fatal("failed withdrawal was accepted") + } + if agent.state.Active != nil { + t.Fatal("failed withdrawal retained revoked access") + } +} + +func TestPendingZoneIsIsolatedFromReadyZone(t *testing.T) { + now := time.Now().UTC() + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + pub := &fakePublisher{} + agent.deps.AckPublisher = pub + now = now.Add(time.Minute) + binding := agent.state.Snapshot.Bindings[0] + binding.Zones = append(binding.Zones, model.ZoneAttachment{ZoneUID: "pending", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + if zones := agent.readyZones(binding); len(zones) != 1 || zones[0].ZoneUID != "zone-a" { + t.Fatalf("pending zone suppressed healthy zone: %#v", zones) + } + candidate := *agent.state.Snapshot + candidate.ConfigurationRevision++ + candidate.Bindings[0] = binding + candidate.Bindings[0].ConfigurationRevision++ + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)); err != nil { + t.Fatal(err) + } + if agent.state.Active == nil || agent.publicationsReady(binding) { + t.Fatal("partial serving state was not distinguished from full readiness") + } + if zone := currentZone(t, agent); zone == "" { + t.Fatal("ready zone was not materialized") + } + verified := false + for _, env := range pub.envs { + var ack model.MemberAck + if err := json.Unmarshal(env.Payload, &ack); err != nil { + t.Fatal(err) + } + if ack.Kind == model.KindServingSnapshot && ack.Revision == candidate.ConfigurationRevision && ack.Phase == model.AckVerified { + verified = true + } + } + if !verified { + t.Fatal("pending zone prevented resolver access from becoming ready") + } +} + +type blockedValidationRunner struct { + recordingRunner + blocked bool + started chan struct{} + release chan struct{} +} + +func (r *blockedValidationRunner) Run(ctx context.Context, command Command) ([]byte, error) { + if command.Path == "validate" && r.blocked { + close(r.started) + select { + case <-r.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return r.recordingRunner.Run(ctx, command) +} + +func TestApprovedConfigurationRemainsInstalledDuringCandidateValidation(t *testing.T) { + now := time.Now().UTC() + runner := &blockedValidationRunner{started: make(chan struct{}), release: make(chan struct{})} + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + runner.blocked = true + candidate := *agent.state.Snapshot + candidate.ConfigurationRevision++ + candidate.Bindings[0].ConfigurationRevision++ + candidate.Bindings[0].Zones = append(candidate.Bindings[0].Zones, model.ZoneAttachment{ZoneUID: "pending", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + done <- agent.Handle(ctx, envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)) + }() + select { + case <-runner.started: + case <-ctx.Done(): + t.Fatal("candidate validation did not start") + } + if !installedFilesMatch(agent.state.Active.Files) || activationPending(cfg.StateDir) { + t.Fatal("candidate validation interrupted the approved configuration") + } + close(runner.release) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestFailedRestartInstallationRequiresActivation(t *testing.T) { + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, time.Now().UTC()) + for path := range agent.state.Active.Files { + if err := os.WriteFile(path, []byte("incomplete configuration"), 0644); err != nil { + t.Fatal(err) + } + break + } + runner := &recordingRunner{failAt: 1} + restarted, err := NewAgent(cfg, Dependencies{Runner: runner}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err == nil { + t.Fatal("failed installation was accepted") + } + if !restarted.needsActivation { + t.Fatal("failed installation must force daemon activation on retry") + } + runner.failAt = 0 + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if restarted.needsActivation || !installedFilesMatch(restarted.state.Active.Files) { + t.Fatal("successful retry did not restore approved configuration") + } +} diff --git a/internal/internaldns/serving/contribution_lifetime_test.go b/internal/internaldns/serving/contribution_lifetime_test.go new file mode 100644 index 00000000..736f9434 --- /dev/null +++ b/internal/internaldns/serving/contribution_lifetime_test.go @@ -0,0 +1,22 @@ +package serving + +import ( + "testing" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +func TestRetiredContributionFenceRejectsEveryReplay(t *testing.T) { + a := &Agent{state: newCheckpoint()} + retirement := model.PublicationPlan{ObservationFences: []model.ObservationFence{{ContributionUID: "endpoint", Generation: 2, Sequence: 2, Retired: true}}} + a.commitContributionFences(retirement) + for _, generation := range []uint64{2, 3} { + plan := model.PublicationPlan{ObservationFences: []model.ObservationFence{{ContributionUID: "endpoint", Generation: generation, Sequence: generation}}} + if err := a.validateContributionFences(plan); err == nil { + t.Fatal("retired contribution returned through a newer snapshot") + } + } + if !a.state.ContributionFences["endpoint"].Retired { + t.Fatal("checkpoint fence lost retirement") + } +} diff --git a/internal/internaldns/serving/coredns_test.go b/internal/internaldns/serving/coredns_test.go new file mode 100644 index 00000000..52649fff --- /dev/null +++ b/internal/internaldns/serving/coredns_test.go @@ -0,0 +1,650 @@ +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +func regionalTestConfig(t *testing.T) Config { + t.Helper() + dir := t.TempDir() + render, _ := renderFixture(t) + render.Role = RenderRegionalCoreDNS + render.ClusterCoreDNS.Path = filepath.Join(dir, "runtime", "regional.conf") + return Config{Region: "east", Shard: "s1", MemberID: "regional-0", ReplicaID: "regional-0", Mode: ModeCombined, StateDir: filepath.Join(dir, "state"), Render: render, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), Validate: []Command{{Path: "validate"}}, Reload: []Command{{Path: "reload"}}, FailClosed: []Command{{Path: "gate"}}}, UnsafeAllowNoDNSVerify: true} +} +func installTestPublication(t *testing.T, a *Agent, now time.Time) { + t.Helper() + ctx := context.Background() + snapshot := testSnapshot(now) + snapshot.Members = []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}} + if err := a.Handle(ctx, envelope(t, model.KindServingSnapshot, "shard", 1, 1, snapshot)); err != nil { + t.Fatal(err) + } + chunk, manifest := testPublication(t, now) + if err := a.Handle(ctx, envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(ctx, envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } +} +func currentZone(t *testing.T, a *Agent) string { + t.Helper() + files, err := renderPublications(a.cfg.Render, *a.state.Snapshot, map[string]bool{}, a.state.Publications) + if err != nil { + t.Fatal(err) + } + for _, data := range files.Files { + var cfg struct { + Contexts []struct{ Zones []struct{ Data string } } + } + if json.Unmarshal(data, &cfg) != nil { + continue + } + for _, entry := range cfg.Contexts { + for _, zone := range entry.Zones { + if zone.Data != "" { + return zone.Data + } + } + } + } + + t.Fatal("no zone") + return "" +} +func TestCoreDNSPublicationActivatesLocalPrivateAuthority(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + pub := &fakePublisher{} + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }, AckPublisher: pub}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + data := currentZone(t, agent) + if !strings.Contains(data, "192.0.2.10") || !strings.Contains(data, publicationMailbox(agent.state.Publications["zone-a"])) { + t.Fatalf("bad zone: %s", data) + } + config, err := os.ReadFile(cfg.Render.ClusterCoreDNS.Path) + if err != nil { + t.Fatal(err) + } + if decodedCoreDNS(t, config).Contexts[0].Zones[0].Data == "" { + t.Fatalf("not direct private authority: %s", config) + } + if !agent.state.Publications["zone-a"].Verified { + t.Fatal("publication not verified") + } +} +func TestCoreDNSPublicationSurvivesBrokerOutageAndRestart(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }, AckPublisher: stalledPublisher{}}) + if err != nil { + t.Fatal(err) + } + // Publish without a transport dependency during setup; partition afterwards. + a.deps.AckPublisher = nil + installTestPublication(t, a, now) + a.deps.AckPublisher = newBoundedAckPublisher(stalledPublisher{}) + a.cfg.AckPublishTimeout = time.Millisecond + clock = now.Add(24 * time.Hour) + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + if data := currentZone(t, a); !strings.Contains(data, "192.0.2.10") { + t.Fatalf("outage removed the published record: %s", data) + } + local := a.state.Publications["zone-a"].LocalRevision + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + if err := restarted.Step(context.Background()); err != nil { + t.Fatal(err) + } + if restarted.state.Publications["zone-a"].LocalRevision != local { + t.Fatal("restart changed the applied publication") + } + if data := currentZone(t, restarted); !strings.Contains(data, "192.0.2.10") { + t.Fatal("restart lost the published record") + } +} +func TestCoreDNSActivationFailurePersistsObservationFence(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + runner := &recordingRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, snapshot)); err != nil { + t.Fatal(err) + } + chunk, m := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + runner.failAt = len(runner.calls) + 1 + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err == nil { + t.Fatal("invalid staged configuration accepted") + } + saved, err := (stateStore{path: filepath.Join(cfg.StateDir, "checkpoint.json")}).Load() + if err != nil { + t.Fatal(err) + } + if f := saved.ContributionFences["contrib-a"]; f.Generation != 1 || f.Sequence != 1 { + t.Fatalf("observation fence lost: %#v", f) + } + if saved.Publications["zone-a"].Verified { + t.Fatal("failed publication acknowledged") + } + if runner.calls[len(runner.calls)-1].Path == "gate" || saved.Active == nil { + t.Fatal("rejected expansion discarded the active configuration") + } + runner.failAt = 0 + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err != nil { + t.Fatalf("equal fenced retry failed: %v", err) + } +} +func TestCoreDNSTombstoneFencesReplayAcrossRestart(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + _, old := testPublication(t, now) + tombstone := model.PublicationManifest{ManifestUID: "deleted", ZoneUID: "zone-a", Apex: "prod.internal", WriterEpoch: 3, Revision: 1, Tombstone: true, GeneratedAt: now, ContentHash: model.EmptyContentHash()} + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 3, 1, tombstone)); err != nil { + t.Fatal(err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if err := restarted.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, old)); err != nil { + t.Fatal(err) + } + if !restarted.state.Publications["zone-a"].Manifest.Tombstone || restarted.publicationsReady(testSnapshot(now).Bindings[0]) { + t.Fatal("old snapshot defeated tombstone") + } +} +func TestNodeRequiresPublicationAndRetainsCacheDuringOutage(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + render, snapshot := renderFixture(t) + render.Role = RenderNode + render.NodeCoreDNS.Path = filepath.Join(t.TempDir(), "node.conf") + render.NodeDNSDist.Path = filepath.Join(filepath.Dir(render.NodeCoreDNS.Path), "dnsdist.conf") + cfg.Render = render + cfg.Mode = ModeResolver + runner := &recordingRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + snapshot = testSnapshot(now) + a.state.Snapshot = &snapshot + a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)}} + if a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("node enabled from ACK without publication plan") + } + chunk, m := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err != nil { + t.Fatal(err) + } + if !a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("complete plan + current proof not eligible") + } + runner.calls = nil + clock = now.Add(24 * time.Hour) + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + +} +func TestCoreDNSPublicationVersionProofIncludesFullContentFingerprint(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + chunk, m := testPublication(t, now) + var plan model.PublicationPlan + if err := json.Unmarshal(chunk.Payload, &plan); err != nil { + t.Fatal(err) + } + p := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Plan: plan} + first := publicationMailbox(p) + p.Plan.RRSets[0].Records[0].Content = "192.0.2.11" + if first == publicationMailbox(p) { + t.Fatal("different installed data shares fingerprint") + } + if len(first) > 255 { + t.Fatal("fingerprint name too long") + } +} + +func TestLegacyBackendCheckpointRejectedWithoutDiscardingFences(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "checkpoint.json") + old := []byte(`{"contributionFences":{"endpoint":{"epoch":7,"sequence":33}}}`) + if err := os.WriteFile(path, old, 0600); err != nil { + t.Fatal(err) + } + if _, err := (stateStore{path: path}).Load(); err == nil { + t.Fatal("old backend checkpoint silently accepted") + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(after) != string(old) { + t.Fatal("unsupported checkpoint discarded observation fences") + } +} + +func TestRegionalPoolRequiresOneMemberWithEveryCurrentZone(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + snapshot := testSnapshot(now) + b := snapshot.Bindings[0] + b.Zones = append(b.Zones, model.ZoneAttachment{ZoneUID: "zone-b", Apex: "custom.internal", RequiredPublicationEpoch: 2, RequiredPublicationRevision: 4}) + snapshot.Bindings = []model.Binding{b} + snapshot.Members = []model.Member{{MemberID: "regional-0", Role: "regional"}, {MemberID: "regional-1", Role: "regional"}} + a := &Agent{cfg: Config{Mode: ModeResolver}, state: newCheckpoint(), deps: Dependencies{Clock: func() time.Time { return now }}} + a.state.Snapshot = &snapshot + a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + for _, z := range b.Zones { + a.state.Publications[z.ZoneUID] = publicationState{Fence: fence{2, 4}, Verified: true} + a.state.Acks[z.ZoneUID] = map[string]replicaAck{} + } + proof := replicaAck{Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)} + a.state.Acks["zone-a"]["regional-0/"] = proof + a.state.Acks["zone-b"]["regional-1/"] = proof + if a.publicationsReady(b) { + t.Fatal("different member per zone incorrectly made whole context ready") + } + a.state.Acks["zone-b"]["regional-0/"] = proof + ready := a.eligibleRegionalMembers(b) + if !ready["regional-0"] || ready["regional-1"] { + t.Fatalf("wrong member eligibility: %#v", ready) + } + c, _ := renderFixture(t) + c.Role = RenderRegionalDNSDist + c.ClusterDNSDist.Backends = []model.Backend{{MemberID: "regional-0", Address: "10.253.0.20", Port: 5300}, {MemberID: "regional-1", Address: "10.253.0.21", Port: 5300}} + c.ReadyMembers = map[string]map[string]bool{b.BindingUID: ready} + rendered, err := Render(c, snapshot, map[string]bool{b.BindingUID: true}) + if err != nil { + t.Fatal(err) + } + conf := string(rendered.Files[c.ClusterDNSDist.Path]) + if strings.Count(conf, "server:addPool(") != 1 { + t.Fatalf("stale live member remains in context pool:\n%s", conf) + } + // A newer ACK without the matching publication plan gates even the formerly + // current member; a resolver must hold the publication version confirmed by the member. + a.state.Acks["zone-a"]["regional-1/"] = replicaAck{Fence: fence{2, 5}, ObservedAt: now, ValidUntil: now.Add(time.Minute)} + if a.publicationsReady(b) { + t.Fatal("ACK ahead of local publication plan activated context") + } +} +func TestMissingOrStalePrivatePublicationHasNoPublicFallback(t *testing.T) { + t.Parallel() + c, snapshot := renderFixture(t) + c.Role = RenderRegionalCoreDNS + publications := map[string]publicationState{"zone-a": {Fence: fence{1, 1}, Plan: model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal"}}} + rendered, err := renderPublications(c, snapshot, map[string]bool{}, publications) + if err != nil { + t.Fatal(err) + } + conf := string(rendered.Files[c.ClusterCoreDNS.Path]) + if !decodedCoreDNS(t, []byte(conf)).Contexts[0].Zones[0].Pending { + t.Fatalf("private zone can recurse publicly: %s", conf) + } + for path := range rendered.Files { + if strings.HasSuffix(path, ".db") { + t.Fatal("stale publication installed as private authority") + } + } +} + +func TestRejectedMemberProofRevokesRoutingAndIgnoresOlderReplay(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + cfg.Mode = ModeResolver + cfg.Render.Role = RenderRegionalDNSDist + cfg.Render.ClusterDNSDist.Path = filepath.Join(t.TempDir(), "frontend.conf") + a, err := NewAgent(cfg, Dependencies{Clock: func() time.Time { return clock }, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + a.state.Snapshot = &snapshot + a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + chunk, _ := testPublication(t, now) + var plan model.PublicationPlan + if err := json.Unmarshal(chunk.Payload, &plan); err != nil { + t.Fatal(err) + } + a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan)} + a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)}} + ack := model.MemberAck{MemberID: "regional-0", ReplicaID: "regional-0", ResourceUID: "zone-a", Kind: model.KindPublicationManifest, Epoch: 2, Revision: 4, Phase: model.AckRejected, ObservedAt: now.Add(-time.Second), ValidUntil: now.Add(time.Minute)} + if err := a.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatal(err) + } + if !a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("older rejection replay revoked fresh proof") + } + clock = now.Add(time.Second) + ack.ObservedAt = clock + if err := a.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatal(err) + } + if a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("new rejection retained stale member routing") + } + ack.Phase = model.AckVerified + ack.ObservedAt = now + if err := a.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatal(err) + } + if a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("old verified ACK resurrected rejected member") + } +} + +func TestPublicationProbeTimeoutDoesNotApproveConfiguration(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = listener.Close() }() + port := listener.Addr().(*net.TCPAddr).Port + cfg.Render.ClusterCoreDNS.ListenAddress = "127.0.0.1" + cfg.Render.ClusterCoreDNS.Port = uint16(port) + cfg.PublicationDNSProbe = PublicationDNSVerifierConfig{Server: listener.Addr().String(), Timeout: 50 * time.Millisecond} + cfg.UnsafeAllowNoDNSVerify = false + a, err := NewAgent(cfg, Dependencies{Clock: func() time.Time { return now }, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, snapshot)); err != nil { + t.Fatal(err) + } + chunk, m := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err == nil { + t.Fatal("timed out publication accepted") + } + if a.state.Publications["zone-a"].Verified { + t.Fatal("failed publication query proof became active") + } +} + +type publicationVerifierFunc func(context.Context, model.PublicationPlan, []PublicationView) error + +func (f publicationVerifierFunc) VerifyPublication(ctx context.Context, p model.PublicationPlan, views []PublicationView) error { + return f(ctx, p, views) +} + +type afterReloadRunner struct { + recordingRunner + after func() +} + +func (r *afterReloadRunner) Run(ctx context.Context, command Command) ([]byte, error) { + data, err := r.recordingRunner.Run(ctx, command) + if err == nil && command.Path == "reload" && r.after != nil { + after := r.after + r.after = nil + after() + } + return data, err +} + +func installedFileVerifier(t *testing.T, configPath string) PublicationVerifier { + t.Helper() + return publicationVerifierFunc(func(_ context.Context, plan model.PublicationPlan, views []PublicationView) error { + config, err := os.ReadFile(configPath) + if err != nil { + return err + } + cfg := decodedCoreDNS(t, config) + var data string + for _, entry := range cfg.Contexts { + for _, zone := range entry.Zones { + if zone.Apex == model.AbsoluteName(plan.Apex) && !zone.Pending { + data = zone.Data + } + } + } + if data == "" { + return errors.New("no activated private zone") + } + parser := dns.NewZoneParser(strings.NewReader(data), model.AbsoluteName(plan.Apex), configPath) + for rr, ok := parser.Next(); ok; rr, ok = parser.Next() { + if soa, ok := rr.(*dns.SOA); ok { + for _, view := range views { + if soa.Serial != view.Serial || soa.Mbox != view.FingerprintMailbox { + return fmt.Errorf("proof expects %d/%s but activated file contains %d/%s", view.Serial, view.FingerprintMailbox, soa.Serial, soa.Mbox) + } + } + return nil + } + } + return errors.New("activated zone lacks SOA") + }) +} + +func TestFailedCoreDNSActivationRetainsPriorInstalledProof(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + runner := &recordingRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + prior := a.installedPublications["zone-a"].Identity + p := a.state.Publications["zone-a"] + p.LocalRevision++ + a.state.Publications["zone-a"] = p + runner.failAt = len(runner.calls) + 1 + if err := a.reconcile(context.Background()); err == nil { + t.Fatal("failed validator was accepted") + } + if a.installedPublications["zone-a"].Identity != prior { + t.Fatal("failed activation advanced installed proof") + } +} + +func TestCoreDNSDoesNotProbeAuthorityBelowRequiredPublicationFence(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + calls := 0 + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }, PublicationVerifier: publicationVerifierFunc(func(context.Context, model.PublicationPlan, []PublicationView) error { + calls++ + return errors.New("unexpected authoritative proof") + })}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + snapshot.Bindings[0].Zones[0].RequiredPublicationEpoch = 2 + snapshot.Bindings[0].Zones[0].RequiredPublicationRevision = 5 + a.state.Snapshot = &snapshot + a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + chunk, manifest := testPublication(t, now) + var plan model.PublicationPlan + if err := json.Unmarshal(chunk.Payload, &plan); err != nil { + t.Fatal(err) + } + a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Manifest: manifest, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan)} + if err := a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + if calls != 0 || len(a.publicationViews(a.state.Publications["zone-a"])) != 0 { + t.Fatal("SERVFAIL zone attempted authoritative publication proof") + } + config, err := os.ReadFile(cfg.Render.ClusterCoreDNS.Path) + if err != nil || !decodedCoreDNS(t, config).Contexts[0].Zones[0].Pending { + t.Fatalf("behind-required private zone was not gated: %s %v", config, err) + } +} + +type partitioningAckPublisher struct { + disconnected atomic.Bool + verified atomic.Int64 +} + +func (p *partitioningAckPublisher) Publish(_ context.Context, _ string, env model.Envelope) (transport.PublishAck, error) { + if p.disconnected.Load() { + return transport.PublishAck{}, errors.New("nats: server is disconnected") + } + var ack model.MemberAck + if err := json.Unmarshal(env.Payload, &ack); err != nil { + return transport.PublishAck{}, err + } + if ack.Kind == model.KindPublicationManifest && ack.Phase == model.AckVerified { + p.verified.Add(1) + } + return transport.PublishAck{}, nil +} + +func TestCoreDNSActivationRacingBrokerDisconnectKeepsServing(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + runner := &afterReloadRunner{} + publisher := &partitioningAckPublisher{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }, AckPublisher: publisher, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterCoreDNS.Path)}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + chunk, manifest := testPublication(t, now) + var plan model.PublicationPlan + if err := json.Unmarshal(chunk.Payload, &plan); err != nil { + t.Fatal(err) + } + plan.ObservationFences[0].Sequence = 2 + plan.RRSets[0].Records[0].Sequence = 2 + plan.RRSets[0].Records[0].Content = "192.0.2.11" + chunk.Payload, err = json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + chunk.ManifestUID = "manifest-b" + chunk.Revision = 5 + chunk.SHA256 = model.Hash(chunk.Payload) + manifest.ManifestUID = chunk.ManifestUID + manifest.PreviousRevision = 4 + manifest.Revision = chunk.Revision + manifest.ContentHash = chunk.SHA256 + manifest.Chunks = []model.ChunkRef{{Index: 0, SHA256: chunk.SHA256, Size: len(chunk.Payload)}} + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 5, chunk)); err != nil { + t.Fatal(err) + } + // Disconnect after CoreDNS reload, exactly before durable/proved activation + // attempts its ACK. A transport error must not become a failed installation. + runner.after = func() { publisher.disconnected.Store(true) } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { + t.Fatalf("safe activation treated broker failure as materialization failure: %v", err) + } + if !a.state.Publications["zone-a"].Verified || a.needsActivation { + t.Fatal("ACK partition revoked proved publication") + } + assertAppliedCheckpoint(t, a) + // Identical manifest redelivery and snapshot delivery must share the same + // best-effort ACK boundary without changing the observation. + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + snapshot.ConfigurationRevision = 2 + snapshot.Bindings[0].Version.Sequence = 2 + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, snapshot)); err != nil { + t.Fatal(err) + } + clock = now.Add(24 * time.Hour) + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + if !strings.Contains(currentZone(t, a), "192.0.2.11") { + t.Fatal("ACK partition removed an applied record") + } + assertAppliedCheckpoint(t, a) + if a.needsActivation { + t.Fatal("healthy serving was gated by ACK outage") + } + before := publisher.verified.Load() + publisher.disconnected.Store(false) + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + if publisher.verified.Load() <= before { + t.Fatal("broker recovery did not retry publication readiness ACK") + } +} + +func TestCoreDNSTombstoneAckFailureDoesNotGateDurableWithdrawal(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + publisher := &partitioningAckPublisher{} + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }, AckPublisher: publisher, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterCoreDNS.Path)}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + publisher.disconnected.Store(true) + manifest := model.PublicationManifest{ManifestUID: "retired-zone", ZoneUID: "zone-a", Apex: "prod.internal", WriterEpoch: 2, Revision: 5, PreviousRevision: 4, ContentHash: model.EmptyContentHash(), GeneratedAt: now, Tombstone: true} + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { + t.Fatal(err) + } + p := a.state.Publications["zone-a"] + if !p.Verified || !p.Manifest.Tombstone || a.needsActivation { + t.Fatal("failed tombstone ACK invalidated safely withdrawn zone") + } + assertAppliedCheckpoint(t, a) +} diff --git a/internal/internaldns/serving/daemons.go b/internal/internaldns/serving/daemons.go new file mode 100644 index 00000000..a11db457 --- /dev/null +++ b/internal/internaldns/serving/daemons.go @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "golang.org/x/sys/unix" +) + +type daemonProcess struct { + name string + config string + args []string + cmd *exec.Cmd + done chan error +} + +func (p *daemonProcess) start() error { + p.cmd = exec.Command(p.name, p.args...) + p.cmd.Stdout, p.cmd.Stderr = os.Stdout, os.Stderr + if err := p.cmd.Start(); err != nil { + p.cmd = nil + return err + } + p.done = make(chan error, 1) + go func(cmd *exec.Cmd, done chan error) { done <- cmd.Wait() }(p.cmd, p.done) + return nil +} + +func (p *daemonProcess) stop() error { + if p.cmd == nil { + return nil + } + _ = p.cmd.Process.Signal(unix.SIGTERM) + select { + case <-p.done: + case <-time.After(time.Second): + _ = p.cmd.Process.Kill() + select { + case <-p.done: + case <-time.After(time.Second): + return fmt.Errorf("DNS process %s did not stop", p.name) + } + } + p.cmd, p.done = nil, nil + return nil +} + +// RunDaemons owns query processes independently of the configuration agent. +// Only incomplete or abandoned activation blocks their lifecycle. +func RunDaemons(ctx context.Context, config Config) error { + if err := os.MkdirAll(config.StateDir, 0o750); err != nil { + return err + } + var processes []*daemonProcess + addBind := func(bind CoreDNSRenderConfig) { + processes = append(processes, &daemonProcess{name: "coredns", config: bind.Path, args: []string{"-config", bind.Path}}) + } + addRouter := func(router DNSDistRenderConfig) { + processes = append(processes, &daemonProcess{name: "dnsdist", config: router.Path, args: []string{"--supervised", "-C", router.Path}}) + } + switch config.Render.Role { + case RenderNode: + addBind(config.Render.NodeCoreDNS) + addRouter(config.Render.NodeDNSDist) + case RenderRegionalCoreDNS: + addBind(config.Render.ClusterCoreDNS) + case RenderRegionalDNSDist: + addRouter(config.Render.ClusterDNSDist) + case RenderCluster: + addBind(config.Render.ClusterCoreDNS) + addRouter(config.Render.ClusterDNSDist) + default: + return fmt.Errorf("unsupported serving process role %q", config.Render.Role) + } + defer func() { + for _, p := range processes { + _ = p.stop() + } + }() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + for { + start, retain, release := daemonPermission(config.StateDir) + for _, p := range processes { + if p.cmd != nil { + select { + case <-p.done: + p.cmd, p.done = nil, nil + default: + } + } + if !retain { + if err := p.stop(); err != nil { + release() + return err + } + continue + } + if p.cmd != nil || !start { + continue + } + if _, err := os.Stat(filepath.Join("/run/internal-dns", "allow-"+p.name)); err != nil { + continue + } + if !activationPending(config.StateDir) { + state, err := (stateStore{path: filepath.Join(config.StateDir, "checkpoint.json")}).Load() + if err != nil || state.Active == nil || !installedFilesMatch(state.Active.Files) { + continue + } + } + if err := p.start(); err != nil && !errors.Is(err, os.ErrNotExist) { + release() + return err + } + } + release() + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} diff --git a/internal/internaldns/serving/readiness.go b/internal/internaldns/serving/readiness.go new file mode 100644 index 00000000..c4d70558 --- /dev/null +++ b/internal/internaldns/serving/readiness.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "time" +) + +const activationMarkerName = "activation.pending" + +func activationMarkerPath(stateDir string) string { + return filepath.Join(stateDir, activationMarkerName) +} + +func activationPending(stateDir string) bool { + _, err := os.Stat(activationMarkerPath(stateDir)) + return !errors.Is(err, os.ErrNotExist) +} + +func installedFilesMatch(files map[string][]byte) bool { + if len(files) == 0 { + return false + } + for path, expected := range files { + actual, err := os.ReadFile(path) + if err != nil || !bytes.Equal(actual, expected) { + return false + } + } + return true +} + +// Readiness verifies the approved files and live DNS processes using local state. +// It does not require the configuration agent or state delivery to be available. +func Readiness(ctx context.Context, config Config) error { + if activationPending(config.StateDir) { + return errors.New("configuration installation is incomplete") + } + state, err := (stateStore{path: filepath.Join(config.StateDir, "checkpoint.json")}).Load() + if err != nil { + return err + } + if state.Active == nil || !installedFilesMatch(state.Active.Files) { + return errors.New("installed configuration does not match the approved checkpoint") + } + if err := state.Active.Snapshot.Validate(); err != nil { + return fmt.Errorf("approved snapshot: %w", err) + } + if config.UnsafeAllowNoDNSVerify { + return errors.New("readiness requires live DNS query verification") + } + return (&Agent{cfg: config}).probeLocalDaemons(ctx) +} + +func (a *Agent) probeLocalDaemons(ctx context.Context) error { + if a.cfg.UnsafeAllowNoDNSVerify { + return nil + } + role := a.cfg.Render.Role + if role == "" { + role = RenderBoth + } + var binds []CoreDNSRenderConfig + var dists []DNSDistRenderConfig + if role == RenderNode || role == RenderBoth { + binds = append(binds, a.cfg.Render.NodeCoreDNS) + dists = append(dists, a.cfg.Render.NodeDNSDist) + } + if role == RenderCluster || role == RenderBoth || role == RenderRegionalCoreDNS { + binds = append(binds, a.cfg.Render.ClusterCoreDNS) + } + if role == RenderCluster || role == RenderBoth || role == RenderRegionalDNSDist { + dists = append(dists, a.cfg.Render.ClusterDNSDist) + } + for _, bind := range binds { + server := net.JoinHostPort(bind.ListenAddress, strconv.Itoa(int(bind.Port))) + view := PublicationView{DestinationAddress: bind.ListenAddress, DestinationPort: bind.Port} + answer, err := exchangePublication(ctx, networkTCP, server, ".", view, 200*time.Millisecond) + if err != nil || answer == nil || !answer.Response { + return fmt.Errorf("local CoreDNS health %s: %w", server, errors.Join(err, errors.New("DNS response required"))) + } + } + for _, dist := range dists { + for _, network := range []string{networkUDP, networkTCP} { + answer, err := exchangeResolverSOA(ctx, network, dist.ListenAddress, ".", 200*time.Millisecond) + if err != nil || answer == nil || !answer.Response { + return fmt.Errorf("local dnsdist health %s: %w", dist.ListenAddress, errors.Join(err, errors.New("DNS response required"))) + } + } + } + return nil +} diff --git a/internal/internaldns/serving/readiness_test.go b/internal/internaldns/serving/readiness_test.go new file mode 100644 index 00000000..060bbedb --- /dev/null +++ b/internal/internaldns/serving/readiness_test.go @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/miekg/dns" +) + +func TestReadinessUsesApprovedConfigurationWithoutAgentOrBroker(t *testing.T) { + tcp, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + udp, err := net.ListenPacket("udp", tcp.Addr().String()) + if err != nil { + t.Fatal(err) + } + handler := dns.HandlerFunc(func(w dns.ResponseWriter, request *dns.Msg) { + response := new(dns.Msg) + response.SetRcode(request, dns.RcodeRefused) + _ = w.WriteMsg(response) + }) + started := make(chan struct{}, 2) + udpServer := &dns.Server{PacketConn: udp, Handler: handler, NotifyStartedFunc: func() { started <- struct{}{} }} + tcpServer := &dns.Server{Listener: tcp, Handler: handler, NotifyStartedFunc: func() { started <- struct{}{} }} + go func() { _ = udpServer.ActivateAndServe() }() + go func() { _ = tcpServer.ActivateAndServe() }() + t.Cleanup(func() { _ = udpServer.Shutdown(); _ = tcpServer.Shutdown() }) + for i := 0; i < 2; i++ { + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("DNS fixture did not start") + } + } + dir := t.TempDir() + configPath := filepath.Join(dir, "dnsdist.conf") + files := map[string][]byte{configPath: []byte("approved configuration")} + if err := os.WriteFile(configPath, files[configPath], 0o644); err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(time.Now().Add(-24 * time.Hour)) + state := newCheckpoint() + state.Snapshot = &snapshot + state.BindingFences, _ = advanceBindingFences(nil, snapshot) + state.Active = &activeConfiguration{Snapshot: snapshot, Files: files} + state.RenderedHash = hashFiles(files) + if err := (stateStore{path: filepath.Join(dir, "checkpoint.json")}).Save(state); err != nil { + t.Fatal(err) + } + config := Config{StateDir: dir, Render: RenderConfig{Role: RenderRegionalDNSDist, ClusterDNSDist: DNSDistRenderConfig{ListenAddress: tcp.Addr().String()}}} + if err := Readiness(context.Background(), config); err != nil { + t.Fatalf("approved DNS required an agent heartbeat or delivery connection: %v", err) + } + if err := os.WriteFile(activationMarkerPath(dir), []byte("installing\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := Readiness(context.Background(), config); err == nil { + t.Fatal("incomplete installation was ready") + } + if err := os.Remove(activationMarkerPath(dir)); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath, []byte("unapproved configuration"), 0o644); err != nil { + t.Fatal(err) + } + if err := Readiness(context.Background(), config); err == nil { + t.Fatal("unapproved files were ready") + } + if err := os.WriteFile(configPath, files[configPath], 0o644); err != nil { + t.Fatal(err) + } + if err := udpServer.Shutdown(); err != nil { + t.Fatal(err) + } + if err := Readiness(context.Background(), config); err == nil { + t.Fatal("TCP-only DNS was ready") + } +} + +func TestAgentRestartAndStopPreserveApprovedConfiguration(t *testing.T) { + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, time.Now().UTC()) + runner := &recordingRunner{} + restarted, err := NewAgent(cfg, Dependencies{Runner: runner}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if len(runner.calls) != 0 { + t.Fatalf("agent restart changed healthy daemon configuration: %#v", runner.calls) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := restarted.Start(ctx); err != nil { + t.Fatal(err) + } + if len(runner.calls) != 0 || !installedFilesMatch(restarted.state.Active.Files) { + t.Fatal("agent stop shut down healthy DNS") + } +} + +func TestFailClosedPersistsIncompleteActivationAcrossRestart(t *testing.T) { + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, time.Now().UTC()) + if err := agent.failClosed(context.Background()); err != nil { + t.Fatal(err) + } + if !activationPending(cfg.StateDir) { + t.Fatal("gated DNS must persist its incomplete activation") + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + if err := Readiness(context.Background(), cfg); err == nil || err.Error() != "configuration installation is incomplete" { + t.Fatalf("gated DNS must not report readiness after restart: %v", err) + } + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if activationPending(cfg.StateDir) { + t.Fatal("successful activation must clear the gate marker") + } +} diff --git a/internal/internaldns/serving/state.go b/internal/internaldns/serving/state.go new file mode 100644 index 00000000..ebc122bc --- /dev/null +++ b/internal/internaldns/serving/state.go @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +type contributionFence struct { + Generation uint64 `json:"generation"` + Sequence uint64 `json:"sequence"` + Retired bool `json:"retired,omitempty"` +} + +type replicaAck struct { + Rejected bool `json:"rejected,omitempty"` + ObservedAt time.Time `json:"observedAt"` + Fence fence `json:"fence"` + ValidUntil time.Time `json:"validUntil"` +} + +const checkpointFormatVersion = 7 + +type checkpoint struct { + Active *activeConfiguration `json:"active,omitempty"` + FormatVersion int `json:"formatVersion"` + ServingFence fence `json:"servingFence"` + Snapshot *model.ServingSnapshot `json:"snapshot,omitempty"` + Publications map[string]publicationState `json:"publications"` + Chunks map[string]map[string]model.PublicationChunk `json:"chunks"` + Acks map[string]map[string]replicaAck `json:"acks"` + BindingFences map[string]bindingFence `json:"bindingFences"` + ContributionFences map[string]contributionFence `json:"contributionFences"` + RenderedHash string `json:"renderedHash,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func newCheckpoint() checkpoint { + return checkpoint{FormatVersion: checkpointFormatVersion, Publications: map[string]publicationState{}, Chunks: map[string]map[string]model.PublicationChunk{}, Acks: map[string]map[string]replicaAck{}, ContributionFences: map[string]contributionFence{}, BindingFences: map[string]bindingFence{}} +} + +type stateStore struct{ path string } + +func (s stateStore) Load() (checkpoint, error) { + state := newCheckpoint() + b, err := os.ReadFile(s.path) + if errors.Is(err, os.ErrNotExist) { + return state, nil + } + if err != nil { + return state, err + } + state.FormatVersion = 0 + state.BindingFences = nil + if err := json.Unmarshal(b, &state); err != nil { + return state, fmt.Errorf("decode checkpoint: %w", err) + } + if state.FormatVersion != checkpointFormatVersion { + return state, fmt.Errorf("unsupported checkpoint format %d: preserve the existing checkpoint; automatic backend or binding authority history migration is not supported", state.FormatVersion) + } + if state.BindingFences == nil { + return state, errors.New("checkpoint lacks required binding authority history") + } + for uid, f := range state.BindingFences { + if uid == "" || f.Identity == "" || f.ConfigurationGeneration == 0 || f.ConfigurationRevision == 0 || f.Version.Epoch == 0 || f.Version.Sequence == 0 { + return state, fmt.Errorf("checkpoint binding %s has incomplete authority history", uid) + } + } + if state.Snapshot != nil { + for _, b := range state.Snapshot.Bindings { + f, ok := state.BindingFences[b.BindingUID] + if !ok || !equalJSON(f, bindingFenceFor(b)) { + return state, fmt.Errorf("checkpoint binding %s does not match its authority history", b.BindingUID) + } + } + } + if state.Publications == nil { + state.Publications = map[string]publicationState{} + } + if state.Chunks == nil { + state.Chunks = map[string]map[string]model.PublicationChunk{} + } + if state.Acks == nil { + state.Acks = map[string]map[string]replicaAck{} + } + if state.ContributionFences == nil { + state.ContributionFences = map[string]contributionFence{} + } + for uid, observation := range state.ContributionFences { + if uid == "" || observation.Generation == 0 || observation.Sequence == 0 { + return state, fmt.Errorf("invalid persisted contribution fence %q", uid) + } + } + if state.Snapshot != nil { + if err := state.Snapshot.Validate(); err != nil { + return state, fmt.Errorf("invalid persisted serving snapshot: %w", err) + } + } + for uid, p := range state.Publications { + if !p.Manifest.Tombstone { + if err := p.Plan.Validate(); err != nil { + return state, fmt.Errorf("invalid persisted publication %s: %w", uid, err) + } + } + } + return state, nil +} + +func (s stateStore) Save(state checkpoint) error { + if state.FormatVersion != checkpointFormatVersion { + return fmt.Errorf("cannot write unsupported checkpoint format %d", state.FormatVersion) + } + state.UpdatedAt = time.Now().UTC() + if err := os.MkdirAll(filepath.Dir(s.path), 0o750); err != nil { + return err + } + b, err := json.MarshalIndent(state, "", " ") + if err != nil { + return err + } + temp := s.path + ".tmp" + if err := writeDurable(temp, b, 0o600); err != nil { + return err + } + if err := os.Rename(temp, s.path); err != nil { + return err + } + dir, err := os.Open(filepath.Dir(s.path)) + if err != nil { + return err + } + defer func() { _ = dir.Close() }() + return dir.Sync() +} + +func chunkKey(index int) string { return fmt.Sprint(index) } diff --git a/internal/internaldns/serving/transaction.go b/internal/internaldns/serving/transaction.go index 467ee010..bd8f756a 100644 --- a/internal/internaldns/serving/transaction.go +++ b/internal/internaldns/serving/transaction.go @@ -1,6 +1,7 @@ package serving import ( + "bytes" "context" "errors" "fmt" @@ -8,7 +9,6 @@ import ( "os/exec" "path/filepath" "sort" - "strconv" "strings" "time" ) @@ -41,8 +41,10 @@ type TransactionConfig struct { } type FileTransaction struct { - Config TransactionConfig - Runner Runner + activationLock *os.File + MarkerPath string + Config TransactionConfig + Runner Runner } type preservedTransactionError struct{ error } @@ -59,18 +61,18 @@ type oldFile struct { existed bool } -func (t FileTransaction) Apply(ctx context.Context, files map[string][]byte) error { +func (t *FileTransaction) Apply(ctx context.Context, files map[string][]byte) error { return t.apply(ctx, files, false) } // ApplyForce validates and reloads every configured daemon even when the // installed bytes already match. It is used after fail-closed startup because // file equality cannot prove that a stopped or firewalled process is active. -func (t FileTransaction) ApplyForce(ctx context.Context, files map[string][]byte) error { +func (t *FileTransaction) ApplyForce(ctx context.Context, files map[string][]byte) error { return t.apply(ctx, files, true) } -func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, force bool) (resultErr error) { +func (t *FileTransaction) apply(ctx context.Context, files map[string][]byte, force bool) (resultErr error) { mutating := false defer func() { if resultErr == nil { @@ -138,20 +140,7 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for for _, path := range paths { stagedPath := filepath.Join(stage, filepath.Base(path)) stagedData := files[path] - if strings.HasSuffix(path, ".conf") { - // Validate the newly staged zone contents, never the previous live files. - commandStage := stage - if t.Config.CommandStageDir != "" { - commandStage = filepath.Join(t.Config.CommandStageDir, filepath.Base(stage)) - } - text := string(stagedData) - for renderedPath := range files { - if strings.HasSuffix(renderedPath, ".db") { - text = strings.ReplaceAll(text, strconv.Quote(renderedPath), strconv.Quote(filepath.Join(commandStage, filepath.Base(renderedPath)))) - } - } - stagedData = []byte(text) - } + if err := writeDurable(stagedPath, stagedData, 0o644); err != nil { return err } @@ -173,6 +162,22 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for if err := t.commandsForChanges(ctx, t.Config.Validate, map[string]string{"{stageDir}": commandStage}, commandChanges); err != nil { return preservedTransactionError{fmt.Errorf("validate staged serving configuration: %w", err)} } + if t.MarkerPath != "" { + structural := force + for _, path := range paths { + structural = structural || changed[path] && routingConfigurationChanged(path, old[path].data, files[path]) + } + if err := t.beginActivation(); err != nil { + return err + } + if structural { + // Restart every owned daemon after changing routing or view configuration. + commandChanges = nil + if err := t.FailClosed(ctx); err != nil { + return uncertainTransactionError{err} + } + } + } installed := make([]string, 0, len(paths)) mutating = true for _, path := range paths { @@ -198,10 +203,31 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for rollbackErr := restoreFiles(installed, old) return errors.Join(err, rollbackErr) } + if err := t.filesInstalled(); err != nil { + return err + } if err := t.commandsForChanges(ctx, t.Config.Reload, nil, commandChanges); err != nil { - rollbackErr := restoreFiles(paths, old) + if t.MarkerPath != "" { + if guardErr := t.beginActivation(); guardErr != nil { + return errors.Join(err, guardErr) + } + commandChanges = nil + if guardErr := t.FailClosed(ctx); guardErr != nil { + return errors.Join(err, guardErr) + } + } + rollbackErr := errors.Join(restoreFiles(paths, old), syncDirs(paths)) + if rollbackErr != nil { + return errors.Join(err, rollbackErr) + } + if guardErr := t.filesInstalled(); guardErr != nil { + return errors.Join(err, guardErr) + } reloadErr := t.commandsForChanges(ctx, t.Config.Reload, nil, commandChanges) if rollbackErr == nil && reloadErr == nil { + if guardErr := t.commitActivation(); guardErr != nil { + return errors.Join(err, guardErr) + } return preservedTransactionError{fmt.Errorf("reload serving configuration: %w", err)} } return errors.Join(fmt.Errorf("reload serving configuration: %w", err), rollbackErr, reloadErr) @@ -272,40 +298,6 @@ func writeDurable(path string, data []byte, mode os.FileMode) error { return f.Close() } -// writeAtomicLease replaces the ephemeral watchdog lease through a unique -// temporary sibling. The file is synced before rename so a reader never accepts -// partially written JSON. The containing directory is deliberately not synced: -// metadata fsync on a container bind mount can block longer than the lease, and -// restart always gates before bootstrap so this lease never needs crash -// survival. -func writeAtomicLease(path string, data []byte, mode os.FileMode) error { - dir := filepath.Dir(path) - temporary, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-") - if err != nil { - return err - } - temporaryPath := temporary.Name() - defer func() { _ = os.Remove(temporaryPath) }() - if err := temporary.Chmod(mode); err != nil { - _ = temporary.Close() - return err - } - if _, err := temporary.Write(data); err != nil { - _ = temporary.Close() - return err - } - if err := temporary.Sync(); err != nil { - _ = temporary.Close() - return err - } - if err := temporary.Close(); err != nil { - return err - } - if err := os.Rename(temporaryPath, path); err != nil { - return err - } - return nil -} func syncDirs(paths []string) error { seen := map[string]bool{} for _, path := range paths { @@ -350,3 +342,7 @@ func restoreFiles(paths []string, old map[string]oldFile) error { } return errors.Join(errs...) } + +func routingConfigurationChanged(path string, before, after []byte) bool { + return strings.HasSuffix(path, ".conf") && !bytes.Equal(before, after) +} diff --git a/internal/internaldns/serving/transaction_test.go b/internal/internaldns/serving/transaction_test.go index 4b55d313..d2df468e 100644 --- a/internal/internaldns/serving/transaction_test.go +++ b/internal/internaldns/serving/transaction_test.go @@ -6,9 +6,9 @@ import ( "errors" "os" "path/filepath" - "strconv" "strings" "testing" + "time" ) type recordingRunner struct { @@ -78,39 +78,6 @@ func TestFileTransactionConfigIsReadableByServiceUID(t *testing.T) { } } -func TestAtomicLeaseWriteReplacesCompleteFile(t *testing.T) { - t.Parallel() - dir := t.TempDir() - path := filepath.Join(dir, "lease.json") - if err := os.WriteFile(path, []byte("old"), 0o600); err != nil { - t.Fatal(err) - } - if err := writeAtomicLease(path, []byte(`{"memberID":"node-0"}`), 0o600); err != nil { - t.Fatal(err) - } - contents, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if string(contents) != `{"memberID":"node-0"}` { - t.Fatalf("atomic durable contents = %q", contents) - } - info, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - if info.Mode().Perm() != 0o600 { - t.Fatalf("atomic durable mode = %#o", info.Mode().Perm()) - } - temporary, err := filepath.Glob(filepath.Join(dir, ".lease.json.tmp-*")) - if err != nil { - t.Fatal(err) - } - if len(temporary) != 0 { - t.Fatalf("temporary lease files remain: %#v", temporary) - } -} - func TestFileTransactionPreservesStagePlaceholderAcrossApplies(t *testing.T) { t.Parallel() dir := t.TempDir() @@ -216,7 +183,7 @@ func TestFileTransactionReloadsOnlyChangedTier(t *testing.T) { t.Fatalf("reload calls = %#v", runner.calls) } if got, err := os.ReadFile(named); err != nil || string(got) != "unchanged" { - t.Fatalf("unchanged BIND file was disturbed: %q, %v", got, err) + t.Fatalf("unchanged CoreDNS file was disturbed: %q, %v", got, err) } } @@ -224,39 +191,158 @@ type transactionRunnerFunc func(context.Context, Command) ([]byte, error) func (f transactionRunnerFunc) Run(ctx context.Context, c Command) ([]byte, error) { return f(ctx, c) } -func TestFileTransactionValidatesFreshStagedZoneReferences(t *testing.T) { +func TestFileTransactionValidatesCompleteStagedSnapshot(t *testing.T) { t.Parallel() dir := t.TempDir() - config := filepath.Join(dir, "live", "named.conf") - zone := filepath.Join(dir, "live", "zone.db") + path := filepath.Join(dir, "contexts.json") + candidate := []byte(`{"contexts":[{"id":"a","data":"new zone"}]}`) runner := transactionRunnerFunc(func(_ context.Context, c Command) ([]byte, error) { - data, err := os.ReadFile(c.Args[0]) - if err != nil { - return nil, err - } - staged := filepath.Join(filepath.Dir(c.Args[0]), "zone.db") - if !strings.Contains(string(data), strconv.Quote(staged)) || strings.Contains(string(data), strconv.Quote(zone)) { - t.Fatalf("validator sees live zone: %s", data) - } - content, err := os.ReadFile(staged) + staged, err := os.ReadFile(c.Args[0]) if err != nil { return nil, err } - if string(content) != "new zone" { - t.Fatalf("staged content=%s", content) + if !bytes.Equal(staged, candidate) || c.Args[0] == path { + t.Fatal("validator did not see complete staged snapshot") } return nil, nil }) - tx := FileTransaction{Config: TransactionConfig{StageDir: filepath.Join(dir, "stage"), Validate: []Command{{Path: "named-checkconf", Args: []string{"{stageDir}/named.conf"}}}}, Runner: runner} - original := []byte("zone \"test.internal\" { type primary; file " + strconv.Quote(zone) + "; };\n") - if err := tx.Apply(context.Background(), map[string][]byte{config: original, zone: []byte("new zone")}); err != nil { + tx := FileTransaction{Config: TransactionConfig{StageDir: filepath.Join(dir, "stage"), Validate: []Command{{Path: "coredns", Args: []string{"{stageDir}/contexts.json"}}}}, Runner: runner} + if err := tx.Apply(context.Background(), map[string][]byte{path: candidate}); err != nil { t.Fatal(err) } - installed, err := os.ReadFile(config) + installed, err := os.ReadFile(path) + if err != nil || !bytes.Equal(installed, candidate) { + t.Fatal("installed snapshot differs", err) + } +} + +func TestActivationGuardsStructuralChangesBeforeInstallingFiles(t *testing.T) { + dir := t.TempDir() + bind := filepath.Join(dir, "bind.conf") + router := filepath.Join(dir, "dnsdist.conf") + for _, path := range []string{bind, router} { + if err := os.WriteFile(path, []byte("approved"), 0o644); err != nil { + t.Fatal(err) + } + } + marker := activationMarkerPath(dir) + runner := &activationBoundaryRunner{t: t, marker: marker, paths: []string{bind, router}} + tx := FileTransaction{MarkerPath: marker, Runner: runner, Config: TransactionConfig{ + StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}, + Reload: []Command{{Path: "reload-bind", WhenChanged: []string{bind}}, {Path: "reload-dnsdist", WhenChanged: []string{router}}}, + }} + if err := tx.Apply(context.Background(), map[string][]byte{bind: []byte("approved"), router: []byte("candidate")}); err != nil { + t.Fatal(err) + } + if err := tx.commitActivation(); err != nil { + t.Fatal(err) + } + if strings.Join(runner.calls, ",") != "gate,reload-bind,reload-dnsdist" { + t.Fatalf("owned daemons were not restarted together: %#v", runner.calls) + } +} + +type activationBoundaryRunner struct { + t *testing.T + marker string + paths []string + calls []string +} + +func (r *activationBoundaryRunner) Run(_ context.Context, command Command) ([]byte, error) { + r.calls = append(r.calls, command.Path) + _, markerErr := os.Stat(r.marker) + if command.Path == "gate" { + if markerErr != nil { + r.t.Fatal("structural gate ran without durable installation guard") + } + for _, path := range r.paths { + data, err := os.ReadFile(path) + if err != nil || string(data) != "approved" { + r.t.Fatal("files changed before DNS was gated") + } + } + } else { + start, retain, release := daemonPermission(filepath.Dir(r.marker)) + defer release() + if !start || !retain { + r.t.Fatal("complete files could not start for validation") + } + } + return nil, nil +} + +type blockedInstallRunner struct { + recordingRunner + block func() error +} + +func (r *blockedInstallRunner) Run(ctx context.Context, command Command) ([]byte, error) { + if command.Path == "gate" && r.block != nil { + if err := r.block(); err != nil { + return nil, err + } + } + return r.recordingRunner.Run(ctx, command) +} + +func TestInterruptedInstallationRequiresLocalRecovery(t *testing.T) { + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) if err != nil { t.Fatal(err) } - if !bytes.Equal(installed, original) { - t.Fatal("installed config retains temporary staging paths") + installTestPublication(t, agent, time.Now().UTC()) + approved := agent.state.Active.Files + badParent := filepath.Join(cfg.StateDir, "not-a-directory") + agent.tx.Runner = &blockedInstallRunner{block: func() error { + return os.WriteFile(badParent, []byte("blocked"), 0o644) + }} + files := map[string][]byte{filepath.Join(badParent, "later.conf"): []byte("candidate")} + for path := range approved { + files[path] = []byte("candidate") + } + if err := agent.tx.Apply(context.Background(), files); err == nil { + t.Fatal("interrupted installation succeeded") + } + if !activationPending(cfg.StateDir) { + t.Fatal("interrupted installation lost its durable guard") + } + if err := agent.tx.abandonActivation(); err != nil { + t.Fatal(err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if activationPending(cfg.StateDir) || !installedFilesMatch(approved) { + t.Fatal("approved configuration could not recover without a broker") + } +} + +func TestContextSnapshotUpdateDoesNotStopDNS(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "contexts.json") + before := []byte(`{"contexts":[{"id":"a","data":"192.0.2.1"}]}`) + after := []byte(`{"contexts":[{"id":"a","data":"192.0.2.2"}]}`) + if err := os.WriteFile(path, before, 0600); err != nil { + t.Fatal(err) + } + runner := &recordingRunner{} + tx := FileTransaction{MarkerPath: activationMarkerPath(dir), Runner: runner, Config: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}, Reload: []Command{{Path: "activate-coredns"}}}} + if err := tx.Apply(context.Background(), map[string][]byte{path: after}); err != nil { + t.Fatal(err) + } + if err := tx.commitActivation(); err != nil { + t.Fatal(err) + } + if len(runner.calls) != 1 || runner.calls[0].Path != "activate-coredns" { + t.Fatal("snapshot update stopped DNS", runner.calls) + } + if !routingConfigurationChanged(filepath.Join(dir, "dnsdist.conf"), before, after) { + t.Fatal("router update did not require a gate") } }