From 0839f10be5090b1f664f19283e7ef6cf37712175 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 12:09:20 -0500 Subject: [PATCH 01/12] feat: Preserve private DNS isolation through serving failures --- internal/internaldns/serving/agent.go | 1578 +++++++++++++++++ internal/internaldns/serving/agent_test.go | 739 ++++++++ internal/internaldns/serving/bind_test.go | 782 ++++++++ .../internaldns/serving/binding_fences.go | 69 + .../serving/binding_fences_test.go | 171 ++ internal/internaldns/serving/bootstrap.go | 146 ++ .../internaldns/serving/bootstrap_test.go | 71 + internal/internaldns/serving/state.go | 210 +++ internal/internaldns/watchdog/watchdog.go | 167 ++ .../internaldns/watchdog/watchdog_test.go | 156 ++ 10 files changed, 4089 insertions(+) create mode 100644 internal/internaldns/serving/agent.go create mode 100644 internal/internaldns/serving/agent_test.go create mode 100644 internal/internaldns/serving/bind_test.go create mode 100644 internal/internaldns/serving/binding_fences.go create mode 100644 internal/internaldns/serving/binding_fences_test.go create mode 100644 internal/internaldns/serving/bootstrap.go create mode 100644 internal/internaldns/serving/bootstrap_test.go create mode 100644 internal/internaldns/serving/state.go create mode 100644 internal/internaldns/watchdog/watchdog.go create mode 100644 internal/internaldns/watchdog/watchdog_test.go diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go new file mode 100644 index 00000000..18c3b9f8 --- /dev/null +++ b/internal/internaldns/serving/agent.go @@ -0,0 +1,1578 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +type Mode string + +const ( + ModeResolver Mode = "resolver" + ModeCombined Mode = "combined" +) + +type Config struct { + Region string `json:"region"` + Shard string `json:"shard"` + MemberID string `json:"memberID"` + ReplicaID string `json:"replicaID,omitempty"` + StateDir string `json:"stateDir"` + Mode Mode `json:"mode"` + Render RenderConfig `json:"render"` + Transaction TransactionConfig `json:"transaction"` + CacheFlush []Command `json:"cacheFlush,omitempty"` + ExpiryInterval time.Duration `json:"expiryInterval,omitempty"` + RetryDelay time.Duration `json:"retryDelay,omitempty"` + AckInterval time.Duration `json:"ackInterval,omitempty"` + AckLease time.Duration `json:"ackLease,omitempty"` + AckPublishTimeout time.Duration `json:"ackPublishTimeout,omitempty"` + LocalServingLease time.Duration `json:"localServingLease,omitempty"` + MaxBindings int `json:"maxBindings,omitempty"` + WatchdogLeasePath string `json:"watchdogLeasePath,omitempty"` + WatchdogLease time.Duration `json:"watchdogLease,omitempty"` + ExternalWatchdog bool `json:"externalWatchdog,omitempty"` + UnsafeAllowNoFailClosed bool `json:"unsafeAllowNoFailClosed,omitempty"` + PublicationDNSProbe PublicationDNSVerifierConfig `json:"publicationDNSProbe,omitempty"` + ResolverDNSProbe DNSVerifierConfig `json:"resolverDNSProbe,omitempty"` + ResolverVerifyConcurrency int `json:"resolverVerifyConcurrency,omitempty"` + UnsafeAllowNoDNSVerify bool `json:"unsafeAllowNoDNSVerify,omitempty"` +} + +func (c Config) Validate() error { + if c.Region == "" || c.Shard == "" || c.MemberID == "" || !filepath.IsAbs(c.StateDir) { + return errors.New("region, shard, member ID, and absolute state directory are required") + } + if c.Mode != ModeResolver && c.Mode != ModeCombined { + return fmt.Errorf("unsupported serving mode %q", c.Mode) + } + if c.Mode == ModeResolver || c.Mode == ModeCombined { + if err := validateRenderConfig(c.Render); err != nil { + return err + } + if !filepath.IsAbs(c.Transaction.StageDir) { + return errors.New("resolver mode requires an absolute transaction stage directory") + } + } + + if c.Mode == ModeResolver && (c.Render.Role == RenderNode || c.Render.Role == RenderBoth) && len(c.CacheFlush) == 0 && !c.UnsafeAllowNoDNSVerify { + return errors.New("node BIND cache withdrawal hooks are required") + } + if c.AckLease > 2*time.Minute { + return errors.New("ack lease may not exceed two minutes") + } + if c.AckPublishTimeout < 0 || c.AckPublishTimeout > 2*time.Second { + return errors.New("ACK publish timeout must be between zero and two seconds") + } + + if c.MaxBindings < 0 { + return errors.New("maximum bindings may not be negative") + } + if c.ResolverVerifyConcurrency < 0 || c.ResolverVerifyConcurrency > 64 { + return errors.New("resolver verification concurrency must be between zero and 64") + } + if !c.UnsafeAllowNoFailClosed && !c.ExternalWatchdog && len(c.Transaction.FailClosed) == 0 { + return errors.New("failClosed command is required; use unsafeAllowNoFailClosed only in tests") + } + if c.WatchdogLeasePath != "" && !filepath.IsAbs(c.WatchdogLeasePath) { + return errors.New("watchdog lease path must be absolute") + } + if c.ExternalWatchdog && c.WatchdogLeasePath == "" { + return errors.New("external watchdog requires watchdogLeasePath") + } + if c.WatchdogLease > 5*time.Second { + return errors.New("watchdog lease may not exceed five seconds") + } + if c.Mode == ModeCombined { + if c.Render.Role != RenderCluster && c.Render.Role != RenderBoth && c.Render.Role != RenderRegionalBIND && c.Render.Role != "" { + return errors.New("combined publication serving requires an owned regional BIND process") + } + if c.PublicationDNSProbe.Server != "" && c.PublicationDNSProbe.Server != net.JoinHostPort(c.Render.ClusterBIND.ListenAddress, strconv.Itoa(int(c.Render.ClusterBIND.Port))) { + return errors.New("publication probe must target this member's protected BIND listener") + } + } + return nil +} + +type EventSource interface { + Next(context.Context) (transport.Message, error) +} + +type BootstrapSource interface { + Load(context.Context) ([]model.Envelope, error) +} + +type Verifier interface { + Verify(context.Context, model.Binding) error +} + +type Dependencies struct { + ServingSource EventSource + RecordSource EventSource + AckSource EventSource + Bootstrap BootstrapSource + AckPublisher transport.ExportPublisher + Runner Runner + Verifier Verifier + Clock func() time.Time + PublicationVerifier PublicationVerifier + OnError func(error) +} + +type ackPublishResult struct { + ack transport.PublishAck + err error +} + +// boundedAckPublisher isolates local serving safety from a transport client +// blocked before it observes the publish context (for example, nats.Conn's +// connection-state mutex during a reconnect dial). There is no queue: at most +// one underlying call exists, and later state-derived ACKs expire at their +// caller deadline until that call returns. The next reconciliation publishes +// current state, naturally superseding intermediate heartbeats. +type boundedAckPublisher struct { + delegate transport.ExportPublisher + inflight chan struct{} +} + +func newBoundedAckPublisher(delegate transport.ExportPublisher) transport.ExportPublisher { + if delegate == nil { + return nil + } + if _, ok := delegate.(*boundedAckPublisher); ok { + return delegate + } + return &boundedAckPublisher{delegate: delegate, inflight: make(chan struct{}, 1)} +} + +func (p *boundedAckPublisher) Publish(ctx context.Context, subject string, env model.Envelope) (transport.PublishAck, error) { + if err := ctx.Err(); err != nil { + return transport.PublishAck{}, err + } + select { + case p.inflight <- struct{}{}: + case <-ctx.Done(): + return transport.PublishAck{}, ctx.Err() + } + if err := ctx.Err(); err != nil { + <-p.inflight + return transport.PublishAck{}, err + } + result := make(chan ackPublishResult, 1) + go func() { + ack, err := p.delegate.Publish(ctx, subject, env) + <-p.inflight + result <- ackPublishResult{ack: ack, err: err} + }() + select { + case result := <-result: + return result.ack, result.err + case <-ctx.Done(): + return transport.PublishAck{}, ctx.Err() + } +} + +type Agent struct { + cfg Config + deps Dependencies + store stateStore + tx FileTransaction + publicationVerifier PublicationVerifier + mu sync.Mutex + state checkpoint + needsActivation bool + lastServingAck map[string]servingAckState + lastPublicationAck map[string]time.Time + installedPublications map[string]installedPublication +} + +// This identity describes the exact files successfully activated, not a +// fingerprint recomputed later when an observation deadline may have passed. +type installedPublication struct { + Identity string + EffectiveHash string + Views []PublicationView +} + +func publicationIdentity(p publicationState) string { + data, _ := json.Marshal(p.Plan) + return fmt.Sprintf("%d/%d/%d/%s", p.Fence.Epoch, p.Fence.Revision, p.LocalRevision, model.Hash(data)) +} + +type servingAckState struct { + phase model.AckPhase + at time.Time +} + +func NewAgent(c Config, d Dependencies) (*Agent, error) { + if err := c.Validate(); err != nil { + return nil, err + } + if d.Clock == nil { + d.Clock = time.Now + } + if d.Runner == nil { + d.Runner = ExecRunner{} + } + d.AckPublisher = newBoundedAckPublisher(d.AckPublisher) + if (c.Mode == ModeResolver || c.Mode == ModeCombined) && c.Render.Role != RenderRegionalBIND && d.Verifier == nil && !c.UnsafeAllowNoDNSVerify { + probeConfig := c.ResolverDNSProbe + if c.Render.Role == RenderCluster || c.Render.Role == RenderRegionalDNSDist { + probeConfig.UseClusterAddress = true + } + d.Verifier = DNSVerifier{Config: probeConfig} + } + publicationVerifier := d.PublicationVerifier + if c.Mode == ModeCombined { + if publicationVerifier == nil && c.PublicationDNSProbe.Server != "" { + publicationVerifier = PublicationDNSVerifier{Config: c.PublicationDNSProbe} + } + if publicationVerifier == nil && !c.UnsafeAllowNoDNSVerify { + return nil, errors.New("regional materialization DNS query verifier is required before readiness ACK") + } + } + store := stateStore{path: filepath.Join(c.StateDir, "checkpoint.json")} + state, err := store.Load() + if err != nil { + return nil, err + } + return &Agent{cfg: c, deps: d, store: store, tx: FileTransaction{Config: c.Transaction, Runner: d.Runner}, publicationVerifier: publicationVerifier, state: state, lastServingAck: map[string]servingAckState{}, lastPublicationAck: map[string]time.Time{}, installedPublications: map[string]installedPublication{}}, nil +} + +func (a *Agent) Start(ctx context.Context) error { + defer func() { _ = a.tx.FailClosed(context.Background()) }() + // A restarted regional materialization must enforce deadlines persisted in its local + // checkpoint before waiting on any remote service. The watchdog lease is not + // renewed here, so an inconsistent or unavailable bootstrap stays gated. + if err := a.expirePersisted(ctx); err != nil { + a.report(fmt.Errorf("enforce persisted expiry at startup: %w", err)) + } + if !a.gateUntilClosed(ctx) { + return nil + } + // FailClosed may stop or firewall the DNS daemon. A persisted content hash + // proves only what was installed before restart, not that the live process + // still serves it, so force one validated apply before renewing the lease. + a.mu.Lock() + a.needsActivation = true + a.mu.Unlock() + for { + err := a.bootstrapAndStep(ctx) + if err == nil { + break + } + if ctx.Err() != nil { + return nil + } + a.report(err) + if !a.gateUntilClosed(ctx) { + return nil + } + retry := a.cfg.RetryDelay + if retry <= 0 { + retry = time.Second + } + if !waitRetry(ctx, retry) { + return nil + } + } + errCh := make(chan error, 3) + start := func(source EventSource) { + if source == nil { + return + } + go func() { errCh <- a.consume(ctx, source) }() + } + start(a.deps.ServingSource) + start(a.deps.RecordSource) + if a.cfg.Mode == ModeResolver || a.cfg.Mode == ModeCombined { + start(a.deps.AckSource) + } + interval := a.cfg.ExpiryInterval + if interval <= 0 { + interval = time.Second + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case err := <-errCh: + if err != nil && !errors.Is(err, context.Canceled) { + a.report(err) + } + case <-ticker.C: + if err := a.Step(ctx); err != nil { + a.report(err) + } + } + } +} + +func (a *Agent) failClosed(ctx context.Context) error { + a.needsActivation = true + // Invalidate independent watchdog readiness immediately, even if the daemon + // gate command itself fails. It must never inherit a prior healthy lease. + if a.cfg.WatchdogLeasePath != "" { + _ = os.Remove(a.cfg.WatchdogLeasePath) + } + return a.tx.FailClosed(ctx) +} + +func (a *Agent) gateUntilClosed(ctx context.Context) bool { + retry := a.cfg.RetryDelay + if retry <= 0 { + retry = time.Second + } + for { + if err := a.failClosed(ctx); err == nil { + return true + } else { + a.report(fmt.Errorf("gate serving during startup: %w", err)) + } + if !waitRetry(ctx, retry) { + return false + } + } +} + +func (a *Agent) expirePersisted(ctx context.Context) error { + if a.cfg.Mode != ModeCombined { + return nil + } + a.mu.Lock() + defer a.mu.Unlock() + return a.expireRecords(ctx) +} + +func (a *Agent) bootstrapAndStep(ctx context.Context) error { + if a.deps.Bootstrap != nil { + envs, err := a.deps.Bootstrap.Load(ctx) + if err != nil { + return fmt.Errorf("load bootstrap snapshot: %w", err) + } + sort.SliceStable(envs, func(i, j int) bool { + order := func(k model.Kind) int { + switch k { + case model.KindPublicationChunk: + return 0 + case model.KindServingSnapshot: + // Context destination mappings are required to query-proof a + // publication before its manifest can be acknowledged. + return 1 + default: + return 2 + } + } + return order(envs[i].Kind) < order(envs[j].Kind) + }) + for _, env := range envs { + if err := a.Handle(ctx, env); err != nil { + return fmt.Errorf("apply bootstrap %s: %w", env.Kind, err) + } + } + } + if err := a.Step(ctx); err != nil { + return fmt.Errorf("initial serving reconciliation: %w", err) + } + return nil +} + +func (a *Agent) consume(ctx context.Context, source EventSource) error { + retry := a.cfg.RetryDelay + if retry <= 0 { + retry = time.Second + } + for { + msg, err := source.Next(ctx) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + a.report(err) + if !waitRetry(ctx, retry) { + return ctx.Err() + } + continue + } + err = a.Handle(ctx, msg.Envelope()) + if err != nil { + env := msg.Envelope() + a.report(fmt.Errorf("apply %s %s at epoch %d revision %d: %w", env.Kind, env.ResourceUID, env.Epoch, env.Revision, err)) + _ = msg.Retry(retry) + continue + } + if err := msg.Ack(ctx); err != nil { + a.report(err) + _ = msg.Retry(retry) + if !waitRetry(ctx, retry) { + return ctx.Err() + } + } + } +} + +func (a *Agent) report(err error) { + if err != nil && a.deps.OnError != nil { + a.deps.OnError(err) + } +} + +func waitRetry(ctx context.Context, d time.Duration) bool { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func (a *Agent) Handle(ctx context.Context, env model.Envelope) error { + if err := env.Validate(); err != nil { + return err + } + if env.Region != a.cfg.Region || env.Shard != a.cfg.Shard { + return fmt.Errorf("event targets %s/%s, agent serves %s/%s", env.Region, env.Shard, a.cfg.Region, a.cfg.Shard) + } + a.mu.Lock() + defer a.mu.Unlock() + switch env.Kind { + case model.KindServingSnapshot: + return a.handleServing(ctx, env) + case model.KindPublicationChunk: + + return a.handleChunk(env) + case model.KindPublicationManifest: + + if err := a.handleManifest(ctx, env); err != nil { + _ = a.failClosed(ctx) + return err + } + return a.finishEvent(ctx) + case model.KindMemberAck: + + return a.handleAck(ctx, env) + default: + return fmt.Errorf("unsupported event kind %q", env.Kind) + } +} + +func (a *Agent) handleServing(ctx context.Context, env model.Envelope) error { + var snapshot model.ServingSnapshot + if err := json.Unmarshal(env.Payload, &snapshot); err != nil { + return err + } + if err := snapshot.Validate(); err != nil { + return err + } + maximum := a.cfg.MaxBindings + if maximum == 0 { + maximum = 4096 + } + if len(snapshot.Bindings) > maximum { + return fmt.Errorf("serving snapshot has %d bindings, exceeding member limit %d", len(snapshot.Bindings), maximum) + } + if snapshot.Region != env.Region || snapshot.Shard != env.Shard || snapshot.ConfigurationEpoch != env.Epoch || snapshot.ConfigurationRevision != env.Revision { + return errors.New("serving envelope identity does not match payload") + } + cmp := model.Compare(env.Epoch, env.Revision, a.state.ServingFence.Epoch, a.state.ServingFence.Revision) + if cmp < 0 { + return nil + } + if cmp == 0 && a.state.Snapshot != nil { + if !equalJSON(snapshot, *a.state.Snapshot) { + return errors.New("conflicting serving snapshot at the same epoch and revision") + } + return a.applyServingSnapshot(ctx, snapshot, false) + } + return a.applyServingSnapshot(ctx, snapshot, true) +} + +func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.ServingSnapshot, advance bool) error { + if advance { + bindingFences, err := advanceBindingFences(a.state.BindingFences, snapshot) + if err != nil { + return err + } + a.state.BindingFences = bindingFences + a.state.ServingFence = fence{snapshot.ConfigurationEpoch, snapshot.ConfigurationRevision} + a.state.Snapshot = &snapshot + if err := a.store.Save(a.state); err != nil { + return err + } + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.finishEvent(ctx) +} + +func (a *Agent) handleChunk(env model.Envelope) error { + var c model.PublicationChunk + if err := json.Unmarshal(env.Payload, &c); err != nil { + return err + } + if err := c.Validate(); err != nil { + return err + } + if c.ZoneUID != env.ResourceUID || c.WriterEpoch != env.Epoch || c.Revision != env.Revision { + return errors.New("chunk envelope identity does not match payload") + } + current := a.state.Publications[c.ZoneUID].Fence + if model.Compare(c.WriterEpoch, c.Revision, current.Epoch, current.Revision) < 0 { + return nil + } + if a.state.Chunks[c.ManifestUID] == nil { + a.state.Chunks[c.ManifestUID] = map[string]model.PublicationChunk{} + } + key := chunkKey(c.Index) + if old, ok := a.state.Chunks[c.ManifestUID][key]; ok && old.SHA256 != c.SHA256 { + return errors.New("conflicting chunk at the same manifest index") + } + a.state.Chunks[c.ManifestUID][key] = c + return a.store.Save(a.state) +} + +func (a *Agent) handleManifest(ctx context.Context, env model.Envelope) error { + var m model.PublicationManifest + if err := json.Unmarshal(env.Payload, &m); err != nil { + return err + } + if err := m.Validate(); err != nil { + return err + } + if m.ZoneUID != env.ResourceUID || m.WriterEpoch != env.Epoch || m.Revision != env.Revision { + return errors.New("manifest envelope identity does not match payload") + } + old := a.state.Publications[m.ZoneUID] + cmp := model.Compare(m.WriterEpoch, m.Revision, old.Fence.Epoch, old.Fence.Revision) + if cmp < 0 { + return nil + } + if cmp == 0 && old.Fence.Revision > 0 && !equalJSON(m, old.Manifest) { + return errors.New("conflicting publication manifest at the same epoch and revision") + } + if cmp == 0 && old.Verified { + if a.cfg.Mode == ModeResolver { + return a.recordResolverPublication(ctx, m.ZoneUID, old, old) + } + if !equalJSON(m, old.Manifest) { + return errors.New("conflicting publication manifest at the same epoch and revision") + } + if err := a.verifyPublication(ctx, old); err != nil { + _ = a.emitAck(ctx, model.KindPublicationManifest, m.ZoneUID, m.WriterEpoch, m.Revision, model.AckRejected, time.Time{}, err.Error()) + return err + } + for _, name := range publicationNames(old.Plan) { + if err := a.flushName(ctx, name); err != nil { + return err + } + } + return a.emitVerifiedPublicationAck(ctx, m.ZoneUID, m.WriterEpoch, m.Revision) + } + // Publications are complete snapshots. The control plane reserves a revision + // before staging immutable artifacts, so a failed or superseded staging pass + // can legitimately leave a gap. Accept a snapshot compiled from a base at or + // ahead of the locally applied revision, while rejecting a newer revision + // compiled from state older than the regional materialization has already committed. + if cmp > 0 && old.Fence.Epoch == m.WriterEpoch && old.Fence.Revision > 0 && m.PreviousRevision < old.Fence.Revision { + return fmt.Errorf("publication revision %d is based on revision %d older than applied revision %d", m.Revision, m.PreviousRevision, old.Fence.Revision) + } + if m.Tombstone { + return a.applyTombstone(ctx, m, old) + } + chunkMap := a.state.Chunks[m.ManifestUID] + chunks := make([]model.PublicationChunk, 0, len(chunkMap)) + for _, c := range chunkMap { + chunks = append(chunks, c) + } + payload, err := model.VerifyManifest(m, chunks) + if err != nil { + return err + } + var plan model.PublicationPlan + if err := json.Unmarshal(payload, &plan); err != nil { + return err + } + if err := plan.Validate(); err != nil { + return err + } + if plan.ZoneUID != m.ZoneUID || model.AbsoluteName(plan.Apex) != model.AbsoluteName(m.Apex) { + return errors.New("publication plan identity does not match manifest") + } + if err := a.validateContributionFences(plan); err != nil { + return err + } + // Persist every accepted observation high-water mark before mutating the + // authoritative server. If the process dies during materialization, restart + // must still reject an older healthy observation that could resurrect data + // withdrawn by this plan. An identical retry remains idempotent. + a.commitContributionFences(plan) + if err := a.store.Save(a.state); err != nil { + return err + } + next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: plan, EffectiveHash: effectivePlanHash(plan, a.deps.Clock())} + if a.cfg.Mode == ModeResolver { + return a.recordResolverPublication(ctx, m.ZoneUID, next, old) + } + return a.activatePublication(ctx, m.ZoneUID, next, old) +} + +func (a *Agent) applyTombstone(ctx context.Context, m model.PublicationManifest, old publicationState) error { + next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: old.Plan} + if a.cfg.Mode == ModeResolver { + return a.recordResolverPublication(ctx, m.ZoneUID, next, old) + } + return a.activatePublication(ctx, m.ZoneUID, next, old) +} + +// Commit the immutable fence before installing files. Failed activation stays +// gated and an identical redelivery retries the accepted snapshot. +func (a *Agent) recordResolverPublication(ctx context.Context, uid string, next, old publicationState) error { + next.Verified = false + a.state.Publications[uid] = next + if err := a.store.Save(a.state); err != nil { + return err + } + for _, name := range append(publicationNames(old.Plan), publicationNames(next.Plan)...) { + if err := a.flushName(ctx, name); err != nil { + _ = a.failClosed(ctx) + return err + } + } + next.Verified = true + a.state.Publications[uid] = next + return a.store.Save(a.state) +} + +func (a *Agent) activatePublication(ctx context.Context, uid string, next, old publicationState) error { + a.state.Publications[uid] = next + if err := a.store.Save(a.state); err != nil { + return err + } + if err := a.reconcile(ctx); err != nil { + return err + } + if !next.Manifest.Tombstone && len(a.publicationViews(next)) == 0 { + return nil + } + if err := a.verifyPublication(ctx, next); err != nil { + _ = a.failClosed(ctx) + return err + } + for _, name := range append(publicationNames(old.Plan), publicationNames(next.Plan)...) { + if err := a.flushName(ctx, name); err != nil { + _ = a.failClosed(ctx) + return err + } + } + next.Verified = true + a.state.Publications[uid] = next + if err := a.store.Save(a.state); err != nil { + return err + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.emitVerifiedPublicationAck(ctx, uid, next.Fence.Epoch, next.Fence.Revision) +} + +func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { + var ack model.MemberAck + if err := json.Unmarshal(env.Payload, &ack); err != nil { + return err + } + if ack.ResourceUID != env.ResourceUID || ack.Epoch != env.Epoch || ack.Revision != env.Revision { + return errors.New("ack envelope identity does not match payload") + } + if ack.ValidUntil.IsZero() || ack.ObservedAt.IsZero() || ack.ValidUntil.After(ack.ObservedAt.Add(2*time.Minute)) { + return errors.New("ack has an invalid serving lease") + } + if a.state.Snapshot == nil { + return nil + } + known := false + for _, m := range a.state.Snapshot.Members { + if m.MemberID == ack.MemberID && m.ReplicaID == ack.ReplicaID { + known = true + break + } + } + if !known { + return fmt.Errorf("ack from unknown member %q replica %q", ack.MemberID, ack.ReplicaID) + } + // A durable ACK consumer can legitimately replay a once-valid lease after a + // backlog. Drain it without changing readiness or renewing the local lease; + // retrying an already expired ACK can otherwise create a permanent NACK loop. + if !a.deps.Clock().Before(ack.ValidUntil) { + return nil + } + if ack.Kind != model.KindPublicationManifest { + return nil + } + if ack.Phase == model.AckExpired { + if err := a.flushPublicationZone(ctx, ack.ResourceUID); err != nil { + return err + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.finishEvent(ctx) + } + if ack.Phase != model.AckVerified { + if ack.Phase != model.AckRejected && ack.Phase != model.AckApplied { + return errors.New("unsupported publication ACK phase") + } + if a.state.Acks[ack.ResourceUID] == nil { + a.state.Acks[ack.ResourceUID] = map[string]replicaAck{} + } + key := ack.MemberID + "/" + ack.ReplicaID + old := a.state.Acks[ack.ResourceUID][key] + cmp := model.Compare(ack.Epoch, ack.Revision, old.Fence.Epoch, old.Fence.Revision) + if cmp < 0 || (cmp == 0 && !ack.ObservedAt.After(old.ObservedAt)) { + return nil + } + a.state.Acks[ack.ResourceUID][key] = replicaAck{Fence: fence{ack.Epoch, ack.Revision}, ObservedAt: ack.ObservedAt} + if err := a.store.Save(a.state); err != nil { + return err + } + if err := a.flushPublicationZone(ctx, ack.ResourceUID); err != nil { + _ = a.failClosed(ctx) + return err + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.finishEvent(ctx) + } + if a.state.Acks[ack.ResourceUID] == nil { + a.state.Acks[ack.ResourceUID] = map[string]replicaAck{} + } + key := ack.MemberID + "/" + ack.ReplicaID + old := a.state.Acks[ack.ResourceUID][key] + comparison := model.Compare(ack.Epoch, ack.Revision, old.Fence.Epoch, old.Fence.Revision) + if comparison < 0 || (comparison == 0 && !ack.ObservedAt.After(old.ObservedAt)) { + return nil + } + highest := fence{} + for _, existing := range a.state.Acks[ack.ResourceUID] { + if model.Compare(existing.Fence.Epoch, existing.Fence.Revision, highest.Epoch, highest.Revision) > 0 { + highest = existing.Fence + } + } + if model.Compare(ack.Epoch, ack.Revision, highest.Epoch, highest.Revision) > 0 { + if err := a.flushPublicationZone(ctx, ack.ResourceUID); err != nil { + return err + } + } + if comparison > 0 || (ack.Epoch == old.Fence.Epoch && ack.Revision == old.Fence.Revision && ack.ValidUntil.After(old.ValidUntil)) { + a.state.Acks[ack.ResourceUID][key] = replicaAck{Fence: fence{ack.Epoch, ack.Revision}, ValidUntil: ack.ValidUntil, ObservedAt: ack.ObservedAt} + if err := a.store.Save(a.state); err != nil { + return err + } + } + if err := a.reconcile(ctx); err != nil { + return err + } + return a.finishEvent(ctx) +} + +func (a *Agent) Step(ctx context.Context) error { + a.mu.Lock() + defer a.mu.Unlock() + if a.cfg.Mode == ModeResolver { + if err := a.retryPendingPublications(ctx); err != nil { + _ = a.failClosed(ctx) + return err + } + if err := a.expireResolverCache(ctx); err != nil { + _ = a.failClosed(ctx) + return err + } + } + if a.cfg.Mode == ModeCombined { + if err := a.retryPendingPublications(ctx); err != nil { + _ = a.failClosed(ctx) + return err + } + if err := a.expireRecords(ctx); err != nil { + _ = a.failClosed(ctx) + return err + } + if a.needsActivation || len(a.installedPublications) == 0 { + if err := a.reconcile(ctx); err != nil { + return err + } + } + if err := a.heartbeatPublications(ctx); err != nil { + return err + } + } + if err := a.reconcile(ctx); err != nil { + return err + } + if a.cfg.Mode == ModeCombined { + if err := a.expireRecords(ctx); err != nil { + _ = a.failClosed(ctx) + return err + } + } + return a.renewSafeWatchdogLease(ctx) +} + +// finishEvent gives event-driven reconciliation the same regional materialization safety +// gate as the periodic Step path before extending the independent watchdog +// lease. This matters when a sustained JetStream backlog repeatedly acquires +// the agent lock and delays the ticker. +func (a *Agent) finishEvent(ctx context.Context) error { + if a.cfg.Mode == ModeResolver { + if err := a.expireResolverCache(ctx); err != nil { + _ = a.failClosed(ctx) + return err + } + } + if a.cfg.Mode == ModeCombined { + if err := a.expireRecords(ctx); err != nil { + _ = a.failClosed(ctx) + return err + } + if err := a.heartbeatPublications(ctx); err != nil { + return err + } + } + return a.renewSafeWatchdogLease(ctx) +} + +func (a *Agent) reconcile(ctx context.Context) error { + if a.state.Snapshot == nil { + return nil + } + + now := a.deps.Clock() + active := map[string]bool{} + filtered := *a.state.Snapshot + filtered.Bindings = nil + for _, b := range a.state.Snapshot.Bindings { + eligible := !b.Tombstone && now.Before(b.Authorization.ValidUntil) && a.publicationsReady(b) + if eligible { + active[b.BindingUID] = true + } + if !b.Tombstone && now.Before(b.Authorization.ValidUntil) { + lease := a.cfg.LocalServingLease + if lease <= 0 { + lease = 90 * time.Second + } + horizon := now.Add(lease) + quantum := a.cfg.AckInterval + if quantum <= 0 { + quantum = 30 * time.Second + } + horizon = horizon.Truncate(quantum).Add(quantum) + if horizon.Before(b.Authorization.ValidUntil) { + b.Authorization.ValidUntil = horizon + } + filtered.Bindings = append(filtered.Bindings, b) + } + } + renderConfig := a.cfg.Render + renderConfig.ReadyMembers = map[string]map[string]bool{} + for _, binding := range filtered.Bindings { + renderConfig.ReadyMembers[binding.BindingUID] = a.eligibleRegionalMembers(binding) + } + rendered, err := renderPublications(renderConfig, filtered, active, a.state.Publications, now) + if err != nil { + return err + } + hash := hashFiles(rendered.Files) + if hash != a.state.RenderedHash || a.needsActivation { + var err error + if a.needsActivation { + err = a.tx.ApplyForce(ctx, rendered.Files) + } else { + err = a.tx.Apply(ctx, rendered.Files) + } + if err != nil { + _ = a.failClosed(ctx) + return err + } + a.state.RenderedHash = hash + if err := a.store.Save(a.state); err != nil { + return err + } + a.needsActivation = false + } + if a.cfg.Mode == ModeCombined { + installed := map[string]installedPublication{} + for uid, p := range a.state.Publications { + installed[uid] = installedPublication{Identity: publicationIdentity(p), EffectiveHash: effectivePlanHash(p.Plan, now), Views: a.publicationViewsAt(p, now)} + } + a.installedPublications = installed + for _, p := range a.state.Publications { + if p.Verified && !p.Manifest.Tombstone && len(a.publicationViews(p)) > 0 { + if err := a.verifyPublication(ctx, p); err != nil { + _ = a.failClosed(ctx) + a.needsActivation = true + return err + } + } + } + } + // End-to-end verification must run after the candidate mapping is live. + // Any binding that fails either UDP or TCP is immediately removed again. + probeFailed := a.verifyResolverBindings(ctx, filtered.Bindings, active) + if probeFailed { + gated, err := renderPublications(renderConfig, filtered, active, a.state.Publications, now) + if err != nil { + return err + } + gatedHash := hashFiles(gated.Files) + if gatedHash != a.state.RenderedHash { + if err := a.tx.Apply(ctx, gated.Files); err != nil { + _ = a.failClosed(ctx) + return err + } + a.state.RenderedHash = gatedHash + if err := a.store.Save(a.state); err != nil { + return err + } + } + } + // Resolver health is a local safety decision. Once every eligible binding + // has passed its end-to-end proof (and every failed binding has been + // durably re-rendered gated), renew the independent watchdog before any + // best-effort broker ACK. A NATS partition must expire remote readiness + // leases, but it must not consume the local five-second watchdog window for + // a dataplane that has just been proved healthy. Combined mode renews later, + // after its regional materialization expiry and health checks also complete. + if a.cfg.Mode == ModeResolver { + if err := a.renewSafeWatchdogLease(ctx); err != nil { + return err + } + } + for _, b := range a.state.Snapshot.Bindings { + if b.Tombstone { + continue + } + p := model.AckVerified + if !active[b.BindingUID] { + p = model.AckApplied + if !now.Before(b.Authorization.ValidUntil) { + p = model.AckExpired + } + } + if err := a.emitServingAck(ctx, b, p); err != nil { + a.report(fmt.Errorf("publish serving ACK %s: %w", b.BindingUID, err)) + break + } + } + return nil +} + +type resolverProbeResult struct { + bindingUID string + duration time.Duration + err error +} + +// verifyResolverBindings probes a shard in a bounded worker pool. All probes +// share one deadline so adding VPCs cannot make reconciliation grow by one +// complete probe timeout per binding. +func (a *Agent) verifyResolverBindings(ctx context.Context, bindings []model.Binding, active map[string]bool) bool { + if a.deps.Verifier == nil { + return false + } + eligible := make([]model.Binding, 0, len(bindings)) + for _, binding := range bindings { + if active[binding.BindingUID] { + eligible = append(eligible, binding) + } + } + if len(eligible) == 0 { + return false + } + + timeout := a.cfg.ResolverDNSProbe.Timeout + if timeout <= 0 { + timeout = 3 * time.Second + } + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + concurrency := a.cfg.ResolverVerifyConcurrency + if concurrency == 0 { + concurrency = 16 + } + concurrency = min(concurrency, len(eligible)) + jobs := make(chan model.Binding) + results := make(chan resolverProbeResult, len(eligible)) + for range concurrency { + go func() { + for binding := range jobs { + started := time.Now() + err := a.deps.Verifier.Verify(probeCtx, binding) + results <- resolverProbeResult{bindingUID: binding.BindingUID, duration: time.Since(started), err: err} + } + }() + } + go func() { + defer close(jobs) + for _, binding := range eligible { + select { + case jobs <- binding: + case <-probeCtx.Done(): + return + } + } + }() + + pending := make(map[string]bool, len(eligible)) + for _, binding := range eligible { + pending[binding.BindingUID] = true + } + probeFailed := false + for len(pending) > 0 { + select { + case result := <-results: + if !pending[result.bindingUID] { + continue + } + delete(pending, result.bindingUID) + if result.err != nil { + a.report(fmt.Errorf("verify resolver binding %s after %s: %w", result.bindingUID, result.duration.Round(time.Millisecond), result.err)) + delete(active, result.bindingUID) + probeFailed = true + } + case <-probeCtx.Done(): + for bindingUID := range pending { + a.report(fmt.Errorf("verify resolver binding %s within shared %s batch deadline: %w", bindingUID, timeout, probeCtx.Err())) + delete(active, bindingUID) + } + return true + } + } + return probeFailed +} + +func (a *Agent) publicationsReady(b model.Binding) bool { + if len(b.Zones) == 0 { + return true + } + if a.cfg.Mode == ModeCombined { + for _, z := range b.Zones { + p, ok := a.state.Publications[z.ZoneUID] + if !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + return false + } + } + return true + } + return len(a.eligibleRegionalMembers(b)) > 0 +} + +// Every selected member must prove every attached zone at the exact publication +// plan version this resolver holds. One zone per different member is unsafe. +func (a *Agent) eligibleRegionalMembers(b model.Binding) map[string]bool { + out := map[string]bool{} + if a.state.Snapshot == nil { + return out + } + if a.cfg.Mode == ModeCombined { + if a.publicationsReady(b) { + out[a.cfg.MemberID] = true + } + return out + } + for _, member := range a.state.Snapshot.Members { + if member.Role != "regional" && member.Role != "cluster" { + continue + } + ready := true + for _, z := range b.Zones { + p, ok := a.state.Publications[z.ZoneUID] + if !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + ready = false + break + } + for _, known := range a.state.Acks[z.ZoneUID] { + if model.Compare(known.Fence.Epoch, known.Fence.Revision, p.Fence.Epoch, p.Fence.Revision) > 0 { + ready = false + } + } + proof := a.state.Acks[z.ZoneUID][member.MemberID+"/"+member.ReplicaID] + if !a.deps.Clock().Before(proof.ValidUntil) || proof.Fence != p.Fence { + ready = false + break + } + } + if ready { + out[member.MemberID] = true + } + } + return out +} + +func (a *Agent) verifyPublication(ctx context.Context, p publicationState) error { + if p.Manifest.Tombstone { + return nil + } + if a.publicationVerifier != nil { + if len(a.publicationViews(p)) == 0 { + return nil + } + installed, ok := a.installedPublications[p.Plan.ZoneUID] + if !ok || installed.Identity != publicationIdentity(p) { + return errors.New("publication has no matching activated file identity") + } + return a.publicationVerifier.VerifyPublication(ctx, p.Plan, installed.Views) + } + return nil +} + +func (a *Agent) publicationViews(p publicationState) []PublicationView { + return a.publicationViewsAt(p, a.deps.Clock()) +} + +func (a *Agent) publicationViewsAt(p publicationState, now time.Time) []PublicationView { + if a.state.Snapshot == nil { + return nil + } + var out []PublicationView + for _, b := range a.state.Snapshot.Bindings { + if b.Tombstone || !now.Before(b.Authorization.ValidUntil) { + continue + } + for _, z := range b.Zones { + if z.ZoneUID == p.Plan.ZoneUID && model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) >= 0 { + out = append(out, PublicationView{View: b.ViewName(), DestinationAddress: b.ClusterAddress, DestinationPort: b.Port, Serial: publicationSerial(p, now), FingerprintMailbox: publicationMailbox(p, now)}) + } + } + } + return out +} + +func (a *Agent) retryPendingPublications(ctx context.Context) error { + for uid, p := range a.state.Publications { + if p.Verified { + continue + } + if a.cfg.Mode == ModeResolver { + if err := a.recordResolverPublication(ctx, uid, p, p); err != nil { + return err + } + continue + } + if !p.Manifest.Tombstone && len(a.publicationViews(p)) == 0 { + continue + } + if err := a.reconcile(ctx); err != nil { + return err + } + if err := a.verifyPublication(ctx, p); err != nil { + return err + } + for _, name := range publicationNames(p.Plan) { + if err := a.flushName(ctx, name); err != nil { + return err + } + } + p.Verified = true + p.EffectiveHash = effectivePlanHash(p.Plan, a.deps.Clock()) + a.state.Publications[uid] = p + if err := a.store.Save(a.state); err != nil { + return err + } + } + return nil +} + +// A node retains the publication deadlines so a broker outage cannot leave +// healthy records in its positive cache after the regional member withdraws. +func (a *Agent) expireResolverCache(ctx context.Context) error { + now := a.deps.Clock() + for uid, p := range a.state.Publications { + if p.Manifest.Tombstone { + continue + } + hash := effectivePlanHash(p.Plan, now) + if hash == p.EffectiveHash { + continue + } + for _, name := range publicationNames(p.Plan) { + if err := a.flushName(ctx, name); err != nil { + return err + } + } + p.EffectiveHash = hash + p.LocalRevision++ + a.state.Publications[uid] = p + if err := a.store.Save(a.state); err != nil { + return err + } + } + return nil +} + +func (a *Agent) expireRecords(ctx context.Context) error { + now := a.deps.Clock() + for uid, p := range a.state.Publications { + if !p.Verified || p.Manifest.Tombstone { + continue + } + hash := effectivePlanHash(p.Plan, now) + if hash == p.EffectiveHash { + continue + } + p.LocalRevision++ + p.EffectiveHash = hash + p.Verified = false + a.state.Publications[uid] = p + if err := a.store.Save(a.state); err != nil { + return err + } + if err := a.reconcile(ctx); err != nil { + return err + } + if len(a.publicationViews(p)) == 0 { + continue + } + if err := a.verifyPublication(ctx, p); err != nil { + return err + } + for _, name := range publicationNames(p.Plan) { + if err := a.flushName(ctx, name); err != nil { + return err + } + } + p.Verified = true + a.state.Publications[uid] = p + if err := a.store.Save(a.state); err != nil { + return err + } + if err := a.reconcile(ctx); err != nil { + return err + } + if err := a.emitAck(ctx, model.KindPublicationManifest, uid, p.Fence.Epoch, p.Fence.Revision, model.AckExpired, time.Time{}, ""); err != nil { + a.report(err) + } + } + return nil +} + +func (a *Agent) flushName(ctx context.Context, name string) error { + + for _, command := range a.cfg.CacheFlush { + command.Args = append([]string(nil), command.Args...) + for i := range command.Args { + command.Args[i] = strings.ReplaceAll(command.Args[i], "{name}", model.AbsoluteName(name)) + } + if _, err := a.deps.Runner.Run(ctx, command); err != nil { + return err + } + } + return nil +} + +func (a *Agent) flushPublicationZone(ctx context.Context, zoneUID string) error { + if a.state.Snapshot == nil { + return nil + } + seen := map[string]bool{} + for _, binding := range a.state.Snapshot.Bindings { + for _, zone := range binding.Zones { + if zone.ZoneUID == zoneUID && !seen[zone.Apex] { + seen[zone.Apex] = true + if err := a.flushName(ctx, zone.Apex); err != nil { + return err + } + } + } + } + return nil +} + +func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, revision uint64, phase model.AckPhase, validUntil time.Time, message string) error { + if a.deps.AckPublisher == nil { + return nil + } + now := a.deps.Clock().UTC() + lease := a.cfg.AckLease + if lease <= 0 { + lease = 90 * time.Second + } + leaseUntil := now.Add(lease) + if !validUntil.IsZero() && validUntil.Before(leaseUntil) { + leaseUntil = validUntil + } + ack := model.MemberAck{MemberID: a.cfg.MemberID, ReplicaID: a.cfg.ReplicaID, ResourceUID: uid, Kind: kind, Epoch: epoch, Revision: revision, Phase: phase, ObservedAt: now, ValidUntil: leaseUntil, Error: message} + eventID := fmt.Sprintf("ack-%s-%s-%d-%d-%s-%d", model.OpaqueToken(a.cfg.MemberID+"/"+a.cfg.ReplicaID), model.OpaqueToken(uid), epoch, revision, phase, now.UnixNano()) + env, err := model.NewEnvelope(model.KindMemberAck, eventID, a.cfg.Region, a.cfg.Shard, uid, epoch, revision, a.deps.Clock(), ack) + if err != nil { + return err + } + publishTimeout := a.cfg.AckPublishTimeout + if publishTimeout <= 0 { + publishTimeout = 500 * time.Millisecond + } + publishCtx, cancel := context.WithTimeout(ctx, publishTimeout) + defer cancel() + _, err = a.deps.AckPublisher.Publish(publishCtx, model.AckSubject(a.cfg.Region, a.cfg.Shard, a.cfg.MemberID), env) + return err +} + +func (a *Agent) emitVerifiedPublicationAck(ctx context.Context, uid string, epoch, revision uint64) error { + // The file transaction, proof and original-deadline enforcement are local + // safety gates. Broker delivery is independent: renew only a safe installed + // state first, then let remote readiness expire if ACK transport is down. + if err := a.renewSafeWatchdogLease(ctx); err != nil { + return err + } + if err := a.emitAck(ctx, model.KindPublicationManifest, uid, epoch, revision, model.AckVerified, time.Time{}, ""); err != nil { + a.report(fmt.Errorf("publish verified publication ACK %s: %w", uid, err)) + return nil + } + // The materialization path has already completed the same staged-file and DNS + // proof required by a periodic heartbeat. Remember it so finishEvent checks + // other due publications without immediately repeating this proof. + a.lastPublicationAck[uid] = a.deps.Clock() + return nil +} + +func (a *Agent) emitServingAck(ctx context.Context, b model.Binding, phase model.AckPhase) error { + now := a.deps.Clock() + interval := a.cfg.AckInterval + if interval <= 0 { + interval = 30 * time.Second + } + last := a.lastServingAck[b.BindingUID] + if last.phase == phase && now.Sub(last.at) < interval { + return nil + } + if err := a.emitAck(ctx, model.KindServingSnapshot, b.BindingUID, a.state.ServingFence.Epoch, a.state.ServingFence.Revision, phase, b.Authorization.ValidUntil, ""); err != nil { + return err + } + a.lastServingAck[b.BindingUID] = servingAckState{phase: phase, at: now} + return nil +} + +func (a *Agent) heartbeatPublications(ctx context.Context) error { + now := a.deps.Clock() + interval := a.cfg.AckInterval + if interval <= 0 { + interval = 30 * time.Second + } + type duePublication struct { + uid string + p publicationState + } + var due []duePublication + for uid, p := range a.state.Publications { + if !p.Verified || (!p.Manifest.Tombstone && len(a.publicationViews(p)) == 0) { + continue + } + if last := a.lastPublicationAck[uid]; !last.IsZero() && now.Sub(last) < interval { + continue + } + if err := a.verifyPublication(ctx, p); err != nil { + _ = a.emitAck(ctx, model.KindPublicationManifest, uid, p.Fence.Epoch, p.Fence.Revision, model.AckRejected, time.Time{}, err.Error()) + return fmt.Errorf("reverify publication %s before ACK renewal: %w", uid, err) + } + due = append(due, duePublication{uid: uid, p: p}) + } + // Local watchdog eligibility depends on the complete local verification + // above, not broker availability. Stop after the first stalled publish so a + // partition consumes at most one bounded publish timeout per Step. Missing + // remote ACK leases still expire independently and gate resolver eligibility. + for _, publication := range due { + p := publication.p + if err := a.emitAck(ctx, model.KindPublicationManifest, publication.uid, p.Fence.Epoch, p.Fence.Revision, model.AckVerified, time.Time{}, ""); err != nil { + a.report(fmt.Errorf("publish verified publication ACK %s: %w", publication.uid, err)) + break + } + a.lastPublicationAck[publication.uid] = now + } + return nil +} + +func hashFiles(files map[string][]byte) string { + keys := make([]string, 0, len(files)) + for k := range files { + keys = append(keys, k) + } + sort.Strings(keys) + size := 0 + for _, key := range keys { + size += len(key) + 1 + len(files[key]) + } + all := make([]byte, 0, size) + for _, k := range keys { + all = append(all, []byte(k)...) + all = append(all, 0) + all = append(all, files[k]...) + } + return model.Hash(all) +} + +func equalJSON(left, right any) bool { + a, errA := json.Marshal(left) + b, errB := json.Marshal(right) + return errA == nil && errB == nil && string(a) == string(b) +} + +func publicationNames(plan model.PublicationPlan) []string { + seen := map[string]bool{} + var out []string + if plan.Apex != "" { + apex := model.AbsoluteName(plan.Apex) + seen[apex] = true + out = append(out, apex) + } + for _, owner := range plan.Owners { + owner = model.AbsoluteName(owner) + if !seen[owner] { + seen[owner] = true + out = append(out, owner) + } + } + for _, rrset := range plan.RRSets { + owner := model.AbsoluteName(rrset.Name) + if !seen[owner] { + seen[owner] = true + out = append(out, owner) + } + } + sort.Strings(out) + return out +} + +func (a *Agent) validateContributionFences(plan model.PublicationPlan) error { + for _, observation := range plan.ObservationFences { + current, ok := a.state.ContributionFences[observation.ContributionUID] + if !ok { + continue + } + comparison := model.Compare(observation.WriterEpoch, observation.Sequence, current.Epoch, current.Sequence) + if comparison < 0 { + return fmt.Errorf("contribution %q regressed from epoch %d sequence %d", observation.ContributionUID, current.Epoch, current.Sequence) + } + if comparison == 0 { + if current.ValidUntil.IsZero() || !current.ValidUntil.Equal(observation.ValidUntil) { + return fmt.Errorf("contribution %q changed the deadline of an existing fence", observation.ContributionUID) + } + if current.GrantUID != "" && current.GrantUID != observation.GrantUID { + return fmt.Errorf("contribution %q changed the grant of an existing fence", observation.ContributionUID) + } + } + } + return nil +} + +func (a *Agent) commitContributionFences(plan model.PublicationPlan) { + for _, observation := range plan.ObservationFences { + next := contributionFence{GrantUID: observation.GrantUID, Epoch: observation.WriterEpoch, Sequence: observation.Sequence, ValidUntil: observation.ValidUntil} + current := a.state.ContributionFences[observation.ContributionUID] + comparison := model.Compare(next.Epoch, next.Sequence, current.Epoch, current.Sequence) + if comparison > 0 || (comparison == 0 && current.GrantUID == "" && !current.ValidUntil.IsZero() && current.ValidUntil.Equal(next.ValidUntil)) { + a.state.ContributionFences[observation.ContributionUID] = next + } + } +} + +type watchdogLease struct { + MemberID string `json:"memberID"` + ReplicaID string `json:"replicaID,omitempty"` + ValidUntil time.Time `json:"validUntil"` +} + +func (a *Agent) watchdogPublicationError() error { + now := a.deps.Clock() + for uid, p := range a.state.Publications { + if p.Manifest.Tombstone { + continue + } + current := effectivePlanHash(p.Plan, now) + if a.cfg.Mode == ModeCombined { + if len(a.publicationViewsAt(p, now)) == 0 { + continue + } + installed, ok := a.installedPublications[uid] + if !ok || installed.Identity != publicationIdentity(p) || installed.EffectiveHash != current { + return fmt.Errorf("publication %s deadline changed after activation; refusing watchdog renewal", uid) + } + } else if p.EffectiveHash != current { + return fmt.Errorf("publication %s deadline changed after cache withdrawal; refusing watchdog renewal", uid) + } + } + return nil +} + +// A deadline can cross during reload or DNS proof. Settle that transition +// locally before renewing readiness, under one budget across every retry. +func (a *Agent) renewSafeWatchdogLease(ctx context.Context) error { + timeout := a.cfg.PublicationDNSProbe.Timeout + if a.cfg.Mode == ModeResolver { + timeout = a.cfg.ResolverDNSProbe.Timeout + } + if timeout <= 0 { + timeout = 3 * time.Second + } + settleCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + for { + if err := settleCtx.Err(); err != nil { + return errors.Join(fmt.Errorf("settle local publication deadlines: %w", err), a.failClosed(ctx)) + } + var err error + if a.cfg.Mode == ModeCombined { + err = a.expireRecords(settleCtx) + } else { + err = a.expireResolverCache(settleCtx) + } + if err != nil { + return errors.Join(err, a.failClosed(ctx)) + } + if a.watchdogPublicationError() == nil { + return a.renewWatchdogLease(settleCtx) + } + if err := a.reconcile(settleCtx); err != nil { + return errors.Join(err, a.failClosed(ctx)) + } + } +} + +func (a *Agent) renewWatchdogLease(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return errors.Join(err, a.failClosed(ctx)) + } + if err := a.watchdogPublicationError(); err != nil { + return errors.Join(err, a.failClosed(ctx)) + } + if a.cfg.WatchdogLeasePath == "" { + return nil + } + lease := a.cfg.WatchdogLease + if lease <= 0 { + lease = 5 * time.Second + } + b, err := json.Marshal(watchdogLease{MemberID: a.cfg.MemberID, ReplicaID: a.cfg.ReplicaID, ValidUntil: a.deps.Clock().UTC().Add(lease)}) + if err != nil { + return err + } + return writeAtomicLease(a.cfg.WatchdogLeasePath, b, 0o600) +} diff --git a/internal/internaldns/serving/agent_test.go b/internal/internaldns/serving/agent_test.go new file mode 100644 index 00000000..9db5bd59 --- /dev/null +++ b/internal/internaldns/serving/agent_test.go @@ -0,0 +1,739 @@ +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +type fakePublisher struct{ envs []model.Envelope } + +func (f *fakePublisher) Publish(_ context.Context, _ string, e model.Envelope) (transport.PublishAck, error) { + f.envs = append(f.envs, e) + return transport.PublishAck{}, nil +} + +type stalledPublisher struct{} + +func (stalledPublisher) Publish(ctx context.Context, _ string, _ model.Envelope) (transport.PublishAck, error) { + <-ctx.Done() + return transport.PublishAck{}, ctx.Err() +} + +type contextIgnoringPublisher struct { + mu sync.Mutex + calls int + started chan<- struct{} + release <-chan struct{} +} + +func (p *contextIgnoringPublisher) Publish(context.Context, string, model.Envelope) (transport.PublishAck, error) { + p.mu.Lock() + p.calls++ + p.mu.Unlock() + select { + case p.started <- struct{}{}: + default: + } + <-p.release + return transport.PublishAck{}, errors.New("nats: server disconnected") +} + +func (p *contextIgnoringPublisher) callCount() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.calls +} + +func TestBoundedAckPublisherDoesNotStartExpiredRequest(t *testing.T) { + t.Parallel() + started := make(chan struct{}, 1) + release := make(chan struct{}) + close(release) + delegate := &contextIgnoringPublisher{started: started, release: release} + publisher := newBoundedAckPublisher(delegate) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := publisher.Publish(ctx, "ack", model.Envelope{}); !errors.Is(err, context.Canceled) { + t.Fatalf("expired ACK request error = %v, want context cancellation", err) + } + if calls := delegate.callCount(); calls != 0 { + t.Fatalf("expired ACK request invoked delegate %d times", calls) + } +} + +type blockingPublisher struct { + started chan<- struct{} + release <-chan struct{} +} + +type resolverVerifierFunc func(context.Context, model.Binding) error + +func (f resolverVerifierFunc) Verify(ctx context.Context, binding model.Binding) error { + return f(ctx, binding) +} + +func (p blockingPublisher) Publish(ctx context.Context, _ string, _ model.Envelope) (transport.PublishAck, error) { + select { + case p.started <- struct{}{}: + default: + } + select { + case <-p.release: + return transport.PublishAck{}, nil + case <-ctx.Done(): + return transport.PublishAck{}, ctx.Err() + } +} + +type commandChannelRunner struct{ calls chan Command } + +func (r *commandChannelRunner) Run(_ context.Context, command Command) ([]byte, error) { + r.calls <- command + return nil, nil +} + +type boundedResolverVerifier struct { + mu sync.Mutex + active int + maximum int + calls int + release <-chan struct{} + started chan<- struct{} + waitForCtx bool +} + +func (v *boundedResolverVerifier) Verify(ctx context.Context, _ model.Binding) error { + v.mu.Lock() + v.active++ + v.calls++ + if v.active > v.maximum { + v.maximum = v.active + } + v.mu.Unlock() + if v.started != nil { + select { + case v.started <- struct{}{}: + default: + } + } + var err error + if v.waitForCtx { + <-ctx.Done() + err = ctx.Err() + } else { + select { + case <-ctx.Done(): + err = ctx.Err() + case <-v.release: + } + } + v.mu.Lock() + v.active-- + v.mu.Unlock() + return err +} + +func (v *boundedResolverVerifier) snapshot() (calls, maximum int) { + v.mu.Lock() + defer v.mu.Unlock() + return v.calls, v.maximum +} + +func envelope(t *testing.T, kind model.Kind, uid string, epoch, revision uint64, payload any) model.Envelope { + t.Helper() + e, err := model.NewEnvelope(kind, "event-"+string(kind)+uid+time.Now().String(), "east", "s1", uid, epoch, revision, time.Now(), payload) + if err != nil { + t.Fatal(err) + } + return e +} + +func testSnapshot(now time.Time) model.ServingSnapshot { + b := model.Binding{BindingUID: "binding-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 2, RequiredPublicationRevision: 4}}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1, ValidUntil: now.Add(10 * time.Minute)}} + return model.ServingSnapshot{Region: "east", Shard: "s1", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []model.Binding{b}, Members: []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}}} +} + +func testPublication(t *testing.T, now time.Time) (model.PublicationChunk, model.PublicationManifest) { + t.Helper() + validUntil := now.Add(time.Minute) + plan := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 1, Sequence: 1, ValidUntil: validUntil}}, Owners: []string{"api.prod.internal"}, RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 1, Sequence: 1, Eligible: true, ValidUntil: validUntil}}}}} + payload, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + chunk := model.PublicationChunk{ManifestUID: "manifest-a", ZoneUID: "zone-a", WriterEpoch: 2, Revision: 4, Index: 0, SHA256: model.Hash(payload), Payload: payload} + manifest := model.PublicationManifest{ManifestUID: "manifest-a", ZoneUID: "zone-a", Apex: "prod.internal", WriterEpoch: 2, Revision: 4, Chunks: []model.ChunkRef{{Index: 0, SHA256: chunk.SHA256, Size: len(payload)}}, ContentHash: model.Hash(payload), GeneratedAt: now} + return chunk, manifest +} + +func TestWithdrawnObservationFencesOlderHealthyRecord(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + agent := &Agent{state: newCheckpoint()} + withdrawn := model.PublicationPlan{ + ZoneUID: "zone-a", Apex: "prod.internal", + ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 33, ValidUntil: now.Add(-time.Minute)}}, + RRSets: []model.RRSet{}, + } + if err := withdrawn.Validate(); err != nil { + t.Fatal(err) + } + agent.commitContributionFences(withdrawn) + olderDeadline := now.Add(time.Minute) + olderHealthy := model.PublicationPlan{ + ZoneUID: "zone-a", Apex: "prod.internal", + ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 32, ValidUntil: olderDeadline}}, + RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{ + Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 7, Sequence: 32, Eligible: true, ValidUntil: olderDeadline, + }}}}, + } + if err := olderHealthy.Validate(); err != nil { + t.Fatal(err) + } + if err := agent.validateContributionFences(olderHealthy); err == nil { + t.Fatal("older healthy observation was accepted after a higher withdrawn observation") + } +} + +func TestObservationFenceReplayCannotExtendDeadline(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + agent := &Agent{state: newCheckpoint()} + original := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 2, Sequence: 4, ValidUntil: now}}, RRSets: []model.RRSet{}} + agent.commitContributionFences(original) + replay := original + replay.ObservationFences = append([]model.ObservationFence(nil), original.ObservationFences...) + replay.ObservationFences[0].ValidUntil = now.Add(time.Hour) + if err := agent.validateContributionFences(replay); err == nil { + t.Fatal("equal observation fence extended its original deadline") + } +} + +func TestClusterRoleAutomaticallyVerifiesClusterVIP(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, _ := renderFixture(t) + renderConfig.Role = RenderCluster + renderConfig.ClusterDNSDist.Path = filepath.Join(dir, "cluster-dnsdist.conf") + renderConfig.ClusterBIND.Path = filepath.Join(dir, "cluster-bind.conf") + renderConfig.NodeDNSDist = DNSDistRenderConfig{} + renderConfig.NodeBIND = BINDRenderConfig{} + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "cluster-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, + UnsafeAllowNoFailClosed: true, + }, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + verifier, ok := agent.deps.Verifier.(DNSVerifier) + if !ok || !verifier.Config.UseClusterAddress { + t.Fatalf("cluster agent verifier = %#v, want cluster VIP", agent.deps.Verifier) + } +} + +func TestStartForceActivatesPersistedUnchangedConfigBeforeLease(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterBIND = BINDRenderConfig{} + rendered, err := Render(renderConfig, snapshot, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + for path, contents := range rendered.Files { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, contents, 0o644); err != nil { + t.Fatal(err) + } + } + stateDir := filepath.Join(dir, "state") + state := newCheckpoint() + state.Snapshot = &snapshot + state.BindingFences, _ = advanceBindingFences(nil, snapshot) + state.ServingFence = fence{Epoch: snapshot.ConfigurationEpoch, Revision: snapshot.ConfigurationRevision} + state.RenderedHash = hashFiles(rendered.Files) + if err := (stateStore{path: filepath.Join(stateDir, "checkpoint.json")}).Save(state); err != nil { + t.Fatal(err) + } + leasePath := filepath.Join(dir, "lease.json") + config := Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: stateDir, Mode: ModeResolver, + Render: renderConfig, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, + WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, ExpiryInterval: time.Hour, + Transaction: TransactionConfig{ + StageDir: filepath.Join(dir, "stage"), + FailClosed: []Command{{Path: "gate"}}, + Validate: []Command{ + {Path: "validate-dnsdist", Args: []string{"{stageDir}/node-dnsdist.conf"}, WhenChanged: []string{renderConfig.NodeDNSDist.Path}}, + {Path: "validate-bind", Args: []string{"{stageDir}/node-bind.conf"}, WhenChanged: []string{renderConfig.NodeBIND.Path}}, + }, + Reload: []Command{ + {Path: "reload-dnsdist", WhenChanged: []string{renderConfig.NodeDNSDist.Path}}, + {Path: "reload-bind", WhenChanged: []string{renderConfig.NodeBIND.Path}}, + }, + }, + } + runner := &commandChannelRunner{calls: make(chan Command, 16)} + agent, err := NewAgent(config, Dependencies{Runner: runner}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- agent.Start(ctx) }() + deadline := time.After(2 * time.Second) + for { + if _, err := os.Stat(leasePath); err == nil { + break + } + select { + case <-deadline: + cancel() + t.Fatal("startup did not activate unchanged config and write its lease") + case <-time.After(5 * time.Millisecond): + } + } + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } + var calls []string + for len(runner.calls) > 0 { + calls = append(calls, (<-runner.calls).Path) + } + wantPrefix := []string{"gate", "validate-dnsdist", "validate-bind", "reload-dnsdist", "reload-bind"} + if len(calls) < len(wantPrefix) { + t.Fatalf("startup calls = %#v", calls) + } + for i, want := range wantPrefix { + if calls[i] != want { + t.Fatalf("startup calls = %#v, want prefix %#v", calls, wantPrefix) + } + } +} + +func TestStepBoundsStalledAckPublishAndRenewsLocalLease(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterBIND = BINDRenderConfig{} + snapshot.Bindings[0].Zones = nil + leasePath := filepath.Join(dir, "lease.json") + var reported []error + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, + UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, + WatchdogLease: 5 * time.Second, AckPublishTimeout: 30 * time.Millisecond, + }, Dependencies{AckPublisher: stalledPublisher{}, Runner: &recordingRunner{}, OnError: func(err error) { reported = append(reported, err) }}) + if err != nil { + t.Fatal(err) + } + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + started := time.Now() + if err := agent.Step(context.Background()); err != nil { + t.Fatal(err) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("Step blocked %s on an unavailable ACK broker", elapsed) + } + if len(reported) == 0 || !strings.Contains(reported[0].Error(), "publish serving ACK") { + t.Fatalf("stalled ACK publish was not reported: %#v", reported) + } + if _, err := os.Stat(leasePath); err != nil { + t.Fatalf("locally healthy Step did not renew watchdog lease: %v", err) + } +} + +// Wait for the ACK boundary rather than imposing a startup SLA on durable +// file staging. Production ACK and watchdog deadlines remain unchanged. +func stepUntilBlockedAck(t *testing.T, agent *Agent, started <-chan struct{}, release chan struct{}) func() error { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + finished := make(chan struct{}) + var stepErr error + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(func() { + unblock() + cancel() + select { + case <-finished: + case <-time.After(10 * time.Second): + t.Error("Step did not finish after unconditional ACK release and cancellation") + } + }) + go func() { + stepErr = agent.Step(ctx) + close(finished) + }() + select { + case <-started: + case <-finished: + t.Fatalf("Step finished before blocked ACK verification: %v", stepErr) + case <-time.After(10 * time.Second): + t.Fatal("resolver did not reach broker ACK publication") + } + return func() error { + unblock() + select { + case <-finished: + return stepErr + case <-time.After(10 * time.Second): + return errors.New("Step did not finish after ACK release") + } + } +} + +func TestResolverRenewsWatchdogBeforeBrokerAckCompletes(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterBIND = BINDRenderConfig{} + snapshot.Bindings[0].Zones = nil + leasePath := filepath.Join(dir, "lease.json") + started := make(chan struct{}, 1) + release := make(chan struct{}) + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, + UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, + WatchdogLease: 5 * time.Second, AckPublishTimeout: time.Second, + }, Dependencies{AckPublisher: blockingPublisher{started: started, release: release}, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + finish := stepUntilBlockedAck(t, agent, started, release) + if _, err := os.Stat(leasePath); err != nil { + t.Fatalf("resolver did not renew watchdog before blocked broker ACK: %v", err) + } + if err := finish(); err != nil { + t.Fatal(err) + } +} + +func TestResolverGatesFailedTenantButRenewsSharedLeaseForHealthyTenant(t *testing.T) { + t.Parallel() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterBIND = BINDRenderConfig{} + snapshot.Bindings[0].Zones = nil + failedUID := snapshot.Bindings[0].BindingUID + healthy := snapshot.Bindings[0] + healthy.BindingUID = "binding-b" + healthy.ProjectUID = "project-b" + healthy.ContextUID = "vpc-b" + healthy.ConsumerAddress = "10.253.0.52" + healthy.ClusterAddress = "10.253.0.42" + snapshot.Bindings = append(snapshot.Bindings, healthy) + leasePath := filepath.Join(dir, "lease.json") + started := make(chan struct{}, 1) + release := make(chan struct{}) + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + CacheFlush: []Command{{Path: "flush"}}, Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, + UnsafeAllowNoFailClosed: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, + WatchdogLease: 5 * time.Second, AckPublishTimeout: time.Second, + }, Dependencies{ + AckPublisher: blockingPublisher{started: started, release: release}, Runner: &recordingRunner{}, + Verifier: resolverVerifierFunc(func(_ context.Context, binding model.Binding) error { + if binding.BindingUID == failedUID { + return errors.New("probe failed") + } + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + finish := stepUntilBlockedAck(t, agent, started, release) + config, err := os.ReadFile(renderConfig.NodeDNSDist.Path) + if err != nil { + t.Fatal(err) + } + if bindingReadyInConfig(string(config), snapshot.Bindings[0].ConsumerAddress) { + t.Fatal("failed tenant remained routed") + } + if !strings.Contains(string(config), healthy.ConsumerAddress) { + t.Fatal("healthy tenant was gated with failed tenant") + } + if _, err := os.Stat(leasePath); err != nil { + t.Fatalf("healthy tenant did not renew shared watchdog after failed tenant was gated: %v", err) + } + if err := finish(); err != nil { + t.Fatal(err) + } +} + +func TestStartContinuouslyRenewsResolverLeaseWhileBrokerDisconnected(t *testing.T) { + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterBIND = BINDRenderConfig{} + snapshot.Bindings[0].Zones = nil + snapshot.Bindings[0].Authorization.ValidUntil = time.Now().Add(10 * time.Second) + leasePath := filepath.Join(dir, "lease.json") + publishStarted := make(chan struct{}, 1) + publishRelease := make(chan struct{}) + publisher := &contextIgnoringPublisher{started: publishStarted, release: publishRelease} + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, + UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, + WatchdogLeasePath: leasePath, WatchdogLease: 500 * time.Millisecond, + ExpiryInterval: 20 * time.Millisecond, AckInterval: 50 * time.Millisecond, + AckPublishTimeout: 10 * time.Millisecond, LocalServingLease: time.Second, + }, Dependencies{AckPublisher: publisher, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- agent.Start(ctx) }() + var stopOnce sync.Once + var stopErr error + stopAgent := func() { + stopOnce.Do(func() { + cancel() + select { + case stopErr = <-done: + case <-time.After(time.Second): + stopErr = errors.New("resolver did not stop after continuous renewal test") + } + }) + } + defer stopAgent() + var releaseOnce sync.Once + releasePublisher := func() { releaseOnce.Do(func() { close(publishRelease) }) } + defer releasePublisher() + select { + case <-publishStarted: + case <-time.After(time.Second): + t.Fatal("resolver did not enter context-ignoring broker publish") + } + + initialDeadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(leasePath); err == nil { + break + } + if time.Now().After(initialDeadline) { + t.Fatal("resolver did not write its initial lease") + } + time.Sleep(5 * time.Millisecond) + } + started := time.Now() + lastValidUntil := time.Time{} + var continuityErr error + for time.Since(started) < 1200*time.Millisecond { + payload, err := os.ReadFile(leasePath) + if err != nil { + continuityErr = fmt.Errorf("read continuously renewed lease: %w", err) + break + } + var lease watchdogLease + if err := json.Unmarshal(payload, &lease); err != nil { + continuityErr = fmt.Errorf("decode continuously renewed lease: %w", err) + break + } + if !lease.ValidUntil.After(time.Now()) { + continuityErr = fmt.Errorf("resolver lease expired during broker partition at %s", lease.ValidUntil) + break + } + if lease.ValidUntil.After(lastValidUntil) { + lastValidUntil = lease.ValidUntil + } + time.Sleep(20 * time.Millisecond) + } + if continuityErr == nil { + if remaining := time.Until(lastValidUntil); remaining < 250*time.Millisecond { + continuityErr = fmt.Errorf("resolver lease was not refreshed across multiple windows: remaining %s", remaining) + } + } + if calls := publisher.callCount(); calls != 1 { + continuityErr = fmt.Errorf("context-ignoring publisher calls = %d, want one bounded inflight call", calls) + } + releasePublisher() + stopAgent() + if stopErr != nil { + t.Fatal(stopErr) + } + if continuityErr != nil { + t.Fatal(continuityErr) + } +} + +func TestResolverVerificationUsesBoundedConcurrentBatch(t *testing.T) { + t.Parallel() + release := make(chan struct{}) + started := make(chan struct{}, 8) + verifier := &boundedResolverVerifier{release: release, started: started} + agent := &Agent{ + cfg: Config{ResolverVerifyConcurrency: 3, ResolverDNSProbe: DNSVerifierConfig{Timeout: time.Second}}, + deps: Dependencies{Verifier: verifier}, + } + bindings := make([]model.Binding, 8) + active := make(map[string]bool, len(bindings)) + for i := range bindings { + bindings[i].BindingUID = fmt.Sprintf("binding-%d", i) + active[bindings[i].BindingUID] = true + } + done := make(chan bool, 1) + go func() { done <- agent.verifyResolverBindings(context.Background(), bindings, active) }() + for range 3 { + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("verification workers did not start concurrently") + } + } + if calls, maximum := verifier.snapshot(); calls != 3 || maximum != 3 { + t.Fatalf("before release calls=%d max=%d, want exactly three workers", calls, maximum) + } + close(release) + if failed := <-done; failed { + t.Fatal("successful resolver proof batch was marked failed") + } + if calls, maximum := verifier.snapshot(); calls != len(bindings) || maximum > 3 { + t.Fatalf("completed calls=%d max=%d, want calls=%d max<=3", calls, maximum, len(bindings)) + } +} + +func TestResolverVerificationHasOneSharedDeadline(t *testing.T) { + t.Parallel() + verifier := &boundedResolverVerifier{waitForCtx: true} + agent := &Agent{ + cfg: Config{ResolverVerifyConcurrency: 2, ResolverDNSProbe: DNSVerifierConfig{Timeout: 40 * time.Millisecond}}, + deps: Dependencies{Verifier: verifier}, + } + bindings := make([]model.Binding, 10) + active := make(map[string]bool, len(bindings)) + for i := range bindings { + bindings[i].BindingUID = fmt.Sprintf("binding-%d", i) + active[bindings[i].BindingUID] = true + } + started := time.Now() + if failed := agent.verifyResolverBindings(context.Background(), bindings, active); !failed { + t.Fatal("timed out resolver proof batch was accepted") + } + if elapsed := time.Since(started); elapsed > 500*time.Millisecond { + t.Fatalf("shared resolver proof deadline took %s", elapsed) + } + if len(active) != 0 { + t.Fatalf("timed out resolver proof batch left %d bindings active", len(active)) + } +} + +func TestAckEventRenewsWatchdogAfterSuccessfulReconcile(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + dir := t.TempDir() + renderConfig, snapshot := renderFixture(t) + renderConfig.Role = RenderNode + renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") + renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.ClusterDNSDist = DNSDistRenderConfig{} + renderConfig.ClusterBIND = BINDRenderConfig{} + snapshot.Members = []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}} + leasePath := filepath.Join(dir, "lease.json") + agent, err := NewAgent(Config{ + Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, + UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, + WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, + }, Dependencies{Clock: func() time.Time { return now }, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", snapshot.ConfigurationEpoch, snapshot.ConfigurationRevision, snapshot)); err != nil { + t.Fatal(err) + } + if err := os.Remove(leasePath); err != nil { + t.Fatal(err) + } + ack := model.MemberAck{MemberID: "regional-0", ReplicaID: "regional-0", ResourceUID: "zone-a", Kind: model.KindPublicationManifest, Epoch: 2, Revision: 4, Phase: model.AckVerified, ObservedAt: now, ValidUntil: now.Add(time.Minute)} + if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(leasePath); err != nil { + t.Fatalf("successful ACK reconcile did not renew watchdog lease: %v", err) + } +} + +func TestExpiredKnownAckReplayDrainsWithoutStateOrLeaseChange(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + dir := t.TempDir() + leasePath := filepath.Join(dir, "lease.json") + agent := &Agent{ + cfg: Config{Region: "east", Shard: "s1", Mode: ModeResolver, WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second}, + deps: Dependencies{Clock: func() time.Time { return now }}, state: newCheckpoint(), + lastServingAck: map[string]servingAckState{}, lastPublicationAck: map[string]time.Time{}, + } + snapshot := testSnapshot(now) + agent.state.Snapshot = &snapshot + agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + ack := model.MemberAck{MemberID: "regional-0", ReplicaID: "regional-0", ResourceUID: "zone-a", Kind: model.KindPublicationManifest, Epoch: 2, Revision: 4, Phase: model.AckVerified, ObservedAt: now.Add(-2 * time.Minute), ValidUntil: now.Add(-time.Minute)} + if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatalf("expired known ACK replay was not drained: %v", err) + } + if len(agent.state.Acks) != 0 { + t.Fatalf("expired ACK changed readiness state: %#v", agent.state.Acks) + } + if _, err := os.Stat(leasePath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expired ACK renewed watchdog lease: %v", err) + } + ack.MemberID = "unknown" + if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err == nil { + t.Fatal("expired ACK from an unknown member was accepted") + } + ack.MemberID = "regional-0" + ack.ValidUntil = ack.ObservedAt.Add(2*time.Minute + time.Nanosecond) + if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err == nil { + t.Fatal("overlong ACK lease was accepted") + } +} diff --git a/internal/internaldns/serving/bind_test.go b/internal/internaldns/serving/bind_test.go new file mode 100644 index 00000000..a12b1fc7 --- /dev/null +++ b/internal/internaldns/serving/bind_test.go @@ -0,0 +1,782 @@ +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "regexp" + + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + "go.miloapis.com/dns-operator/internal/internaldns/transport" +) + +func regionalTestConfig(t *testing.T) Config { + t.Helper() + dir := t.TempDir() + render, _ := renderFixture(t) + render.Role = RenderRegionalBIND + render.ClusterBIND.Path = filepath.Join(dir, "runtime", "regional.conf") + return Config{Region: "east", Shard: "s1", MemberID: "regional-0", ReplicaID: "regional-0", Mode: ModeCombined, StateDir: filepath.Join(dir, "state"), Render: render, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), Validate: []Command{{Path: "validate"}}, Reload: []Command{{Path: "reload"}}, FailClosed: []Command{{Path: "gate"}}}, UnsafeAllowNoDNSVerify: true, WatchdogLeasePath: filepath.Join(dir, "lease.json")} +} +func installTestPublication(t *testing.T, a *Agent, now time.Time) { + t.Helper() + ctx := context.Background() + snapshot := testSnapshot(now) + snapshot.Members = []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}} + if err := a.Handle(ctx, envelope(t, model.KindServingSnapshot, "shard", 1, 1, snapshot)); err != nil { + t.Fatal(err) + } + chunk, manifest := testPublication(t, now) + if err := a.Handle(ctx, envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(ctx, envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } +} +func currentZone(t *testing.T, a *Agent) string { + t.Helper() + files, err := renderPublications(a.cfg.Render, *a.state.Snapshot, map[string]bool{}, a.state.Publications, a.deps.Clock()) + if err != nil { + t.Fatal(err) + } + for path, data := range files.Files { + if strings.HasSuffix(path, ".db") { + return string(data) + } + } + t.Fatal("no zone") + return "" +} +func TestBINDPublicationActivatesLocalPrivateAuthority(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + pub := &fakePublisher{} + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }, AckPublisher: pub}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + data := currentZone(t, agent) + if !strings.Contains(data, "192.0.2.10") || !strings.Contains(data, publicationMailbox(agent.state.Publications["zone-a"], now)) { + t.Fatalf("bad zone: %s", data) + } + config, err := os.ReadFile(cfg.Render.ClusterBIND.Path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(config), "type primary") || strings.Contains(string(config), "static-stub") || strings.Contains(string(config), "query-source") { + t.Fatalf("not direct private authority: %s", config) + } + if !agent.state.Publications["zone-a"].Verified { + t.Fatal("publication not verified") + } +} +func TestBINDExpirySurvivesBrokerOutageAndRestart(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }, AckPublisher: stalledPublisher{}}) + if err != nil { + t.Fatal(err) + } + // Publish without a transport dependency during setup; partition afterwards. + a.deps.AckPublisher = nil + installTestPublication(t, a, now) + a.deps.AckPublisher = newBoundedAckPublisher(stalledPublisher{}) + a.cfg.AckPublishTimeout = time.Millisecond + clock = now.Add(61 * time.Second) + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + if data := currentZone(t, a); strings.Contains(data, "192.0.2.10") || !strings.Contains(data, "api.prod.internal. 5 IN "+model.OwnershipMarkerType) { + t.Fatalf("expiry did not preserve NODATA ownership: %s", data) + } + local := a.state.Publications["zone-a"].LocalRevision + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + if err := restarted.Step(context.Background()); err != nil { + t.Fatal(err) + } + if restarted.state.Publications["zone-a"].LocalRevision != local { + t.Fatal("restart renewed expired observation") + } + if data := currentZone(t, restarted); strings.Contains(data, "192.0.2.10") { + t.Fatal("restart resurrected record") + } +} +func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + runner := &recordingRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, snapshot)); err != nil { + t.Fatal(err) + } + chunk, m := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + runner.failAt = len(runner.calls) + 1 + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err == nil { + t.Fatal("invalid staged configuration accepted") + } + saved, err := (stateStore{path: filepath.Join(cfg.StateDir, "checkpoint.json")}).Load() + if err != nil { + t.Fatal(err) + } + if f := saved.ContributionFences["contrib-a"]; f.Epoch != 1 || f.Sequence != 1 || !f.ValidUntil.Equal(now.Add(time.Minute)) { + t.Fatalf("observation fence lost: %#v", f) + } + if saved.Publications["zone-a"].Verified { + t.Fatal("failed publication acknowledged") + } + if runner.calls[len(runner.calls)-1].Path != "gate" { + t.Fatal("failed activation did not fail closed") + } + runner.failAt = 0 + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err != nil { + t.Fatalf("equal fenced retry failed: %v", err) + } +} +func TestBINDFailedExpiryDoesNotRenewWatchdog(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + runner := &recordingRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + if err := os.Remove(cfg.WatchdogLeasePath); err != nil { + t.Fatal(err) + } + clock = now.Add(61 * time.Second) + runner.failAt = len(runner.calls) + 1 + if err := a.Step(context.Background()); err == nil { + t.Fatal("expiry validation failure accepted") + } + if _, err := os.Stat(cfg.WatchdogLeasePath); !errors.Is(err, os.ErrNotExist) { + t.Fatal("failed expiry renewed watchdog") + } + runner.failAt = 0 + if err := a.Step(context.Background()); err != nil { + t.Fatalf("pending expiry retry failed: %v", err) + } + if strings.Contains(currentZone(t, a), "192.0.2.10") { + t.Fatal("retry resurrected expired record") + } +} +func TestBINDTombstoneFencesReplayAcrossRestart(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + _, old := testPublication(t, now) + tombstone := model.PublicationManifest{ManifestUID: "deleted", ZoneUID: "zone-a", Apex: "prod.internal", WriterEpoch: 3, Revision: 1, Tombstone: true, GeneratedAt: now, ContentHash: model.EmptyContentHash()} + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 3, 1, tombstone)); err != nil { + t.Fatal(err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if err := restarted.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, old)); err != nil { + t.Fatal(err) + } + if !restarted.state.Publications["zone-a"].Manifest.Tombstone || restarted.publicationsReady(testSnapshot(now).Bindings[0]) { + t.Fatal("old snapshot defeated tombstone") + } +} +func TestNodeRequiresDeadlinePlanAndExpiresCacheLocally(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + render, snapshot := renderFixture(t) + render.Role = RenderNode + render.NodeBIND.Path = filepath.Join(t.TempDir(), "node.conf") + render.NodeDNSDist.Path = filepath.Join(filepath.Dir(render.NodeBIND.Path), "dnsdist.conf") + cfg.Render = render + cfg.Mode = ModeResolver + cfg.CacheFlush = []Command{{Path: "flush", Args: []string{"{name}"}}} + runner := &recordingRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + snapshot = testSnapshot(now) + a.state.Snapshot = &snapshot + a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ValidUntil: now.Add(time.Minute)}} + if a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("node enabled from ACK without deadline plan") + } + chunk, m := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err != nil { + t.Fatal(err) + } + if !a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("complete plan + current proof not eligible") + } + runner.calls = nil + clock = now.Add(61 * time.Second) + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + flushed := false + for _, call := range runner.calls { + if call.Path == "flush" && call.Args[0] == "api.prod.internal." { + flushed = true + } + } + if !flushed { + t.Fatal("node did not flush original-deadline record without broker") + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + if restarted.state.Publications["zone-a"].Plan.ObservationFences[0].ValidUntil != now.Add(time.Minute) { + t.Fatal("restart changed original deadline") + } +} +func TestBINDPublicationVersionProofIncludesFullContentFingerprint(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + chunk, m := testPublication(t, now) + var plan model.PublicationPlan + if err := json.Unmarshal(chunk.Payload, &plan); err != nil { + t.Fatal(err) + } + p := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Plan: plan} + first := publicationMailbox(p, now) + p.Plan.RRSets[0].Records[0].Content = "192.0.2.11" + if first == publicationMailbox(p, now) { + t.Fatal("different installed data shares fingerprint") + } + if len(first) > 255 { + t.Fatal("fingerprint name too long") + } +} + +func TestLegacyBackendCheckpointRejectedWithoutDiscardingFences(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "checkpoint.json") + old := []byte(`{"contributionFences":{"endpoint":{"epoch":7,"sequence":33}}}`) + if err := os.WriteFile(path, old, 0600); err != nil { + t.Fatal(err) + } + if _, err := (stateStore{path: path}).Load(); err == nil { + t.Fatal("old backend checkpoint silently accepted") + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(after) != string(old) { + t.Fatal("unsupported checkpoint discarded observation fences") + } +} + +func TestRegionalPoolRequiresOneMemberWithEveryCurrentZone(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + snapshot := testSnapshot(now) + b := snapshot.Bindings[0] + b.Zones = append(b.Zones, model.ZoneAttachment{ZoneUID: "zone-b", Apex: "custom.internal", RequiredPublicationEpoch: 2, RequiredPublicationRevision: 4}) + snapshot.Bindings = []model.Binding{b} + snapshot.Members = []model.Member{{MemberID: "regional-0", Role: "regional"}, {MemberID: "regional-1", Role: "regional"}} + a := &Agent{cfg: Config{Mode: ModeResolver}, state: newCheckpoint(), deps: Dependencies{Clock: func() time.Time { return now }}} + a.state.Snapshot = &snapshot + a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + for _, z := range b.Zones { + a.state.Publications[z.ZoneUID] = publicationState{Fence: fence{2, 4}, Verified: true} + a.state.Acks[z.ZoneUID] = map[string]replicaAck{} + } + proof := replicaAck{Fence: fence{2, 4}, ValidUntil: now.Add(time.Minute)} + a.state.Acks["zone-a"]["regional-0/"] = proof + a.state.Acks["zone-b"]["regional-1/"] = proof + if a.publicationsReady(b) { + t.Fatal("different member per zone incorrectly made whole context ready") + } + a.state.Acks["zone-b"]["regional-0/"] = proof + ready := a.eligibleRegionalMembers(b) + if !ready["regional-0"] || ready["regional-1"] { + t.Fatalf("wrong member eligibility: %#v", ready) + } + c, _ := renderFixture(t) + c.Role = RenderRegionalDNSDist + c.ClusterDNSDist.Backends = []model.Backend{{MemberID: "regional-0", Address: "10.253.0.20", Port: 5300}, {MemberID: "regional-1", Address: "10.253.0.21", Port: 5300}} + c.ReadyMembers = map[string]map[string]bool{b.BindingUID: ready} + rendered, err := Render(c, snapshot, map[string]bool{b.BindingUID: true}) + if err != nil { + t.Fatal(err) + } + conf := string(rendered.Files[c.ClusterDNSDist.Path]) + if strings.Count(conf, "server:addPool(") != 1 { + t.Fatalf("stale live member remains in context pool:\n%s", conf) + } + // A newer ACK without the matching deadline plan gates even the formerly + // current member; a missed record stream cannot serve data with old leases. + a.state.Acks["zone-a"]["regional-1/"] = replicaAck{Fence: fence{2, 5}, ValidUntil: now.Add(time.Minute)} + if a.publicationsReady(b) { + t.Fatal("ACK ahead of local deadline plan activated context") + } +} +func TestMissingOrStalePrivatePublicationHasNoPublicFallback(t *testing.T) { + t.Parallel() + c, snapshot := renderFixture(t) + c.Role = RenderRegionalBIND + publications := map[string]publicationState{"zone-a": {Fence: fence{1, 1}, Plan: model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal"}}} + rendered, err := renderPublications(c, snapshot, map[string]bool{}, publications, time.Now()) + if err != nil { + t.Fatal(err) + } + conf := string(rendered.Files[c.ClusterBIND.Path]) + if !strings.Contains(conf, "type forward; forward only; forwarders { 127.0.0.1 port 9; }") { + t.Fatalf("private zone can recurse publicly: %s", conf) + } + for path := range rendered.Files { + if strings.HasSuffix(path, ".db") { + t.Fatal("stale publication installed as private authority") + } + } +} + +func TestRejectedMemberProofRevokesRoutingAndIgnoresOlderReplay(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + cfg.Mode = ModeResolver + cfg.Render.Role = RenderRegionalDNSDist + cfg.Render.ClusterDNSDist.Path = filepath.Join(t.TempDir(), "frontend.conf") + a, err := NewAgent(cfg, Dependencies{Clock: func() time.Time { return clock }, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + a.state.Snapshot = &snapshot + a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + chunk, _ := testPublication(t, now) + var plan model.PublicationPlan + if err := json.Unmarshal(chunk.Payload, &plan); err != nil { + t.Fatal(err) + } + a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan, now)} + a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)}} + ack := model.MemberAck{MemberID: "regional-0", ReplicaID: "regional-0", ResourceUID: "zone-a", Kind: model.KindPublicationManifest, Epoch: 2, Revision: 4, Phase: model.AckRejected, ObservedAt: now.Add(-time.Second), ValidUntil: now.Add(time.Minute)} + if err := a.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatal(err) + } + if !a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("older rejection replay revoked fresh proof") + } + clock = now.Add(time.Second) + ack.ObservedAt = clock + if err := a.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatal(err) + } + if a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("new rejection retained stale member routing") + } + ack.Phase = model.AckVerified + ack.ObservedAt = now + if err := a.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { + t.Fatal(err) + } + if a.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("old verified ACK resurrected rejected member") + } +} + +func TestPublicationProbeTimeoutDoesNotRenewWatchdog(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + udp, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = udp.Close() }() + port := udp.LocalAddr().(*net.UDPAddr).Port + cfg.Render.ClusterBIND.ListenAddress = "127.0.0.1" + cfg.Render.ClusterBIND.Port = uint16(port) + cfg.PublicationDNSProbe = PublicationDNSVerifierConfig{Server: udp.LocalAddr().String(), Timeout: 50 * time.Millisecond} + cfg.UnsafeAllowNoDNSVerify = false + a, err := NewAgent(cfg, Dependencies{Clock: func() time.Time { return now }, Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, snapshot)); err != nil { + t.Fatal(err) + } + chunk, m := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err == nil { + t.Fatal("timed out publication accepted") + } + if _, err := os.Stat(cfg.WatchdogLeasePath); !errors.Is(err, os.ErrNotExist) { + t.Fatal("failed publication query proof renewed watchdog") + } +} + +type publicationVerifierFunc func(context.Context, model.PublicationPlan, []PublicationView) error + +func (f publicationVerifierFunc) VerifyPublication(ctx context.Context, p model.PublicationPlan, views []PublicationView) error { + return f(ctx, p, views) +} + +type afterReloadRunner struct { + recordingRunner + after func() +} + +func (r *afterReloadRunner) Run(ctx context.Context, command Command) ([]byte, error) { + data, err := r.recordingRunner.Run(ctx, command) + if err == nil && command.Path == "reload" && r.after != nil { + after := r.after + r.after = nil + after() + } + return data, err +} + +func installedFileVerifier(t *testing.T, configPath string) PublicationVerifier { + t.Helper() + return publicationVerifierFunc(func(_ context.Context, plan model.PublicationPlan, views []PublicationView) error { + config, err := os.ReadFile(configPath) + if err != nil { + return err + } + pattern := regexp.MustCompile(`zone "` + regexp.QuoteMeta(strings.TrimSuffix(plan.Apex, ".")) + `" \{ type primary; file "([^"]+)"`) + match := pattern.FindSubmatch(config) + if len(match) != 2 { + return errors.New("no activated primary zone") + } + data, err := os.ReadFile(string(match[1])) + if err != nil { + return err + } + parser := dns.NewZoneParser(strings.NewReader(string(data)), model.AbsoluteName(plan.Apex), string(match[1])) + for rr, ok := parser.Next(); ok; rr, ok = parser.Next() { + if soa, ok := rr.(*dns.SOA); ok { + for _, view := range views { + if soa.Serial != view.Serial || soa.Mbox != view.FingerprintMailbox { + return fmt.Errorf("proof expects %d/%s but activated file contains %d/%s", view.Serial, view.FingerprintMailbox, soa.Serial, soa.Mbox) + } + } + return nil + } + } + return errors.New("activated zone lacks SOA") + }) +} + +func TestBINDProofPinsActivatedFilesWhenDeadlineCrossesReload(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + runner := &afterReloadRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + clock = now.Add(59 * time.Second) + a.needsActivation = true + runner.after = func() { clock = now.Add(61 * time.Second) } + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + if a.state.Publications["zone-a"].LocalRevision == 0 || strings.Contains(currentZone(t, a), "192.0.2.10") { + t.Fatal("expiry crossing reload was not materialized before watchdog renewal") + } + if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { + t.Fatalf("safe expired authority did not renew watchdog: %v", err) + } + // Restart must reconstruct file proofs through activation, never trust the + // earlier process's in-memory proof or its persisted rendered hash alone. + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) + if err != nil { + t.Fatal(err) + } + if len(restarted.installedPublications) != 0 { + t.Fatal("restart inherited unverified process file proofs") + } + restarted.needsActivation = true + if err := restarted.Step(context.Background()); err != nil { + t.Fatal(err) + } + if len(restarted.installedPublications["zone-a"].Views) == 0 { + t.Fatal("restart did not reconstruct exact activated zone proof") + } +} + +func TestFailedBINDActivationRetainsPriorInstalledProof(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + runner := &recordingRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + prior := a.installedPublications["zone-a"].Identity + p := a.state.Publications["zone-a"] + p.LocalRevision++ + a.state.Publications["zone-a"] = p + runner.failAt = len(runner.calls) + 1 + if err := a.reconcile(context.Background()); err == nil { + t.Fatal("failed validator was accepted") + } + if a.installedPublications["zone-a"].Identity != prior { + t.Fatal("failed activation advanced installed proof") + } +} + +func TestBINDDoesNotProbeAuthorityBelowRequiredPublicationFence(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + calls := 0 + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }, PublicationVerifier: publicationVerifierFunc(func(context.Context, model.PublicationPlan, []PublicationView) error { + calls++ + return errors.New("unexpected authoritative proof") + })}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + snapshot.Bindings[0].Zones[0].RequiredPublicationEpoch = 2 + snapshot.Bindings[0].Zones[0].RequiredPublicationRevision = 5 + a.state.Snapshot = &snapshot + a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) + chunk, manifest := testPublication(t, now) + var plan model.PublicationPlan + if err := json.Unmarshal(chunk.Payload, &plan); err != nil { + t.Fatal(err) + } + a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Manifest: manifest, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan, now)} + if err := a.reconcile(context.Background()); err != nil { + t.Fatal(err) + } + if calls != 0 || len(a.publicationViews(a.state.Publications["zone-a"])) != 0 { + t.Fatal("SERVFAIL zone attempted authoritative publication proof") + } + config, err := os.ReadFile(cfg.Render.ClusterBIND.Path) + if err != nil || !strings.Contains(string(config), "forward only; forwarders { 127.0.0.1 port 9; }") { + t.Fatalf("behind-required private zone was not gated: %s %v", config, err) + } +} + +func TestBINDWatchdogCannotRenewExpiredInstalledSnapshot(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + clock = now.Add(61 * time.Second) + if err := a.renewWatchdogLease(context.Background()); err == nil { + t.Fatal("expired installed snapshot renewed watchdog") + } + if _, err := os.Stat(cfg.WatchdogLeasePath); !errors.Is(err, os.ErrNotExist) || !a.needsActivation { + t.Fatalf("expired snapshot was not gated: %v", err) + } +} + +func TestBINDExpiryCrossingProofSettlesBeforeWatchdogRenewal(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + files := installedFileVerifier(t, cfg.Render.ClusterBIND.Path) + advance := false + verifier := publicationVerifierFunc(func(ctx context.Context, plan model.PublicationPlan, views []PublicationView) error { + err := files.VerifyPublication(ctx, plan, views) + if advance { + advance = false + clock = now.Add(61 * time.Second) + } + return err + }) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }, PublicationVerifier: verifier}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + clock = now.Add(59 * time.Second) + advance = true + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + if strings.Contains(currentZone(t, a), "192.0.2.10") || a.state.Publications["zone-a"].LocalRevision == 0 { + t.Fatal("deadline crossing proof renewed stale authority") + } + if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { + t.Fatalf("settled expiry unnecessarily gated process: %v", err) + } +} + +type partitioningAckPublisher struct { + disconnected atomic.Bool + verified atomic.Int64 +} + +func (p *partitioningAckPublisher) Publish(_ context.Context, _ string, env model.Envelope) (transport.PublishAck, error) { + if p.disconnected.Load() { + return transport.PublishAck{}, errors.New("nats: server is disconnected") + } + var ack model.MemberAck + if err := json.Unmarshal(env.Payload, &ack); err != nil { + return transport.PublishAck{}, err + } + if ack.Kind == model.KindPublicationManifest && ack.Phase == model.AckVerified { + p.verified.Add(1) + } + return transport.PublishAck{}, nil +} + +func TestBINDActivationRacingBrokerDisconnectKeepsSafeLeaseAndOriginalDeadline(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + runner := &afterReloadRunner{} + publisher := &partitioningAckPublisher{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }, AckPublisher: publisher, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + chunk, manifest := testPublication(t, now) + var plan model.PublicationPlan + if err := json.Unmarshal(chunk.Payload, &plan); err != nil { + t.Fatal(err) + } + plan.ObservationFences[0].Sequence = 2 + plan.RRSets[0].Records[0].Sequence = 2 + plan.RRSets[0].Records[0].Content = "192.0.2.11" + chunk.Payload, err = json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + chunk.ManifestUID = "manifest-b" + chunk.Revision = 5 + chunk.SHA256 = model.Hash(chunk.Payload) + manifest.ManifestUID = chunk.ManifestUID + manifest.PreviousRevision = 4 + manifest.Revision = chunk.Revision + manifest.ContentHash = chunk.SHA256 + manifest.Chunks = []model.ChunkRef{{Index: 0, SHA256: chunk.SHA256, Size: len(chunk.Payload)}} + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 5, chunk)); err != nil { + t.Fatal(err) + } + // Disconnect after BIND reload, exactly before durable/proved activation + // attempts its ACK. A transport error must not become a failed installation. + runner.after = func() { publisher.disconnected.Store(true) } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { + t.Fatalf("safe activation treated broker failure as materialization failure: %v", err) + } + if !a.state.Publications["zone-a"].Verified || a.needsActivation { + t.Fatal("ACK partition revoked proved publication") + } + if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { + t.Fatalf("ACK partition removed safe local lease: %v", err) + } + // Identical manifest redelivery and snapshot delivery must share the same + // best-effort ACK boundary without extending observation deadlines. + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + snapshot.ConfigurationRevision = 2 + snapshot.Bindings[0].Authorization.Revision = 2 + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, snapshot)); err != nil { + t.Fatal(err) + } + clock = now.Add(61 * time.Second) + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + if strings.Contains(currentZone(t, a), "192.0.2.11") { + t.Fatal("ACK partition left records past their original deadline") + } + if !a.state.Publications["zone-a"].Plan.ObservationFences[0].ValidUntil.Equal(now.Add(time.Minute)) { + t.Fatal("activation or ACK retry renewed the observation deadline") + } + if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil || a.needsActivation { + t.Fatalf("safe local expiry was gated by ACK outage: %v", err) + } + before := publisher.verified.Load() + publisher.disconnected.Store(false) + if err := a.Step(context.Background()); err != nil { + t.Fatal(err) + } + if publisher.verified.Load() <= before { + t.Fatal("broker recovery did not retry publication readiness ACK") + } +} + +func TestBINDTombstoneAckFailureDoesNotGateDurableWithdrawal(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + publisher := &partitioningAckPublisher{} + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }, AckPublisher: publisher, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, a, now) + publisher.disconnected.Store(true) + manifest := model.PublicationManifest{ManifestUID: "retired-zone", ZoneUID: "zone-a", Apex: "prod.internal", WriterEpoch: 2, Revision: 5, PreviousRevision: 4, ContentHash: model.EmptyContentHash(), GeneratedAt: now, Tombstone: true} + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { + t.Fatal(err) + } + p := a.state.Publications["zone-a"] + if !p.Verified || !p.Manifest.Tombstone || a.needsActivation { + t.Fatal("failed tombstone ACK invalidated safely withdrawn zone") + } + if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { + t.Fatalf("withdrawal lost safe lease because broker was unavailable: %v", err) + } +} diff --git a/internal/internaldns/serving/binding_fences.go b/internal/internaldns/serving/binding_fences.go new file mode 100644 index 00000000..18bb83d6 --- /dev/null +++ b/internal/internaldns/serving/binding_fences.go @@ -0,0 +1,69 @@ +package serving + +import ( + "encoding/json" + "fmt" + "sort" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +// bindingFence retains source authority independently of shard snapshot ordering. +// Omission withdraws access without inventing a source-issued sequence number. +type bindingFence struct { + Identity string `json:"identity"` + ConfigurationGeneration uint64 `json:"configurationGeneration"` + ConfigurationRevision uint64 `json:"configurationRevision"` + Authorization model.Authorization `json:"authorization"` + Tombstone bool `json:"tombstone"` +} + +func bindingIdentity(b model.Binding) string { + transports := append([]model.Transport(nil), b.Transports...) + sort.Slice(transports, func(i, j int) bool { return transports[i] < transports[j] }) + data, _ := json.Marshal(struct { + Project, Context, Node, Regional string + Port uint16 + Transports []model.Transport + }{b.ProjectUID, b.ContextUID, b.ConsumerAddress, b.ClusterAddress, b.Port, transports}) + return model.Hash(data) +} + +func bindingFenceFor(b model.Binding) bindingFence { + return bindingFence{Identity: bindingIdentity(b), ConfigurationGeneration: b.BindingGeneration, ConfigurationRevision: b.ConfigurationRevision, Authorization: b.Authorization, Tombstone: b.Tombstone} +} + +func advanceBindingFences(prior map[string]bindingFence, snapshot model.ServingSnapshot) (map[string]bindingFence, error) { + next := make(map[string]bindingFence, len(prior)+len(snapshot.Bindings)) + for uid, old := range prior { + withdrawn := old + withdrawn.Tombstone = true + next[uid] = withdrawn + } + for _, b := range snapshot.Bindings { + candidate := bindingFenceFor(b) + if old, ok := prior[b.BindingUID]; ok { + if candidate.Identity != old.Identity { + return nil, fmt.Errorf("binding %s changed its UID-pinned source or listeners", b.BindingUID) + } + if candidate.ConfigurationGeneration < old.ConfigurationGeneration || candidate.ConfigurationRevision < old.ConfigurationRevision { + return nil, fmt.Errorf("binding %s configuration regressed", b.BindingUID) + } + cmp := model.Compare(candidate.Authorization.IssuerEpoch, candidate.Authorization.Revision, old.Authorization.IssuerEpoch, old.Authorization.Revision) + if cmp < 0 { + return nil, fmt.Errorf("binding %s source authorization regressed", b.BindingUID) + } + if cmp == 0 && !candidate.Authorization.ValidUntil.Equal(old.Authorization.ValidUntil) { + return nil, fmt.Errorf("binding %s changed the original deadline at the same source fence", b.BindingUID) + } + if old.Tombstone && !candidate.Tombstone && cmp <= 0 { + return nil, fmt.Errorf("binding %s reactivation requires a newer source authorization", b.BindingUID) + } + if old.Tombstone != candidate.Tombstone && candidate.ConfigurationRevision <= old.ConfigurationRevision { + return nil, fmt.Errorf("binding %s withdrawal or reactivation requires a newer configuration revision", b.BindingUID) + } + } + next[b.BindingUID] = candidate + } + return next, nil +} diff --git a/internal/internaldns/serving/binding_fences_test.go b/internal/internaldns/serving/binding_fences_test.go new file mode 100644 index 00000000..b398cc6e --- /dev/null +++ b/internal/internaldns/serving/binding_fences_test.go @@ -0,0 +1,171 @@ +package serving + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +func TestBindingWithdrawalPreservesSourceSequenceAndAllowsNormalRenewal(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + active := testSnapshot(now) + active.Bindings[0].Authorization.Revision = 2 + active.Bindings[0].Authorization.ValidUntil = now.Add(10 * time.Second) + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + chunk, manifest := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } + clock = now.Add(11 * time.Second) + withdrawn := active + withdrawn.Bindings = append([]model.Binding(nil), active.Bindings...) + withdrawn.ConfigurationRevision = 2 + withdrawn.Bindings[0].ConfigurationRevision = 2 + withdrawn.Bindings[0].Tombstone = true + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, withdrawn)); err != nil { + t.Fatal(err) + } + if f := a.state.BindingFences["binding-a"]; !f.Tombstone || f.Authorization.Revision != 2 || !f.Authorization.ValidUntil.Equal(active.Bindings[0].Authorization.ValidUntil) { + t.Fatal("withdrawal fabricated source authorization") + } + // Source2 -> Source3 is the next legitimate renewal; DNS must not have + // consumed sequence3 itself while withdrawing the expired authorization. + renewed := withdrawn + renewed.Bindings = append([]model.Binding(nil), withdrawn.Bindings...) + renewed.ConfigurationRevision = 3 + renewed.Bindings[0].ConfigurationRevision = 3 + renewed.Bindings[0].Tombstone = false + renewed.Bindings[0].Authorization.Revision = 3 + renewed.Bindings[0].Authorization.ValidUntil = now.Add(2 * time.Minute) + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, renewed)); err != nil { + t.Fatalf("normal source2 -> source3 renewal rejected: %v", err) + } + config, err := os.ReadFile(cfg.Render.ClusterBIND.Path) + if err != nil || !strings.Contains(string(config), renewed.Bindings[0].ViewName()) { + t.Fatalf("renewal did not restore context: %v", err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, withdrawn)); err != nil { + t.Fatal(err) + } + if a.state.Snapshot.Bindings[0].Tombstone || a.state.ServingFence.Revision != 3 { + t.Fatal("stale expiry replay withdrew current renewal") + } + // A higher shard revision cannot legitimize an older source lease. + stale := withdrawn + stale.ConfigurationRevision = 4 + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 4, stale)); err == nil { + t.Fatal("new shard revision accepted stale binding authority") + } + if a.state.ServingFence.Revision != 3 { + t.Fatal("rejected binding authority advanced shard fence") + } +} + +func TestBindingOmissionFenceSurvivesRestartAndRequiresNewSourceAuthority(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + active := testSnapshot(now) + active.Bindings[0].Authorization.Revision = 2 + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + omitted := active + omitted.ConfigurationRevision = 2 + omitted.Bindings = nil + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, omitted)); err != nil { + t.Fatal(err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if !restarted.state.BindingFences["binding-a"].Tombstone { + t.Fatal("restart forgot omitted UID withdrawal") + } + candidate := active + candidate.Bindings = append([]model.Binding(nil), active.Bindings...) + candidate.ConfigurationRevision = 3 + candidate.Bindings[0].ConfigurationRevision = 2 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err == nil { + t.Fatal("same source sequence resurrected omitted binding") + } + candidate.Bindings[0].Authorization.ValidUntil = now.Add(20 * time.Minute) + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err == nil { + t.Fatal("same source sequence changed its original deadline") + } + candidate.Bindings[0].Authorization.Revision = 3 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err != nil { + t.Fatalf("new source sequence could not reactivate omitted binding: %v", err) + } + // A replacement resource has a new UID and independent source sequence. + replacement := candidate + replacement.Bindings = append([]model.Binding(nil), candidate.Bindings...) + replacement.ConfigurationRevision = 4 + replacement.Bindings[0].BindingUID = "replacement-binding" + replacement.Bindings[0].ConfigurationRevision = 1 + replacement.Bindings[0].Authorization.Revision = 1 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 4, replacement)); err != nil { + t.Fatal(err) + } + if !restarted.state.BindingFences["binding-a"].Tombstone || restarted.state.BindingFences["replacement-binding"].Authorization.Revision != 1 { + t.Fatal("replacement UID discarded old source fences or inherited its sequence") + } + candidate.ConfigurationRevision = 5 + candidate.Bindings[0].ConfigurationRevision = 3 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 5, candidate)); err == nil { + t.Fatal("retired UID replay replaced newly installed resource") + } + mutated := replacement + mutated.Bindings = append([]model.Binding(nil), replacement.Bindings...) + mutated.ConfigurationRevision = 5 + mutated.Bindings[0].ProjectUID = "different-project" + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 5, mutated)); err == nil { + t.Fatal("same binding UID changed pinned project identity") + } +} + +func TestCheckpointRejectsPreBindingFenceFormatWithoutDiscardingIt(t *testing.T) { + t.Parallel() + for _, version := range []int{2, checkpointFormatVersion} { + t.Run(string(rune('0'+version)), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "checkpoint.json") + data, err := json.Marshal(map[string]any{"formatVersion": version, "contributionFences": map[string]any{"original-observation": map[string]any{"epoch": 1, "sequence": 2}}}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + if _, err := (stateStore{path: path}).Load(); err == nil { + t.Fatal("checkpoint without complete binding authority history accepted") + } + retained, err := os.ReadFile(path) + if err != nil || string(retained) != string(data) { + t.Fatal("unsupported checkpoint was modified") + } + }) + } +} diff --git a/internal/internaldns/serving/bootstrap.go b/internal/internaldns/serving/bootstrap.go new file mode 100644 index 00000000..030f20ad --- /dev/null +++ b/internal/internaldns/serving/bootstrap.go @@ -0,0 +1,146 @@ +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +type SnapshotLister interface { + List(context.Context, string) ([]model.Envelope, error) +} +type SnapshotBootstrap struct { + Store SnapshotLister + Region, Shard string + Attempts int + RetryDelay time.Duration +} + +// Load obtains a self-consistent bootstrap cut. Chunk keys are immutable and +// revision-qualified; stable manifest heads are accepted only when all of their +// referenced chunks are present and hash correctly in the same listing. +func (b SnapshotBootstrap) Load(ctx context.Context) ([]model.Envelope, error) { + if b.Store == nil || b.Region == "" || b.Shard == "" { + return nil, errors.New("snapshot store, region, and shard are required") + } + attempts := b.Attempts + if attempts <= 0 { + attempts = 4 + } + delay := b.RetryDelay + if delay <= 0 { + delay = 100 * time.Millisecond + } + var last error + for attempt := 0; attempt < attempts; attempt++ { + envs, err := b.Store.List(ctx, model.SnapshotPrefix(b.Region, b.Shard)) + if err == nil { + err = verifyBootstrap(envs, b.Region, b.Shard) + } + if err == nil { + envs, err = currentBootstrapEnvelopes(envs) + } + if err == nil { + sort.SliceStable(envs, func(i, j int) bool { return bootstrapOrder(envs[i].Kind) < bootstrapOrder(envs[j].Kind) }) + return envs, nil + } + last = err + if attempt+1 < attempts { + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + } + return nil, fmt.Errorf("consistent internal DNS bootstrap unavailable: %w", last) +} + +// currentBootstrapEnvelopes drops immutable chunk sets that are no longer +// referenced by a stable manifest head. The KV listing may retain them for +// replay/rollback safety, but a cold member must not checkpoint or apply them. +func currentBootstrapEnvelopes(envs []model.Envelope) ([]model.Envelope, error) { + manifests := map[string]bool{} + for _, env := range envs { + if env.Kind != model.KindPublicationManifest { + continue + } + var manifest model.PublicationManifest + if err := json.Unmarshal(env.Payload, &manifest); err != nil { + return nil, err + } + manifests[manifest.ManifestUID] = true + } + out := make([]model.Envelope, 0, len(envs)) + for _, env := range envs { + if env.Kind == model.KindPublicationChunk { + var chunk model.PublicationChunk + if err := json.Unmarshal(env.Payload, &chunk); err != nil { + return nil, err + } + if !manifests[chunk.ManifestUID] { + continue + } + } + out = append(out, env) + } + return out, nil +} + +func verifyBootstrap(envs []model.Envelope, region, shard string) error { + chunks := map[string][]model.PublicationChunk{} + var manifests []model.PublicationManifest + serving := false + for _, env := range envs { + if err := env.Validate(); err != nil { + return err + } + if env.Region != region || env.Shard != shard { + return errors.New("bootstrap contains a foreign region or shard") + } + switch env.Kind { + case model.KindPublicationChunk: + var c model.PublicationChunk + if err := json.Unmarshal(env.Payload, &c); err != nil { + return err + } + chunks[c.ManifestUID] = append(chunks[c.ManifestUID], c) + case model.KindPublicationManifest: + var m model.PublicationManifest + if err := json.Unmarshal(env.Payload, &m); err != nil { + return err + } + manifests = append(manifests, m) + case model.KindServingSnapshot: + serving = true + default: + return fmt.Errorf("bootstrap contains unsupported kind %q", env.Kind) + } + } + if !serving { + return errors.New("bootstrap has no complete serving snapshot") + } + for _, m := range manifests { + if _, err := model.VerifyManifest(m, chunks[m.ManifestUID]); err != nil { + return err + } + } + return nil +} +func bootstrapOrder(k model.Kind) int { + switch k { + case model.KindPublicationChunk: + return 0 + case model.KindServingSnapshot: + return 1 + default: + return 2 + } +} diff --git a/internal/internaldns/serving/bootstrap_test.go b/internal/internaldns/serving/bootstrap_test.go new file mode 100644 index 00000000..3d7e4ac7 --- /dev/null +++ b/internal/internaldns/serving/bootstrap_test.go @@ -0,0 +1,71 @@ +package serving + +import ( + "context" + "errors" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +type bootstrapLister struct { + calls int + results [][]model.Envelope +} + +func (l *bootstrapLister) List(context.Context, string) ([]model.Envelope, error) { + l.calls++ + if len(l.results) == 0 { + return nil, errors.New("empty") + } + i := l.calls - 1 + if i >= len(l.results) { + i = len(l.results) - 1 + } + return l.results[i], nil +} + +func TestSnapshotBootstrapRetriesRacingManifestHead(t *testing.T) { + t.Parallel() + now := testSnapshotNow() + chunk, manifest := testPublication(t, now) + serving := testSnapshot(now) + chunkEnv := envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk) + manifestEnv := envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest) + servingEnv := envelope(t, model.KindServingSnapshot, "shard", 1, 1, serving) + lister := &bootstrapLister{results: [][]model.Envelope{{manifestEnv, servingEnv}, {manifestEnv, chunkEnv, servingEnv}}} + got, err := (SnapshotBootstrap{Store: lister, Region: "east", Shard: "s1", Attempts: 2}).Load(context.Background()) + if err != nil { + t.Fatal(err) + } + if lister.calls != 2 || len(got) != 3 || got[0].Kind != model.KindPublicationChunk { + t.Fatalf("calls=%d events=%v", lister.calls, got) + } +} + +func TestSnapshotBootstrapIgnoresUnreferencedHistoricalChunks(t *testing.T) { + t.Parallel() + now := testSnapshotNow() + chunk, manifest := testPublication(t, now) + historical := chunk + historical.ManifestUID = "old-manifest" + events := []model.Envelope{ + envelope(t, model.KindPublicationChunk, "zone-a", 2, 3, historical), + envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk), + envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest), + envelope(t, model.KindServingSnapshot, "shard", 1, 1, testSnapshot(now)), + } + lister := &bootstrapLister{results: [][]model.Envelope{events}} + got, err := (SnapshotBootstrap{Store: lister, Region: "east", Shard: "s1"}).Load(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("cold bootstrap retained %d events, want current chunk+manifest+serving", len(got)) + } + if got[0].Kind != model.KindPublicationChunk || got[1].Kind != model.KindServingSnapshot || got[2].Kind != model.KindPublicationManifest { + t.Fatalf("unsafe bootstrap order: %#v", got) + } +} +func testSnapshotNow() time.Time { return time.Now().UTC() } diff --git a/internal/internaldns/serving/state.go b/internal/internaldns/serving/state.go new file mode 100644 index 00000000..52a4faf5 --- /dev/null +++ b/internal/internaldns/serving/state.go @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +const legacyObservationGrantUID = "legacy-checkpoint" + +// contributionFence persists the complete immutable observation identity. +// LegacyRevision only exists to decode checkpoints written before Sequence +// and the original deadline were persisted. +type contributionFence struct { + GrantUID string `json:"grantUID,omitempty"` + Epoch uint64 `json:"epoch"` + Sequence uint64 `json:"sequence,omitempty"` + ValidUntil time.Time `json:"validUntil,omitempty"` + LegacyRevision uint64 `json:"revision,omitempty"` +} + +type replicaAck struct { + ObservedAt time.Time `json:"observedAt"` + Fence fence `json:"fence"` + ValidUntil time.Time `json:"validUntil"` +} + +const checkpointFormatVersion = 3 + +type checkpoint struct { + FormatVersion int `json:"formatVersion"` + ServingFence fence `json:"servingFence"` + Snapshot *model.ServingSnapshot `json:"snapshot,omitempty"` + Publications map[string]publicationState `json:"publications"` + Chunks map[string]map[string]model.PublicationChunk `json:"chunks"` + Acks map[string]map[string]replicaAck `json:"acks"` + BindingFences map[string]bindingFence `json:"bindingFences"` + ContributionFences map[string]contributionFence `json:"contributionFences"` + RenderedHash string `json:"renderedHash,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func newCheckpoint() checkpoint { + return checkpoint{FormatVersion: checkpointFormatVersion, Publications: map[string]publicationState{}, Chunks: map[string]map[string]model.PublicationChunk{}, Acks: map[string]map[string]replicaAck{}, ContributionFences: map[string]contributionFence{}, BindingFences: map[string]bindingFence{}} +} + +type stateStore struct{ path string } + +func (s stateStore) Load() (checkpoint, error) { + state := newCheckpoint() + b, err := os.ReadFile(s.path) + if errors.Is(err, os.ErrNotExist) { + return state, nil + } + if err != nil { + return state, err + } + state.FormatVersion = 0 + state.BindingFences = nil + if err := json.Unmarshal(b, &state); err != nil { + return state, fmt.Errorf("decode checkpoint: %w", err) + } + if state.FormatVersion != checkpointFormatVersion { + return state, fmt.Errorf("unsupported checkpoint format %d: preserve the existing checkpoint; automatic backend or binding authority history migration is not supported", state.FormatVersion) + } + if state.BindingFences == nil { + return state, errors.New("checkpoint lacks required binding authority history") + } + for uid, f := range state.BindingFences { + if uid == "" || f.Identity == "" || f.ConfigurationGeneration == 0 || f.ConfigurationRevision == 0 || f.Authorization.IssuerEpoch == 0 || f.Authorization.Revision == 0 || f.Authorization.ValidUntil.IsZero() { + return state, fmt.Errorf("checkpoint binding %s has incomplete authority history", uid) + } + } + if state.Snapshot != nil { + for _, b := range state.Snapshot.Bindings { + f, ok := state.BindingFences[b.BindingUID] + if !ok || !equalJSON(f, bindingFenceFor(b)) { + return state, fmt.Errorf("checkpoint binding %s does not match its authority history", b.BindingUID) + } + } + } + if state.Publications == nil { + state.Publications = map[string]publicationState{} + } + if state.Chunks == nil { + state.Chunks = map[string]map[string]model.PublicationChunk{} + } + if state.Acks == nil { + state.Acks = map[string]map[string]replicaAck{} + } + if state.ContributionFences == nil { + state.ContributionFences = map[string]contributionFence{} + } + if state.Snapshot != nil { + if err := state.Snapshot.Validate(); err != nil { + return state, fmt.Errorf("invalid persisted serving snapshot: %w", err) + } + } + for uid, p := range state.Publications { + if !p.Manifest.Tombstone { + var err error + p.Plan, err = migrateLegacyObservationFences(p.Plan) + if err != nil { + return state, fmt.Errorf("migrate persisted publication %s: %w", uid, err) + } + state.Publications[uid] = p + if err := p.Plan.Validate(); err != nil { + return state, fmt.Errorf("invalid persisted publication %s: %w", uid, err) + } + } + } + if err := migrateContributionHighWater(&state); err != nil { + return state, err + } + return state, nil +} + +func migrateLegacyObservationFences(plan model.PublicationPlan) (model.PublicationPlan, error) { + if len(plan.ObservationFences) != 0 { + return plan, nil + } + byUID := map[string]model.ObservationFence{} + for _, rrset := range plan.RRSets { + for _, record := range rrset.Records { + if record.ContributionUID == "" { + continue + } + next := model.ObservationFence{ContributionUID: record.ContributionUID, GrantUID: legacyObservationGrantUID, WriterEpoch: record.WriterEpoch, Sequence: record.Sequence, ValidUntil: record.ValidUntil} + if old, ok := byUID[record.ContributionUID]; ok && (old.WriterEpoch != next.WriterEpoch || old.Sequence != next.Sequence || !old.ValidUntil.Equal(next.ValidUntil)) { + return plan, fmt.Errorf("legacy contribution %q has inconsistent record fences", record.ContributionUID) + } + byUID[record.ContributionUID] = next + } + } + for _, observation := range byUID { + plan.ObservationFences = append(plan.ObservationFences, observation) + } + sort.Slice(plan.ObservationFences, func(i, j int) bool { + return plan.ObservationFences[i].ContributionUID < plan.ObservationFences[j].ContributionUID + }) + return plan, nil +} + +func migrateContributionHighWater(state *checkpoint) error { + proofs := map[string]model.ObservationFence{} + for _, publication := range state.Publications { + for _, observation := range publication.Plan.ObservationFences { + old, ok := proofs[observation.ContributionUID] + if !ok || model.Compare(observation.WriterEpoch, observation.Sequence, old.WriterEpoch, old.Sequence) > 0 { + proofs[observation.ContributionUID] = observation + } + } + } + for uid, current := range state.ContributionFences { + if current.Sequence == 0 { + current.Sequence = current.LegacyRevision + } + current.LegacyRevision = 0 + if proof, ok := proofs[uid]; ok && proof.WriterEpoch == current.Epoch && proof.Sequence == current.Sequence { + if current.ValidUntil.IsZero() { + current.ValidUntil = proof.ValidUntil + } + if current.GrantUID == "" && proof.GrantUID != legacyObservationGrantUID { + current.GrantUID = proof.GrantUID + } + } + if current.Epoch == 0 || current.Sequence == 0 { + return fmt.Errorf("invalid persisted contribution fence %q", uid) + } + state.ContributionFences[uid] = current + } + return nil +} + +func (s stateStore) Save(state checkpoint) error { + if state.FormatVersion != checkpointFormatVersion { + return fmt.Errorf("cannot write unsupported checkpoint format %d", state.FormatVersion) + } + state.UpdatedAt = time.Now().UTC() + if err := os.MkdirAll(filepath.Dir(s.path), 0o750); err != nil { + return err + } + b, err := json.MarshalIndent(state, "", " ") + if err != nil { + return err + } + temp := s.path + ".tmp" + if err := writeDurable(temp, b, 0o600); err != nil { + return err + } + if err := os.Rename(temp, s.path); err != nil { + return err + } + dir, err := os.Open(filepath.Dir(s.path)) + if err != nil { + return err + } + defer func() { _ = dir.Close() }() + return dir.Sync() +} + +func chunkKey(index int) string { return fmt.Sprint(index) } diff --git a/internal/internaldns/watchdog/watchdog.go b/internal/internaldns/watchdog/watchdog.go new file mode 100644 index 00000000..f45a0b03 --- /dev/null +++ b/internal/internaldns/watchdog/watchdog.go @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package watchdog supervises a serving agent from an independent process. An +// agent that is killed or hung cannot renew this lease; the supervisor gates its +// DNS processes without depending on Kubernetes or a remote control plane. +package watchdog + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/serving" +) + +type Config struct { + LeasePath string `json:"leasePath"` + MemberID string `json:"memberID"` + ReplicaID string `json:"replicaID,omitempty"` + MaxLeaseSeconds int `json:"maxLeaseSeconds,omitempty"` + StartupGraceSeconds int `json:"startupGraceSeconds,omitempty"` + IntervalMillis int `json:"intervalMillis,omitempty"` + FailClosed []serving.Command `json:"failClosed"` +} + +type Lease struct { + MemberID string `json:"memberID"` + ReplicaID string `json:"replicaID,omitempty"` + ValidUntil time.Time `json:"validUntil"` +} + +type Watchdog struct { + Config Config + Runner serving.Runner + Now func() time.Time + ReadFile func(string) ([]byte, error) +} + +func (c Config) Validate() error { + if !filepath.IsAbs(c.LeasePath) || c.MemberID == "" || len(c.FailClosed) == 0 { + return errors.New("watchdog requires an absolute lease path, member identity, and fail-closed commands") + } + if c.MaxLeaseSeconds < 0 || c.MaxLeaseSeconds > 30 || c.StartupGraceSeconds < 0 || c.StartupGraceSeconds > 60 { + return errors.New("watchdog lease and startup grace exceed platform bounds") + } + if c.IntervalMillis < 0 || c.IntervalMillis > 1000 { + return errors.New("watchdog checks must run at least once per second") + } + for _, command := range c.FailClosed { + if command.Path == "" { + return errors.New("watchdog fail-closed command path is required") + } + } + return nil +} + +func (w *Watchdog) Check(now time.Time) error { + readFile := w.ReadFile + if readFile == nil { + readFile = os.ReadFile + } + var lease Lease + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + b, err := readFile(w.Config.LeasePath) + if err != nil { + lastErr = fmt.Errorf("read serving lease: %w", err) + } else if len(b) > 4096 { + // A decoded identity or deadline is never retried, and neither is an + // oversized file: only a transient read or partial JSON observation + // can be repaired by the writer's atomic replacement becoming visible. + return errors.New("serving lease exceeds size limit") + } else { + lease = Lease{} + if err := json.Unmarshal(b, &lease); err != nil { + lastErr = fmt.Errorf("decode serving lease: %w", err) + } else { + lastErr = nil + break + } + } + if attempt < 2 { + time.Sleep(10 * time.Millisecond) + } + } + if lastErr != nil { + return lastErr + } + if lease.MemberID != w.Config.MemberID || w.Config.ReplicaID != "" && lease.ReplicaID != w.Config.ReplicaID { + return errors.New("serving lease belongs to another member") + } + if w.Now != nil { + // A successful retry may observe a lease written after Check's caller + // sampled now. Validate that replacement against the current clock so a + // fresh maximum-length lease is not misclassified as overlong. + now = w.Now() + } + max := w.Config.MaxLeaseSeconds + if max == 0 { + max = 5 + } + if lease.ValidUntil.IsZero() || !lease.ValidUntil.After(now) { + return errors.New("serving lease expired") + } + if lease.ValidUntil.After(now.Add(time.Duration(max) * time.Second)) { + return errors.New("serving lease deadline exceeds the watchdog bound") + } + return nil +} + +func (w *Watchdog) Start(ctx context.Context) error { + if err := w.Config.Validate(); err != nil { + return err + } + if w.Runner == nil { + w.Runner = serving.ExecRunner{} + } + if w.Now == nil { + w.Now = time.Now + } + start := w.Now() + last := start + interval := w.Config.IntervalMillis + if interval == 0 { + interval = 250 + } + ticker := time.NewTicker(time.Duration(interval) * time.Millisecond) + defer ticker.Stop() + for { + now := w.Now() + err := w.Check(now) + if now.Before(last.Add(-time.Second)) { + err = errors.New("watchdog clock moved backwards") + } + last = now + // Grace only applies to a new member without any checkpoint. An existing + // expired or malformed lease is gated immediately even during startup. + if err != nil && errors.Is(err, os.ErrNotExist) && now.Sub(start) < time.Duration(w.Config.StartupGraceSeconds)*time.Second { + err = nil + } + if err != nil { + var failures []error + for _, command := range w.Config.FailClosed { + commandCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + out, runErr := w.Runner.Run(commandCtx, command) + cancel() + if runErr != nil { + failures = append(failures, fmt.Errorf("fail closed %s: %w: %s", command.Path, runErr, out)) + } + } + if len(failures) == 0 { + return fmt.Errorf("serving member gated: %w", err) + } + // A failed gate is retried; returning success or letting a remote + // controller handle it would allow stale local DNS to keep serving. + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} diff --git a/internal/internaldns/watchdog/watchdog_test.go b/internal/internaldns/watchdog/watchdog_test.go new file mode 100644 index 00000000..6db9c180 --- /dev/null +++ b/internal/internaldns/watchdog/watchdog_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package watchdog + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/serving" +) + +type recordingRunner struct{ calls int } + +func (r *recordingRunner) Run(context.Context, serving.Command) ([]byte, error) { + r.calls++ + return nil, nil +} + +func TestDeadOrForgedAgentIsGatedWithoutRemoteState(t *testing.T) { + now := time.Date(2026, 10, 6, 18, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + lease Lease + valid bool + }{ + {"fresh", Lease{MemberID: "auth-1", ValidUntil: now.Add(5 * time.Second)}, true}, + {"expired", Lease{MemberID: "auth-1", ValidUntil: now.Add(-time.Second)}, false}, + {"another member", Lease{MemberID: "auth-2", ValidUntil: now.Add(time.Second)}, false}, + {"overlong lease", Lease{MemberID: "auth-1", ValidUntil: now.Add(time.Minute)}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "lease.json") + b, _ := json.Marshal(tc.lease) + if err := os.WriteFile(path, b, 0600); err != nil { + t.Fatal(err) + } + w := Watchdog{Config: Config{LeasePath: path, MemberID: "auth-1", FailClosed: []serving.Command{{Path: "stop-private-dns"}}}, Now: func() time.Time { return now }} + if got := w.Check(now) == nil; got != tc.valid { + t.Fatalf("valid=%v want=%v", got, tc.valid) + } + if !tc.valid { + runner := &recordingRunner{} + w.Runner = runner + if err := w.Start(context.Background()); err == nil || runner.calls != 1 { + t.Fatalf("dead agent was not gated: calls=%d err=%v", runner.calls, err) + } + } + }) + } +} + +func TestExpiredCheckpointCannotUseStartupGrace(t *testing.T) { + now := time.Now() + path := filepath.Join(t.TempDir(), "lease.json") + b, _ := json.Marshal(Lease{MemberID: "auth-1", ValidUntil: now.Add(-time.Second)}) + if err := os.WriteFile(path, b, 0600); err != nil { + t.Fatal(err) + } + runner := &recordingRunner{} + w := Watchdog{Config: Config{LeasePath: path, MemberID: "auth-1", StartupGraceSeconds: 60, FailClosed: []serving.Command{{Path: "stop"}}}, Runner: runner, Now: func() time.Time { return now }} + if err := w.Start(context.Background()); err == nil || runner.calls != 1 { + t.Fatal("expired checkpoint survived restart") + } +} + +func TestCheckRereadsTransientPartialLease(t *testing.T) { + now := time.Now().UTC() + valid, err := json.Marshal(Lease{MemberID: "auth-1", ValidUntil: now.Add(5 * time.Second)}) + if err != nil { + t.Fatal(err) + } + reads := 0 + w := Watchdog{ + Config: Config{LeasePath: "/lease.json", MemberID: "auth-1", MaxLeaseSeconds: 5}, + Now: func() time.Time { return now }, + ReadFile: func(string) ([]byte, error) { + reads++ + if reads == 1 { + return []byte{}, nil + } + return valid, nil + }, + } + if err := w.Check(now); err != nil { + t.Fatalf("transient partial lease was gated: %v", err) + } + if reads != 2 { + t.Fatalf("lease reads = %d, want 2", reads) + } +} + +func TestCheckRetriesOnlyReadAndDecodeFailures(t *testing.T) { + now := time.Now().UTC() + tests := []struct { + name string + lease Lease + }{ + {name: "wrong identity", lease: Lease{MemberID: "other", ValidUntil: now.Add(time.Second)}}, + {name: "expired", lease: Lease{MemberID: "auth-1", ValidUntil: now.Add(-time.Second)}}, + {name: "overlong", lease: Lease{MemberID: "auth-1", ValidUntil: now.Add(6 * time.Second)}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + payload, err := json.Marshal(test.lease) + if err != nil { + t.Fatal(err) + } + reads := 0 + w := Watchdog{ + Config: Config{LeasePath: "/lease.json", MemberID: "auth-1", MaxLeaseSeconds: 5}, + Now: func() time.Time { return now }, + ReadFile: func(string) ([]byte, error) { + reads++ + return payload, nil + }, + } + if err := w.Check(now); err == nil { + t.Fatal("unsafe decoded lease was accepted") + } + if reads != 1 { + t.Fatalf("unsafe decoded lease was retried %d times", reads) + } + }) + } + + for _, failure := range []struct { + name string + read func(string) ([]byte, error) + }{ + {name: "read", read: func(string) ([]byte, error) { return nil, errors.New("short read") }}, + {name: "decode", read: func(string) ([]byte, error) { return []byte("{"), nil }}, + } { + t.Run("persistent "+failure.name, func(t *testing.T) { + reads := 0 + w := Watchdog{ + Config: Config{LeasePath: "/lease.json", MemberID: "auth-1", MaxLeaseSeconds: 5}, + Now: func() time.Time { return now }, + ReadFile: func(path string) ([]byte, error) { + reads++ + return failure.read(path) + }, + } + if err := w.Check(now); err == nil { + t.Fatalf("persistent %s failure was accepted", failure.name) + } + if reads != 3 { + t.Fatalf("persistent %s failure reads = %d, want 3", failure.name, reads) + } + }) + } +} From e3a2b91b5bc3968c952ca4dc16005e9ae2c047d4 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 17:08:43 -0500 Subject: [PATCH 02/12] Keep validated DNS available during agent restarts --- internal/internaldns/serving/activation.go | 140 +++++++++++++++ .../internaldns/serving/activation_test.go | 46 +++++ internal/internaldns/serving/agent.go | 130 +++----------- internal/internaldns/serving/agent_test.go | 144 ++++++--------- internal/internaldns/serving/bind_test.go | 21 +-- internal/internaldns/serving/continuity.go | 120 +++---------- .../internaldns/serving/continuity_test.go | 66 +------ internal/internaldns/serving/daemons.go | 131 ++++++++++++++ internal/internaldns/serving/readiness.go | 102 +++++++++++ .../internaldns/serving/readiness_test.go | 114 ++++++++++++ internal/internaldns/serving/transaction.go | 99 ++++++----- .../internaldns/serving/transaction_test.go | 155 ++++++++++++---- internal/internaldns/watchdog/watchdog.go | 167 ------------------ .../internaldns/watchdog/watchdog_test.go | 156 ---------------- 14 files changed, 825 insertions(+), 766 deletions(-) create mode 100644 internal/internaldns/serving/activation.go create mode 100644 internal/internaldns/serving/activation_test.go create mode 100644 internal/internaldns/serving/daemons.go create mode 100644 internal/internaldns/serving/readiness.go create mode 100644 internal/internaldns/serving/readiness_test.go delete mode 100644 internal/internaldns/watchdog/watchdog.go delete mode 100644 internal/internaldns/watchdog/watchdog_test.go diff --git a/internal/internaldns/serving/activation.go b/internal/internaldns/serving/activation.go new file mode 100644 index 00000000..3d27f556 --- /dev/null +++ b/internal/internaldns/serving/activation.go @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +const ( + activationInstalling = "installing" + activationVerifying = "verifying" +) + +type activationState struct { + Phase string `json:"phase"` +} + +func (t *FileTransaction) beginActivation() error { + if t.MarkerPath == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(t.MarkerPath), 0o750); err != nil { + return err + } + if t.activationLock == nil { + lock, err := os.OpenFile(t.MarkerPath+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil { + _ = lock.Close() + return err + } + t.activationLock = lock + } + if err := unix.Flock(int(t.activationLock.Fd()), unix.LOCK_EX); err != nil { + return err + } + return t.writeActivation(activationState{Phase: activationInstalling}) +} + +func (t *FileTransaction) writeActivation(state activationState) error { + if t.MarkerPath == "" { + return nil + } + data, err := json.Marshal(state) + if err != nil { + return err + } + temporary := t.MarkerPath + ".tmp" + if err := writeDurable(temporary, data, 0o644); err != nil { + return err + } + if err := os.Rename(temporary, t.MarkerPath); err != nil { + return err + } + return syncDirs([]string{t.MarkerPath}) +} + +func (t *FileTransaction) filesInstalled() error { + if err := t.writeActivation(activationState{Phase: activationVerifying}); err != nil { + return err + } + if t.activationLock != nil { + return unix.Flock(int(t.activationLock.Fd()), unix.LOCK_SH) + } + return nil +} + +func (t *FileTransaction) commitActivation() error { + if t.MarkerPath != "" { + if err := os.Remove(t.MarkerPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := syncDirs([]string{t.MarkerPath}); err != nil { + return err + } + } + return t.abandonActivation() +} + +// The kernel releases activation ownership when the agent process exits. +func (t *FileTransaction) abandonActivation() error { + if t.activationLock == nil { + return nil + } + err := t.activationLock.Close() + t.activationLock = nil + return err +} + +// daemonPermission holds the shared activation lock through daemon startup so +// installation cannot race with a process loading its configuration files. +func daemonPermission(stateDir string) (start, retain bool, release func()) { + marker := activationMarkerPath(stateDir) + lock, err := os.OpenFile(marker+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return false, false, func() {} + } + release = func() { _ = lock.Close() } + if err := unix.Flock(int(lock.Fd()), unix.LOCK_SH|unix.LOCK_NB); err != nil { + if errors.Is(err, unix.EWOULDBLOCK) { + // Running processes retain their in-memory configuration while the + // installer writes files. New processes must wait. + return false, true, release + } + return false, false, release + } + upgradeErr := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB) + data, readErr := os.ReadFile(marker) + if errors.Is(readErr, os.ErrNotExist) { + return true, true, release + } + if upgradeErr == nil { + // A pending marker without an installer is an abandoned activation. + return false, false, release + } + if !errors.Is(upgradeErr, unix.EWOULDBLOCK) { + return false, false, release + } + // A failed upgrade may release the shared lock. Reacquire it before + // reading the phase or allowing a daemon to load its configuration. + if err := unix.Flock(int(lock.Fd()), unix.LOCK_SH|unix.LOCK_NB); err != nil { + return false, true, release + } + data, readErr = os.ReadFile(marker) + if errors.Is(readErr, os.ErrNotExist) { + return true, true, release + } + var state activationState + if readErr != nil || json.Unmarshal(data, &state) != nil || state.Phase != activationVerifying { + return false, false, release + } + return true, true, release +} diff --git a/internal/internaldns/serving/activation_test.go b/internal/internaldns/serving/activation_test.go new file mode 100644 index 00000000..bd69c071 --- /dev/null +++ b/internal/internaldns/serving/activation_test.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "os" + "testing" +) + +func TestActivationOwnershipGatesAbandonedUpdatesWithoutHeartbeat(t *testing.T) { + dir := t.TempDir() + tx := FileTransaction{MarkerPath: activationMarkerPath(dir)} + if err := tx.beginActivation(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = tx.abandonActivation() }) + start, retain, release := daemonPermission(dir) + release() + if start || !retain { + t.Fatal("installation allowed a new process or stopped approved in-memory DNS") + } + if err := tx.filesInstalled(); err != nil { + t.Fatal(err) + } + start, retain, release = daemonPermission(dir) + release() + if !start || !retain { + t.Fatal("live installer could not verify its complete configuration") + } + if err := tx.abandonActivation(); err != nil { + t.Fatal(err) + } + start, retain, release = daemonPermission(dir) + release() + if start || retain { + t.Fatal("abandoned activation retained query processes") + } + if err := os.Remove(tx.MarkerPath); err != nil { + t.Fatal(err) + } + start, retain, release = daemonPermission(dir) + release() + if !start || !retain { + t.Fatal("completed configuration required agent ownership") + } +} diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index 9620d3b6..01636b09 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "net" - "os" "path/filepath" "sort" "strconv" @@ -43,9 +42,6 @@ type Config struct { AckLease time.Duration `json:"ackLease,omitempty"` AckPublishTimeout time.Duration `json:"ackPublishTimeout,omitempty"` MaxBindings int `json:"maxBindings,omitempty"` - WatchdogLeasePath string `json:"watchdogLeasePath,omitempty"` - WatchdogLease time.Duration `json:"watchdogLease,omitempty"` - ExternalWatchdog bool `json:"externalWatchdog,omitempty"` UnsafeAllowNoFailClosed bool `json:"unsafeAllowNoFailClosed,omitempty"` PublicationDNSProbe PublicationDNSVerifierConfig `json:"publicationDNSProbe,omitempty"` ResolverDNSProbe DNSVerifierConfig `json:"resolverDNSProbe,omitempty"` @@ -85,18 +81,9 @@ func (c Config) Validate() error { if c.ResolverVerifyConcurrency < 0 || c.ResolverVerifyConcurrency > 64 { return errors.New("resolver verification concurrency must be between zero and 64") } - if !c.UnsafeAllowNoFailClosed && !c.ExternalWatchdog && len(c.Transaction.FailClosed) == 0 { + if !c.UnsafeAllowNoFailClosed && len(c.Transaction.FailClosed) == 0 { return errors.New("failClosed command is required; use unsafeAllowNoFailClosed only in tests") } - if c.WatchdogLeasePath != "" && !filepath.IsAbs(c.WatchdogLeasePath) { - return errors.New("watchdog lease path must be absolute") - } - if c.ExternalWatchdog && c.WatchdogLeasePath == "" { - return errors.New("external watchdog requires watchdogLeasePath") - } - if c.WatchdogLease > 5*time.Second { - return errors.New("watchdog lease may not exceed five seconds") - } if c.Mode == ModeCombined { if c.Render.Role != RenderCluster && c.Render.Role != RenderBoth && c.Render.Role != RenderRegionalBIND && c.Render.Role != "" { return errors.New("combined publication serving requires an owned regional BIND process") @@ -193,8 +180,6 @@ type Agent struct { tx FileTransaction publicationVerifier PublicationVerifier mu sync.Mutex - healthMu sync.Mutex - healthEnabled bool state checkpoint needsActivation bool lastServingAck map[string]servingAckState @@ -249,31 +234,20 @@ func NewAgent(c Config, d Dependencies) (*Agent, error) { store := stateStore{path: filepath.Join(c.StateDir, "checkpoint.json")} state, err := store.Load() if err != nil { - return nil, err + gateErr := (FileTransaction{Config: c.Transaction, Runner: d.Runner}).FailClosed(context.Background()) + return nil, errors.Join(err, gateErr) } - return &Agent{cfg: c, deps: d, store: store, tx: FileTransaction{Config: c.Transaction, Runner: d.Runner}, publicationVerifier: publicationVerifier, state: state, lastServingAck: map[string]servingAckState{}, lastPublicationAck: map[string]time.Time{}, installedPublications: map[string]installedPublication{}}, nil + return &Agent{cfg: c, deps: d, store: store, tx: FileTransaction{Config: c.Transaction, Runner: d.Runner, MarkerPath: activationMarkerPath(c.StateDir)}, publicationVerifier: publicationVerifier, state: state, lastServingAck: map[string]servingAckState{}, lastPublicationAck: map[string]time.Time{}, installedPublications: map[string]installedPublication{}}, nil } func (a *Agent) Start(ctx context.Context) error { - defer func() { - a.mu.Lock() - defer a.mu.Unlock() - _ = a.failClosed(context.Background()) - }() - // Verify the persisted configuration locally before serving after restart. - if !a.gateUntilClosed(ctx) { - return nil - } - // FailClosed may stop or firewall the DNS daemon. A persisted content hash - // proves only what was installed before restart, not that the live process - // still serves it, so force one validated apply before renewing the lease. + defer func() { _ = a.tx.abandonActivation() }() + // Restore locally validated configuration without depending on state delivery. a.mu.Lock() - a.needsActivation = true if err := a.restoreActive(ctx); err != nil { a.report(err) } a.mu.Unlock() - go a.runActiveHealth(ctx) for { err := a.bootstrapAndStep(ctx) if err == nil { @@ -326,34 +300,10 @@ func (a *Agent) Start(ctx context.Context) error { } func (a *Agent) failClosed(ctx context.Context) error { - a.suspendActiveHealth() a.needsActivation = true - // Invalidate independent watchdog readiness immediately, even if the daemon - // gate command itself fails. It must never inherit a prior healthy lease. - if a.cfg.WatchdogLeasePath != "" { - _ = os.Remove(a.cfg.WatchdogLeasePath) - } - return a.tx.FailClosed(ctx) -} - -func (a *Agent) gateUntilClosed(ctx context.Context) bool { - retry := a.cfg.RetryDelay - if retry <= 0 { - retry = time.Second - } - for { - a.mu.Lock() - err := a.failClosed(ctx) - a.mu.Unlock() - if err == nil { - return true - } else { - a.report(fmt.Errorf("gate serving during startup: %w", err)) - } - if !waitRetry(ctx, retry) { - return false - } - } + gateCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + return errors.Join(a.tx.FailClosed(gateCtx), a.tx.abandonActivation()) } func (a *Agent) bootstrapAndStep(ctx context.Context) error { @@ -538,7 +488,6 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving a.state.Snapshot = &snapshot if !a.canRetainActive() { a.state.Active = nil - a.suspendActiveHealth() } if err := a.store.Save(a.state); err != nil { _ = a.failClosed(ctx) @@ -777,7 +726,7 @@ func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { return fmt.Errorf("ack from unknown member %q replica %q", ack.MemberID, ack.ReplicaID) } // A durable ACK consumer can legitimately replay a once-valid lease after a - // backlog. Drain it without changing readiness or renewing the local lease; + // backlog. Drain it without changing applied readiness; // retrying an already expired ACK can otherwise create a permanent NACK loop. if !a.deps.Clock().Before(ack.ValidUntil) { return nil @@ -881,20 +830,17 @@ func (a *Agent) Step(ctx context.Context) error { if err := a.reconcile(ctx); err != nil { return err } - return a.renewSafeWatchdogLease(ctx) + return a.verifyAppliedConfiguration(ctx) } -// finishEvent gives event-driven reconciliation the same regional materialization safety -// gate as the periodic Step path before extending the independent watchdog -// lease. This matters when a sustained JetStream backlog repeatedly acquires -// the agent lock and delays the ticker. +// finishEvent verifies regional materialization before acknowledging an update. func (a *Agent) finishEvent(ctx context.Context) error { if a.cfg.Mode == ModeCombined { if err := a.heartbeatPublications(ctx); err != nil { return err } } - return a.renewSafeWatchdogLease(ctx) + return a.verifyAppliedConfiguration(ctx) } func (a *Agent) reconcile(ctx context.Context) error { @@ -935,7 +881,6 @@ func (a *Agent) reconcile(ctx context.Context) error { hash := hashFiles(rendered.Files) if hash != a.state.RenderedHash || a.needsActivation { if !a.canRetainActive() { - a.suspendActiveHealth() a.state.Active = nil if err := a.store.Save(a.state); err != nil { return err @@ -997,9 +942,9 @@ func (a *Agent) reconcile(ctx context.Context) error { if err := a.approveConfiguration(rendered.Files); err != nil { return err } - // Local DNS proof renews the watchdog independently of broker acknowledgements. + // Applied configuration remains independent of broker acknowledgements. if a.cfg.Mode == ModeResolver { - if err := a.renewSafeWatchdogLease(ctx); err != nil { + if err := a.verifyAppliedConfiguration(ctx); err != nil { return err } } @@ -1310,8 +1255,8 @@ func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, } func (a *Agent) emitVerifiedPublicationAck(ctx context.Context, uid string, epoch, revision uint64) error { - // Renew local readiness before best-effort acknowledgement delivery. - if err := a.renewSafeWatchdogLease(ctx); err != nil { + // Validate local activation before best-effort acknowledgement delivery. + if err := a.verifyAppliedConfiguration(ctx); err != nil { return err } if err := a.emitAck(ctx, model.KindPublicationManifest, uid, epoch, revision, model.AckVerified, ""); err != nil { @@ -1458,13 +1403,7 @@ func (a *Agent) commitContributionFences(plan model.PublicationPlan) { } } -type watchdogLease struct { - MemberID string `json:"memberID"` - ReplicaID string `json:"replicaID,omitempty"` - ValidUntil time.Time `json:"validUntil"` -} - -func (a *Agent) watchdogPublicationError() error { +func (a *Agent) appliedPublicationError() error { for uid, p := range a.state.Publications { if p.Manifest.Tombstone { continue @@ -1476,44 +1415,21 @@ func (a *Agent) watchdogPublicationError() error { } installed, ok := a.installedPublications[uid] if !ok || installed.Identity != publicationIdentity(p) || installed.EffectiveHash != current { - return fmt.Errorf("publication %s configuration differs from activation; refusing watchdog renewal", uid) + return fmt.Errorf("publication %s configuration differs from activation; refusing activation", uid) } } else if p.EffectiveHash != current { - return fmt.Errorf("publication %s configuration differs from the applied checkpoint; refusing watchdog renewal", uid) + return fmt.Errorf("publication %s configuration differs from the applied checkpoint; refusing activation", uid) } } return nil } -func (a *Agent) renewWatchdogLease(ctx context.Context) error { +func (a *Agent) verifyAppliedConfiguration(ctx context.Context) error { if err := ctx.Err(); err != nil { - return errors.Join(err, a.failClosed(ctx)) - } - if err := a.watchdogPublicationError(); err != nil { - return errors.Join(err, a.failClosed(ctx)) - } - if a.cfg.WatchdogLeasePath == "" { - return nil - } - return a.writeWatchdogLease() -} - -func (a *Agent) writeWatchdogLease() error { - lease := a.cfg.WatchdogLease - if lease <= 0 { - lease = 5 * time.Second - } - b, err := json.Marshal(watchdogLease{MemberID: a.cfg.MemberID, ReplicaID: a.cfg.ReplicaID, ValidUntil: a.deps.Clock().UTC().Add(lease)}) - if err != nil { return err } - return writeAtomicLease(a.cfg.WatchdogLeasePath, b, 0o600) -} - -// Renew the local watchdog only when the activated publication matches its checkpoint. -func (a *Agent) renewSafeWatchdogLease(ctx context.Context) error { - if err := a.watchdogPublicationError(); err != nil { + if err := a.appliedPublicationError(); err != nil { return errors.Join(err, a.failClosed(ctx)) } - return a.renewWatchdogLease(ctx) + return ctx.Err() } diff --git a/internal/internaldns/serving/agent_test.go b/internal/internaldns/serving/agent_test.go index 4e4e3791..c0ffb6d7 100644 --- a/internal/internaldns/serving/agent_test.go +++ b/internal/internaldns/serving/agent_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "sync" "testing" @@ -212,7 +213,7 @@ func TestClusterRoleAutomaticallyVerifiesClusterVIP(t *testing.T) { renderConfig.NodeBIND = BINDRenderConfig{} agent, err := NewAgent(Config{ Region: "east", Shard: "s1", MemberID: "cluster-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, - Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, UnsafeAllowNoFailClosed: true, }, Dependencies{Runner: &recordingRunner{}}) if err != nil { @@ -224,7 +225,7 @@ func TestClusterRoleAutomaticallyVerifiesClusterVIP(t *testing.T) { } } -func TestStartForceActivatesPersistedUnchangedConfigBeforeLease(t *testing.T) { +func TestStartActivatesPersistedDesiredConfiguration(t *testing.T) { t.Parallel() dir := t.TempDir() renderConfig, snapshot := renderFixture(t) @@ -254,11 +255,10 @@ func TestStartForceActivatesPersistedUnchangedConfigBeforeLease(t *testing.T) { if err := (stateStore{path: filepath.Join(stateDir, "checkpoint.json")}).Save(state); err != nil { t.Fatal(err) } - leasePath := filepath.Join(dir, "lease.json") config := Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: stateDir, Mode: ModeResolver, - Render: renderConfig, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, - WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, ReconcileInterval: time.Hour, + Render: renderConfig, UnsafeAllowNoDNSVerify: true, + ReconcileInterval: time.Hour, Transaction: TransactionConfig{ StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}, @@ -282,13 +282,13 @@ func TestStartForceActivatesPersistedUnchangedConfigBeforeLease(t *testing.T) { go func() { done <- agent.Start(ctx) }() deadline := time.After(2 * time.Second) for { - if _, err := os.Stat(leasePath); err == nil { + if state, err := agent.store.Load(); err == nil && state.Active != nil { break } select { case <-deadline: cancel() - t.Fatal("startup did not activate unchanged config and write its lease") + t.Fatal("startup did not approve persisted desired configuration") case <-time.After(5 * time.Millisecond): } } @@ -300,18 +300,16 @@ func TestStartForceActivatesPersistedUnchangedConfigBeforeLease(t *testing.T) { for len(runner.calls) > 0 { calls = append(calls, (<-runner.calls).Path) } - wantPrefix := []string{"gate", "validate-dnsdist", "validate-bind", "reload-dnsdist", "reload-bind"} - if len(calls) < len(wantPrefix) { - t.Fatalf("startup calls = %#v", calls) - } - for i, want := range wantPrefix { - if calls[i] != want { - t.Fatalf("startup calls = %#v, want prefix %#v", calls, wantPrefix) + for _, required := range []string{"validate-dnsdist", "validate-bind", "reload-dnsdist", "reload-bind"} { + if !slices.Contains(calls, required) { + t.Fatalf("startup omitted %s: %#v", required, calls) } } + assertAppliedCheckpoint(t, agent) + } -func TestStepBoundsStalledAckPublishAndRenewsLocalLease(t *testing.T) { +func TestStepBoundsStalledAckPublishAndRetainsAppliedConfiguration(t *testing.T) { t.Parallel() dir := t.TempDir() renderConfig, snapshot := renderFixture(t) @@ -321,13 +319,11 @@ func TestStepBoundsStalledAckPublishAndRenewsLocalLease(t *testing.T) { renderConfig.ClusterDNSDist = DNSDistRenderConfig{} renderConfig.ClusterBIND = BINDRenderConfig{} snapshot.Bindings[0].Zones = nil - leasePath := filepath.Join(dir, "lease.json") var reported []error agent, err := NewAgent(Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, - Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, - UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, - WatchdogLease: 5 * time.Second, AckPublishTimeout: 30 * time.Millisecond, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoDNSVerify: true, AckPublishTimeout: 30 * time.Millisecond, }, Dependencies{Runner: &recordingRunner{}, OnError: func(err error) { reported = append(reported, err) }}) if err != nil { t.Fatal(err) @@ -351,13 +347,9 @@ func TestStepBoundsStalledAckPublishAndRenewsLocalLease(t *testing.T) { if len(reported) == 0 || !strings.Contains(reported[0].Error(), "publish serving ACK") { t.Fatalf("stalled ACK publish was not reported: %#v", reported) } - if _, err := os.Stat(leasePath); err != nil { - t.Fatalf("locally healthy Step did not renew watchdog lease: %v", err) - } + assertAppliedCheckpoint(t, agent) } -// Wait for the ACK boundary rather than imposing a startup SLA on durable -// file staging. Production ACK and watchdog deadlines remain unchanged. func stepUntilBlockedAck(t *testing.T, agent *Agent, started <-chan struct{}, release chan struct{}) func() error { t.Helper() ctx, cancel := context.WithCancel(context.Background()) @@ -396,7 +388,7 @@ func stepUntilBlockedAck(t *testing.T, agent *Agent, started <-chan struct{}, re } } -func TestResolverRenewsWatchdogBeforeBrokerAckCompletes(t *testing.T) { +func TestResolverApprovesConfigurationBeforeBrokerAckCompletes(t *testing.T) { t.Parallel() dir := t.TempDir() renderConfig, snapshot := renderFixture(t) @@ -406,14 +398,12 @@ func TestResolverRenewsWatchdogBeforeBrokerAckCompletes(t *testing.T) { renderConfig.ClusterDNSDist = DNSDistRenderConfig{} renderConfig.ClusterBIND = BINDRenderConfig{} snapshot.Bindings[0].Zones = nil - leasePath := filepath.Join(dir, "lease.json") started := make(chan struct{}, 1) release := make(chan struct{}) agent, err := NewAgent(Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, - Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, - UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, - WatchdogLease: 5 * time.Second, AckPublishTimeout: time.Second, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoDNSVerify: true, AckPublishTimeout: time.Second, }, Dependencies{AckPublisher: blockingPublisher{started: started, release: release}, Runner: &recordingRunner{}}) if err != nil { t.Fatal(err) @@ -422,15 +412,13 @@ func TestResolverRenewsWatchdogBeforeBrokerAckCompletes(t *testing.T) { agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) agent.state.ServingFence = fence{Epoch: 1, Revision: 1} finish := stepUntilBlockedAck(t, agent, started, release) - if _, err := os.Stat(leasePath); err != nil { - t.Fatalf("resolver did not renew watchdog before blocked broker ACK: %v", err) - } + assertAppliedCheckpoint(t, agent) if err := finish(); err != nil { t.Fatal(err) } } -func TestResolverGatesFailedTenantButRenewsSharedLeaseForHealthyTenant(t *testing.T) { +func TestResolverGatesFailedTenantAndRetainsHealthyTenant(t *testing.T) { t.Parallel() dir := t.TempDir() renderConfig, snapshot := renderFixture(t) @@ -448,14 +436,12 @@ func TestResolverGatesFailedTenantButRenewsSharedLeaseForHealthyTenant(t *testin healthy.ConsumerAddress = "10.253.0.52" healthy.ClusterAddress = "10.253.0.42" snapshot.Bindings = append(snapshot.Bindings, healthy) - leasePath := filepath.Join(dir, "lease.json") started := make(chan struct{}, 1) release := make(chan struct{}) agent, err := NewAgent(Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, - CacheFlush: []Command{{Path: "flush"}}, Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, - UnsafeAllowNoFailClosed: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, - WatchdogLease: 5 * time.Second, AckPublishTimeout: time.Second, + CacheFlush: []Command{{Path: "flush"}}, Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoFailClosed: true, AckPublishTimeout: time.Second, }, Dependencies{ AckPublisher: blockingPublisher{started: started, release: release}, Runner: &recordingRunner{}, Verifier: resolverVerifierFunc(func(_ context.Context, binding model.Binding) error { @@ -482,15 +468,13 @@ func TestResolverGatesFailedTenantButRenewsSharedLeaseForHealthyTenant(t *testin if !strings.Contains(string(config), healthy.ConsumerAddress) { t.Fatal("healthy tenant was gated with failed tenant") } - if _, err := os.Stat(leasePath); err != nil { - t.Fatalf("healthy tenant did not renew shared watchdog after failed tenant was gated: %v", err) - } + assertAppliedCheckpoint(t, agent) if err := finish(); err != nil { t.Fatal(err) } } -func TestStartContinuouslyRenewsResolverLeaseWhileBrokerDisconnected(t *testing.T) { +func TestStartPreservesAppliedConfigurationWhileBrokerDisconnected(t *testing.T) { dir := t.TempDir() renderConfig, snapshot := renderFixture(t) renderConfig.Role = RenderNode @@ -499,15 +483,13 @@ func TestStartContinuouslyRenewsResolverLeaseWhileBrokerDisconnected(t *testing. renderConfig.ClusterDNSDist = DNSDistRenderConfig{} renderConfig.ClusterBIND = BINDRenderConfig{} snapshot.Bindings[0].Zones = nil - leasePath := filepath.Join(dir, "lease.json") publishStarted := make(chan struct{}, 1) publishRelease := make(chan struct{}) publisher := &contextIgnoringPublisher{started: publishStarted, release: publishRelease} agent, err := NewAgent(Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, - Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, - UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, - WatchdogLeasePath: leasePath, WatchdogLease: 500 * time.Millisecond, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ReconcileInterval: 20 * time.Millisecond, AckInterval: 50 * time.Millisecond, AckPublishTimeout: 10 * time.Millisecond, }, Dependencies{AckPublisher: publisher, Runner: &recordingRunner{}}) @@ -528,7 +510,7 @@ func TestStartContinuouslyRenewsResolverLeaseWhileBrokerDisconnected(t *testing. select { case stopErr = <-done: case <-time.After(time.Second): - stopErr = errors.New("resolver did not stop after continuous renewal test") + stopErr = errors.New("resolver did not stop after continuity test") } }) } @@ -542,44 +524,16 @@ func TestStartContinuouslyRenewsResolverLeaseWhileBrokerDisconnected(t *testing. t.Fatal("resolver did not enter context-ignoring broker publish") } - initialDeadline := time.Now().Add(2 * time.Second) - for { - if _, err := os.Stat(leasePath); err == nil { - break - } - if time.Now().After(initialDeadline) { - t.Fatal("resolver did not write its initial lease") - } - time.Sleep(5 * time.Millisecond) - } started := time.Now() - lastValidUntil := time.Time{} var continuityErr error for time.Since(started) < 1200*time.Millisecond { - payload, err := os.ReadFile(leasePath) - if err != nil { - continuityErr = fmt.Errorf("read continuously renewed lease: %w", err) - break - } - var lease watchdogLease - if err := json.Unmarshal(payload, &lease); err != nil { - continuityErr = fmt.Errorf("decode continuously renewed lease: %w", err) - break - } - if !lease.ValidUntil.After(time.Now()) { - continuityErr = fmt.Errorf("resolver lease expired during broker partition at %s", lease.ValidUntil) + state, err := agent.store.Load() + if err != nil || state.Active == nil || !installedFilesMatch(state.Active.Files) || activationPending(agent.cfg.StateDir) { + continuityErr = fmt.Errorf("applied configuration lost during broker partition: %w", err) break } - if lease.ValidUntil.After(lastValidUntil) { - lastValidUntil = lease.ValidUntil - } time.Sleep(20 * time.Millisecond) } - if continuityErr == nil { - if remaining := time.Until(lastValidUntil); remaining < 250*time.Millisecond { - continuityErr = fmt.Errorf("resolver lease was not refreshed across multiple windows: remaining %s", remaining) - } - } if calls := publisher.callCount(); calls != 1 { continuityErr = fmt.Errorf("context-ignoring publisher calls = %d, want one bounded inflight call", calls) } @@ -654,7 +608,7 @@ func TestResolverVerificationHasOneSharedDeadline(t *testing.T) { } } -func TestAckEventRenewsWatchdogAfterSuccessfulReconcile(t *testing.T) { +func TestAckEventPreservesAppliedConfigurationAfterReconcile(t *testing.T) { t.Parallel() now := time.Now().UTC() dir := t.TempDir() @@ -665,12 +619,10 @@ func TestAckEventRenewsWatchdogAfterSuccessfulReconcile(t *testing.T) { renderConfig.ClusterDNSDist = DNSDistRenderConfig{} renderConfig.ClusterBIND = BINDRenderConfig{} snapshot.Members = []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}} - leasePath := filepath.Join(dir, "lease.json") agent, err := NewAgent(Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, - Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, - UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, - WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, }, Dependencies{Clock: func() time.Time { return now }, Runner: &recordingRunner{}}) if err != nil { t.Fatal(err) @@ -678,25 +630,18 @@ func TestAckEventRenewsWatchdogAfterSuccessfulReconcile(t *testing.T) { if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", snapshot.ConfigurationEpoch, snapshot.ConfigurationRevision, snapshot)); err != nil { t.Fatal(err) } - if err := os.Remove(leasePath); err != nil { - t.Fatal(err) - } ack := model.MemberAck{MemberID: "regional-0", ReplicaID: "regional-0", ResourceUID: "zone-a", Kind: model.KindPublicationManifest, Epoch: 2, Revision: 4, Phase: model.AckVerified, ObservedAt: now, ValidUntil: now.Add(time.Minute)} if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { t.Fatal(err) } - if _, err := os.Stat(leasePath); err != nil { - t.Fatalf("successful ACK reconcile did not renew watchdog lease: %v", err) - } + assertAppliedCheckpoint(t, agent) } -func TestExpiredKnownAckReplayDrainsWithoutStateOrLeaseChange(t *testing.T) { +func TestExpiredKnownAckReplayDrainsWithoutStateChange(t *testing.T) { t.Parallel() now := time.Now().UTC() - dir := t.TempDir() - leasePath := filepath.Join(dir, "lease.json") agent := &Agent{ - cfg: Config{Region: "east", Shard: "s1", Mode: ModeResolver, WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second}, + cfg: Config{Region: "east", Shard: "s1", Mode: ModeResolver}, deps: Dependencies{Clock: func() time.Time { return now }}, state: newCheckpoint(), lastServingAck: map[string]servingAckState{}, lastPublicationAck: map[string]time.Time{}, } @@ -710,8 +655,8 @@ func TestExpiredKnownAckReplayDrainsWithoutStateOrLeaseChange(t *testing.T) { if len(agent.state.Acks) != 0 { t.Fatalf("expired ACK changed readiness state: %#v", agent.state.Acks) } - if _, err := os.Stat(leasePath); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("expired ACK renewed watchdog lease: %v", err) + if agent.state.Active != nil { + t.Fatal("expired ACK changed applied configuration") } ack.MemberID = "unknown" if err := agent.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err == nil { @@ -723,3 +668,14 @@ func TestExpiredKnownAckReplayDrainsWithoutStateOrLeaseChange(t *testing.T) { t.Fatal("overlong ACK lease was accepted") } } + +func assertAppliedCheckpoint(t *testing.T, agent *Agent) { + t.Helper() + state, err := agent.store.Load() + if err != nil { + t.Fatal(err) + } + if state.Active == nil || !installedFilesMatch(state.Active.Files) || activationPending(agent.cfg.StateDir) { + t.Fatal("approved configuration is not durably active") + } +} diff --git a/internal/internaldns/serving/bind_test.go b/internal/internaldns/serving/bind_test.go index 4c6c2aae..fd517351 100644 --- a/internal/internaldns/serving/bind_test.go +++ b/internal/internaldns/serving/bind_test.go @@ -27,7 +27,7 @@ func regionalTestConfig(t *testing.T) Config { render, _ := renderFixture(t) render.Role = RenderRegionalBIND render.ClusterBIND.Path = filepath.Join(dir, "runtime", "regional.conf") - return Config{Region: "east", Shard: "s1", MemberID: "regional-0", ReplicaID: "regional-0", Mode: ModeCombined, StateDir: filepath.Join(dir, "state"), Render: render, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), Validate: []Command{{Path: "validate"}}, Reload: []Command{{Path: "reload"}}, FailClosed: []Command{{Path: "gate"}}}, UnsafeAllowNoDNSVerify: true, WatchdogLeasePath: filepath.Join(dir, "lease.json")} + return Config{Region: "east", Shard: "s1", MemberID: "regional-0", ReplicaID: "regional-0", Mode: ModeCombined, StateDir: filepath.Join(dir, "state"), Render: render, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), Validate: []Command{{Path: "validate"}}, Reload: []Command{{Path: "reload"}}, FailClosed: []Command{{Path: "gate"}}}, UnsafeAllowNoDNSVerify: true} } func installTestPublication(t *testing.T, a *Agent, now time.Time) { t.Helper() @@ -390,7 +390,7 @@ func TestRejectedMemberProofRevokesRoutingAndIgnoresOlderReplay(t *testing.T) { } } -func TestPublicationProbeTimeoutDoesNotRenewWatchdog(t *testing.T) { +func TestPublicationProbeTimeoutDoesNotApproveConfiguration(t *testing.T) { t.Parallel() now := time.Now().UTC() cfg := regionalTestConfig(t) @@ -419,8 +419,8 @@ func TestPublicationProbeTimeoutDoesNotRenewWatchdog(t *testing.T) { if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err == nil { t.Fatal("timed out publication accepted") } - if _, err := os.Stat(cfg.WatchdogLeasePath); !errors.Is(err, os.ErrNotExist) { - t.Fatal("failed publication query proof renewed watchdog") + if a.state.Publications["zone-a"].Verified { + t.Fatal("failed publication query proof became active") } } @@ -597,9 +597,7 @@ func TestBINDActivationRacingBrokerDisconnectKeepsServing(t *testing.T) { if !a.state.Publications["zone-a"].Verified || a.needsActivation { t.Fatal("ACK partition revoked proved publication") } - if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { - t.Fatalf("ACK partition removed safe local lease: %v", err) - } + assertAppliedCheckpoint(t, a) // Identical manifest redelivery and snapshot delivery must share the same // best-effort ACK boundary without changing the observation. if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { @@ -618,8 +616,9 @@ func TestBINDActivationRacingBrokerDisconnectKeepsServing(t *testing.T) { if !strings.Contains(currentZone(t, a), "192.0.2.11") { t.Fatal("ACK partition removed an applied record") } - if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil || a.needsActivation { - t.Fatalf("healthy local serving was gated by ACK outage: %v", err) + assertAppliedCheckpoint(t, a) + if a.needsActivation { + t.Fatal("healthy serving was gated by ACK outage") } before := publisher.verified.Load() publisher.disconnected.Store(false) @@ -650,7 +649,5 @@ func TestBINDTombstoneAckFailureDoesNotGateDurableWithdrawal(t *testing.T) { if !p.Verified || !p.Manifest.Tombstone || a.needsActivation { t.Fatal("failed tombstone ACK invalidated safely withdrawn zone") } - if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { - t.Fatalf("withdrawal lost safe lease because broker was unavailable: %v", err) - } + assertAppliedCheckpoint(t, a) } diff --git a/internal/internaldns/serving/continuity.go b/internal/internaldns/serving/continuity.go index 3a790eee..b3228f1e 100644 --- a/internal/internaldns/serving/continuity.go +++ b/internal/internaldns/serving/continuity.go @@ -6,10 +6,6 @@ import ( "context" "encoding/json" "errors" - "fmt" - "net" - "strconv" - "time" "go.miloapis.com/dns-operator/internal/internaldns/model" ) @@ -93,113 +89,39 @@ func (a *Agent) approveConfiguration(files map[string][]byte) error { return err } if a.state.Active != nil && equalJSON(*a.state.Active, active) { - a.healthMu.Lock() - a.healthEnabled = true - a.healthMu.Unlock() - return nil + return a.tx.commitActivation() } + previous := a.state.Active a.state.Active = &active if err := a.store.Save(a.state); err != nil { + a.state.Active = previous return err } - a.healthMu.Lock() - a.healthEnabled = true - a.healthMu.Unlock() - return nil -} - -func (a *Agent) suspendActiveHealth() { - a.healthMu.Lock() - defer a.healthMu.Unlock() - a.healthEnabled = false - + return a.tx.commitActivation() } +// restoreActive preserves running daemons when their installed configuration +// matches the approved checkpoint. Incomplete installation requires recovery. func (a *Agent) restoreActive(ctx context.Context) error { if !a.canRetainActive() { - return nil - } - if err := a.tx.ApplyForce(ctx, a.state.Active.Files); err != nil { - return err - } - a.healthMu.Lock() - a.healthEnabled = true - a.healthMu.Unlock() - return nil -} - -func (a *Agent) runActiveHealth(ctx context.Context) { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if err := a.refreshActiveHealth(ctx); err != nil { - a.report(err) - } + return a.failClosed(ctx) + } + files := a.state.Active.Files + if !activationPending(a.cfg.StateDir) && installedFilesMatch(files) { + if err := a.probeLocalDaemons(ctx); err == nil { + a.needsActivation = false + return nil + } else if ctx.Err() != nil { + return ctx.Err() } } -} - -func (a *Agent) refreshActiveHealth(ctx context.Context) error { - a.healthMu.Lock() - enabled := a.healthEnabled - a.healthMu.Unlock() - if !enabled || a.cfg.WatchdogLeasePath == "" { - return nil - } - probeCtx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - if err := a.probeLocalDaemons(probeCtx); err != nil { + if err := a.tx.ApplyForce(ctx, files); err != nil { return err } - a.healthMu.Lock() - defer a.healthMu.Unlock() - if !a.healthEnabled { - return nil - } - return a.writeWatchdogLease() -} - -func (a *Agent) probeLocalDaemons(ctx context.Context) error { - if a.cfg.UnsafeAllowNoDNSVerify { - return nil - } - role := a.cfg.Render.Role - if role == "" { - role = RenderBoth - } - var binds []BINDRenderConfig - var dists []DNSDistRenderConfig - if role == RenderNode || role == RenderBoth { - binds = append(binds, a.cfg.Render.NodeBIND) - dists = append(dists, a.cfg.Render.NodeDNSDist) - } - if role == RenderCluster || role == RenderBoth || role == RenderRegionalBIND { - binds = append(binds, a.cfg.Render.ClusterBIND) - } - if role == RenderCluster || role == RenderBoth || role == RenderRegionalDNSDist { - dists = append(dists, a.cfg.Render.ClusterDNSDist) - } - for _, bind := range binds { - server := net.JoinHostPort(bind.ListenAddress, strconv.Itoa(int(bind.Port))) - view := PublicationView{DestinationAddress: bind.ListenAddress, DestinationPort: bind.Port} - for _, network := range []string{networkUDP, networkTCP} { - answer, err := exchangePublication(ctx, network, server, ".", view, 200*time.Millisecond) - if err != nil || answer == nil || !answer.Response { - return fmt.Errorf("local BIND health %s: %w", server, errors.Join(err, errors.New("DNS response required"))) - } - } - } - for _, dist := range dists { - for _, network := range []string{networkUDP, networkTCP} { - answer, err := exchangeResolverSOA(ctx, network, dist.ListenAddress, ".", 200*time.Millisecond) - if err != nil || answer == nil || !answer.Response { - return fmt.Errorf("local dnsdist health %s: %w", dist.ListenAddress, errors.Join(err, errors.New("DNS response required"))) - } - } + a.state.RenderedHash = hashFiles(files) + a.needsActivation = false + if err := a.store.Save(a.state); err != nil { + return err } - return nil + return a.tx.commitActivation() } diff --git a/internal/internaldns/serving/continuity_test.go b/internal/internaldns/serving/continuity_test.go index a99fc60b..471a1528 100644 --- a/internal/internaldns/serving/continuity_test.go +++ b/internal/internaldns/serving/continuity_test.go @@ -3,10 +3,8 @@ package serving import ( - "bytes" "context" "encoding/json" - "os" "testing" "time" @@ -31,26 +29,18 @@ func TestRejectedExpansionPreservesServingAndRestarts(t *testing.T) { if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)); err == nil { t.Fatal("invalid expansion was accepted") } - if agent.state.RenderedHash != activeHash || agent.state.Active == nil || !agent.healthEnabled { + if agent.state.RenderedHash != activeHash || agent.state.Active == nil { t.Fatal("invalid expansion interrupted the active configuration") } now = now.Add(time.Hour) - if err := agent.refreshActiveHealth(context.Background()); err != nil { - t.Fatal(err) - } - data, err := os.ReadFile(cfg.WatchdogLeasePath) - if err != nil { - t.Fatal(err) - } - var lease watchdogLease - if err := json.Unmarshal(data, &lease); err != nil || !lease.ValidUntil.After(now) { - t.Fatalf("active health stopped during rejected update: %v", err) + if !installedFilesMatch(agent.state.Active.Files) { + t.Fatal("rejected update changed the approved files") } restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) if err != nil { t.Fatal(err) } - if err := restarted.restoreActive(context.Background()); err != nil || !restarted.healthEnabled { + if err := restarted.restoreActive(context.Background()); err != nil { t.Fatalf("restart could not recover active configuration: %v", err) } withdrawn := candidate @@ -60,7 +50,7 @@ func TestRejectedExpansionPreservesServingAndRestarts(t *testing.T) { if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", withdrawn.ConfigurationEpoch, withdrawn.ConfigurationRevision, withdrawn)); err == nil { t.Fatal("failed withdrawal was accepted") } - if agent.state.Active != nil || agent.healthEnabled { + if agent.state.Active != nil { t.Fatal("failed withdrawal retained revoked access") } } @@ -128,7 +118,7 @@ func (r *blockedValidationRunner) Run(ctx context.Context, command Command) ([]b return r.recordingRunner.Run(ctx, command) } -func TestActiveHealthDoesNotWaitForCandidateValidation(t *testing.T) { +func TestApprovedConfigurationRemainsInstalledDuringCandidateValidation(t *testing.T) { now := time.Now().UTC() runner := &blockedValidationRunner{started: make(chan struct{}), release: make(chan struct{})} cfg := regionalTestConfig(t) @@ -153,51 +143,11 @@ func TestActiveHealthDoesNotWaitForCandidateValidation(t *testing.T) { case <-ctx.Done(): t.Fatal("candidate validation did not start") } - healthDone := make(chan error, 1) - go func() { healthDone <- agent.refreshActiveHealth(ctx) }() - select { - case err := <-healthDone: - if err != nil { - t.Fatal(err) - } - case <-time.After(time.Second): - t.Fatal("active health waited for the candidate transaction") + if !installedFilesMatch(agent.state.Active.Files) || activationPending(cfg.StateDir) { + t.Fatal("candidate validation interrupted the approved configuration") } close(runner.release) if err := <-done; err != nil { t.Fatal(err) } } - -func TestWithdrawalTransactionRetainsOnlyExistingWatchdogDeadline(t *testing.T) { - now := time.Now().UTC() - cfg := regionalTestConfig(t) - agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) - if err != nil { - t.Fatal(err) - } - installTestPublication(t, agent, now) - before, err := os.ReadFile(cfg.WatchdogLeasePath) - if err != nil { - t.Fatal(err) - } - agent.suspendActiveHealth() - now = now.Add(time.Hour) - if err := agent.refreshActiveHealth(context.Background()); err != nil { - t.Fatal(err) - } - after, err := os.ReadFile(cfg.WatchdogLeasePath) - if err != nil || !bytes.Equal(before, after) { - t.Fatal("transaction changed the existing watchdog deadline") - } - var lease watchdogLease - if err := json.Unmarshal(after, &lease); err != nil || lease.ValidUntil.After(now) { - t.Fatal("suspended health extended serving authority") - } - if err := agent.failClosed(context.Background()); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(cfg.WatchdogLeasePath); !os.IsNotExist(err) { - t.Fatal("failed withdrawal retained watchdog readiness") - } -} diff --git a/internal/internaldns/serving/daemons.go b/internal/internaldns/serving/daemons.go new file mode 100644 index 00000000..4ae33a4a --- /dev/null +++ b/internal/internaldns/serving/daemons.go @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + "golang.org/x/sys/unix" +) + +type daemonProcess struct { + name string + config string + args []string + cmd *exec.Cmd + done chan error +} + +func (p *daemonProcess) start() error { + p.cmd = exec.Command(p.name, p.args...) + p.cmd.Stdout, p.cmd.Stderr = os.Stdout, os.Stderr + if err := p.cmd.Start(); err != nil { + p.cmd = nil + return err + } + p.done = make(chan error, 1) + go func(cmd *exec.Cmd, done chan error) { done <- cmd.Wait() }(p.cmd, p.done) + return nil +} + +func (p *daemonProcess) stop() error { + if p.cmd == nil { + return nil + } + _ = p.cmd.Process.Signal(unix.SIGTERM) + select { + case <-p.done: + case <-time.After(time.Second): + _ = p.cmd.Process.Kill() + select { + case <-p.done: + case <-time.After(time.Second): + return fmt.Errorf("DNS process %s did not stop", p.name) + } + } + p.cmd, p.done = nil, nil + return nil +} + +// RunDaemons owns query processes independently of the configuration agent. +// Only incomplete or abandoned activation blocks their lifecycle. +func RunDaemons(ctx context.Context, config Config) error { + if err := os.MkdirAll(config.StateDir, 0o750); err != nil { + return err + } + var processes []*daemonProcess + addBind := func(bind BINDRenderConfig) { + processes = append(processes, &daemonProcess{name: "named", config: bind.Path, args: []string{"-g", "-c", bind.Path, "-n", "2"}}) + } + addRouter := func(router DNSDistRenderConfig) { + processes = append(processes, &daemonProcess{name: "dnsdist", config: router.Path, args: []string{"--supervised", "-C", router.Path}}) + } + switch config.Render.Role { + case RenderNode: + addBind(config.Render.NodeBIND) + addRouter(config.Render.NodeDNSDist) + case RenderRegionalBIND: + addBind(config.Render.ClusterBIND) + case RenderRegionalDNSDist: + addRouter(config.Render.ClusterDNSDist) + case RenderCluster: + addBind(config.Render.ClusterBIND) + addRouter(config.Render.ClusterDNSDist) + default: + return fmt.Errorf("unsupported serving process role %q", config.Render.Role) + } + defer func() { + for _, p := range processes { + _ = p.stop() + } + }() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + for { + start, retain, release := daemonPermission(config.StateDir) + for _, p := range processes { + if p.cmd != nil { + select { + case <-p.done: + p.cmd, p.done = nil, nil + default: + } + } + if !retain { + if err := p.stop(); err != nil { + release() + return err + } + continue + } + if p.cmd != nil || !start { + continue + } + if _, err := os.Stat(filepath.Join("/run/internal-dns", "allow-"+p.name)); err != nil { + continue + } + if !activationPending(config.StateDir) { + state, err := (stateStore{path: filepath.Join(config.StateDir, "checkpoint.json")}).Load() + if err != nil || state.Active == nil || !installedFilesMatch(state.Active.Files) { + continue + } + } + if err := p.start(); err != nil && !errors.Is(err, os.ErrNotExist) { + release() + return err + } + } + release() + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} diff --git a/internal/internaldns/serving/readiness.go b/internal/internaldns/serving/readiness.go new file mode 100644 index 00000000..15fb0449 --- /dev/null +++ b/internal/internaldns/serving/readiness.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "time" +) + +const activationMarkerName = "activation.pending" + +func activationMarkerPath(stateDir string) string { + return filepath.Join(stateDir, activationMarkerName) +} + +func activationPending(stateDir string) bool { + _, err := os.Stat(activationMarkerPath(stateDir)) + return !errors.Is(err, os.ErrNotExist) +} + +func installedFilesMatch(files map[string][]byte) bool { + if len(files) == 0 { + return false + } + for path, expected := range files { + actual, err := os.ReadFile(path) + if err != nil || !bytes.Equal(actual, expected) { + return false + } + } + return true +} + +// Readiness verifies the approved files and live DNS processes using local state. +// It does not require the configuration agent or state delivery to be available. +func Readiness(ctx context.Context, config Config) error { + if activationPending(config.StateDir) { + return errors.New("configuration installation is incomplete") + } + state, err := (stateStore{path: filepath.Join(config.StateDir, "checkpoint.json")}).Load() + if err != nil { + return err + } + if state.Active == nil || !installedFilesMatch(state.Active.Files) { + return errors.New("installed configuration does not match the approved checkpoint") + } + if err := state.Active.Snapshot.Validate(); err != nil { + return fmt.Errorf("approved snapshot: %w", err) + } + if config.UnsafeAllowNoDNSVerify { + return errors.New("readiness requires live DNS query verification") + } + return (&Agent{cfg: config}).probeLocalDaemons(ctx) +} + +func (a *Agent) probeLocalDaemons(ctx context.Context) error { + if a.cfg.UnsafeAllowNoDNSVerify { + return nil + } + role := a.cfg.Render.Role + if role == "" { + role = RenderBoth + } + var binds []BINDRenderConfig + var dists []DNSDistRenderConfig + if role == RenderNode || role == RenderBoth { + binds = append(binds, a.cfg.Render.NodeBIND) + dists = append(dists, a.cfg.Render.NodeDNSDist) + } + if role == RenderCluster || role == RenderBoth || role == RenderRegionalBIND { + binds = append(binds, a.cfg.Render.ClusterBIND) + } + if role == RenderCluster || role == RenderBoth || role == RenderRegionalDNSDist { + dists = append(dists, a.cfg.Render.ClusterDNSDist) + } + for _, bind := range binds { + server := net.JoinHostPort(bind.ListenAddress, strconv.Itoa(int(bind.Port))) + view := PublicationView{DestinationAddress: bind.ListenAddress, DestinationPort: bind.Port} + for _, network := range []string{networkUDP, networkTCP} { + answer, err := exchangePublication(ctx, network, server, ".", view, 200*time.Millisecond) + if err != nil || answer == nil || !answer.Response { + return fmt.Errorf("local BIND health %s: %w", server, errors.Join(err, errors.New("DNS response required"))) + } + } + } + for _, dist := range dists { + for _, network := range []string{networkUDP, networkTCP} { + answer, err := exchangeResolverSOA(ctx, network, dist.ListenAddress, ".", 200*time.Millisecond) + if err != nil || answer == nil || !answer.Response { + return fmt.Errorf("local dnsdist health %s: %w", dist.ListenAddress, errors.Join(err, errors.New("DNS response required"))) + } + } + } + return nil +} diff --git a/internal/internaldns/serving/readiness_test.go b/internal/internaldns/serving/readiness_test.go new file mode 100644 index 00000000..d42d912a --- /dev/null +++ b/internal/internaldns/serving/readiness_test.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/miekg/dns" +) + +func TestReadinessUsesApprovedConfigurationWithoutAgentOrBroker(t *testing.T) { + tcp, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + udp, err := net.ListenPacket("udp", tcp.Addr().String()) + if err != nil { + t.Fatal(err) + } + handler := dns.HandlerFunc(func(w dns.ResponseWriter, request *dns.Msg) { + response := new(dns.Msg) + response.SetRcode(request, dns.RcodeRefused) + _ = w.WriteMsg(response) + }) + started := make(chan struct{}, 2) + udpServer := &dns.Server{PacketConn: udp, Handler: handler, NotifyStartedFunc: func() { started <- struct{}{} }} + tcpServer := &dns.Server{Listener: tcp, Handler: handler, NotifyStartedFunc: func() { started <- struct{}{} }} + go func() { _ = udpServer.ActivateAndServe() }() + go func() { _ = tcpServer.ActivateAndServe() }() + t.Cleanup(func() { _ = udpServer.Shutdown(); _ = tcpServer.Shutdown() }) + for i := 0; i < 2; i++ { + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("DNS fixture did not start") + } + } + dir := t.TempDir() + configPath := filepath.Join(dir, "dnsdist.conf") + files := map[string][]byte{configPath: []byte("approved configuration")} + if err := os.WriteFile(configPath, files[configPath], 0o644); err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(time.Now().Add(-24 * time.Hour)) + state := newCheckpoint() + state.Snapshot = &snapshot + state.BindingFences, _ = advanceBindingFences(nil, snapshot) + state.Active = &activeConfiguration{Snapshot: snapshot, Files: files} + state.RenderedHash = hashFiles(files) + if err := (stateStore{path: filepath.Join(dir, "checkpoint.json")}).Save(state); err != nil { + t.Fatal(err) + } + config := Config{StateDir: dir, Render: RenderConfig{Role: RenderRegionalDNSDist, ClusterDNSDist: DNSDistRenderConfig{ListenAddress: tcp.Addr().String()}}} + if err := Readiness(context.Background(), config); err != nil { + t.Fatalf("approved DNS required an agent heartbeat or delivery connection: %v", err) + } + if err := os.WriteFile(activationMarkerPath(dir), []byte("installing\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := Readiness(context.Background(), config); err == nil { + t.Fatal("incomplete installation was ready") + } + if err := os.Remove(activationMarkerPath(dir)); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath, []byte("unapproved configuration"), 0o644); err != nil { + t.Fatal(err) + } + if err := Readiness(context.Background(), config); err == nil { + t.Fatal("unapproved files were ready") + } + if err := os.WriteFile(configPath, files[configPath], 0o644); err != nil { + t.Fatal(err) + } + if err := udpServer.Shutdown(); err != nil { + t.Fatal(err) + } + if err := Readiness(context.Background(), config); err == nil { + t.Fatal("TCP-only DNS was ready") + } +} + +func TestAgentRestartAndStopPreserveApprovedConfiguration(t *testing.T) { + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, time.Now().UTC()) + runner := &recordingRunner{} + restarted, err := NewAgent(cfg, Dependencies{Runner: runner}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if len(runner.calls) != 0 { + t.Fatalf("agent restart changed healthy daemon configuration: %#v", runner.calls) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := restarted.Start(ctx); err != nil { + t.Fatal(err) + } + if len(runner.calls) != 0 || !installedFilesMatch(restarted.state.Active.Files) { + t.Fatal("agent stop shut down healthy DNS") + } +} diff --git a/internal/internaldns/serving/transaction.go b/internal/internaldns/serving/transaction.go index 467ee010..b203a3f0 100644 --- a/internal/internaldns/serving/transaction.go +++ b/internal/internaldns/serving/transaction.go @@ -1,12 +1,14 @@ package serving import ( + "bytes" "context" "errors" "fmt" "os" "os/exec" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -41,8 +43,10 @@ type TransactionConfig struct { } type FileTransaction struct { - Config TransactionConfig - Runner Runner + activationLock *os.File + MarkerPath string + Config TransactionConfig + Runner Runner } type preservedTransactionError struct{ error } @@ -59,18 +63,18 @@ type oldFile struct { existed bool } -func (t FileTransaction) Apply(ctx context.Context, files map[string][]byte) error { +func (t *FileTransaction) Apply(ctx context.Context, files map[string][]byte) error { return t.apply(ctx, files, false) } // ApplyForce validates and reloads every configured daemon even when the // installed bytes already match. It is used after fail-closed startup because // file equality cannot prove that a stopped or firewalled process is active. -func (t FileTransaction) ApplyForce(ctx context.Context, files map[string][]byte) error { +func (t *FileTransaction) ApplyForce(ctx context.Context, files map[string][]byte) error { return t.apply(ctx, files, true) } -func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, force bool) (resultErr error) { +func (t *FileTransaction) apply(ctx context.Context, files map[string][]byte, force bool) (resultErr error) { mutating := false defer func() { if resultErr == nil { @@ -173,6 +177,22 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for if err := t.commandsForChanges(ctx, t.Config.Validate, map[string]string{"{stageDir}": commandStage}, commandChanges); err != nil { return preservedTransactionError{fmt.Errorf("validate staged serving configuration: %w", err)} } + if t.MarkerPath != "" { + structural := force + for _, path := range paths { + structural = structural || changed[path] && routingConfigurationChanged(path, old[path].data, files[path]) + } + if err := t.beginActivation(); err != nil { + return err + } + if structural { + // Restart every owned daemon after changing routing or view configuration. + commandChanges = nil + if err := t.FailClosed(ctx); err != nil { + return uncertainTransactionError{err} + } + } + } installed := make([]string, 0, len(paths)) mutating = true for _, path := range paths { @@ -198,10 +218,31 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for rollbackErr := restoreFiles(installed, old) return errors.Join(err, rollbackErr) } + if err := t.filesInstalled(); err != nil { + return err + } if err := t.commandsForChanges(ctx, t.Config.Reload, nil, commandChanges); err != nil { - rollbackErr := restoreFiles(paths, old) + if t.MarkerPath != "" { + if guardErr := t.beginActivation(); guardErr != nil { + return errors.Join(err, guardErr) + } + commandChanges = nil + if guardErr := t.FailClosed(ctx); guardErr != nil { + return errors.Join(err, guardErr) + } + } + rollbackErr := errors.Join(restoreFiles(paths, old), syncDirs(paths)) + if rollbackErr != nil { + return errors.Join(err, rollbackErr) + } + if guardErr := t.filesInstalled(); guardErr != nil { + return errors.Join(err, guardErr) + } reloadErr := t.commandsForChanges(ctx, t.Config.Reload, nil, commandChanges) if rollbackErr == nil && reloadErr == nil { + if guardErr := t.commitActivation(); guardErr != nil { + return errors.Join(err, guardErr) + } return preservedTransactionError{fmt.Errorf("reload serving configuration: %w", err)} } return errors.Join(fmt.Errorf("reload serving configuration: %w", err), rollbackErr, reloadErr) @@ -272,40 +313,6 @@ func writeDurable(path string, data []byte, mode os.FileMode) error { return f.Close() } -// writeAtomicLease replaces the ephemeral watchdog lease through a unique -// temporary sibling. The file is synced before rename so a reader never accepts -// partially written JSON. The containing directory is deliberately not synced: -// metadata fsync on a container bind mount can block longer than the lease, and -// restart always gates before bootstrap so this lease never needs crash -// survival. -func writeAtomicLease(path string, data []byte, mode os.FileMode) error { - dir := filepath.Dir(path) - temporary, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-") - if err != nil { - return err - } - temporaryPath := temporary.Name() - defer func() { _ = os.Remove(temporaryPath) }() - if err := temporary.Chmod(mode); err != nil { - _ = temporary.Close() - return err - } - if _, err := temporary.Write(data); err != nil { - _ = temporary.Close() - return err - } - if err := temporary.Sync(); err != nil { - _ = temporary.Close() - return err - } - if err := temporary.Close(); err != nil { - return err - } - if err := os.Rename(temporaryPath, path); err != nil { - return err - } - return nil -} func syncDirs(paths []string) error { seen := map[string]bool{} for _, path := range paths { @@ -350,3 +357,15 @@ func restoreFiles(paths []string, old map[string]oldFile) error { } return errors.Join(errs...) } + +var immutableZoneFile = regexp.MustCompile(`file "([^"\n]*/zone-[a-f0-9]{24})-[a-f0-9]{16}\.db"`) + +// Content-addressed zone files change record data without changing view routing. +func routingConfigurationChanged(path string, before, after []byte) bool { + if !strings.HasSuffix(path, ".conf") { + return false + } + before = immutableZoneFile.ReplaceAll(before, []byte(`file "${1}.db"`)) + after = immutableZoneFile.ReplaceAll(after, []byte(`file "${1}.db"`)) + return !bytes.Equal(before, after) +} diff --git a/internal/internaldns/serving/transaction_test.go b/internal/internaldns/serving/transaction_test.go index 4b55d313..ebb3181d 100644 --- a/internal/internaldns/serving/transaction_test.go +++ b/internal/internaldns/serving/transaction_test.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" "testing" + "time" ) type recordingRunner struct { @@ -78,39 +79,6 @@ func TestFileTransactionConfigIsReadableByServiceUID(t *testing.T) { } } -func TestAtomicLeaseWriteReplacesCompleteFile(t *testing.T) { - t.Parallel() - dir := t.TempDir() - path := filepath.Join(dir, "lease.json") - if err := os.WriteFile(path, []byte("old"), 0o600); err != nil { - t.Fatal(err) - } - if err := writeAtomicLease(path, []byte(`{"memberID":"node-0"}`), 0o600); err != nil { - t.Fatal(err) - } - contents, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if string(contents) != `{"memberID":"node-0"}` { - t.Fatalf("atomic durable contents = %q", contents) - } - info, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - if info.Mode().Perm() != 0o600 { - t.Fatalf("atomic durable mode = %#o", info.Mode().Perm()) - } - temporary, err := filepath.Glob(filepath.Join(dir, ".lease.json.tmp-*")) - if err != nil { - t.Fatal(err) - } - if len(temporary) != 0 { - t.Fatalf("temporary lease files remain: %#v", temporary) - } -} - func TestFileTransactionPreservesStagePlaceholderAcrossApplies(t *testing.T) { t.Parallel() dir := t.TempDir() @@ -260,3 +228,124 @@ func TestFileTransactionValidatesFreshStagedZoneReferences(t *testing.T) { t.Fatal("installed config retains temporary staging paths") } } + +func TestActivationGuardsStructuralChangesBeforeInstallingFiles(t *testing.T) { + dir := t.TempDir() + bind := filepath.Join(dir, "bind.conf") + router := filepath.Join(dir, "dnsdist.conf") + for _, path := range []string{bind, router} { + if err := os.WriteFile(path, []byte("approved"), 0o644); err != nil { + t.Fatal(err) + } + } + marker := activationMarkerPath(dir) + runner := &activationBoundaryRunner{t: t, marker: marker, paths: []string{bind, router}} + tx := FileTransaction{MarkerPath: marker, Runner: runner, Config: TransactionConfig{ + StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}, + Reload: []Command{{Path: "reload-bind", WhenChanged: []string{bind}}, {Path: "reload-dnsdist", WhenChanged: []string{router}}}, + }} + if err := tx.Apply(context.Background(), map[string][]byte{bind: []byte("approved"), router: []byte("candidate")}); err != nil { + t.Fatal(err) + } + if err := tx.commitActivation(); err != nil { + t.Fatal(err) + } + if strings.Join(runner.calls, ",") != "gate,reload-bind,reload-dnsdist" { + t.Fatalf("owned daemons were not restarted together: %#v", runner.calls) + } +} + +type activationBoundaryRunner struct { + t *testing.T + marker string + paths []string + calls []string +} + +func (r *activationBoundaryRunner) Run(_ context.Context, command Command) ([]byte, error) { + r.calls = append(r.calls, command.Path) + _, markerErr := os.Stat(r.marker) + if command.Path == "gate" { + if markerErr != nil { + r.t.Fatal("structural gate ran without durable installation guard") + } + for _, path := range r.paths { + data, err := os.ReadFile(path) + if err != nil || string(data) != "approved" { + r.t.Fatal("files changed before DNS was gated") + } + } + } else { + start, retain, release := daemonPermission(filepath.Dir(r.marker)) + defer release() + if !start || !retain { + r.t.Fatal("complete files could not start for validation") + } + } + return nil, nil +} + +func TestInterruptedInstallationRequiresLocalRecovery(t *testing.T) { + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, time.Now().UTC()) + approved := agent.state.Active.Files + badParent := filepath.Join(cfg.StateDir, "not-a-directory") + if err := os.WriteFile(badParent, []byte("blocked"), 0o644); err != nil { + t.Fatal(err) + } + files := map[string][]byte{filepath.Join(badParent, "later.conf"): []byte("candidate")} + for path := range approved { + files[path] = []byte("candidate") + } + if err := agent.tx.Apply(context.Background(), files); err == nil { + t.Fatal("interrupted installation succeeded") + } + if !activationPending(cfg.StateDir) { + t.Fatal("interrupted installation lost its durable guard") + } + if err := agent.tx.abandonActivation(); err != nil { + t.Fatal(err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if activationPending(cfg.StateDir) || !installedFilesMatch(approved) { + t.Fatal("approved configuration could not recover without a broker") + } +} + +func TestRecordUpdateReloadsImmutableZoneWithoutStoppingDNS(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "bind.conf") + zoneToken := "0123456789abcdef01234567" + before := []byte(`view "tenant-a" { zone "prod.internal" { file "` + dir + `/zone-` + zoneToken + `-0123456789abcdef.db"; }; };`) + after := []byte(`view "tenant-a" { zone "prod.internal" { file "` + dir + `/zone-` + zoneToken + `-abcdef0123456789.db"; }; };`) + if err := os.WriteFile(configPath, before, 0o644); err != nil { + t.Fatal(err) + } + runner := &recordingRunner{} + tx := FileTransaction{MarkerPath: activationMarkerPath(dir), Runner: runner, Config: TransactionConfig{ + StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}, Reload: []Command{{Path: "reload-bind"}}, + }} + if err := tx.Apply(context.Background(), map[string][]byte{configPath: after}); err != nil { + t.Fatal(err) + } + if err := tx.commitActivation(); err != nil { + t.Fatal(err) + } + if len(runner.calls) != 1 || runner.calls[0].Path != "reload-bind" { + t.Fatalf("record update stopped healthy DNS: %#v", runner.calls) + } + newTenant := bytes.ReplaceAll(after, []byte("tenant-a"), []byte("tenant-b")) + if !routingConfigurationChanged(configPath, after, newTenant) { + t.Fatal("tenant routing change did not require a gate") + } +} diff --git a/internal/internaldns/watchdog/watchdog.go b/internal/internaldns/watchdog/watchdog.go deleted file mode 100644 index f45a0b03..00000000 --- a/internal/internaldns/watchdog/watchdog.go +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -// Package watchdog supervises a serving agent from an independent process. An -// agent that is killed or hung cannot renew this lease; the supervisor gates its -// DNS processes without depending on Kubernetes or a remote control plane. -package watchdog - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "time" - - "go.miloapis.com/dns-operator/internal/internaldns/serving" -) - -type Config struct { - LeasePath string `json:"leasePath"` - MemberID string `json:"memberID"` - ReplicaID string `json:"replicaID,omitempty"` - MaxLeaseSeconds int `json:"maxLeaseSeconds,omitempty"` - StartupGraceSeconds int `json:"startupGraceSeconds,omitempty"` - IntervalMillis int `json:"intervalMillis,omitempty"` - FailClosed []serving.Command `json:"failClosed"` -} - -type Lease struct { - MemberID string `json:"memberID"` - ReplicaID string `json:"replicaID,omitempty"` - ValidUntil time.Time `json:"validUntil"` -} - -type Watchdog struct { - Config Config - Runner serving.Runner - Now func() time.Time - ReadFile func(string) ([]byte, error) -} - -func (c Config) Validate() error { - if !filepath.IsAbs(c.LeasePath) || c.MemberID == "" || len(c.FailClosed) == 0 { - return errors.New("watchdog requires an absolute lease path, member identity, and fail-closed commands") - } - if c.MaxLeaseSeconds < 0 || c.MaxLeaseSeconds > 30 || c.StartupGraceSeconds < 0 || c.StartupGraceSeconds > 60 { - return errors.New("watchdog lease and startup grace exceed platform bounds") - } - if c.IntervalMillis < 0 || c.IntervalMillis > 1000 { - return errors.New("watchdog checks must run at least once per second") - } - for _, command := range c.FailClosed { - if command.Path == "" { - return errors.New("watchdog fail-closed command path is required") - } - } - return nil -} - -func (w *Watchdog) Check(now time.Time) error { - readFile := w.ReadFile - if readFile == nil { - readFile = os.ReadFile - } - var lease Lease - var lastErr error - for attempt := 0; attempt < 3; attempt++ { - b, err := readFile(w.Config.LeasePath) - if err != nil { - lastErr = fmt.Errorf("read serving lease: %w", err) - } else if len(b) > 4096 { - // A decoded identity or deadline is never retried, and neither is an - // oversized file: only a transient read or partial JSON observation - // can be repaired by the writer's atomic replacement becoming visible. - return errors.New("serving lease exceeds size limit") - } else { - lease = Lease{} - if err := json.Unmarshal(b, &lease); err != nil { - lastErr = fmt.Errorf("decode serving lease: %w", err) - } else { - lastErr = nil - break - } - } - if attempt < 2 { - time.Sleep(10 * time.Millisecond) - } - } - if lastErr != nil { - return lastErr - } - if lease.MemberID != w.Config.MemberID || w.Config.ReplicaID != "" && lease.ReplicaID != w.Config.ReplicaID { - return errors.New("serving lease belongs to another member") - } - if w.Now != nil { - // A successful retry may observe a lease written after Check's caller - // sampled now. Validate that replacement against the current clock so a - // fresh maximum-length lease is not misclassified as overlong. - now = w.Now() - } - max := w.Config.MaxLeaseSeconds - if max == 0 { - max = 5 - } - if lease.ValidUntil.IsZero() || !lease.ValidUntil.After(now) { - return errors.New("serving lease expired") - } - if lease.ValidUntil.After(now.Add(time.Duration(max) * time.Second)) { - return errors.New("serving lease deadline exceeds the watchdog bound") - } - return nil -} - -func (w *Watchdog) Start(ctx context.Context) error { - if err := w.Config.Validate(); err != nil { - return err - } - if w.Runner == nil { - w.Runner = serving.ExecRunner{} - } - if w.Now == nil { - w.Now = time.Now - } - start := w.Now() - last := start - interval := w.Config.IntervalMillis - if interval == 0 { - interval = 250 - } - ticker := time.NewTicker(time.Duration(interval) * time.Millisecond) - defer ticker.Stop() - for { - now := w.Now() - err := w.Check(now) - if now.Before(last.Add(-time.Second)) { - err = errors.New("watchdog clock moved backwards") - } - last = now - // Grace only applies to a new member without any checkpoint. An existing - // expired or malformed lease is gated immediately even during startup. - if err != nil && errors.Is(err, os.ErrNotExist) && now.Sub(start) < time.Duration(w.Config.StartupGraceSeconds)*time.Second { - err = nil - } - if err != nil { - var failures []error - for _, command := range w.Config.FailClosed { - commandCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - out, runErr := w.Runner.Run(commandCtx, command) - cancel() - if runErr != nil { - failures = append(failures, fmt.Errorf("fail closed %s: %w: %s", command.Path, runErr, out)) - } - } - if len(failures) == 0 { - return fmt.Errorf("serving member gated: %w", err) - } - // A failed gate is retried; returning success or letting a remote - // controller handle it would allow stale local DNS to keep serving. - } - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - } - } -} diff --git a/internal/internaldns/watchdog/watchdog_test.go b/internal/internaldns/watchdog/watchdog_test.go deleted file mode 100644 index 6db9c180..00000000 --- a/internal/internaldns/watchdog/watchdog_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package watchdog - -import ( - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - "testing" - "time" - - "go.miloapis.com/dns-operator/internal/internaldns/serving" -) - -type recordingRunner struct{ calls int } - -func (r *recordingRunner) Run(context.Context, serving.Command) ([]byte, error) { - r.calls++ - return nil, nil -} - -func TestDeadOrForgedAgentIsGatedWithoutRemoteState(t *testing.T) { - now := time.Date(2026, 10, 6, 18, 0, 0, 0, time.UTC) - for _, tc := range []struct { - name string - lease Lease - valid bool - }{ - {"fresh", Lease{MemberID: "auth-1", ValidUntil: now.Add(5 * time.Second)}, true}, - {"expired", Lease{MemberID: "auth-1", ValidUntil: now.Add(-time.Second)}, false}, - {"another member", Lease{MemberID: "auth-2", ValidUntil: now.Add(time.Second)}, false}, - {"overlong lease", Lease{MemberID: "auth-1", ValidUntil: now.Add(time.Minute)}, false}, - } { - t.Run(tc.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "lease.json") - b, _ := json.Marshal(tc.lease) - if err := os.WriteFile(path, b, 0600); err != nil { - t.Fatal(err) - } - w := Watchdog{Config: Config{LeasePath: path, MemberID: "auth-1", FailClosed: []serving.Command{{Path: "stop-private-dns"}}}, Now: func() time.Time { return now }} - if got := w.Check(now) == nil; got != tc.valid { - t.Fatalf("valid=%v want=%v", got, tc.valid) - } - if !tc.valid { - runner := &recordingRunner{} - w.Runner = runner - if err := w.Start(context.Background()); err == nil || runner.calls != 1 { - t.Fatalf("dead agent was not gated: calls=%d err=%v", runner.calls, err) - } - } - }) - } -} - -func TestExpiredCheckpointCannotUseStartupGrace(t *testing.T) { - now := time.Now() - path := filepath.Join(t.TempDir(), "lease.json") - b, _ := json.Marshal(Lease{MemberID: "auth-1", ValidUntil: now.Add(-time.Second)}) - if err := os.WriteFile(path, b, 0600); err != nil { - t.Fatal(err) - } - runner := &recordingRunner{} - w := Watchdog{Config: Config{LeasePath: path, MemberID: "auth-1", StartupGraceSeconds: 60, FailClosed: []serving.Command{{Path: "stop"}}}, Runner: runner, Now: func() time.Time { return now }} - if err := w.Start(context.Background()); err == nil || runner.calls != 1 { - t.Fatal("expired checkpoint survived restart") - } -} - -func TestCheckRereadsTransientPartialLease(t *testing.T) { - now := time.Now().UTC() - valid, err := json.Marshal(Lease{MemberID: "auth-1", ValidUntil: now.Add(5 * time.Second)}) - if err != nil { - t.Fatal(err) - } - reads := 0 - w := Watchdog{ - Config: Config{LeasePath: "/lease.json", MemberID: "auth-1", MaxLeaseSeconds: 5}, - Now: func() time.Time { return now }, - ReadFile: func(string) ([]byte, error) { - reads++ - if reads == 1 { - return []byte{}, nil - } - return valid, nil - }, - } - if err := w.Check(now); err != nil { - t.Fatalf("transient partial lease was gated: %v", err) - } - if reads != 2 { - t.Fatalf("lease reads = %d, want 2", reads) - } -} - -func TestCheckRetriesOnlyReadAndDecodeFailures(t *testing.T) { - now := time.Now().UTC() - tests := []struct { - name string - lease Lease - }{ - {name: "wrong identity", lease: Lease{MemberID: "other", ValidUntil: now.Add(time.Second)}}, - {name: "expired", lease: Lease{MemberID: "auth-1", ValidUntil: now.Add(-time.Second)}}, - {name: "overlong", lease: Lease{MemberID: "auth-1", ValidUntil: now.Add(6 * time.Second)}}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - payload, err := json.Marshal(test.lease) - if err != nil { - t.Fatal(err) - } - reads := 0 - w := Watchdog{ - Config: Config{LeasePath: "/lease.json", MemberID: "auth-1", MaxLeaseSeconds: 5}, - Now: func() time.Time { return now }, - ReadFile: func(string) ([]byte, error) { - reads++ - return payload, nil - }, - } - if err := w.Check(now); err == nil { - t.Fatal("unsafe decoded lease was accepted") - } - if reads != 1 { - t.Fatalf("unsafe decoded lease was retried %d times", reads) - } - }) - } - - for _, failure := range []struct { - name string - read func(string) ([]byte, error) - }{ - {name: "read", read: func(string) ([]byte, error) { return nil, errors.New("short read") }}, - {name: "decode", read: func(string) ([]byte, error) { return []byte("{"), nil }}, - } { - t.Run("persistent "+failure.name, func(t *testing.T) { - reads := 0 - w := Watchdog{ - Config: Config{LeasePath: "/lease.json", MemberID: "auth-1", MaxLeaseSeconds: 5}, - Now: func() time.Time { return now }, - ReadFile: func(path string) ([]byte, error) { - reads++ - return failure.read(path) - }, - } - if err := w.Check(now); err == nil { - t.Fatalf("persistent %s failure was accepted", failure.name) - } - if reads != 3 { - t.Fatalf("persistent %s failure reads = %d, want 3", failure.name, reads) - } - }) - } -} From a6f3cb6f8588971edac67a8ee1078e20e3a8d08f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 17:12:20 -0500 Subject: [PATCH 03/12] Keep incomplete DNS publication activation gated until proof --- .../internaldns/serving/activation_test.go | 88 +++++++++++++++++++ internal/internaldns/serving/agent.go | 9 ++ internal/internaldns/serving/continuity.go | 4 + 3 files changed, 101 insertions(+) diff --git a/internal/internaldns/serving/activation_test.go b/internal/internaldns/serving/activation_test.go index bd69c071..86a6e2f2 100644 --- a/internal/internaldns/serving/activation_test.go +++ b/internal/internaldns/serving/activation_test.go @@ -3,7 +3,12 @@ package serving import ( + "context" + "errors" "os" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" "testing" ) @@ -44,3 +49,86 @@ func TestActivationOwnershipGatesAbandonedUpdatesWithoutHeartbeat(t *testing.T) t.Fatal("completed configuration required agent ownership") } } + +func TestPublicationActivationRemainsOwnedUntilDNSProof(t *testing.T) { + now := time.Now().UTC() + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + snapshot := testSnapshot(now) + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, snapshot)); err != nil { + t.Fatal(err) + } + approved := agent.state.Active + verified := false + agent.deps.PublicationVerifier = publicationVerifierFunc(func(context.Context, model.PublicationPlan, []PublicationView) error { + verified = true + if !activationPending(cfg.StateDir) { + t.Fatal("activation committed before DNS proof") + } + if agent.state.Active != approved { + t.Fatal("unverified publication replaced approved checkpoint") + } + start, retain, release := daemonPermission(cfg.StateDir) + release() + if !start || !retain { + t.Fatal("live verification cannot query installed configuration") + } + if err := agent.tx.abandonActivation(); err != nil { + t.Fatal(err) + } + start, retain, release = daemonPermission(cfg.StateDir) + release() + if start || retain { + t.Fatal("abandoned unverified publication remains queryable") + } + return errors.New("publication proof interrupted") + }) + chunk, manifest := testPublication(t, now) + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err == nil { + t.Fatal("unverified publication accepted") + } + if !verified { + t.Fatal("publication proof was not attempted") + } + if !activationPending(cfg.StateDir) { + t.Fatal("interrupted proof lost recovery marker") + } +} + +func TestRestoreActiveUsesInstalledConfigurationHash(t *testing.T) { + now := time.Now().UTC() + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + actual := hashFiles(agent.state.Active.Files) + agent.state.RenderedHash = "unapproved-candidate" + if err := agent.store.Save(agent.state); err != nil { + t.Fatal(err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if restarted.state.RenderedHash != actual { + t.Fatal("restart retained candidate hash for approved files") + } + saved, err := restarted.store.Load() + if err != nil { + t.Fatal(err) + } + if saved.RenderedHash != actual { + t.Fatal("approved installed hash was not checkpointed") + } +} diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index 01636b09..63a1f8cc 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -939,6 +939,15 @@ func (a *Agent) reconcile(ctx context.Context) error { } } } + // Keep activation owned until every materialized publication has query proof + // and its affected names have been flushed from resolver caches. + if a.cfg.Mode == ModeCombined { + for _, p := range a.state.Publications { + if !p.Verified && (p.Manifest.Tombstone || len(a.publicationViews(p)) > 0) { + return nil + } + } + } if err := a.approveConfiguration(rendered.Files); err != nil { return err } diff --git a/internal/internaldns/serving/continuity.go b/internal/internaldns/serving/continuity.go index b3228f1e..26035e80 100644 --- a/internal/internaldns/serving/continuity.go +++ b/internal/internaldns/serving/continuity.go @@ -110,6 +110,10 @@ func (a *Agent) restoreActive(ctx context.Context) error { if !activationPending(a.cfg.StateDir) && installedFilesMatch(files) { if err := a.probeLocalDaemons(ctx); err == nil { a.needsActivation = false + if actual := hashFiles(files); a.state.RenderedHash != actual { + a.state.RenderedHash = actual + return a.store.Save(a.state) + } return nil } else if ctx.Err() != nil { return ctx.Err() From 95b7c8f3ac7d694ba807c1ca02b30923c96b8314 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 17:17:22 -0500 Subject: [PATCH 04/12] Exercise owned DNS activation and interrupted installation --- internal/internaldns/serving/activation.go | 4 ++-- .../internaldns/serving/activation_test.go | 4 ++-- .../internaldns/serving/transaction_test.go | 20 ++++++++++++++++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/internal/internaldns/serving/activation.go b/internal/internaldns/serving/activation.go index 3d27f556..74a2c6dd 100644 --- a/internal/internaldns/serving/activation.go +++ b/internal/internaldns/serving/activation.go @@ -112,7 +112,7 @@ func daemonPermission(stateDir string) (start, retain bool, release func()) { return false, false, release } upgradeErr := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB) - data, readErr := os.ReadFile(marker) + _, readErr := os.ReadFile(marker) if errors.Is(readErr, os.ErrNotExist) { return true, true, release } @@ -128,7 +128,7 @@ func daemonPermission(stateDir string) (start, retain bool, release func()) { if err := unix.Flock(int(lock.Fd()), unix.LOCK_SH|unix.LOCK_NB); err != nil { return false, true, release } - data, readErr = os.ReadFile(marker) + data, readErr := os.ReadFile(marker) if errors.Is(readErr, os.ErrNotExist) { return true, true, release } diff --git a/internal/internaldns/serving/activation_test.go b/internal/internaldns/serving/activation_test.go index 86a6e2f2..6ee5421f 100644 --- a/internal/internaldns/serving/activation_test.go +++ b/internal/internaldns/serving/activation_test.go @@ -6,10 +6,10 @@ import ( "context" "errors" "os" + "testing" "time" "go.miloapis.com/dns-operator/internal/internaldns/model" - "testing" ) func TestActivationOwnershipGatesAbandonedUpdatesWithoutHeartbeat(t *testing.T) { @@ -63,7 +63,7 @@ func TestPublicationActivationRemainsOwnedUntilDNSProof(t *testing.T) { } approved := agent.state.Active verified := false - agent.deps.PublicationVerifier = publicationVerifierFunc(func(context.Context, model.PublicationPlan, []PublicationView) error { + agent.publicationVerifier = publicationVerifierFunc(func(context.Context, model.PublicationPlan, []PublicationView) error { verified = true if !activationPending(cfg.StateDir) { t.Fatal("activation committed before DNS proof") diff --git a/internal/internaldns/serving/transaction_test.go b/internal/internaldns/serving/transaction_test.go index ebb3181d..151058e4 100644 --- a/internal/internaldns/serving/transaction_test.go +++ b/internal/internaldns/serving/transaction_test.go @@ -285,6 +285,20 @@ func (r *activationBoundaryRunner) Run(_ context.Context, command Command) ([]by return nil, nil } +type blockedInstallRunner struct { + recordingRunner + block func() error +} + +func (r *blockedInstallRunner) Run(ctx context.Context, command Command) ([]byte, error) { + if command.Path == "gate" && r.block != nil { + if err := r.block(); err != nil { + return nil, err + } + } + return r.recordingRunner.Run(ctx, command) +} + func TestInterruptedInstallationRequiresLocalRecovery(t *testing.T) { cfg := regionalTestConfig(t) agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) @@ -294,9 +308,9 @@ func TestInterruptedInstallationRequiresLocalRecovery(t *testing.T) { installTestPublication(t, agent, time.Now().UTC()) approved := agent.state.Active.Files badParent := filepath.Join(cfg.StateDir, "not-a-directory") - if err := os.WriteFile(badParent, []byte("blocked"), 0o644); err != nil { - t.Fatal(err) - } + agent.tx.Runner = &blockedInstallRunner{block: func() error { + return os.WriteFile(badParent, []byte("blocked"), 0o644) + }} files := map[string][]byte{filepath.Join(badParent, "later.conf"): []byte("candidate")} for path := range approved { files[path] = []byte("candidate") From c30bcf7e4f95ffb1bbcee19f8d85f8fe8bb9969d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 16:59:43 -0500 Subject: [PATCH 05/12] feat: recover CoreDNS from accepted snapshots --- internal/internaldns/serving/agent.go | 147 +----------------- internal/internaldns/serving/agent_test.go | 34 ++-- .../serving/binding_fences_test.go | 83 +++++++++- .../serving/{bind_test.go => coredns_test.go} | 105 ++++++------- internal/internaldns/serving/daemons.go | 12 +- internal/internaldns/serving/readiness.go | 10 +- internal/internaldns/serving/transaction.go | 27 +--- .../internaldns/serving/transaction_test.go | 63 +++----- 8 files changed, 191 insertions(+), 290 deletions(-) rename internal/internaldns/serving/{bind_test.go => coredns_test.go} (89%) diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index 63a1f8cc..34280df0 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -11,7 +11,6 @@ import ( "path/filepath" "sort" "strconv" - "strings" "sync" "time" @@ -35,7 +34,6 @@ type Config struct { Mode Mode `json:"mode"` Render RenderConfig `json:"render"` Transaction TransactionConfig `json:"transaction"` - CacheFlush []Command `json:"cacheFlush,omitempty"` ReconcileInterval time.Duration `json:"reconcileInterval,omitempty"` RetryDelay time.Duration `json:"retryDelay,omitempty"` AckInterval time.Duration `json:"ackInterval,omitempty"` @@ -65,9 +63,6 @@ func (c Config) Validate() error { } } - if c.Mode == ModeResolver && (c.Render.Role == RenderNode || c.Render.Role == RenderBoth) && len(c.CacheFlush) == 0 && !c.UnsafeAllowNoDNSVerify { - return errors.New("node BIND cache withdrawal hooks are required") - } if c.AckLease > 2*time.Minute { return errors.New("ack lease may not exceed two minutes") } @@ -85,11 +80,11 @@ func (c Config) Validate() error { return errors.New("failClosed command is required; use unsafeAllowNoFailClosed only in tests") } if c.Mode == ModeCombined { - if c.Render.Role != RenderCluster && c.Render.Role != RenderBoth && c.Render.Role != RenderRegionalBIND && c.Render.Role != "" { - return errors.New("combined publication serving requires an owned regional BIND process") + if c.Render.Role != RenderCluster && c.Render.Role != RenderBoth && c.Render.Role != RenderRegionalCoreDNS && c.Render.Role != "" { + return errors.New("combined publication serving requires an owned regional CoreDNS process") } - if c.PublicationDNSProbe.Server != "" && c.PublicationDNSProbe.Server != net.JoinHostPort(c.Render.ClusterBIND.ListenAddress, strconv.Itoa(int(c.Render.ClusterBIND.Port))) { - return errors.New("publication probe must target this member's protected BIND listener") + if c.PublicationDNSProbe.Server != "" && c.PublicationDNSProbe.Server != net.JoinHostPort(c.Render.ClusterCoreDNS.ListenAddress, strconv.Itoa(int(c.Render.ClusterCoreDNS.Port))) { + return errors.New("publication probe must target this member's protected CoreDNS listener") } } return nil @@ -215,7 +210,7 @@ func NewAgent(c Config, d Dependencies) (*Agent, error) { d.Runner = ExecRunner{} } d.AckPublisher = newBoundedAckPublisher(d.AckPublisher) - if (c.Mode == ModeResolver || c.Mode == ModeCombined) && c.Render.Role != RenderRegionalBIND && d.Verifier == nil && !c.UnsafeAllowNoDNSVerify { + if (c.Mode == ModeResolver || c.Mode == ModeCombined) && c.Render.Role != RenderRegionalCoreDNS && d.Verifier == nil && !c.UnsafeAllowNoDNSVerify { probeConfig := c.ResolverDNSProbe if c.Render.Role == RenderCluster || c.Render.Role == RenderRegionalDNSDist { probeConfig.UseClusterAddress = true @@ -482,7 +477,6 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving if err != nil { return err } - previous := a.state.Snapshot a.state.BindingFences = bindingFences a.state.ServingFence = fence{snapshot.ConfigurationEpoch, snapshot.ConfigurationRevision} a.state.Snapshot = &snapshot @@ -493,10 +487,6 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving _ = a.failClosed(ctx) return err } - if err := a.flushRemovedBindingZones(ctx, previous, snapshot); err != nil { - _ = a.failClosed(ctx) - return err - } } if err := a.reconcile(ctx); err != nil { return err @@ -504,33 +494,6 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving return a.finishEvent(ctx) } -// A configuration change can detach a zone without changing its publication. -// Clear its old cache tree before accepting the mapping, including when the -// context is temporarily pending. The cache hook must flush descendants too. -func (a *Agent) flushRemovedBindingZones(ctx context.Context, previous *model.ServingSnapshot, snapshot model.ServingSnapshot) error { - if previous == nil { - return nil - } - attached := map[string]bool{} - for _, binding := range snapshot.Bindings { - for _, zone := range binding.Zones { - attached[binding.BindingUID+"/"+zone.ZoneUID+"/"+zone.Apex] = true - } - } - flushed := map[string]bool{} - for _, binding := range previous.Bindings { - for _, zone := range binding.Zones { - if !attached[binding.BindingUID+"/"+zone.ZoneUID+"/"+zone.Apex] && !flushed[zone.Apex] { - if err := a.flushName(ctx, zone.Apex); err != nil { - return err - } - flushed[zone.Apex] = true - } - } - } - return nil -} - func (a *Agent) handleChunk(env model.Envelope) error { var c model.PublicationChunk if err := json.Unmarshal(env.Payload, &c); err != nil { @@ -587,11 +550,6 @@ func (a *Agent) handleManifest(ctx context.Context, env model.Envelope) error { _ = a.emitAck(ctx, model.KindPublicationManifest, m.ZoneUID, m.WriterEpoch, m.Revision, model.AckRejected, err.Error()) return err } - for _, name := range publicationNames(old.Plan) { - if err := a.flushName(ctx, name); err != nil { - return err - } - } return a.emitVerifiedPublicationAck(ctx, m.ZoneUID, m.WriterEpoch, m.Revision) } // Publications are complete snapshots. The control plane reserves a revision @@ -658,12 +616,6 @@ func (a *Agent) recordResolverPublication(ctx context.Context, uid string, next, if err := a.store.Save(a.state); err != nil { return err } - for _, name := range append(publicationNames(old.Plan), publicationNames(next.Plan)...) { - if err := a.flushName(ctx, name); err != nil { - _ = a.failClosed(ctx) - return err - } - } next.Verified = true a.state.Publications[uid] = next return a.store.Save(a.state) @@ -684,12 +636,6 @@ func (a *Agent) activatePublication(ctx context.Context, uid string, next, old p _ = a.failClosed(ctx) return err } - for _, name := range append(publicationNames(old.Plan), publicationNames(next.Plan)...) { - if err := a.flushName(ctx, name); err != nil { - _ = a.failClosed(ctx) - return err - } - } next.Verified = true a.state.Publications[uid] = next if err := a.store.Save(a.state); err != nil { @@ -735,9 +681,6 @@ func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { return nil } if ack.Phase == model.AckExpired { - if err := a.flushPublicationZone(ctx, ack.ResourceUID); err != nil { - return err - } if err := a.reconcile(ctx); err != nil { return err } @@ -760,10 +703,6 @@ func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { if err := a.store.Save(a.state); err != nil { return err } - if err := a.flushPublicationZone(ctx, ack.ResourceUID); err != nil { - _ = a.failClosed(ctx) - return err - } if err := a.reconcile(ctx); err != nil { return err } @@ -778,17 +717,6 @@ func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { if comparison < 0 || (comparison == 0 && !ack.ObservedAt.After(old.ObservedAt)) { return nil } - highest := fence{} - for _, existing := range a.state.Acks[ack.ResourceUID] { - if model.Compare(existing.Fence.Epoch, existing.Fence.Revision, highest.Epoch, highest.Revision) > 0 { - highest = existing.Fence - } - } - if model.Compare(ack.Epoch, ack.Revision, highest.Epoch, highest.Revision) > 0 { - if err := a.flushPublicationZone(ctx, ack.ResourceUID); err != nil { - return err - } - } if comparison > 0 || (ack.Epoch == old.Fence.Epoch && ack.Revision == old.Fence.Revision && ack.ValidUntil.After(old.ValidUntil)) { a.state.Acks[ack.ResourceUID][key] = replicaAck{Fence: fence{ack.Epoch, ack.Revision}, ValidUntil: ack.ValidUntil, ObservedAt: ack.ObservedAt} if err := a.store.Save(a.state); err != nil { @@ -940,7 +868,7 @@ func (a *Agent) reconcile(ctx context.Context) error { } } // Keep activation owned until every materialized publication has query proof - // and its affected names have been flushed from resolver caches. + // and publication content has passed query verification. if a.cfg.Mode == ModeCombined { for _, p := range a.state.Publications { if !p.Verified && (p.Manifest.Tombstone || len(a.publicationViews(p)) > 0) { @@ -1190,11 +1118,6 @@ func (a *Agent) retryPendingPublications(ctx context.Context) error { if err := a.verifyPublication(ctx, p); err != nil { return err } - for _, name := range publicationNames(p.Plan) { - if err := a.flushName(ctx, name); err != nil { - return err - } - } p.Verified = true p.EffectiveHash = effectivePlanHash(p.Plan) a.state.Publications[uid] = p @@ -1205,38 +1128,6 @@ func (a *Agent) retryPendingPublications(ctx context.Context) error { return nil } -func (a *Agent) flushName(ctx context.Context, name string) error { - - for _, command := range a.cfg.CacheFlush { - command.Args = append([]string(nil), command.Args...) - for i := range command.Args { - command.Args[i] = strings.ReplaceAll(command.Args[i], "{name}", model.AbsoluteName(name)) - } - if _, err := a.deps.Runner.Run(ctx, command); err != nil { - return err - } - } - return nil -} - -func (a *Agent) flushPublicationZone(ctx context.Context, zoneUID string) error { - if a.state.Snapshot == nil { - return nil - } - seen := map[string]bool{} - for _, binding := range a.state.Snapshot.Bindings { - for _, zone := range binding.Zones { - if zone.ZoneUID == zoneUID && !seen[zone.Apex] { - seen[zone.Apex] = true - if err := a.flushName(ctx, zone.Apex); err != nil { - return err - } - } - } - } - return nil -} - func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, revision uint64, phase model.AckPhase, message string) error { if a.deps.AckPublisher == nil { return nil @@ -1357,32 +1248,6 @@ func equalJSON(left, right any) bool { return errA == nil && errB == nil && string(a) == string(b) } -func publicationNames(plan model.PublicationPlan) []string { - seen := map[string]bool{} - var out []string - if plan.Apex != "" { - apex := model.AbsoluteName(plan.Apex) - seen[apex] = true - out = append(out, apex) - } - for _, owner := range plan.Owners { - owner = model.AbsoluteName(owner) - if !seen[owner] { - seen[owner] = true - out = append(out, owner) - } - } - for _, rrset := range plan.RRSets { - owner := model.AbsoluteName(rrset.Name) - if !seen[owner] { - seen[owner] = true - out = append(out, owner) - } - } - sort.Strings(out) - return out -} - func (a *Agent) validateContributionFences(plan model.PublicationPlan) error { for _, observation := range plan.ObservationFences { current, ok := a.state.ContributionFences[observation.ContributionUID] diff --git a/internal/internaldns/serving/agent_test.go b/internal/internaldns/serving/agent_test.go index c0ffb6d7..a6995ede 100644 --- a/internal/internaldns/serving/agent_test.go +++ b/internal/internaldns/serving/agent_test.go @@ -208,9 +208,9 @@ func TestClusterRoleAutomaticallyVerifiesClusterVIP(t *testing.T) { renderConfig, _ := renderFixture(t) renderConfig.Role = RenderCluster renderConfig.ClusterDNSDist.Path = filepath.Join(dir, "cluster-dnsdist.conf") - renderConfig.ClusterBIND.Path = filepath.Join(dir, "cluster-bind.conf") + renderConfig.ClusterCoreDNS.Path = filepath.Join(dir, "cluster-coredns.json") renderConfig.NodeDNSDist = DNSDistRenderConfig{} - renderConfig.NodeBIND = BINDRenderConfig{} + renderConfig.NodeCoreDNS = CoreDNSRenderConfig{} agent, err := NewAgent(Config{ Region: "east", Shard: "s1", MemberID: "cluster-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, @@ -231,9 +231,9 @@ func TestStartActivatesPersistedDesiredConfiguration(t *testing.T) { renderConfig, snapshot := renderFixture(t) renderConfig.Role = RenderNode renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") - renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") renderConfig.ClusterDNSDist = DNSDistRenderConfig{} - renderConfig.ClusterBIND = BINDRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} rendered, err := Render(renderConfig, snapshot, map[string]bool{}) if err != nil { t.Fatal(err) @@ -264,11 +264,11 @@ func TestStartActivatesPersistedDesiredConfiguration(t *testing.T) { FailClosed: []Command{{Path: "gate"}}, Validate: []Command{ {Path: "validate-dnsdist", Args: []string{"{stageDir}/node-dnsdist.conf"}, WhenChanged: []string{renderConfig.NodeDNSDist.Path}}, - {Path: "validate-bind", Args: []string{"{stageDir}/node-bind.conf"}, WhenChanged: []string{renderConfig.NodeBIND.Path}}, + {Path: "validate-bind", Args: []string{"{stageDir}/node-coredns.json"}, WhenChanged: []string{renderConfig.NodeCoreDNS.Path}}, }, Reload: []Command{ {Path: "reload-dnsdist", WhenChanged: []string{renderConfig.NodeDNSDist.Path}}, - {Path: "reload-bind", WhenChanged: []string{renderConfig.NodeBIND.Path}}, + {Path: "reload-bind", WhenChanged: []string{renderConfig.NodeCoreDNS.Path}}, }, }, } @@ -315,9 +315,9 @@ func TestStepBoundsStalledAckPublishAndRetainsAppliedConfiguration(t *testing.T) renderConfig, snapshot := renderFixture(t) renderConfig.Role = RenderNode renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") - renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") renderConfig.ClusterDNSDist = DNSDistRenderConfig{} - renderConfig.ClusterBIND = BINDRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} snapshot.Bindings[0].Zones = nil var reported []error agent, err := NewAgent(Config{ @@ -394,9 +394,9 @@ func TestResolverApprovesConfigurationBeforeBrokerAckCompletes(t *testing.T) { renderConfig, snapshot := renderFixture(t) renderConfig.Role = RenderNode renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") - renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") renderConfig.ClusterDNSDist = DNSDistRenderConfig{} - renderConfig.ClusterBIND = BINDRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} snapshot.Bindings[0].Zones = nil started := make(chan struct{}, 1) release := make(chan struct{}) @@ -424,9 +424,9 @@ func TestResolverGatesFailedTenantAndRetainsHealthyTenant(t *testing.T) { renderConfig, snapshot := renderFixture(t) renderConfig.Role = RenderNode renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") - renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") renderConfig.ClusterDNSDist = DNSDistRenderConfig{} - renderConfig.ClusterBIND = BINDRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} snapshot.Bindings[0].Zones = nil failedUID := snapshot.Bindings[0].BindingUID healthy := snapshot.Bindings[0] @@ -440,7 +440,7 @@ func TestResolverGatesFailedTenantAndRetainsHealthyTenant(t *testing.T) { release := make(chan struct{}) agent, err := NewAgent(Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, - CacheFlush: []Command{{Path: "flush"}}, Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, + Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}}, UnsafeAllowNoFailClosed: true, AckPublishTimeout: time.Second, }, Dependencies{ AckPublisher: blockingPublisher{started: started, release: release}, Runner: &recordingRunner{}, @@ -479,9 +479,9 @@ func TestStartPreservesAppliedConfigurationWhileBrokerDisconnected(t *testing.T) renderConfig, snapshot := renderFixture(t) renderConfig.Role = RenderNode renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") - renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") renderConfig.ClusterDNSDist = DNSDistRenderConfig{} - renderConfig.ClusterBIND = BINDRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} snapshot.Bindings[0].Zones = nil publishStarted := make(chan struct{}, 1) publishRelease := make(chan struct{}) @@ -615,9 +615,9 @@ func TestAckEventPreservesAppliedConfigurationAfterReconcile(t *testing.T) { renderConfig, snapshot := renderFixture(t) renderConfig.Role = RenderNode renderConfig.NodeDNSDist.Path = filepath.Join(dir, "runtime", "node-dnsdist.conf") - renderConfig.NodeBIND.Path = filepath.Join(dir, "runtime", "node-bind.conf") + renderConfig.NodeCoreDNS.Path = filepath.Join(dir, "runtime", "node-coredns.json") renderConfig.ClusterDNSDist = DNSDistRenderConfig{} - renderConfig.ClusterBIND = BINDRenderConfig{} + renderConfig.ClusterCoreDNS = CoreDNSRenderConfig{} snapshot.Members = []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}} agent, err := NewAgent(Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: ModeResolver, diff --git a/internal/internaldns/serving/binding_fences_test.go b/internal/internaldns/serving/binding_fences_test.go index a09ca066..663c46d2 100644 --- a/internal/internaldns/serving/binding_fences_test.go +++ b/internal/internaldns/serving/binding_fences_test.go @@ -55,7 +55,7 @@ func TestBindingWithdrawalPreservesSourceSequenceAndAllowsNormalRenewal(t *testi if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, renewed)); err != nil { t.Fatalf("normal source2 -> source3 renewal rejected: %v", err) } - config, err := os.ReadFile(cfg.Render.ClusterBIND.Path) + config, err := os.ReadFile(cfg.Render.ClusterCoreDNS.Path) if err != nil || !strings.Contains(string(config), renewed.Bindings[0].ViewName()) { t.Fatalf("renewal did not restore context: %v", err) } @@ -145,7 +145,7 @@ func TestBindingOmissionFenceSurvivesRestartAndRequiresNewSourceAuthority(t *tes func TestCheckpointRejectsPreBindingFenceFormatWithoutDiscardingIt(t *testing.T) { t.Parallel() - for _, version := range []int{2, checkpointFormatVersion} { + for _, version := range []int{2, 3, checkpointFormatVersion} { t.Run(string(rune('0'+version)), func(t *testing.T) { path := filepath.Join(t.TempDir(), "checkpoint.json") data, err := json.Marshal(map[string]any{"formatVersion": version, "contributionFences": map[string]any{"original-observation": map[string]any{"epoch": 1, "sequence": 2}}}) @@ -165,3 +165,82 @@ func TestCheckpointRejectsPreBindingFenceFormatWithoutDiscardingIt(t *testing.T) }) } } + +func TestPendingConfigurationGatesQueriesAndRecoversWithoutAuthorizationRenewal(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + runner := &recordingRunner{} + dependencies := Dependencies{Runner: runner, Clock: func() time.Time { return now }} + agent, err := NewAgent(cfg, dependencies) + if err != nil { + t.Fatal(err) + } + active := testSnapshot(now) + healthy := active.Bindings[0] + healthy.BindingUID, healthy.ProjectUID, healthy.ContextUID = "binding-b", "project-b", "context-b" + healthy.ConsumerAddress, healthy.ClusterAddress = "10.253.0.52", "10.253.0.42" + healthy.Zones = nil + active.Bindings = append(active.Bindings, healthy) + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + chunk, manifest := testPublication(t, now) + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } + original := active.Bindings[0].Version + pending := active + pending.Bindings = append([]model.Binding(nil), active.Bindings...) + pending.ConfigurationRevision = 2 + pending.Bindings[0].ConfigurationRevision = 2 + pending.Bindings[0].ConfigurationPending = true + pending.Bindings[0].Zones = nil + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, pending)); err != nil { + t.Fatal(err) + } + if agent.state.BindingFences["binding-a"].Tombstone || agent.publicationsReady(pending.Bindings[0]) || !agent.publicationsReady(healthy) { + t.Fatal("pending configuration revoked access, activated incomplete data, or gated another context") + } + config, err := os.ReadFile(cfg.Render.ClusterCoreDNS.Path) + if err != nil { + t.Fatal(err) + } + contexts := decodedCoreDNS(t, config).Contexts + found := false + for _, context := range contexts { + if context.ID == pending.Bindings[0].ViewName() { + found = true + if !context.Pending || len(context.Zones) > 0 { + t.Fatal("pending context can serve private data") + } + } + } + if !found { + t.Fatal("pending context missing") + } + restarted, err := NewAgent(cfg, dependencies) + if err != nil { + t.Fatal(err) + } + ready := active + ready.Bindings = append([]model.Binding(nil), active.Bindings...) + ready.ConfigurationRevision = 3 + ready.Bindings[0].ConfigurationRevision = 2 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, ready)); err != nil { + t.Fatalf("completion required a new authorization: %v", err) + } + if restarted.state.BindingFences["binding-a"].Version != original || !restarted.publicationsReady(ready.Bindings[0]) { + t.Fatal("completion changed authority or remained gated") + } + // The older pending snapshot cannot undo the completed configuration. + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, pending)); err != nil { + t.Fatal(err) + } + if restarted.state.Snapshot.Bindings[0].ConfigurationPending { + t.Fatal("stale pending replay gated the recovered context") + } +} diff --git a/internal/internaldns/serving/bind_test.go b/internal/internaldns/serving/coredns_test.go similarity index 89% rename from internal/internaldns/serving/bind_test.go rename to internal/internaldns/serving/coredns_test.go index fd517351..7631257f 100644 --- a/internal/internaldns/serving/bind_test.go +++ b/internal/internaldns/serving/coredns_test.go @@ -8,7 +8,6 @@ import ( "net" "os" "path/filepath" - "regexp" "strings" "sync/atomic" @@ -25,8 +24,8 @@ func regionalTestConfig(t *testing.T) Config { t.Helper() dir := t.TempDir() render, _ := renderFixture(t) - render.Role = RenderRegionalBIND - render.ClusterBIND.Path = filepath.Join(dir, "runtime", "regional.conf") + render.Role = RenderRegionalCoreDNS + render.ClusterCoreDNS.Path = filepath.Join(dir, "runtime", "regional.conf") return Config{Region: "east", Shard: "s1", MemberID: "regional-0", ReplicaID: "regional-0", Mode: ModeCombined, StateDir: filepath.Join(dir, "state"), Render: render, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage"), Validate: []Command{{Path: "validate"}}, Reload: []Command{{Path: "reload"}}, FailClosed: []Command{{Path: "gate"}}}, UnsafeAllowNoDNSVerify: true} } func installTestPublication(t *testing.T, a *Agent, now time.Time) { @@ -51,15 +50,26 @@ func currentZone(t *testing.T, a *Agent) string { if err != nil { t.Fatal(err) } - for path, data := range files.Files { - if strings.HasSuffix(path, ".db") { - return string(data) + for _, data := range files.Files { + var cfg struct { + Contexts []struct{ Zones []struct{ Data string } } + } + if json.Unmarshal(data, &cfg) != nil { + continue + } + for _, context := range cfg.Contexts { + for _, zone := range context.Zones { + if zone.Data != "" { + return zone.Data + } + } } } + t.Fatal("no zone") return "" } -func TestBINDPublicationActivatesLocalPrivateAuthority(t *testing.T) { +func TestCoreDNSPublicationActivatesLocalPrivateAuthority(t *testing.T) { t.Parallel() now := time.Now().UTC() cfg := regionalTestConfig(t) @@ -73,18 +83,18 @@ func TestBINDPublicationActivatesLocalPrivateAuthority(t *testing.T) { if !strings.Contains(data, "192.0.2.10") || !strings.Contains(data, publicationMailbox(agent.state.Publications["zone-a"])) { t.Fatalf("bad zone: %s", data) } - config, err := os.ReadFile(cfg.Render.ClusterBIND.Path) + config, err := os.ReadFile(cfg.Render.ClusterCoreDNS.Path) if err != nil { t.Fatal(err) } - if !strings.Contains(string(config), "type primary") || strings.Contains(string(config), "static-stub") || strings.Contains(string(config), "query-source") { + if decodedCoreDNS(t, config).Contexts[0].Zones[0].Data == "" { t.Fatalf("not direct private authority: %s", config) } if !agent.state.Publications["zone-a"].Verified { t.Fatal("publication not verified") } } -func TestBINDPublicationSurvivesBrokerOutageAndRestart(t *testing.T) { +func TestCoreDNSPublicationSurvivesBrokerOutageAndRestart(t *testing.T) { t.Parallel() now := time.Now().UTC() clock := now @@ -120,7 +130,7 @@ func TestBINDPublicationSurvivesBrokerOutageAndRestart(t *testing.T) { t.Fatal("restart lost the published record") } } -func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { +func TestCoreDNSActivationFailurePersistsObservationFence(t *testing.T) { t.Parallel() now := time.Now().UTC() cfg := regionalTestConfig(t) @@ -159,7 +169,7 @@ func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { t.Fatalf("equal fenced retry failed: %v", err) } } -func TestBINDTombstoneFencesReplayAcrossRestart(t *testing.T) { +func TestCoreDNSTombstoneFencesReplayAcrossRestart(t *testing.T) { t.Parallel() now := time.Now().UTC() cfg := regionalTestConfig(t) @@ -191,11 +201,10 @@ func TestNodeRequiresPublicationAndRetainsCacheDuringOutage(t *testing.T) { cfg := regionalTestConfig(t) render, snapshot := renderFixture(t) render.Role = RenderNode - render.NodeBIND.Path = filepath.Join(t.TempDir(), "node.conf") - render.NodeDNSDist.Path = filepath.Join(filepath.Dir(render.NodeBIND.Path), "dnsdist.conf") + render.NodeCoreDNS.Path = filepath.Join(t.TempDir(), "node.conf") + render.NodeDNSDist.Path = filepath.Join(filepath.Dir(render.NodeCoreDNS.Path), "dnsdist.conf") cfg.Render = render cfg.Mode = ModeResolver - cfg.CacheFlush = []Command{{Path: "flush", Args: []string{"{name}"}}} runner := &recordingRunner{} a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }}) if err != nil { @@ -223,24 +232,9 @@ func TestNodeRequiresPublicationAndRetainsCacheDuringOutage(t *testing.T) { if err := a.Step(context.Background()); err != nil { t.Fatal(err) } - flushed := false - for _, call := range runner.calls { - if call.Path == "flush" && call.Args[0] == "api.prod.internal." { - flushed = true - } - } - if flushed { - t.Fatal("node flushed a valid record during the outage") - } - restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) - if err != nil { - t.Fatal(err) - } - if !restarted.publicationsReady(snapshot.Bindings[0]) { - t.Fatal("restart lost the installed publication proof") - } + } -func TestBINDPublicationVersionProofIncludesFullContentFingerprint(t *testing.T) { +func TestCoreDNSPublicationVersionProofIncludesFullContentFingerprint(t *testing.T) { t.Parallel() now := time.Now().UTC() chunk, m := testPublication(t, now) @@ -326,14 +320,14 @@ func TestRegionalPoolRequiresOneMemberWithEveryCurrentZone(t *testing.T) { func TestMissingOrStalePrivatePublicationHasNoPublicFallback(t *testing.T) { t.Parallel() c, snapshot := renderFixture(t) - c.Role = RenderRegionalBIND + c.Role = RenderRegionalCoreDNS publications := map[string]publicationState{"zone-a": {Fence: fence{1, 1}, Plan: model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal"}}} rendered, err := renderPublications(c, snapshot, map[string]bool{}, publications) if err != nil { t.Fatal(err) } - conf := string(rendered.Files[c.ClusterBIND.Path]) - if !strings.Contains(conf, "type forward; forward only; forwarders { 127.0.0.1 port 9; }") { + conf := string(rendered.Files[c.ClusterCoreDNS.Path]) + if !decodedCoreDNS(t, []byte(conf)).Contexts[0].Zones[0].Pending { t.Fatalf("private zone can recurse publicly: %s", conf) } for path := range rendered.Files { @@ -400,8 +394,8 @@ func TestPublicationProbeTimeoutDoesNotApproveConfiguration(t *testing.T) { } defer func() { _ = udp.Close() }() port := udp.LocalAddr().(*net.UDPAddr).Port - cfg.Render.ClusterBIND.ListenAddress = "127.0.0.1" - cfg.Render.ClusterBIND.Port = uint16(port) + cfg.Render.ClusterCoreDNS.ListenAddress = "127.0.0.1" + cfg.Render.ClusterCoreDNS.Port = uint16(port) cfg.PublicationDNSProbe = PublicationDNSVerifierConfig{Server: udp.LocalAddr().String(), Timeout: 50 * time.Millisecond} cfg.UnsafeAllowNoDNSVerify = false a, err := NewAgent(cfg, Dependencies{Clock: func() time.Time { return now }, Runner: &recordingRunner{}}) @@ -452,16 +446,19 @@ func installedFileVerifier(t *testing.T, configPath string) PublicationVerifier if err != nil { return err } - pattern := regexp.MustCompile(`zone "` + regexp.QuoteMeta(strings.TrimSuffix(plan.Apex, ".")) + `" \{ type primary; file "([^"]+)"`) - match := pattern.FindSubmatch(config) - if len(match) != 2 { - return errors.New("no activated primary zone") + cfg := decodedCoreDNS(t, config) + var data string + for _, context := range cfg.Contexts { + for _, zone := range context.Zones { + if zone.Apex == model.AbsoluteName(plan.Apex) && !zone.Pending { + data = zone.Data + } + } } - data, err := os.ReadFile(string(match[1])) - if err != nil { - return err + if data == "" { + return errors.New("no activated private zone") } - parser := dns.NewZoneParser(strings.NewReader(string(data)), model.AbsoluteName(plan.Apex), string(match[1])) + parser := dns.NewZoneParser(strings.NewReader(data), model.AbsoluteName(plan.Apex), configPath) for rr, ok := parser.Next(); ok; rr, ok = parser.Next() { if soa, ok := rr.(*dns.SOA); ok { for _, view := range views { @@ -476,7 +473,7 @@ func installedFileVerifier(t *testing.T, configPath string) PublicationVerifier }) } -func TestFailedBINDActivationRetainsPriorInstalledProof(t *testing.T) { +func TestFailedCoreDNSActivationRetainsPriorInstalledProof(t *testing.T) { t.Parallel() now := time.Now().UTC() cfg := regionalTestConfig(t) @@ -499,7 +496,7 @@ func TestFailedBINDActivationRetainsPriorInstalledProof(t *testing.T) { } } -func TestBINDDoesNotProbeAuthorityBelowRequiredPublicationFence(t *testing.T) { +func TestCoreDNSDoesNotProbeAuthorityBelowRequiredPublicationFence(t *testing.T) { t.Parallel() now := time.Now().UTC() cfg := regionalTestConfig(t) @@ -528,8 +525,8 @@ func TestBINDDoesNotProbeAuthorityBelowRequiredPublicationFence(t *testing.T) { if calls != 0 || len(a.publicationViews(a.state.Publications["zone-a"])) != 0 { t.Fatal("SERVFAIL zone attempted authoritative publication proof") } - config, err := os.ReadFile(cfg.Render.ClusterBIND.Path) - if err != nil || !strings.Contains(string(config), "forward only; forwarders { 127.0.0.1 port 9; }") { + config, err := os.ReadFile(cfg.Render.ClusterCoreDNS.Path) + if err != nil || !decodedCoreDNS(t, config).Contexts[0].Zones[0].Pending { t.Fatalf("behind-required private zone was not gated: %s %v", config, err) } } @@ -553,14 +550,14 @@ func (p *partitioningAckPublisher) Publish(_ context.Context, _ string, env mode return transport.PublishAck{}, nil } -func TestBINDActivationRacingBrokerDisconnectKeepsServing(t *testing.T) { +func TestCoreDNSActivationRacingBrokerDisconnectKeepsServing(t *testing.T) { t.Parallel() now := time.Now().UTC() clock := now cfg := regionalTestConfig(t) runner := &afterReloadRunner{} publisher := &partitioningAckPublisher{} - a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }, AckPublisher: publisher, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }, AckPublisher: publisher, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterCoreDNS.Path)}) if err != nil { t.Fatal(err) } @@ -588,7 +585,7 @@ func TestBINDActivationRacingBrokerDisconnectKeepsServing(t *testing.T) { if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 5, chunk)); err != nil { t.Fatal(err) } - // Disconnect after BIND reload, exactly before durable/proved activation + // Disconnect after CoreDNS reload, exactly before durable/proved activation // attempts its ACK. A transport error must not become a failed installation. runner.after = func() { publisher.disconnected.Store(true) } if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { @@ -630,12 +627,12 @@ func TestBINDActivationRacingBrokerDisconnectKeepsServing(t *testing.T) { } } -func TestBINDTombstoneAckFailureDoesNotGateDurableWithdrawal(t *testing.T) { +func TestCoreDNSTombstoneAckFailureDoesNotGateDurableWithdrawal(t *testing.T) { t.Parallel() now := time.Now().UTC() cfg := regionalTestConfig(t) publisher := &partitioningAckPublisher{} - a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }, AckPublisher: publisher, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) + a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }, AckPublisher: publisher, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterCoreDNS.Path)}) if err != nil { t.Fatal(err) } diff --git a/internal/internaldns/serving/daemons.go b/internal/internaldns/serving/daemons.go index 4ae33a4a..a11db457 100644 --- a/internal/internaldns/serving/daemons.go +++ b/internal/internaldns/serving/daemons.go @@ -60,22 +60,22 @@ func RunDaemons(ctx context.Context, config Config) error { return err } var processes []*daemonProcess - addBind := func(bind BINDRenderConfig) { - processes = append(processes, &daemonProcess{name: "named", config: bind.Path, args: []string{"-g", "-c", bind.Path, "-n", "2"}}) + addBind := func(bind CoreDNSRenderConfig) { + processes = append(processes, &daemonProcess{name: "coredns", config: bind.Path, args: []string{"-config", bind.Path}}) } addRouter := func(router DNSDistRenderConfig) { processes = append(processes, &daemonProcess{name: "dnsdist", config: router.Path, args: []string{"--supervised", "-C", router.Path}}) } switch config.Render.Role { case RenderNode: - addBind(config.Render.NodeBIND) + addBind(config.Render.NodeCoreDNS) addRouter(config.Render.NodeDNSDist) - case RenderRegionalBIND: - addBind(config.Render.ClusterBIND) + case RenderRegionalCoreDNS: + addBind(config.Render.ClusterCoreDNS) case RenderRegionalDNSDist: addRouter(config.Render.ClusterDNSDist) case RenderCluster: - addBind(config.Render.ClusterBIND) + addBind(config.Render.ClusterCoreDNS) addRouter(config.Render.ClusterDNSDist) default: return fmt.Errorf("unsupported serving process role %q", config.Render.Role) diff --git a/internal/internaldns/serving/readiness.go b/internal/internaldns/serving/readiness.go index 15fb0449..46a591ff 100644 --- a/internal/internaldns/serving/readiness.go +++ b/internal/internaldns/serving/readiness.go @@ -68,14 +68,14 @@ func (a *Agent) probeLocalDaemons(ctx context.Context) error { if role == "" { role = RenderBoth } - var binds []BINDRenderConfig + var binds []CoreDNSRenderConfig var dists []DNSDistRenderConfig if role == RenderNode || role == RenderBoth { - binds = append(binds, a.cfg.Render.NodeBIND) + binds = append(binds, a.cfg.Render.NodeCoreDNS) dists = append(dists, a.cfg.Render.NodeDNSDist) } - if role == RenderCluster || role == RenderBoth || role == RenderRegionalBIND { - binds = append(binds, a.cfg.Render.ClusterBIND) + if role == RenderCluster || role == RenderBoth || role == RenderRegionalCoreDNS { + binds = append(binds, a.cfg.Render.ClusterCoreDNS) } if role == RenderCluster || role == RenderBoth || role == RenderRegionalDNSDist { dists = append(dists, a.cfg.Render.ClusterDNSDist) @@ -86,7 +86,7 @@ func (a *Agent) probeLocalDaemons(ctx context.Context) error { for _, network := range []string{networkUDP, networkTCP} { answer, err := exchangePublication(ctx, network, server, ".", view, 200*time.Millisecond) if err != nil || answer == nil || !answer.Response { - return fmt.Errorf("local BIND health %s: %w", server, errors.Join(err, errors.New("DNS response required"))) + return fmt.Errorf("local CoreDNS health %s: %w", server, errors.Join(err, errors.New("DNS response required"))) } } } diff --git a/internal/internaldns/serving/transaction.go b/internal/internaldns/serving/transaction.go index b203a3f0..bd8f756a 100644 --- a/internal/internaldns/serving/transaction.go +++ b/internal/internaldns/serving/transaction.go @@ -8,9 +8,7 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "sort" - "strconv" "strings" "time" ) @@ -142,20 +140,7 @@ func (t *FileTransaction) apply(ctx context.Context, files map[string][]byte, fo for _, path := range paths { stagedPath := filepath.Join(stage, filepath.Base(path)) stagedData := files[path] - if strings.HasSuffix(path, ".conf") { - // Validate the newly staged zone contents, never the previous live files. - commandStage := stage - if t.Config.CommandStageDir != "" { - commandStage = filepath.Join(t.Config.CommandStageDir, filepath.Base(stage)) - } - text := string(stagedData) - for renderedPath := range files { - if strings.HasSuffix(renderedPath, ".db") { - text = strings.ReplaceAll(text, strconv.Quote(renderedPath), strconv.Quote(filepath.Join(commandStage, filepath.Base(renderedPath)))) - } - } - stagedData = []byte(text) - } + if err := writeDurable(stagedPath, stagedData, 0o644); err != nil { return err } @@ -358,14 +343,6 @@ func restoreFiles(paths []string, old map[string]oldFile) error { return errors.Join(errs...) } -var immutableZoneFile = regexp.MustCompile(`file "([^"\n]*/zone-[a-f0-9]{24})-[a-f0-9]{16}\.db"`) - -// Content-addressed zone files change record data without changing view routing. func routingConfigurationChanged(path string, before, after []byte) bool { - if !strings.HasSuffix(path, ".conf") { - return false - } - before = immutableZoneFile.ReplaceAll(before, []byte(`file "${1}.db"`)) - after = immutableZoneFile.ReplaceAll(after, []byte(`file "${1}.db"`)) - return !bytes.Equal(before, after) + return strings.HasSuffix(path, ".conf") && !bytes.Equal(before, after) } diff --git a/internal/internaldns/serving/transaction_test.go b/internal/internaldns/serving/transaction_test.go index 151058e4..d2df468e 100644 --- a/internal/internaldns/serving/transaction_test.go +++ b/internal/internaldns/serving/transaction_test.go @@ -6,7 +6,6 @@ import ( "errors" "os" "path/filepath" - "strconv" "strings" "testing" "time" @@ -184,7 +183,7 @@ func TestFileTransactionReloadsOnlyChangedTier(t *testing.T) { t.Fatalf("reload calls = %#v", runner.calls) } if got, err := os.ReadFile(named); err != nil || string(got) != "unchanged" { - t.Fatalf("unchanged BIND file was disturbed: %q, %v", got, err) + t.Fatalf("unchanged CoreDNS file was disturbed: %q, %v", got, err) } } @@ -192,40 +191,28 @@ type transactionRunnerFunc func(context.Context, Command) ([]byte, error) func (f transactionRunnerFunc) Run(ctx context.Context, c Command) ([]byte, error) { return f(ctx, c) } -func TestFileTransactionValidatesFreshStagedZoneReferences(t *testing.T) { +func TestFileTransactionValidatesCompleteStagedSnapshot(t *testing.T) { t.Parallel() dir := t.TempDir() - config := filepath.Join(dir, "live", "named.conf") - zone := filepath.Join(dir, "live", "zone.db") + path := filepath.Join(dir, "contexts.json") + candidate := []byte(`{"contexts":[{"id":"a","data":"new zone"}]}`) runner := transactionRunnerFunc(func(_ context.Context, c Command) ([]byte, error) { - data, err := os.ReadFile(c.Args[0]) + staged, err := os.ReadFile(c.Args[0]) if err != nil { return nil, err } - staged := filepath.Join(filepath.Dir(c.Args[0]), "zone.db") - if !strings.Contains(string(data), strconv.Quote(staged)) || strings.Contains(string(data), strconv.Quote(zone)) { - t.Fatalf("validator sees live zone: %s", data) - } - content, err := os.ReadFile(staged) - if err != nil { - return nil, err - } - if string(content) != "new zone" { - t.Fatalf("staged content=%s", content) + if !bytes.Equal(staged, candidate) || c.Args[0] == path { + t.Fatal("validator did not see complete staged snapshot") } return nil, nil }) - tx := FileTransaction{Config: TransactionConfig{StageDir: filepath.Join(dir, "stage"), Validate: []Command{{Path: "named-checkconf", Args: []string{"{stageDir}/named.conf"}}}}, Runner: runner} - original := []byte("zone \"test.internal\" { type primary; file " + strconv.Quote(zone) + "; };\n") - if err := tx.Apply(context.Background(), map[string][]byte{config: original, zone: []byte("new zone")}); err != nil { + tx := FileTransaction{Config: TransactionConfig{StageDir: filepath.Join(dir, "stage"), Validate: []Command{{Path: "coredns", Args: []string{"{stageDir}/contexts.json"}}}}, Runner: runner} + if err := tx.Apply(context.Background(), map[string][]byte{path: candidate}); err != nil { t.Fatal(err) } - installed, err := os.ReadFile(config) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(installed, original) { - t.Fatal("installed config retains temporary staging paths") + installed, err := os.ReadFile(path) + if err != nil || !bytes.Equal(installed, candidate) { + t.Fatal("installed snapshot differs", err) } } @@ -336,30 +323,26 @@ func TestInterruptedInstallationRequiresLocalRecovery(t *testing.T) { } } -func TestRecordUpdateReloadsImmutableZoneWithoutStoppingDNS(t *testing.T) { +func TestContextSnapshotUpdateDoesNotStopDNS(t *testing.T) { dir := t.TempDir() - configPath := filepath.Join(dir, "bind.conf") - zoneToken := "0123456789abcdef01234567" - before := []byte(`view "tenant-a" { zone "prod.internal" { file "` + dir + `/zone-` + zoneToken + `-0123456789abcdef.db"; }; };`) - after := []byte(`view "tenant-a" { zone "prod.internal" { file "` + dir + `/zone-` + zoneToken + `-abcdef0123456789.db"; }; };`) - if err := os.WriteFile(configPath, before, 0o644); err != nil { + path := filepath.Join(dir, "contexts.json") + before := []byte(`{"contexts":[{"id":"a","data":"192.0.2.1"}]}`) + after := []byte(`{"contexts":[{"id":"a","data":"192.0.2.2"}]}`) + if err := os.WriteFile(path, before, 0600); err != nil { t.Fatal(err) } runner := &recordingRunner{} - tx := FileTransaction{MarkerPath: activationMarkerPath(dir), Runner: runner, Config: TransactionConfig{ - StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}, Reload: []Command{{Path: "reload-bind"}}, - }} - if err := tx.Apply(context.Background(), map[string][]byte{configPath: after}); err != nil { + tx := FileTransaction{MarkerPath: activationMarkerPath(dir), Runner: runner, Config: TransactionConfig{StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}, Reload: []Command{{Path: "activate-coredns"}}}} + if err := tx.Apply(context.Background(), map[string][]byte{path: after}); err != nil { t.Fatal(err) } if err := tx.commitActivation(); err != nil { t.Fatal(err) } - if len(runner.calls) != 1 || runner.calls[0].Path != "reload-bind" { - t.Fatalf("record update stopped healthy DNS: %#v", runner.calls) + if len(runner.calls) != 1 || runner.calls[0].Path != "activate-coredns" { + t.Fatal("snapshot update stopped DNS", runner.calls) } - newTenant := bytes.ReplaceAll(after, []byte("tenant-a"), []byte("tenant-b")) - if !routingConfigurationChanged(configPath, after, newTenant) { - t.Fatal("tenant routing change did not require a gate") + if !routingConfigurationChanged(filepath.Join(dir, "dnsdist.conf"), before, after) { + t.Fatal("router update did not require a gate") } } From ee5789a9418298c28a58b592500a734fbf5a924e Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:34:41 -0500 Subject: [PATCH 06/12] refactor: simplify publication activation --- internal/internaldns/serving/agent.go | 16 ++++++++-------- .../internaldns/serving/binding_fences_test.go | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index 34280df0..ad627a4f 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -541,7 +541,7 @@ func (a *Agent) handleManifest(ctx context.Context, env model.Envelope) error { } if cmp == 0 && old.Verified { if a.cfg.Mode == ModeResolver { - return a.recordResolverPublication(ctx, m.ZoneUID, old, old) + return a.recordResolverPublication(m.ZoneUID, old) } if !equalJSON(m, old.Manifest) { return errors.New("conflicting publication manifest at the same epoch and revision") @@ -595,22 +595,22 @@ func (a *Agent) handleManifest(ctx context.Context, env model.Envelope) error { } next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: plan, EffectiveHash: effectivePlanHash(plan)} if a.cfg.Mode == ModeResolver { - return a.recordResolverPublication(ctx, m.ZoneUID, next, old) + return a.recordResolverPublication(m.ZoneUID, next) } - return a.activatePublication(ctx, m.ZoneUID, next, old) + return a.activatePublication(ctx, m.ZoneUID, next) } func (a *Agent) applyTombstone(ctx context.Context, m model.PublicationManifest, old publicationState) error { next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: old.Plan} if a.cfg.Mode == ModeResolver { - return a.recordResolverPublication(ctx, m.ZoneUID, next, old) + return a.recordResolverPublication(m.ZoneUID, next) } - return a.activatePublication(ctx, m.ZoneUID, next, old) + return a.activatePublication(ctx, m.ZoneUID, next) } // Commit the immutable fence before installing files. Failed activation stays // gated and an identical redelivery retries the accepted snapshot. -func (a *Agent) recordResolverPublication(ctx context.Context, uid string, next, old publicationState) error { +func (a *Agent) recordResolverPublication(uid string, next publicationState) error { next.Verified = false a.state.Publications[uid] = next if err := a.store.Save(a.state); err != nil { @@ -621,7 +621,7 @@ func (a *Agent) recordResolverPublication(ctx context.Context, uid string, next, return a.store.Save(a.state) } -func (a *Agent) activatePublication(ctx context.Context, uid string, next, old publicationState) error { +func (a *Agent) activatePublication(ctx context.Context, uid string, next publicationState) error { a.state.Publications[uid] = next if err := a.store.Save(a.state); err != nil { return err @@ -1104,7 +1104,7 @@ func (a *Agent) retryPendingPublications(ctx context.Context) error { continue } if a.cfg.Mode == ModeResolver { - if err := a.recordResolverPublication(ctx, uid, p, p); err != nil { + if err := a.recordResolverPublication(uid, p); err != nil { return err } continue diff --git a/internal/internaldns/serving/binding_fences_test.go b/internal/internaldns/serving/binding_fences_test.go index 663c46d2..d0f2ee28 100644 --- a/internal/internaldns/serving/binding_fences_test.go +++ b/internal/internaldns/serving/binding_fences_test.go @@ -211,10 +211,10 @@ func TestPendingConfigurationGatesQueriesAndRecoversWithoutAuthorizationRenewal( } contexts := decodedCoreDNS(t, config).Contexts found := false - for _, context := range contexts { - if context.ID == pending.Bindings[0].ViewName() { + for _, entry := range contexts { + if entry.ID == pending.Bindings[0].ViewName() { found = true - if !context.Pending || len(context.Zones) > 0 { + if !entry.Pending || len(entry.Zones) > 0 { t.Fatal("pending context can serve private data") } } From 0a56c872456aa66ac48dc9baec33052236e73fc6 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:35:09 -0500 Subject: [PATCH 07/12] style: clarify the context test fixtures --- internal/internaldns/serving/coredns_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/internaldns/serving/coredns_test.go b/internal/internaldns/serving/coredns_test.go index 7631257f..f2942faf 100644 --- a/internal/internaldns/serving/coredns_test.go +++ b/internal/internaldns/serving/coredns_test.go @@ -57,8 +57,8 @@ func currentZone(t *testing.T, a *Agent) string { if json.Unmarshal(data, &cfg) != nil { continue } - for _, context := range cfg.Contexts { - for _, zone := range context.Zones { + for _, entry := range cfg.Contexts { + for _, zone := range entry.Zones { if zone.Data != "" { return zone.Data } @@ -448,8 +448,8 @@ func installedFileVerifier(t *testing.T, configPath string) PublicationVerifier } cfg := decodedCoreDNS(t, config) var data string - for _, context := range cfg.Contexts { - for _, zone := range context.Zones { + for _, entry := range cfg.Contexts { + for _, zone := range entry.Zones { if zone.Apex == model.AbsoluteName(plan.Apex) && !zone.Pending { data = zone.Data } From 7c99760a0e4305610f84f8c771c344cdf7224b08 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:39:32 -0500 Subject: [PATCH 08/12] test: bound resolver setup independently Allow disk-backed resolver activation to complete before measuring broker partition continuity on shared CI runners. --- internal/internaldns/serving/agent_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/internaldns/serving/agent_test.go b/internal/internaldns/serving/agent_test.go index a6995ede..332aa506 100644 --- a/internal/internaldns/serving/agent_test.go +++ b/internal/internaldns/serving/agent_test.go @@ -520,7 +520,7 @@ func TestStartPreservesAppliedConfigurationWhileBrokerDisconnected(t *testing.T) defer releasePublisher() select { case <-publishStarted: - case <-time.After(time.Second): + case <-time.After(10 * time.Second): t.Fatal("resolver did not enter context-ignoring broker publish") } From e74eaa2261d2cbf36002f1ef0cfbff1f4adf4680 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 21:03:08 -0500 Subject: [PATCH 09/12] fix: retry DNS activation after restart installation errors --- internal/internaldns/serving/continuity.go | 8 +++-- .../internaldns/serving/continuity_test.go | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/internal/internaldns/serving/continuity.go b/internal/internaldns/serving/continuity.go index 26035e80..74e70f90 100644 --- a/internal/internaldns/serving/continuity.go +++ b/internal/internaldns/serving/continuity.go @@ -119,13 +119,17 @@ func (a *Agent) restoreActive(ctx context.Context) error { return ctx.Err() } } + a.needsActivation = true if err := a.tx.ApplyForce(ctx, files); err != nil { return err } a.state.RenderedHash = hashFiles(files) - a.needsActivation = false if err := a.store.Save(a.state); err != nil { return err } - return a.tx.commitActivation() + if err := a.tx.commitActivation(); err != nil { + return err + } + a.needsActivation = false + return nil } diff --git a/internal/internaldns/serving/continuity_test.go b/internal/internaldns/serving/continuity_test.go index 471a1528..948c2845 100644 --- a/internal/internaldns/serving/continuity_test.go +++ b/internal/internaldns/serving/continuity_test.go @@ -5,6 +5,7 @@ package serving import ( "context" "encoding/json" + "os" "testing" "time" @@ -151,3 +152,36 @@ func TestApprovedConfigurationRemainsInstalledDuringCandidateValidation(t *testi t.Fatal(err) } } + +func TestFailedRestartInstallationRequiresActivation(t *testing.T) { + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, time.Now().UTC()) + for path := range agent.state.Active.Files { + if err := os.WriteFile(path, []byte("incomplete configuration"), 0644); err != nil { + t.Fatal(err) + } + break + } + runner := &recordingRunner{failAt: 1} + restarted, err := NewAgent(cfg, Dependencies{Runner: runner}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err == nil { + t.Fatal("failed installation was accepted") + } + if !restarted.needsActivation { + t.Fatal("failed installation must force daemon activation on retry") + } + runner.failAt = 0 + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if restarted.needsActivation || !installedFilesMatch(restarted.state.Active.Files) { + t.Fatal("successful retry did not restore approved configuration") + } +} From 0f08c61df1543d9dcc923251fdda6d6c21122182 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 21:14:05 -0500 Subject: [PATCH 10/12] fix: persist DNS readiness gate across agent restarts --- internal/internaldns/serving/agent.go | 2 +- .../internaldns/serving/readiness_test.go | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index ad627a4f..163fe39f 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -298,7 +298,7 @@ func (a *Agent) failClosed(ctx context.Context) error { a.needsActivation = true gateCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() - return errors.Join(a.tx.FailClosed(gateCtx), a.tx.abandonActivation()) + return errors.Join(a.tx.beginActivation(), a.tx.FailClosed(gateCtx), a.tx.abandonActivation()) } func (a *Agent) bootstrapAndStep(ctx context.Context) error { diff --git a/internal/internaldns/serving/readiness_test.go b/internal/internaldns/serving/readiness_test.go index d42d912a..060bbedb 100644 --- a/internal/internaldns/serving/readiness_test.go +++ b/internal/internaldns/serving/readiness_test.go @@ -112,3 +112,31 @@ func TestAgentRestartAndStopPreserveApprovedConfiguration(t *testing.T) { t.Fatal("agent stop shut down healthy DNS") } } + +func TestFailClosedPersistsIncompleteActivationAcrossRestart(t *testing.T) { + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, time.Now().UTC()) + if err := agent.failClosed(context.Background()); err != nil { + t.Fatal(err) + } + if !activationPending(cfg.StateDir) { + t.Fatal("gated DNS must persist its incomplete activation") + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}}) + if err != nil { + t.Fatal(err) + } + if err := Readiness(context.Background(), cfg); err == nil || err.Error() != "configuration installation is incomplete" { + t.Fatalf("gated DNS must not report readiness after restart: %v", err) + } + if err := restarted.restoreActive(context.Background()); err != nil { + t.Fatal(err) + } + if activationPending(cfg.StateDir) { + t.Fatal("successful activation must clear the gate marker") + } +} From 3f8fa1af88a00d96b61b08223b6e2a35aaabf57a Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 22 Oct 2026 14:46:27 -0500 Subject: [PATCH 11/12] fix: gate serving when a withdrawal cannot be installed `Agent.reconcile` only failed closed when `Apply` was rejected. If the desired state withdrew something the approved configuration serves (a revoked binding, a detached zone or a deleted record) and reconciliation failed earlier, for example in `renderPublications` or the checkpoint save after clearing `Active`, it returned before any gate. The daemons kept answering with the withdrawn data and every tick retried the same error. Reconcile now records whether the approved configuration can be retained before it starts, and fails closed on any reconcile error when it cannot. A shutdown is not treated as a failed install, since `restoreActive` gates again on restart. A render failure that withdraws nothing still keeps the approved configuration. A new test injects a render error on a binding withdrawal and asserts that serving is gated, and that the same error without a withdrawal does not gate. Co-Authored-By: Claude Opus 5.5 --- internal/internaldns/serving/agent.go | 28 ++++++- .../serving/binding_fences_test.go | 78 +++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index 163fe39f..24f8b7bc 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -180,6 +180,8 @@ type Agent struct { lastServingAck map[string]servingAckState lastPublicationAck map[string]time.Time installedPublications map[string]installedPublication + // render is renderPublications unless a test injects a failure. + render func(RenderConfig, model.ServingSnapshot, map[string]bool, map[string]publicationState) (Rendered, error) } // installedPublication identifies the exact files activated by a validated transaction. @@ -771,10 +773,32 @@ func (a *Agent) finishEvent(ctx context.Context) error { return a.verifyAppliedConfiguration(ctx) } +// reconcile installs the desired configuration. When the desired state +// withdraws anything the approved configuration serves (a revoked binding, a +// detached zone or a deleted record), any failure to install it gates serving: +// returning early would leave the daemons answering with withdrawn data while +// every retry hits the same error. func (a *Agent) reconcile(ctx context.Context) error { if a.state.Snapshot == nil { return nil } + withdrawing := !a.canRetainActive() + err := a.reconcileDesired(ctx) + // Shutdown is not a failed install: restart gates again via restoreActive. + if err != nil && withdrawing && ctx.Err() == nil { + return errors.Join(err, a.failClosed(ctx)) + } + return err +} + +func (a *Agent) renderDesired(c RenderConfig, snapshot model.ServingSnapshot, active map[string]bool) (Rendered, error) { + if a.render != nil { + return a.render(c, snapshot, active, a.state.Publications) + } + return renderPublications(c, snapshot, active, a.state.Publications) +} + +func (a *Agent) reconcileDesired(ctx context.Context) error { active := map[string]bool{} filtered := *a.state.Snapshot @@ -802,7 +826,7 @@ func (a *Agent) reconcile(ctx context.Context) error { renderConfig.ReadyZoneMembers[binding.BindingUID][zone.ZoneUID] = a.eligibleRegionalMembers(private) } } - rendered, err := renderPublications(renderConfig, filtered, active, a.state.Publications) + rendered, err := a.renderDesired(renderConfig, filtered, active) if err != nil { return err } @@ -850,7 +874,7 @@ func (a *Agent) reconcile(ctx context.Context) error { // Any binding that fails either UDP or TCP is immediately removed again. probeFailed := a.verifyResolverBindings(ctx, filtered.Bindings, active) if probeFailed { - gated, err := renderPublications(renderConfig, filtered, active, a.state.Publications) + gated, err := a.renderDesired(renderConfig, filtered, active) if err != nil { return err } diff --git a/internal/internaldns/serving/binding_fences_test.go b/internal/internaldns/serving/binding_fences_test.go index d0f2ee28..5f46cd47 100644 --- a/internal/internaldns/serving/binding_fences_test.go +++ b/internal/internaldns/serving/binding_fences_test.go @@ -3,6 +3,7 @@ package serving import ( "context" "encoding/json" + "errors" "os" "path/filepath" "strings" @@ -244,3 +245,80 @@ func TestPendingConfigurationGatesQueriesAndRecoversWithoutAuthorizationRenewal( t.Fatal("stale pending replay gated the recovered context") } } + +func TestWithdrawalRenderFailureStopsServing(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + clock := now + cfg := regionalTestConfig(t) + runner := &recordingRunner{} + a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }}) + if err != nil { + t.Fatal(err) + } + active := testSnapshot(now) + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + chunk, manifest := testPublication(t, now) + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } + if a.state.Active == nil || a.needsActivation { + t.Fatalf("fixture did not approve serving: active=%t needsActivation=%t", a.state.Active != nil, a.needsActivation) + } + gatesBefore := gateCalls(runner) + + a.render = func(RenderConfig, model.ServingSnapshot, map[string]bool, map[string]publicationState) (Rendered, error) { + return Rendered{}, errors.New("injected render failure") + } + clock = now.Add(11 * time.Second) + withdrawn := active + withdrawn.Bindings = append([]model.Binding(nil), active.Bindings...) + withdrawn.ConfigurationRevision = 2 + withdrawn.Bindings[0].ConfigurationRevision = 2 + withdrawn.Bindings[0].Tombstone = true + if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, withdrawn)); err == nil { + t.Fatal("withdrawal with a failed render reported success") + } + if gateCalls(runner) == gatesBefore || !a.needsActivation { + t.Fatalf("withdrawal with a failed render kept serving withdrawn data: gates=%d needsActivation=%t", gateCalls(runner)-gatesBefore, a.needsActivation) + } + + // A render failure that withdraws nothing keeps the approved configuration. + retained := &recordingRunner{} + b, err := NewAgent(regionalTestConfig(t), Dependencies{Runner: retained, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if err := b.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + if err := b.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := b.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } + retainedBefore := gateCalls(retained) + b.render = a.render + if err := b.reconcile(context.Background()); err == nil { + t.Fatal("injected render failure was not returned") + } + if gateCalls(retained) != retainedBefore || b.needsActivation { + t.Fatal("a render failure that withdraws nothing gated serving") + } +} + +func gateCalls(r *recordingRunner) int { + n := 0 + for _, c := range r.calls { + if c.Path == "gate" { + n++ + } + } + return n +} From 50d1585f4e3d7e7ace8c2950639dfad4a16848ba Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 22 Oct 2026 14:46:51 -0500 Subject: [PATCH 12/12] fix: probe local CoreDNS health over TCP only The local daemon readiness probe queried CoreDNS over UDP and TCP, but CoreDNS has listened only on TCP since the shared query serving change, with dnsdist in front of it for UDP. The UDP query was refused, so `internal-dns-node` never became Ready and the E2E environment timed out in `env:internal-dns:up`. The probe now checks CoreDNS over TCP only and still checks dnsdist over both UDP and TCP, matching what each daemon listens on. The publication probe timeout test now listens on TCP accordingly. Co-Authored-By: Claude Opus 5.5 --- internal/internaldns/serving/coredns_test.go | 8 ++++---- internal/internaldns/serving/readiness.go | 8 +++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/internaldns/serving/coredns_test.go b/internal/internaldns/serving/coredns_test.go index f2942faf..52649fff 100644 --- a/internal/internaldns/serving/coredns_test.go +++ b/internal/internaldns/serving/coredns_test.go @@ -388,15 +388,15 @@ func TestPublicationProbeTimeoutDoesNotApproveConfiguration(t *testing.T) { t.Parallel() now := time.Now().UTC() cfg := regionalTestConfig(t) - udp, err := net.ListenPacket("udp", "127.0.0.1:0") + listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } - defer func() { _ = udp.Close() }() - port := udp.LocalAddr().(*net.UDPAddr).Port + defer func() { _ = listener.Close() }() + port := listener.Addr().(*net.TCPAddr).Port cfg.Render.ClusterCoreDNS.ListenAddress = "127.0.0.1" cfg.Render.ClusterCoreDNS.Port = uint16(port) - cfg.PublicationDNSProbe = PublicationDNSVerifierConfig{Server: udp.LocalAddr().String(), Timeout: 50 * time.Millisecond} + cfg.PublicationDNSProbe = PublicationDNSVerifierConfig{Server: listener.Addr().String(), Timeout: 50 * time.Millisecond} cfg.UnsafeAllowNoDNSVerify = false a, err := NewAgent(cfg, Dependencies{Clock: func() time.Time { return now }, Runner: &recordingRunner{}}) if err != nil { diff --git a/internal/internaldns/serving/readiness.go b/internal/internaldns/serving/readiness.go index 46a591ff..c4d70558 100644 --- a/internal/internaldns/serving/readiness.go +++ b/internal/internaldns/serving/readiness.go @@ -83,11 +83,9 @@ func (a *Agent) probeLocalDaemons(ctx context.Context) error { for _, bind := range binds { server := net.JoinHostPort(bind.ListenAddress, strconv.Itoa(int(bind.Port))) view := PublicationView{DestinationAddress: bind.ListenAddress, DestinationPort: bind.Port} - for _, network := range []string{networkUDP, networkTCP} { - answer, err := exchangePublication(ctx, network, server, ".", view, 200*time.Millisecond) - if err != nil || answer == nil || !answer.Response { - return fmt.Errorf("local CoreDNS health %s: %w", server, errors.Join(err, errors.New("DNS response required"))) - } + answer, err := exchangePublication(ctx, networkTCP, server, ".", view, 200*time.Millisecond) + if err != nil || answer == nil || !answer.Response { + return fmt.Errorf("local CoreDNS health %s: %w", server, errors.Join(err, errors.New("DNS response required"))) } } for _, dist := range dists {