feat: Preserve private DNS isolation through serving failures

internal/internaldns/serving/activation.go

@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package serving
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	activationInstalling = "installing"
+	activationVerifying  = "verifying"
+)
+
+type activationState struct {
+	Phase string `json:"phase"`
+}
+
+func (t *FileTransaction) beginActivation() error {
+	if t.MarkerPath == "" {
+		return nil
+	}
+	if err := os.MkdirAll(filepath.Dir(t.MarkerPath), 0o750); err != nil {
+		return err
+	}
+	if t.activationLock == nil {
+		lock, err := os.OpenFile(t.MarkerPath+".lock", os.O_CREATE|os.O_RDWR, 0o600)
+		if err != nil {
+			return err
+		}
+		if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil {
+			_ = lock.Close()
+			return err
+		}
+		t.activationLock = lock
+	}
+	if err := unix.Flock(int(t.activationLock.Fd()), unix.LOCK_EX); err != nil {
+		return err
+	}
+	return t.writeActivation(activationState{Phase: activationInstalling})
+}
+
+func (t *FileTransaction) writeActivation(state activationState) error {
+	if t.MarkerPath == "" {
+		return nil
+	}
+	data, err := json.Marshal(state)
+	if err != nil {
+		return err
+	}
+	temporary := t.MarkerPath + ".tmp"
+	if err := writeDurable(temporary, data, 0o644); err != nil {
+		return err
+	}
+	if err := os.Rename(temporary, t.MarkerPath); err != nil {
+		return err
+	}
+	return syncDirs([]string{t.MarkerPath})
+}
+
+func (t *FileTransaction) filesInstalled() error {
+	if err := t.writeActivation(activationState{Phase: activationVerifying}); err != nil {
+		return err
+	}
+	if t.activationLock != nil {
+		return unix.Flock(int(t.activationLock.Fd()), unix.LOCK_SH)
+	}
+	return nil
+}
+
+func (t *FileTransaction) commitActivation() error {
+	if t.MarkerPath != "" {
+		if err := os.Remove(t.MarkerPath); err != nil && !errors.Is(err, os.ErrNotExist) {
+			return err
+		}
+		if err := syncDirs([]string{t.MarkerPath}); err != nil {
+			return err
+		}
+	}
+	return t.abandonActivation()
+}
+
+// The kernel releases activation ownership when the agent process exits.
+func (t *FileTransaction) abandonActivation() error {
+	if t.activationLock == nil {
+		return nil
+	}
+	err := t.activationLock.Close()
+	t.activationLock = nil
+	return err
+}
+
+// daemonPermission holds the shared activation lock through daemon startup so
+// installation cannot race with a process loading its configuration files.
+func daemonPermission(stateDir string) (start, retain bool, release func()) {
+	marker := activationMarkerPath(stateDir)
+	lock, err := os.OpenFile(marker+".lock", os.O_CREATE|os.O_RDWR, 0o600)
+	if err != nil {
+		return false, false, func() {}
+	}
+	release = func() { _ = lock.Close() }
+	if err := unix.Flock(int(lock.Fd()), unix.LOCK_SH|unix.LOCK_NB); err != nil {
+		if errors.Is(err, unix.EWOULDBLOCK) {
+			// Running processes retain their in-memory configuration while the
+			// installer writes files. New processes must wait.
+			return false, true, release
+		}
+		return false, false, release
+	}
+	upgradeErr := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB)
+	_, readErr := os.ReadFile(marker)
+	if errors.Is(readErr, os.ErrNotExist) {
+		return true, true, release
+	}
+	if upgradeErr == nil {
+		// A pending marker without an installer is an abandoned activation.
+		return false, false, release
+	}
+	if !errors.Is(upgradeErr, unix.EWOULDBLOCK) {
+		return false, false, release
+	}
+	// A failed upgrade may release the shared lock. Reacquire it before
+	// reading the phase or allowing a daemon to load its configuration.
+	if err := unix.Flock(int(lock.Fd()), unix.LOCK_SH|unix.LOCK_NB); err != nil {
+		return false, true, release
+	}
+	data, readErr := os.ReadFile(marker)
+	if errors.Is(readErr, os.ErrNotExist) {
+		return true, true, release
+	}
+	var state activationState
+	if readErr != nil || json.Unmarshal(data, &state) != nil || state.Phase != activationVerifying {
+		return false, false, release
+	}
+	return true, true, release
+}

internal/internaldns/serving/activation_test.go

@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package serving
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+	"time"
+
+	"go.miloapis.com/dns-operator/internal/internaldns/model"
+)
+
+func TestActivationOwnershipGatesAbandonedUpdatesWithoutHeartbeat(t *testing.T) {
+	dir := t.TempDir()
+	tx := FileTransaction{MarkerPath: activationMarkerPath(dir)}
+	if err := tx.beginActivation(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = tx.abandonActivation() })
+	start, retain, release := daemonPermission(dir)
+	release()
+	if start || !retain {
+		t.Fatal("installation allowed a new process or stopped approved in-memory DNS")
+	}
+	if err := tx.filesInstalled(); err != nil {
+		t.Fatal(err)
+	}
+	start, retain, release = daemonPermission(dir)
+	release()
+	if !start || !retain {
+		t.Fatal("live installer could not verify its complete configuration")
+	}
+	if err := tx.abandonActivation(); err != nil {
+		t.Fatal(err)
+	}
+	start, retain, release = daemonPermission(dir)
+	release()
+	if start || retain {
+		t.Fatal("abandoned activation retained query processes")
+	}
+	if err := os.Remove(tx.MarkerPath); err != nil {
+		t.Fatal(err)
+	}
+	start, retain, release = daemonPermission(dir)
+	release()
+	if !start || !retain {
+		t.Fatal("completed configuration required agent ownership")
+	}
+}
+
+func TestPublicationActivationRemainsOwnedUntilDNSProof(t *testing.T) {
+	now := time.Now().UTC()
+	cfg := regionalTestConfig(t)
+	agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }})
+	if err != nil {
+		t.Fatal(err)
+	}
+	snapshot := testSnapshot(now)
+	if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, snapshot)); err != nil {
+		t.Fatal(err)
+	}
+	approved := agent.state.Active
+	verified := false
+	agent.publicationVerifier = publicationVerifierFunc(func(context.Context, model.PublicationPlan, []PublicationView) error {
+		verified = true
+		if !activationPending(cfg.StateDir) {
+			t.Fatal("activation committed before DNS proof")
+		}
+		if agent.state.Active != approved {
+			t.Fatal("unverified publication replaced approved checkpoint")
+		}
+		start, retain, release := daemonPermission(cfg.StateDir)
+		release()
+		if !start || !retain {
+			t.Fatal("live verification cannot query installed configuration")
+		}
+		if err := agent.tx.abandonActivation(); err != nil {
+			t.Fatal(err)
+		}
+		start, retain, release = daemonPermission(cfg.StateDir)
+		release()
+		if start || retain {
+			t.Fatal("abandoned unverified publication remains queryable")
+		}
+		return errors.New("publication proof interrupted")
+	})
+	chunk, manifest := testPublication(t, now)
+	if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil {
+		t.Fatal(err)
+	}
+	if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err == nil {
+		t.Fatal("unverified publication accepted")
+	}
+	if !verified {
+		t.Fatal("publication proof was not attempted")
+	}
+	if !activationPending(cfg.StateDir) {
+		t.Fatal("interrupted proof lost recovery marker")
+	}
+}
+
+func TestRestoreActiveUsesInstalledConfigurationHash(t *testing.T) {
+	now := time.Now().UTC()
+	cfg := regionalTestConfig(t)
+	agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }})
+	if err != nil {
+		t.Fatal(err)
+	}
+	installTestPublication(t, agent, now)
+	actual := hashFiles(agent.state.Active.Files)
+	agent.state.RenderedHash = "unapproved-candidate"
+	if err := agent.store.Save(agent.state); err != nil {
+		t.Fatal(err)
+	}
+	restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := restarted.restoreActive(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if restarted.state.RenderedHash != actual {
+		t.Fatal("restart retained candidate hash for approved files")
+	}
+	saved, err := restarted.store.Load()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if saved.RenderedHash != actual {
+		t.Fatal("approved installed hash was not checkpointed")
+	}
+}