diff --git a/files/homelab/cluster/README.md b/files/homelab/cluster/README.md index d8767e36..90cb984f 100644 --- a/files/homelab/cluster/README.md +++ b/files/homelab/cluster/README.md @@ -111,7 +111,10 @@ and the `admin.conf` server (`.local:6443`). The node writes a `kubeadm join --config`; a failed join is reset and retried with a fresh token. k0s: `k0s token create --role=worker --expiry=15m0s` (embeds the CA and server); the node writes `/etc/k0s/token` (0600) and starts -`k0s-first-boot.service`, which starts `k0sworker`. +`k0s-first-boot.service`, which starts `k0sworker`. A restarted `join` +counts the node as joined only by the marker or a bootstrapped kubelet +kubeconfig (`/etc/kubernetes/kubelet.conf`, `/var/lib/k0s/kubelet.conf`), +never by the token alone. **Rate limits.** Server: every PAKE run is charged before `Exchange`, under one lock with the limit check, to the source (IPv4 address, or IPv6 /64) @@ -136,7 +139,9 @@ runs a rogue client or a rogue control plane; no access to either host. 2^17.4 per year against 2^51.7 for a generated passphrase). It never receives sealed data or causes a mint without tagC. - Rogue control plane: one guess per node attempt, bounded by the node's - budget however many instances it advertises; the node never sends tagC + budget however many instances it advertises (the node skips a candidate + that cannot prove the passphrase and tries the next, so a rogue instance + listed first cannot hide the real one); the node never sends tagC unless tagS verified, so it cannot be steered to a cluster that does not know the passphrase. A spoofed `.local` cannot impersonate the API server: kubeadm pins the CA by hash; k0s's token carries the CA. diff --git a/files/homelab/cluster/cmd/bluefin-cluster/main.go b/files/homelab/cluster/cmd/bluefin-cluster/main.go index 203198a3..50dbc8ea 100644 --- a/files/homelab/cluster/cmd/bluefin-cluster/main.go +++ b/files/homelab/cluster/cmd/bluefin-cluster/main.go @@ -193,7 +193,7 @@ func controlPlanePassphrase(c conf.Config) (string, error) { log.Warn("using the operator-chosen join passphrase from " + passKey + "; a generated one is stronger") return p, ops.WriteFile(passFile, []byte(p+"\n"), 0o600) } - if data, err := os.ReadFile(passFile); err == nil { + if data, err := os.ReadFile(filepath.Join(ops.Root, passFile)); err == nil { return passphrase.Normalize(string(data)) } p, bits, err := passphrase.Generate(passphrase.DefaultWords) @@ -274,10 +274,16 @@ func serve(ctx context.Context) error { return err } -// publish advertises the join service; the TXT record carries no secrets. -func publish(cluster, runtime, port string) error { - body := fmt.Sprintf("# bluefin-cluster serve: the homelab join service (no secrets here).\n[Service]\nName=%%H\nType=%s\nPort=%s\nTxtText=v=1 cluster=%s runtime=%s\n", +// dnssdBody is the join service's DNS-SD file; the TXT record carries no +// secrets. +func dnssdBody(cluster, runtime, port string) string { + return fmt.Sprintf("# bluefin-cluster serve: the homelab join service (no secrets here).\n[Service]\nName=%%H\nType=%s\nPort=%s\nTxtText=v=1 cluster=%s runtime=%s\n", discover.ServiceType, port, cluster, runtime) +} + +// publish advertises the join service. +func publish(cluster, runtime, port string) error { + body := dnssdBody(cluster, runtime, port) // Explicit modes: the unit's UMask=0077 would hide both from // systemd-resolved, which runs as its own user. dir := filepath.Dir(dnssdFile) @@ -329,9 +335,14 @@ func nodePassphrase(c conf.Config) (string, error) { return passphrase.Normalize(raw) } +// joined is true once this node is in a cluster: the marker, or the kubelet +// kubeconfig kubeadm join or the k0s worker writes after bootstrapping. A +// k0s join token alone is no evidence: Join writes it before it starts +// k0s-first-boot.service, so a failed start, or a token that expired before +// k0sworker used it, would otherwise pass for a join and stop retrying. func joined() bool { - for _, p := range []string{joinedMarker, ops.KubeadmKubelet, ops.K0sTokenFile} { - if _, err := os.Stat(p); err == nil { + for _, p := range []string{joinedMarker, ops.KubeadmKubelet, ops.K0sKubelet} { + if _, err := os.Stat(filepath.Join(ops.Root, p)); err == nil { return true } } @@ -438,18 +449,31 @@ func attempt(ctx context.Context, cl *join.Client, manual string) (join.Payload, return join.Payload{}, fmt.Errorf("no %s control plane advertised %s on the local network yet", cl.Runtime, discover.ServiceType) } } + return tryCandidates(ctx, cl.Join, cands) +} + +// tryCandidates asks each candidate in turn. One that cannot prove the +// passphrase (an impostor, or another cluster's control plane) is skipped +// rather than ending the round: its other addresses would only spend guesses +// on the same answer, and stopping would let whichever instance is listed +// first hide the real control plane for good. Only the guess budget, shared +// by every candidate, ends the round early. +func tryCandidates(ctx context.Context, joinAt func(context.Context, netip.AddrPort) (join.Payload, error), cands []discover.Candidate) (join.Payload, error) { var errs []error for _, cand := range cands { for _, addr := range cand.Addrs { - p, err := cl.Join(ctx, addr) + p, err := joinAt(ctx, addr) if err == nil { return p, nil } errs = append(errs, fmt.Errorf("%s (%s): %w", cand.Name, addr, err)) var be *join.ErrBudget - if errors.As(err, &be) || errors.Is(err, join.ErrNotProven) { + if errors.As(err, &be) { return join.Payload{}, errors.Join(errs...) } + if errors.Is(err, join.ErrNotProven) { + break + } } } return join.Payload{}, errors.Join(errs...) diff --git a/files/homelab/cluster/cmd/bluefin-cluster/main_test.go b/files/homelab/cluster/cmd/bluefin-cluster/main_test.go new file mode 100644 index 00000000..11c8d64e --- /dev/null +++ b/files/homelab/cluster/cmd/bluefin-cluster/main_test.go @@ -0,0 +1,228 @@ +package main + +import ( + "context" + "errors" + "net/netip" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/projectbluefin/server/files/homelab/cluster/internal/conf" + "github.com/projectbluefin/server/files/homelab/cluster/internal/discover" + "github.com/projectbluefin/server/files/homelab/cluster/internal/join" + "github.com/projectbluefin/server/files/homelab/cluster/internal/ops" + "github.com/projectbluefin/server/files/homelab/cluster/internal/passphrase" +) + +// withRoot points ops.Root (every host path) at a fresh directory and +// creates files under it. +func withRoot(t *testing.T, files ...string) string { + t.Helper() + root := t.TempDir() + orig := ops.Root + ops.Root = root + t.Cleanup(func() { ops.Root = orig }) + for _, f := range files { + p := filepath.Join(root, f) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, nil, 0o600); err != nil { + t.Fatal(err) + } + } + return root +} + +func TestJoined(t *testing.T) { + for _, tc := range []struct { + name string + files []string + want bool + }{ + {"fresh node", nil, false}, + {"marker", []string{joinedMarker}, true}, + {"kubeadm join bootstrapped kubelet", []string{ops.KubeadmKubelet}, true}, + {"k0s worker bootstrapped kubelet", []string{ops.K0sTokenFile, ops.K0sKubelet}, true}, + // Written before k0s-first-boot.service starts: if that failed, or + // the token expired unused, the node must ask for a fresh one. + {"k0s token alone", []string{ops.K0sTokenFile}, false}, + {"kubeadm bootstrap kubeconfig alone", []string{"/etc/kubernetes/bootstrap-kubelet.conf"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + withRoot(t, tc.files...) + if got := joined(); got != tc.want { + t.Errorf("joined() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestTryCandidates(t *testing.T) { + a1 := netip.MustParseAddrPort("192.0.2.1:6447") + a2 := netip.MustParseAddrPort("[2001:db8::1]:6447") + b1 := netip.MustParseAddrPort("192.0.2.2:6447") + c1 := netip.MustParseAddrPort("192.0.2.3:6447") + cands := []discover.Candidate{ + {Name: "a", Addrs: []netip.AddrPort{a1, a2}}, + {Name: "b", Addrs: []netip.AddrPort{b1}}, + {Name: "c", Addrs: []netip.AddrPort{c1}}, + } + refused := errors.New("connection refused") + budget := &join.ErrBudget{RetryAfter: time.Minute} + for _, tc := range []struct { + name string + results map[netip.AddrPort]error // nil: joins + tried []netip.AddrPort + joined bool + budget bool + }{ + {"first address joins", map[netip.AddrPort]error{a1: nil}, []netip.AddrPort{a1}, true, false}, + {"unreachable address falls through to the next one", map[netip.AddrPort]error{a1: refused, a2: nil}, []netip.AddrPort{a1, a2}, true, false}, + // An impostor (or another cluster) listed first must not hide the + // control plane: skip the rest of it, go on with the next. + {"not proven skips that candidate", map[netip.AddrPort]error{a1: join.ErrNotProven, b1: nil}, []netip.AddrPort{a1, b1}, true, false}, + {"every candidate not proven", map[netip.AddrPort]error{a1: join.ErrNotProven, b1: join.ErrNotProven, c1: join.ErrNotProven}, []netip.AddrPort{a1, b1, c1}, false, false}, + {"exhausted budget ends the round", map[netip.AddrPort]error{a1: join.ErrNotProven, b1: budget}, []netip.AddrPort{a1, b1}, false, true}, + {"nothing answers", map[netip.AddrPort]error{a1: refused, a2: refused, b1: refused, c1: refused}, []netip.AddrPort{a1, a2, b1, c1}, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var tried []netip.AddrPort + p, err := tryCandidates(context.Background(), func(_ context.Context, addr netip.AddrPort) (join.Payload, error) { + tried = append(tried, addr) + err, ok := tc.results[addr] + if !ok { + t.Errorf("unexpected address %s", addr) + return join.Payload{}, errors.New("unexpected address") + } + if err != nil { + return join.Payload{}, err + } + return join.Payload{Cluster: addr.String()}, nil + }, cands) + if (err == nil) != tc.joined { + t.Fatalf("joined %v, err %v", err == nil, err) + } + if tc.joined && p.Cluster != tried[len(tried)-1].String() { + t.Errorf("payload from %s, last tried %s", p.Cluster, tried[len(tried)-1]) + } + var be *join.ErrBudget + if errors.As(err, &be) != tc.budget { + t.Errorf("budget error %v: %v", tc.budget, err) + } + if len(tried) != len(tc.tried) { + t.Fatalf("tried %v, want %v", tried, tc.tried) + } + for i := range tried { + if tried[i] != tc.tried[i] { + t.Fatalf("tried %v, want %v", tried, tc.tried) + } + } + }) + } +} + +func TestControlPlanePassphrase(t *testing.T) { + const strong = "Correct Horse battery_staple-orbit" + for _, tc := range []struct { + name string + conf conf.Config + stored string // existing passFile content, "" for none + want string // "" without err: a generated one + err bool + }{ + {"operator-chosen, normalised and stored", conf.Config{passKey: strong}, "", "correct-horse-battery-staple-orbit", false}, + {"operator-chosen beats the stored one", conf.Config{passKey: strong}, "old-words-here-now\n", "correct-horse-battery-staple-orbit", false}, + {"operator-chosen too weak", conf.Config{passKey: "short pass"}, "", "", true}, + {"operator-chosen not ASCII", conf.Config{passKey: "café crème brûlée soufflé"}, "", "", true}, + {"stored one kept", conf.Config{}, "Alpha Bravo charlie delta\n", "alpha-bravo-charlie-delta", false}, + {"generated on first start", conf.Config{}, "", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + root := withRoot(t) + file := filepath.Join(root, passFile) + if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil { + t.Fatal(err) + } + if tc.stored != "" { + if err := os.WriteFile(file, []byte(tc.stored), 0o600); err != nil { + t.Fatal(err) + } + } + got, err := controlPlanePassphrase(tc.conf) + if (err != nil) != tc.err { + t.Fatalf("err %v, want error %v", err, tc.err) + } + if tc.err { + if err != nil && strings.Contains(err.Error(), tc.conf[passKey]) { + t.Errorf("error leaks the passphrase: %v", err) + } + return + } + if tc.want != "" && got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + if tc.want == "" && len(strings.Split(got, "-")) != passphrase.DefaultWords { + t.Errorf("generated %q: want %d words", got, passphrase.DefaultWords) + } + // What serve hands out is what a restart reads back; a stored + // one is left as it is. + want := got + "\n" + if tc.stored != "" && tc.conf[passKey] == "" { + want = tc.stored + } + if data, err := os.ReadFile(file); err != nil || string(data) != want { + t.Errorf("stored %q (%v), want %q", data, err, want) + } + fi, err := os.Stat(file) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Errorf("passphrase file mode %v, want 0600", fi.Mode().Perm()) + } + }) + } +} + +func TestNodePassphrase(t *testing.T) { + for _, tc := range []struct { + name string + conf conf.Config + credential string // "" for none + want string + err bool + }{ + {"from homelab.conf", conf.Config{passKey: "Four Words From_Console"}, "", "four-words-from-console", false}, + {"homelab.conf beats the credential", conf.Config{passKey: "four words from console"}, "other words entirely here", "four-words-from-console", false}, + {"from the credential", conf.Config{}, "four words from console\n", "four-words-from-console", false}, + {"none", conf.Config{}, "", "", true}, + {"not ASCII", conf.Config{passKey: "naïve words"}, "", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + t.Setenv("CREDENTIALS_DIRECTORY", dir) + if tc.credential != "" { + if err := os.WriteFile(filepath.Join(dir, credentialKey), []byte(tc.credential), 0o600); err != nil { + t.Fatal(err) + } + } + got, err := nodePassphrase(tc.conf) + if (err != nil) != tc.err || got != tc.want { + t.Errorf("got %q, %v; want %q, error %v", got, err, tc.want, tc.err) + } + }) + } +} + +func TestDNSSDBodyCarriesNoSecrets(t *testing.T) { + got := dnssdBody("lab", "kubeadm", "6447") + want := "# bluefin-cluster serve: the homelab join service (no secrets here).\n" + + "[Service]\nName=%H\nType=" + discover.ServiceType + "\nPort=6447\nTxtText=v=1 cluster=lab runtime=kubeadm\n" + if got != want { + t.Errorf("dnssd body:\n%s\nwant:\n%s", got, want) + } +} diff --git a/files/homelab/cluster/internal/ops/ops.go b/files/homelab/cluster/internal/ops/ops.go index 488e2910..228450ee 100644 --- a/files/homelab/cluster/internal/ops/ops.go +++ b/files/homelab/cluster/internal/ops/ops.go @@ -29,6 +29,7 @@ const ( KubeadmKubelet = "/etc/kubernetes/kubelet.conf" K0sAdminConf = "/var/lib/k0s/pki/admin.conf" K0sTokenFile = "/etc/k0s/token" + K0sKubelet = "/var/lib/k0s/kubelet.conf" // k0s worker, once bootstrapped CRISocket = "unix:///run/containerd/containerd.sock" RuntimeKubeadm = "kubeadm" diff --git a/tests/unit/test_homelab_cluster.py b/tests/unit/test_homelab_cluster.py index 4807174c..e36271b5 100644 --- a/tests/unit/test_homelab_cluster.py +++ b/tests/unit/test_homelab_cluster.py @@ -107,9 +107,12 @@ def test_applier_does_nothing_on_a_node() -> None: def test_dnssd_advertisement_carries_no_secrets() -> None: main = MAIN.read_text() publish = main.split("func publish(", 1)[1].split("\n}\n", 1)[0] - txt = re.search(r"TxtText=([^\\]*)\\n", publish).group(1) + body = main.split("func dnssdBody(", 1)[1].split("\n}\n", 1)[0] + assert "dnssdBody(cluster, runtime, port)" in publish + txt = re.search(r"TxtText=([^\\]*)\\n", body).group(1) assert txt == "v=1 cluster=%s runtime=%s" - assert "pass" not in publish.lower() and "token" not in publish.lower() + for fn in (publish, body): + assert "pass" not in fn.lower() and "token" not in fn.lower() assert 'dnssdFile = "/run/systemd/dnssd/bluefin-cluster.dnssd"' in main assert "*.dnssd" in install_script(), "the build refuses a static advertisement in the sysext" # serve runs with UMask=0077; systemd-resolved (its own user) must read it.