From bfc69ddfb6ef84d181d5017c500cfbb972dfca35 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Fri, 30 Oct 2026 22:49:09 -0500 Subject: [PATCH 1/3] fix(homelab): skip a join candidate that cannot prove the passphrase attempt() ended the whole round on the first ErrNotProven. mDNS answers are unauthenticated, so a rogue advertiser, or a second cluster's control plane on the same LAN, listed before the real control plane made every round stop there: the node backed off and asked the same peer first again, never reaching the control plane that knows its passphrase. The loop (now tryCandidates, with the join call injected for tests) skips the rest of a candidate that failed to prove the passphrase and moves on to the next one. Its other addresses are not tried: they would only spend guesses on the same answer. The guess budget is unchanged and still shared by every candidate (charged before each PAKE, refunded only after the peer proved the passphrase), so ErrBudget still ends the round and a rogue advertiser gets no more guesses than before. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- files/homelab/cluster/README.md | 4 +- .../cluster/cmd/bluefin-cluster/main.go | 17 ++++- .../cluster/cmd/bluefin-cluster/main_test.go | 76 +++++++++++++++++++ 3 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 files/homelab/cluster/cmd/bluefin-cluster/main_test.go diff --git a/files/homelab/cluster/README.md b/files/homelab/cluster/README.md index d8767e36..f38a18f9 100644 --- a/files/homelab/cluster/README.md +++ b/files/homelab/cluster/README.md @@ -136,7 +136,9 @@ runs a rogue client or a rogue control plane; no access to either host. 2^17.4 per year against 2^51.7 for a generated passphrase). It never receives sealed data or causes a mint without tagC. - Rogue control plane: one guess per node attempt, bounded by the node's - budget however many instances it advertises; the node never sends tagC + budget however many instances it advertises (the node skips a candidate + that cannot prove the passphrase and tries the next, so a rogue instance + listed first cannot hide the real one); the node never sends tagC unless tagS verified, so it cannot be steered to a cluster that does not know the passphrase. A spoofed `.local` cannot impersonate the API server: kubeadm pins the CA by hash; k0s's token carries the CA. diff --git a/files/homelab/cluster/cmd/bluefin-cluster/main.go b/files/homelab/cluster/cmd/bluefin-cluster/main.go index 203198a3..1840d3a8 100644 --- a/files/homelab/cluster/cmd/bluefin-cluster/main.go +++ b/files/homelab/cluster/cmd/bluefin-cluster/main.go @@ -438,18 +438,31 @@ func attempt(ctx context.Context, cl *join.Client, manual string) (join.Payload, return join.Payload{}, fmt.Errorf("no %s control plane advertised %s on the local network yet", cl.Runtime, discover.ServiceType) } } + return tryCandidates(ctx, cl.Join, cands) +} + +// tryCandidates asks each candidate in turn. One that cannot prove the +// passphrase (an impostor, or another cluster's control plane) is skipped +// rather than ending the round: its other addresses would only spend guesses +// on the same answer, and stopping would let whichever instance is listed +// first hide the real control plane for good. Only the guess budget, shared +// by every candidate, ends the round early. +func tryCandidates(ctx context.Context, joinAt func(context.Context, netip.AddrPort) (join.Payload, error), cands []discover.Candidate) (join.Payload, error) { var errs []error for _, cand := range cands { for _, addr := range cand.Addrs { - p, err := cl.Join(ctx, addr) + p, err := joinAt(ctx, addr) if err == nil { return p, nil } errs = append(errs, fmt.Errorf("%s (%s): %w", cand.Name, addr, err)) var be *join.ErrBudget - if errors.As(err, &be) || errors.Is(err, join.ErrNotProven) { + if errors.As(err, &be) { return join.Payload{}, errors.Join(errs...) } + if errors.Is(err, join.ErrNotProven) { + break + } } } return join.Payload{}, errors.Join(errs...) diff --git a/files/homelab/cluster/cmd/bluefin-cluster/main_test.go b/files/homelab/cluster/cmd/bluefin-cluster/main_test.go new file mode 100644 index 00000000..f2c3121c --- /dev/null +++ b/files/homelab/cluster/cmd/bluefin-cluster/main_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "errors" + "net/netip" + "testing" + "time" + + "github.com/projectbluefin/server/files/homelab/cluster/internal/discover" + "github.com/projectbluefin/server/files/homelab/cluster/internal/join" +) + +func TestTryCandidates(t *testing.T) { + a1 := netip.MustParseAddrPort("192.0.2.1:6447") + a2 := netip.MustParseAddrPort("[2001:db8::1]:6447") + b1 := netip.MustParseAddrPort("192.0.2.2:6447") + c1 := netip.MustParseAddrPort("192.0.2.3:6447") + cands := []discover.Candidate{ + {Name: "a", Addrs: []netip.AddrPort{a1, a2}}, + {Name: "b", Addrs: []netip.AddrPort{b1}}, + {Name: "c", Addrs: []netip.AddrPort{c1}}, + } + refused := errors.New("connection refused") + budget := &join.ErrBudget{RetryAfter: time.Minute} + for _, tc := range []struct { + name string + results map[netip.AddrPort]error // nil: joins + tried []netip.AddrPort + joined bool + budget bool + }{ + {"first address joins", map[netip.AddrPort]error{a1: nil}, []netip.AddrPort{a1}, true, false}, + {"unreachable address falls through to the next one", map[netip.AddrPort]error{a1: refused, a2: nil}, []netip.AddrPort{a1, a2}, true, false}, + // An impostor (or another cluster) listed first must not hide the + // control plane: skip the rest of it, go on with the next. + {"not proven skips that candidate", map[netip.AddrPort]error{a1: join.ErrNotProven, b1: nil}, []netip.AddrPort{a1, b1}, true, false}, + {"every candidate not proven", map[netip.AddrPort]error{a1: join.ErrNotProven, b1: join.ErrNotProven, c1: join.ErrNotProven}, []netip.AddrPort{a1, b1, c1}, false, false}, + {"exhausted budget ends the round", map[netip.AddrPort]error{a1: join.ErrNotProven, b1: budget}, []netip.AddrPort{a1, b1}, false, true}, + {"nothing answers", map[netip.AddrPort]error{a1: refused, a2: refused, b1: refused, c1: refused}, []netip.AddrPort{a1, a2, b1, c1}, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var tried []netip.AddrPort + p, err := tryCandidates(context.Background(), func(_ context.Context, addr netip.AddrPort) (join.Payload, error) { + tried = append(tried, addr) + err, ok := tc.results[addr] + if !ok { + t.Errorf("unexpected address %s", addr) + return join.Payload{}, errors.New("unexpected address") + } + if err != nil { + return join.Payload{}, err + } + return join.Payload{Cluster: addr.String()}, nil + }, cands) + if (err == nil) != tc.joined { + t.Fatalf("joined %v, err %v", err == nil, err) + } + if tc.joined && p.Cluster != tried[len(tried)-1].String() { + t.Errorf("payload from %s, last tried %s", p.Cluster, tried[len(tried)-1]) + } + var be *join.ErrBudget + if errors.As(err, &be) != tc.budget { + t.Errorf("budget error %v: %v", tc.budget, err) + } + if len(tried) != len(tc.tried) { + t.Fatalf("tried %v, want %v", tried, tc.tried) + } + for i := range tried { + if tried[i] != tc.tried[i] { + t.Fatalf("tried %v, want %v", tried, tc.tried) + } + } + }) + } +} From f6460aa145289e073471633517012d6a0cab2c04 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Fri, 30 Oct 2026 22:53:57 -0500 Subject: [PATCH 2/3] fix(homelab): do not take a k0s join token as proof of a join joined() returned true as soon as /etc/k0s/token existed. ops.Join writes that token before it starts k0s-first-boot.service, so when the start failed, or k0sworker could not use the token before its 15-minute expiry, a restarted bluefin-cluster join logged "already joined", wrote the joined marker and never asked for a fresh token: the node stayed out of the cluster for good. A node now counts as joined only on the marker or on the kubelet kubeconfig written after bootstrapping: /etc/kubernetes/kubelet.conf for kubeadm (unchanged) and /var/lib/k0s/kubelet.conf for a k0s worker (k0s's KubeletAuthConfigPath, which k0s itself checks first). joined() honours ops.Root so it can be tested. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- files/homelab/cluster/README.md | 5 +- .../cluster/cmd/bluefin-cluster/main.go | 9 +++- .../cluster/cmd/bluefin-cluster/main_test.go | 47 +++++++++++++++++++ files/homelab/cluster/internal/ops/ops.go | 1 + 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/files/homelab/cluster/README.md b/files/homelab/cluster/README.md index f38a18f9..90cb984f 100644 --- a/files/homelab/cluster/README.md +++ b/files/homelab/cluster/README.md @@ -111,7 +111,10 @@ and the `admin.conf` server (`.local:6443`). The node writes a `kubeadm join --config`; a failed join is reset and retried with a fresh token. k0s: `k0s token create --role=worker --expiry=15m0s` (embeds the CA and server); the node writes `/etc/k0s/token` (0600) and starts -`k0s-first-boot.service`, which starts `k0sworker`. +`k0s-first-boot.service`, which starts `k0sworker`. A restarted `join` +counts the node as joined only by the marker or a bootstrapped kubelet +kubeconfig (`/etc/kubernetes/kubelet.conf`, `/var/lib/k0s/kubelet.conf`), +never by the token alone. **Rate limits.** Server: every PAKE run is charged before `Exchange`, under one lock with the limit check, to the source (IPv4 address, or IPv6 /64) diff --git a/files/homelab/cluster/cmd/bluefin-cluster/main.go b/files/homelab/cluster/cmd/bluefin-cluster/main.go index 1840d3a8..41299721 100644 --- a/files/homelab/cluster/cmd/bluefin-cluster/main.go +++ b/files/homelab/cluster/cmd/bluefin-cluster/main.go @@ -329,9 +329,14 @@ func nodePassphrase(c conf.Config) (string, error) { return passphrase.Normalize(raw) } +// joined is true once this node is in a cluster: the marker, or the kubelet +// kubeconfig kubeadm join or the k0s worker writes after bootstrapping. A +// k0s join token alone is no evidence: Join writes it before it starts +// k0s-first-boot.service, so a failed start, or a token that expired before +// k0sworker used it, would otherwise pass for a join and stop retrying. func joined() bool { - for _, p := range []string{joinedMarker, ops.KubeadmKubelet, ops.K0sTokenFile} { - if _, err := os.Stat(p); err == nil { + for _, p := range []string{joinedMarker, ops.KubeadmKubelet, ops.K0sKubelet} { + if _, err := os.Stat(filepath.Join(ops.Root, p)); err == nil { return true } } diff --git a/files/homelab/cluster/cmd/bluefin-cluster/main_test.go b/files/homelab/cluster/cmd/bluefin-cluster/main_test.go index f2c3121c..cc819db0 100644 --- a/files/homelab/cluster/cmd/bluefin-cluster/main_test.go +++ b/files/homelab/cluster/cmd/bluefin-cluster/main_test.go @@ -4,13 +4,60 @@ import ( "context" "errors" "net/netip" + "os" + "path/filepath" "testing" "time" "github.com/projectbluefin/server/files/homelab/cluster/internal/discover" "github.com/projectbluefin/server/files/homelab/cluster/internal/join" + "github.com/projectbluefin/server/files/homelab/cluster/internal/ops" ) +// withRoot points ops.Root (every host path) at a fresh directory and +// creates files under it. +func withRoot(t *testing.T, files ...string) string { + t.Helper() + root := t.TempDir() + orig := ops.Root + ops.Root = root + t.Cleanup(func() { ops.Root = orig }) + for _, f := range files { + p := filepath.Join(root, f) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, nil, 0o600); err != nil { + t.Fatal(err) + } + } + return root +} + +func TestJoined(t *testing.T) { + for _, tc := range []struct { + name string + files []string + want bool + }{ + {"fresh node", nil, false}, + {"marker", []string{joinedMarker}, true}, + {"kubeadm join bootstrapped kubelet", []string{ops.KubeadmKubelet}, true}, + {"k0s worker bootstrapped kubelet", []string{ops.K0sTokenFile, ops.K0sKubelet}, true}, + // Written before k0s-first-boot.service starts: if that failed, or + // the token expired unused, the node must ask for a fresh one. + {"k0s token alone", []string{ops.K0sTokenFile}, false}, + {"kubeadm bootstrap kubeconfig alone", []string{"/etc/kubernetes/bootstrap-kubelet.conf"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + withRoot(t, tc.files...) + if got := joined(); got != tc.want { + t.Errorf("joined() = %v, want %v", got, tc.want) + } + }) + } +} + func TestTryCandidates(t *testing.T) { a1 := netip.MustParseAddrPort("192.0.2.1:6447") a2 := netip.MustParseAddrPort("[2001:db8::1]:6447") diff --git a/files/homelab/cluster/internal/ops/ops.go b/files/homelab/cluster/internal/ops/ops.go index 488e2910..228450ee 100644 --- a/files/homelab/cluster/internal/ops/ops.go +++ b/files/homelab/cluster/internal/ops/ops.go @@ -29,6 +29,7 @@ const ( KubeadmKubelet = "/etc/kubernetes/kubelet.conf" K0sAdminConf = "/var/lib/k0s/pki/admin.conf" K0sTokenFile = "/etc/k0s/token" + K0sKubelet = "/var/lib/k0s/kubelet.conf" // k0s worker, once bootstrapped CRISocket = "unix:///run/containerd/containerd.sock" RuntimeKubeadm = "kubeadm" From 9d934597d2a5515f2862074a6a79e2d6a3116743 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Fri, 30 Oct 2026 22:55:19 -0500 Subject: [PATCH 3/3] test(homelab): cover bluefin-cluster's passphrases and DNS-SD record The cmd package had no Go tests; the only checks on its passphrase handling and its mDNS advertisement were source-text assertions in test_homelab_cluster.py. Add table-driven tests for: - controlPlanePassphrase: operator-chosen passphrase normalised, stored 0600 and preferred over a stored one; weak or non-ASCII ones refused without echoing them; a stored one reused unchanged; a 4-word one generated on first start and stored for the next. - nodePassphrase: homelab.conf before the credential, the credential, neither, and non-ASCII input. - the DNS-SD body, exactly, so the TXT record keeps carrying no secrets. Minimal seams: the DNS-SD text moves into dnssdBody(), and the stored passphrase is read under ops.Root like every other ops path. The Python check now follows the body into dnssdBody() and still asserts the same TXT format and that neither function mentions a passphrase or token. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cluster/cmd/bluefin-cluster/main.go | 14 ++- .../cluster/cmd/bluefin-cluster/main_test.go | 105 ++++++++++++++++++ tests/unit/test_homelab_cluster.py | 7 +- 3 files changed, 120 insertions(+), 6 deletions(-) diff --git a/files/homelab/cluster/cmd/bluefin-cluster/main.go b/files/homelab/cluster/cmd/bluefin-cluster/main.go index 41299721..50dbc8ea 100644 --- a/files/homelab/cluster/cmd/bluefin-cluster/main.go +++ b/files/homelab/cluster/cmd/bluefin-cluster/main.go @@ -193,7 +193,7 @@ func controlPlanePassphrase(c conf.Config) (string, error) { log.Warn("using the operator-chosen join passphrase from " + passKey + "; a generated one is stronger") return p, ops.WriteFile(passFile, []byte(p+"\n"), 0o600) } - if data, err := os.ReadFile(passFile); err == nil { + if data, err := os.ReadFile(filepath.Join(ops.Root, passFile)); err == nil { return passphrase.Normalize(string(data)) } p, bits, err := passphrase.Generate(passphrase.DefaultWords) @@ -274,10 +274,16 @@ func serve(ctx context.Context) error { return err } -// publish advertises the join service; the TXT record carries no secrets. -func publish(cluster, runtime, port string) error { - body := fmt.Sprintf("# bluefin-cluster serve: the homelab join service (no secrets here).\n[Service]\nName=%%H\nType=%s\nPort=%s\nTxtText=v=1 cluster=%s runtime=%s\n", +// dnssdBody is the join service's DNS-SD file; the TXT record carries no +// secrets. +func dnssdBody(cluster, runtime, port string) string { + return fmt.Sprintf("# bluefin-cluster serve: the homelab join service (no secrets here).\n[Service]\nName=%%H\nType=%s\nPort=%s\nTxtText=v=1 cluster=%s runtime=%s\n", discover.ServiceType, port, cluster, runtime) +} + +// publish advertises the join service. +func publish(cluster, runtime, port string) error { + body := dnssdBody(cluster, runtime, port) // Explicit modes: the unit's UMask=0077 would hide both from // systemd-resolved, which runs as its own user. dir := filepath.Dir(dnssdFile) diff --git a/files/homelab/cluster/cmd/bluefin-cluster/main_test.go b/files/homelab/cluster/cmd/bluefin-cluster/main_test.go index cc819db0..11c8d64e 100644 --- a/files/homelab/cluster/cmd/bluefin-cluster/main_test.go +++ b/files/homelab/cluster/cmd/bluefin-cluster/main_test.go @@ -6,12 +6,15 @@ import ( "net/netip" "os" "path/filepath" + "strings" "testing" "time" + "github.com/projectbluefin/server/files/homelab/cluster/internal/conf" "github.com/projectbluefin/server/files/homelab/cluster/internal/discover" "github.com/projectbluefin/server/files/homelab/cluster/internal/join" "github.com/projectbluefin/server/files/homelab/cluster/internal/ops" + "github.com/projectbluefin/server/files/homelab/cluster/internal/passphrase" ) // withRoot points ops.Root (every host path) at a fresh directory and @@ -121,3 +124,105 @@ func TestTryCandidates(t *testing.T) { }) } } + +func TestControlPlanePassphrase(t *testing.T) { + const strong = "Correct Horse battery_staple-orbit" + for _, tc := range []struct { + name string + conf conf.Config + stored string // existing passFile content, "" for none + want string // "" without err: a generated one + err bool + }{ + {"operator-chosen, normalised and stored", conf.Config{passKey: strong}, "", "correct-horse-battery-staple-orbit", false}, + {"operator-chosen beats the stored one", conf.Config{passKey: strong}, "old-words-here-now\n", "correct-horse-battery-staple-orbit", false}, + {"operator-chosen too weak", conf.Config{passKey: "short pass"}, "", "", true}, + {"operator-chosen not ASCII", conf.Config{passKey: "café crème brûlée soufflé"}, "", "", true}, + {"stored one kept", conf.Config{}, "Alpha Bravo charlie delta\n", "alpha-bravo-charlie-delta", false}, + {"generated on first start", conf.Config{}, "", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + root := withRoot(t) + file := filepath.Join(root, passFile) + if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil { + t.Fatal(err) + } + if tc.stored != "" { + if err := os.WriteFile(file, []byte(tc.stored), 0o600); err != nil { + t.Fatal(err) + } + } + got, err := controlPlanePassphrase(tc.conf) + if (err != nil) != tc.err { + t.Fatalf("err %v, want error %v", err, tc.err) + } + if tc.err { + if err != nil && strings.Contains(err.Error(), tc.conf[passKey]) { + t.Errorf("error leaks the passphrase: %v", err) + } + return + } + if tc.want != "" && got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + if tc.want == "" && len(strings.Split(got, "-")) != passphrase.DefaultWords { + t.Errorf("generated %q: want %d words", got, passphrase.DefaultWords) + } + // What serve hands out is what a restart reads back; a stored + // one is left as it is. + want := got + "\n" + if tc.stored != "" && tc.conf[passKey] == "" { + want = tc.stored + } + if data, err := os.ReadFile(file); err != nil || string(data) != want { + t.Errorf("stored %q (%v), want %q", data, err, want) + } + fi, err := os.Stat(file) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Errorf("passphrase file mode %v, want 0600", fi.Mode().Perm()) + } + }) + } +} + +func TestNodePassphrase(t *testing.T) { + for _, tc := range []struct { + name string + conf conf.Config + credential string // "" for none + want string + err bool + }{ + {"from homelab.conf", conf.Config{passKey: "Four Words From_Console"}, "", "four-words-from-console", false}, + {"homelab.conf beats the credential", conf.Config{passKey: "four words from console"}, "other words entirely here", "four-words-from-console", false}, + {"from the credential", conf.Config{}, "four words from console\n", "four-words-from-console", false}, + {"none", conf.Config{}, "", "", true}, + {"not ASCII", conf.Config{passKey: "naïve words"}, "", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + t.Setenv("CREDENTIALS_DIRECTORY", dir) + if tc.credential != "" { + if err := os.WriteFile(filepath.Join(dir, credentialKey), []byte(tc.credential), 0o600); err != nil { + t.Fatal(err) + } + } + got, err := nodePassphrase(tc.conf) + if (err != nil) != tc.err || got != tc.want { + t.Errorf("got %q, %v; want %q, error %v", got, err, tc.want, tc.err) + } + }) + } +} + +func TestDNSSDBodyCarriesNoSecrets(t *testing.T) { + got := dnssdBody("lab", "kubeadm", "6447") + want := "# bluefin-cluster serve: the homelab join service (no secrets here).\n" + + "[Service]\nName=%H\nType=" + discover.ServiceType + "\nPort=6447\nTxtText=v=1 cluster=lab runtime=kubeadm\n" + if got != want { + t.Errorf("dnssd body:\n%s\nwant:\n%s", got, want) + } +} diff --git a/tests/unit/test_homelab_cluster.py b/tests/unit/test_homelab_cluster.py index 4807174c..e36271b5 100644 --- a/tests/unit/test_homelab_cluster.py +++ b/tests/unit/test_homelab_cluster.py @@ -107,9 +107,12 @@ def test_applier_does_nothing_on_a_node() -> None: def test_dnssd_advertisement_carries_no_secrets() -> None: main = MAIN.read_text() publish = main.split("func publish(", 1)[1].split("\n}\n", 1)[0] - txt = re.search(r"TxtText=([^\\]*)\\n", publish).group(1) + body = main.split("func dnssdBody(", 1)[1].split("\n}\n", 1)[0] + assert "dnssdBody(cluster, runtime, port)" in publish + txt = re.search(r"TxtText=([^\\]*)\\n", body).group(1) assert txt == "v=1 cluster=%s runtime=%s" - assert "pass" not in publish.lower() and "token" not in publish.lower() + for fn in (publish, body): + assert "pass" not in fn.lower() and "token" not in fn.lower() assert 'dnssdFile = "/run/systemd/dnssd/bluefin-cluster.dnssd"' in main assert "*.dnssd" in install_script(), "the build refuses a static advertisement in the sysext" # serve runs with UMask=0077; systemd-resolved (its own user) must read it.