Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions files/homelab/cluster/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,10 @@ and the `admin.conf` server (`<cp>.local:6443`). The node writes a
`kubeadm join --config`; a failed join is reset and retried with a fresh
token. k0s: `k0s token create --role=worker --expiry=15m0s` (embeds the CA
and server); the node writes `/etc/k0s/token` (0600) and starts
`k0s-first-boot.service`, which starts `k0sworker`.
`k0s-first-boot.service`, which starts `k0sworker`. A restarted `join`
counts the node as joined only by the marker or a bootstrapped kubelet
kubeconfig (`/etc/kubernetes/kubelet.conf`, `/var/lib/k0s/kubelet.conf`),
never by the token alone.

**Rate limits.** Server: every PAKE run is charged before `Exchange`, under
one lock with the limit check, to the source (IPv4 address, or IPv6 /64)
Expand All @@ -136,7 +139,9 @@ runs a rogue client or a rogue control plane; no access to either host.
2^17.4 per year against 2^51.7 for a generated passphrase). It never
receives sealed data or causes a mint without tagC.
- Rogue control plane: one guess per node attempt, bounded by the node's
budget however many instances it advertises; the node never sends tagC
budget however many instances it advertises (the node skips a candidate
that cannot prove the passphrase and tries the next, so a rogue instance
listed first cannot hide the real one); the node never sends tagC
unless tagS verified, so it cannot be steered to a cluster that does not
know the passphrase. A spoofed `<cp>.local` cannot impersonate the API
server: kubeadm pins the CA by hash; k0s's token carries the CA.
Expand Down
40 changes: 32 additions & 8 deletions files/homelab/cluster/cmd/bluefin-cluster/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ func controlPlanePassphrase(c conf.Config) (string, error) {
log.Warn("using the operator-chosen join passphrase from " + passKey + "; a generated one is stronger")
return p, ops.WriteFile(passFile, []byte(p+"\n"), 0o600)
}
if data, err := os.ReadFile(passFile); err == nil {
if data, err := os.ReadFile(filepath.Join(ops.Root, passFile)); err == nil {
return passphrase.Normalize(string(data))
}
p, bits, err := passphrase.Generate(passphrase.DefaultWords)
Expand Down Expand Up @@ -274,10 +274,16 @@ func serve(ctx context.Context) error {
return err
}

// publish advertises the join service; the TXT record carries no secrets.
func publish(cluster, runtime, port string) error {
body := fmt.Sprintf("# bluefin-cluster serve: the homelab join service (no secrets here).\n[Service]\nName=%%H\nType=%s\nPort=%s\nTxtText=v=1 cluster=%s runtime=%s\n",
// dnssdBody is the join service's DNS-SD file; the TXT record carries no
// secrets.
func dnssdBody(cluster, runtime, port string) string {
return fmt.Sprintf("# bluefin-cluster serve: the homelab join service (no secrets here).\n[Service]\nName=%%H\nType=%s\nPort=%s\nTxtText=v=1 cluster=%s runtime=%s\n",
discover.ServiceType, port, cluster, runtime)
}

// publish advertises the join service.
func publish(cluster, runtime, port string) error {
body := dnssdBody(cluster, runtime, port)
// Explicit modes: the unit's UMask=0077 would hide both from
// systemd-resolved, which runs as its own user.
dir := filepath.Dir(dnssdFile)
Expand Down Expand Up @@ -329,9 +335,14 @@ func nodePassphrase(c conf.Config) (string, error) {
return passphrase.Normalize(raw)
}

// joined is true once this node is in a cluster: the marker, or the kubelet
// kubeconfig kubeadm join or the k0s worker writes after bootstrapping. A
// k0s join token alone is no evidence: Join writes it before it starts
// k0s-first-boot.service, so a failed start, or a token that expired before
// k0sworker used it, would otherwise pass for a join and stop retrying.
func joined() bool {
for _, p := range []string{joinedMarker, ops.KubeadmKubelet, ops.K0sTokenFile} {
if _, err := os.Stat(p); err == nil {
for _, p := range []string{joinedMarker, ops.KubeadmKubelet, ops.K0sKubelet} {
if _, err := os.Stat(filepath.Join(ops.Root, p)); err == nil {
return true
}
}
Expand Down Expand Up @@ -438,18 +449,31 @@ func attempt(ctx context.Context, cl *join.Client, manual string) (join.Payload,
return join.Payload{}, fmt.Errorf("no %s control plane advertised %s on the local network yet", cl.Runtime, discover.ServiceType)
}
}
return tryCandidates(ctx, cl.Join, cands)
}

// tryCandidates asks each candidate in turn. One that cannot prove the
// passphrase (an impostor, or another cluster's control plane) is skipped
// rather than ending the round: its other addresses would only spend guesses
// on the same answer, and stopping would let whichever instance is listed
// first hide the real control plane for good. Only the guess budget, shared
// by every candidate, ends the round early.
func tryCandidates(ctx context.Context, joinAt func(context.Context, netip.AddrPort) (join.Payload, error), cands []discover.Candidate) (join.Payload, error) {
var errs []error
for _, cand := range cands {
for _, addr := range cand.Addrs {
p, err := cl.Join(ctx, addr)
p, err := joinAt(ctx, addr)
if err == nil {
return p, nil
}
errs = append(errs, fmt.Errorf("%s (%s): %w", cand.Name, addr, err))
var be *join.ErrBudget
if errors.As(err, &be) || errors.Is(err, join.ErrNotProven) {
if errors.As(err, &be) {
return join.Payload{}, errors.Join(errs...)
}
if errors.Is(err, join.ErrNotProven) {
break
}
}
}
return join.Payload{}, errors.Join(errs...)
Expand Down
228 changes: 228 additions & 0 deletions files/homelab/cluster/cmd/bluefin-cluster/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
package main

import (
"context"
"errors"
"net/netip"
"os"
"path/filepath"
"strings"
"testing"
"time"

"github.com/projectbluefin/server/files/homelab/cluster/internal/conf"
"github.com/projectbluefin/server/files/homelab/cluster/internal/discover"
"github.com/projectbluefin/server/files/homelab/cluster/internal/join"
"github.com/projectbluefin/server/files/homelab/cluster/internal/ops"
"github.com/projectbluefin/server/files/homelab/cluster/internal/passphrase"
)

// withRoot points ops.Root (every host path) at a fresh directory and
// creates files under it.
func withRoot(t *testing.T, files ...string) string {
t.Helper()
root := t.TempDir()
orig := ops.Root
ops.Root = root
t.Cleanup(func() { ops.Root = orig })
for _, f := range files {
p := filepath.Join(root, f)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, nil, 0o600); err != nil {
t.Fatal(err)
}
}
return root
}

func TestJoined(t *testing.T) {
for _, tc := range []struct {
name string
files []string
want bool
}{
{"fresh node", nil, false},
{"marker", []string{joinedMarker}, true},
{"kubeadm join bootstrapped kubelet", []string{ops.KubeadmKubelet}, true},
{"k0s worker bootstrapped kubelet", []string{ops.K0sTokenFile, ops.K0sKubelet}, true},
// Written before k0s-first-boot.service starts: if that failed, or
// the token expired unused, the node must ask for a fresh one.
{"k0s token alone", []string{ops.K0sTokenFile}, false},
{"kubeadm bootstrap kubeconfig alone", []string{"/etc/kubernetes/bootstrap-kubelet.conf"}, false},
} {
t.Run(tc.name, func(t *testing.T) {
withRoot(t, tc.files...)
if got := joined(); got != tc.want {
t.Errorf("joined() = %v, want %v", got, tc.want)
}
})
}
}

func TestTryCandidates(t *testing.T) {
a1 := netip.MustParseAddrPort("192.0.2.1:6447")
a2 := netip.MustParseAddrPort("[2001:db8::1]:6447")
b1 := netip.MustParseAddrPort("192.0.2.2:6447")
c1 := netip.MustParseAddrPort("192.0.2.3:6447")
cands := []discover.Candidate{
{Name: "a", Addrs: []netip.AddrPort{a1, a2}},
{Name: "b", Addrs: []netip.AddrPort{b1}},
{Name: "c", Addrs: []netip.AddrPort{c1}},
}
refused := errors.New("connection refused")
budget := &join.ErrBudget{RetryAfter: time.Minute}
for _, tc := range []struct {
name string
results map[netip.AddrPort]error // nil: joins
tried []netip.AddrPort
joined bool
budget bool
}{
{"first address joins", map[netip.AddrPort]error{a1: nil}, []netip.AddrPort{a1}, true, false},
{"unreachable address falls through to the next one", map[netip.AddrPort]error{a1: refused, a2: nil}, []netip.AddrPort{a1, a2}, true, false},
// An impostor (or another cluster) listed first must not hide the
// control plane: skip the rest of it, go on with the next.
{"not proven skips that candidate", map[netip.AddrPort]error{a1: join.ErrNotProven, b1: nil}, []netip.AddrPort{a1, b1}, true, false},
{"every candidate not proven", map[netip.AddrPort]error{a1: join.ErrNotProven, b1: join.ErrNotProven, c1: join.ErrNotProven}, []netip.AddrPort{a1, b1, c1}, false, false},
{"exhausted budget ends the round", map[netip.AddrPort]error{a1: join.ErrNotProven, b1: budget}, []netip.AddrPort{a1, b1}, false, true},
{"nothing answers", map[netip.AddrPort]error{a1: refused, a2: refused, b1: refused, c1: refused}, []netip.AddrPort{a1, a2, b1, c1}, false, false},
} {
t.Run(tc.name, func(t *testing.T) {
var tried []netip.AddrPort
p, err := tryCandidates(context.Background(), func(_ context.Context, addr netip.AddrPort) (join.Payload, error) {
tried = append(tried, addr)
err, ok := tc.results[addr]
if !ok {
t.Errorf("unexpected address %s", addr)
return join.Payload{}, errors.New("unexpected address")
}
if err != nil {
return join.Payload{}, err
}
return join.Payload{Cluster: addr.String()}, nil
}, cands)
if (err == nil) != tc.joined {
t.Fatalf("joined %v, err %v", err == nil, err)
}
if tc.joined && p.Cluster != tried[len(tried)-1].String() {
t.Errorf("payload from %s, last tried %s", p.Cluster, tried[len(tried)-1])
}
var be *join.ErrBudget
if errors.As(err, &be) != tc.budget {
t.Errorf("budget error %v: %v", tc.budget, err)
}
if len(tried) != len(tc.tried) {
t.Fatalf("tried %v, want %v", tried, tc.tried)
}
for i := range tried {
if tried[i] != tc.tried[i] {
t.Fatalf("tried %v, want %v", tried, tc.tried)
}
}
})
}
}

func TestControlPlanePassphrase(t *testing.T) {
const strong = "Correct Horse battery_staple-orbit"
for _, tc := range []struct {
name string
conf conf.Config
stored string // existing passFile content, "" for none
want string // "" without err: a generated one
err bool
}{
{"operator-chosen, normalised and stored", conf.Config{passKey: strong}, "", "correct-horse-battery-staple-orbit", false},
{"operator-chosen beats the stored one", conf.Config{passKey: strong}, "old-words-here-now\n", "correct-horse-battery-staple-orbit", false},
{"operator-chosen too weak", conf.Config{passKey: "short pass"}, "", "", true},
{"operator-chosen not ASCII", conf.Config{passKey: "café crème brûlée soufflé"}, "", "", true},
{"stored one kept", conf.Config{}, "Alpha Bravo charlie delta\n", "alpha-bravo-charlie-delta", false},
{"generated on first start", conf.Config{}, "", "", false},
} {
t.Run(tc.name, func(t *testing.T) {
root := withRoot(t)
file := filepath.Join(root, passFile)
if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil {
t.Fatal(err)
}
if tc.stored != "" {
if err := os.WriteFile(file, []byte(tc.stored), 0o600); err != nil {
t.Fatal(err)
}
}
got, err := controlPlanePassphrase(tc.conf)
if (err != nil) != tc.err {
t.Fatalf("err %v, want error %v", err, tc.err)
}
if tc.err {
if err != nil && strings.Contains(err.Error(), tc.conf[passKey]) {
t.Errorf("error leaks the passphrase: %v", err)
}
return
}
if tc.want != "" && got != tc.want {
t.Errorf("got %q, want %q", got, tc.want)
}
if tc.want == "" && len(strings.Split(got, "-")) != passphrase.DefaultWords {
t.Errorf("generated %q: want %d words", got, passphrase.DefaultWords)
}
// What serve hands out is what a restart reads back; a stored
// one is left as it is.
want := got + "\n"
if tc.stored != "" && tc.conf[passKey] == "" {
want = tc.stored
}
if data, err := os.ReadFile(file); err != nil || string(data) != want {
t.Errorf("stored %q (%v), want %q", data, err, want)
}
fi, err := os.Stat(file)
if err != nil {
t.Fatal(err)
}
if fi.Mode().Perm() != 0o600 {
t.Errorf("passphrase file mode %v, want 0600", fi.Mode().Perm())
}
})
}
}

func TestNodePassphrase(t *testing.T) {
for _, tc := range []struct {
name string
conf conf.Config
credential string // "" for none
want string
err bool
}{
{"from homelab.conf", conf.Config{passKey: "Four Words From_Console"}, "", "four-words-from-console", false},
{"homelab.conf beats the credential", conf.Config{passKey: "four words from console"}, "other words entirely here", "four-words-from-console", false},
{"from the credential", conf.Config{}, "four words from console\n", "four-words-from-console", false},
{"none", conf.Config{}, "", "", true},
{"not ASCII", conf.Config{passKey: "naïve words"}, "", "", true},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
t.Setenv("CREDENTIALS_DIRECTORY", dir)
if tc.credential != "" {
if err := os.WriteFile(filepath.Join(dir, credentialKey), []byte(tc.credential), 0o600); err != nil {
t.Fatal(err)
}
}
got, err := nodePassphrase(tc.conf)
if (err != nil) != tc.err || got != tc.want {
t.Errorf("got %q, %v; want %q, error %v", got, err, tc.want, tc.err)
}
})
}
}

func TestDNSSDBodyCarriesNoSecrets(t *testing.T) {
got := dnssdBody("lab", "kubeadm", "6447")
want := "# bluefin-cluster serve: the homelab join service (no secrets here).\n" +
"[Service]\nName=%H\nType=" + discover.ServiceType + "\nPort=6447\nTxtText=v=1 cluster=lab runtime=kubeadm\n"
if got != want {
t.Errorf("dnssd body:\n%s\nwant:\n%s", got, want)
}
}
1 change: 1 addition & 0 deletions files/homelab/cluster/internal/ops/ops.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const (
KubeadmKubelet = "/etc/kubernetes/kubelet.conf"
K0sAdminConf = "/var/lib/k0s/pki/admin.conf"
K0sTokenFile = "/etc/k0s/token"
K0sKubelet = "/var/lib/k0s/kubelet.conf" // k0s worker, once bootstrapped
CRISocket = "unix:///run/containerd/containerd.sock"

RuntimeKubeadm = "kubeadm"
Expand Down
7 changes: 5 additions & 2 deletions tests/unit/test_homelab_cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,12 @@ def test_applier_does_nothing_on_a_node() -> None:
def test_dnssd_advertisement_carries_no_secrets() -> None:
main = MAIN.read_text()
publish = main.split("func publish(", 1)[1].split("\n}\n", 1)[0]
txt = re.search(r"TxtText=([^\\]*)\\n", publish).group(1)
body = main.split("func dnssdBody(", 1)[1].split("\n}\n", 1)[0]
assert "dnssdBody(cluster, runtime, port)" in publish
txt = re.search(r"TxtText=([^\\]*)\\n", body).group(1)
assert txt == "v=1 cluster=%s runtime=%s"
assert "pass" not in publish.lower() and "token" not in publish.lower()
for fn in (publish, body):
assert "pass" not in fn.lower() and "token" not in fn.lower()
assert 'dnssdFile = "/run/systemd/dnssd/bluefin-cluster.dnssd"' in main
assert "*.dnssd" in install_script(), "the build refuses a static advertisement in the sysext"
# serve runs with UMask=0077; systemd-resolved (its own user) must read it.
Expand Down
Loading