diff --git a/app.go b/app.go index 4d9dd8f..5b8fae8 100644 --- a/app.go +++ b/app.go @@ -191,6 +191,10 @@ func (a *App) startCore(ctx context.Context) error { docker := runtime.NewDockerProvider() docker.SetIsolationSetting(func() string { return a.cfg.Config().Isolation }) + docker.SetFenceGated(func(slug string) bool { + svc, ok := cat.Get(slug) + return ok && svc.Docker.FenceGated + }) // A failed write is retried on every check, so it is reported once per failure // rather than every five minutes. Docker's own evidence still counts meanwhile. var memoryFailing atomic.Bool diff --git a/app_proxies_test.go b/app_proxies_test.go index cb3881a..f10ebbe 100644 --- a/app_proxies_test.go +++ b/app_proxies_test.go @@ -40,7 +40,7 @@ func TestProxiesStateNeverCarriesSecretsAndSaysWhatCannotRun(t *testing.T) { t.Fatalf("honeygain can run behind a proxy, got %q", refusals["honeygain"]) } if !strings.Contains(refusals["mysterium"], "cannot") { - t.Fatalf("mysterium needs the host's network and cannot, got %q", refusals["mysterium"]) + t.Fatalf("mysterium runs only behind the LAN fence and cannot, got %q", refusals["mysterium"]) } if state.TunnelImage != "" || !strings.Contains(state.TunnelImageError, "CASHPILOT_TUNNEL_IMAGE") { t.Fatalf("a refused tunnel image is said: %+v", state) diff --git a/catalog-overlay/README.md b/catalog-overlay/README.md index 776de81..5a661fe 100644 --- a/catalog-overlay/README.md +++ b/catalog-overlay/README.md @@ -66,10 +66,16 @@ the entry on its own, which is how a Desktop-only service would be carried. | Delta | Where | Why | | --- | --- | --- | -| 15 image digest pins | `image-pins.yml` | Desktop's digest-pin rule; the web catalog ships floating references. | +| 14 image digest pins | `image-pins.yml` | Desktop's digest-pin rule; the web catalog ships floating references. | | `native:` block on mysterium | `append/bandwidth/mysterium.yml` | Desktop can run a service as a supervised native process with no container runtime. The web app deploys through Docker on a worker and has no such runtime. | | `native:` schema documentation | `append/_schema.yml` | Documents the field above. | +## Fields Desktop reads differently + +| Field | What Desktop does | Why | +| --- | --- | --- | +| `docker.fence_gated` | A deploy is refused unless the earner joins the fenced network (`internal/runtime/lanfence.go`), and the compose export refuses the service. Turning isolation off stops the node before the fence's drops go, and Start refuses it until isolation is on again. The container keeps the restart policy every earner gets, `unless-stopped`. | The web worker sets restart policy `no` and starts the node only after it has read the fence back, at boot and after a crash. Desktop has no such starter yet; with `no` the node would stay down after every reboot until someone redeployed it. The gap that leaves: after a reboot, Docker starts the node before the app writes the fence again. In that window the fenced network has no masquerade, so the node has no internet route, but nothing drops what it sends to this machine itself. The node's own `--firewall.protected.networks` still blackholes private ranges inside its namespace. | + ## Files Desktop reads beside the services | File | Why | diff --git a/catalog-overlay/append/bandwidth/mysterium.yml b/catalog-overlay/append/bandwidth/mysterium.yml index c97b6ca..4048f94 100644 --- a/catalog-overlay/append/bandwidth/mysterium.yml +++ b/catalog-overlay/append/bandwidth/mysterium.yml @@ -2,13 +2,14 @@ # Native (no-Docker) execution: run the official Mysterium headless node binary as a # supervised child process. Pinned to release 1.38.5; sha256 is the SHA-256 of each # published release artifact. There is no linux/arm64 tarball upstream (only .deb), so -# arm64 Linux hosts fall back to the docker: image above. +# arm64 Linux hosts fall back to the docker: image above, which deploys only behind +# the LAN fence (docker.fence_gated). # -# The launch args deliberately do NOT mirror docker.command any more. The container -# binds its UI and Tequila API to 0.0.0.0 because it runs inside a container with -# host networking on a worker box; a native process binds them to 127.0.0.1 because -# it runs directly on the user's own machine, where 0.0.0.0 would publish an -# unauthenticated node admin API to the local network. Port 4449 and the accepted +# The launch args deliberately do NOT mirror docker.command. The container binds its +# UI to 0.0.0.0 inside its own network namespace, which reaches the host only where +# docker.ui_port publishes it; a native process binds it to 127.0.0.1 because it runs +# directly on the user's own machine, where 0.0.0.0 would publish an unauthenticated +# node admin API to the local network. Port 4449 and the accepted # terms are unchanged, so the earnings collector at # http://localhost:4449/tequilapi still works. # diff --git a/catalog-overlay/image-pins.yml b/catalog-overlay/image-pins.yml index e0cc80c..e6bc259 100644 --- a/catalog-overlay/image-pins.yml +++ b/catalog-overlay/image-pins.yml @@ -41,9 +41,6 @@ pins: iproyal: image: "iproyal/pawns-cli@sha256:b8bd21d755a7bd33cdde59da31f9b39d13e67fc67d00cb5f895d1c606eb5a95d" why: "Upstream ships no tag at all on this entry." - mysterium: - image: "mysteriumnetwork/myst@sha256:1b68b8336112567a2b4596eccebd53cb704bfa37630dd3e2aef63d8a0eca2b2a" - why: "Upstream ships no tag at all on this entry." packetstream: image: "packetstream/psclient@sha256:e5015592031fb6f3876582117f47995ecec8157a639e9fc3fcfe4af28b2c497a" why: "Upstream ships no tag at all on this entry." diff --git a/e2e/fence_gated_test.go b/e2e/fence_gated_test.go new file mode 100644 index 0000000..58172cf --- /dev/null +++ b/e2e/fence_gated_test.go @@ -0,0 +1,132 @@ +package e2e + +import ( + "strings" + "testing" + + "github.com/GeiserX/CashPilot-Desktop/internal/runtime" +) + +// gatedProvider is a Docker provider whose isolation setting the test changes, with +// the catalog's fence_gated services wired in the way app.go does. Mysterium is +// deployed behind the fence and running. +func gatedProvider(t *testing.T, e *env, setting *string) *runtime.DockerProvider { + t.Helper() + provider := runtime.NewDockerProvider() + provider.SetIsolationSetting(func() string { return *setting }) + provider.SetFenceGated(func(slug string) bool { + svc, ok := e.Catalog.Get(slug) + return ok && svc.Docker.FenceGated + }) + svc, _ := e.Catalog.Get("mysterium") + if _, err := provider.Deploy(e.ctx(), runtime.DeploySpec{Slug: "mysterium", Service: svc}, nil); err != nil { + t.Fatalf("deploy mysterium: %v", err) + } + return provider +} + +func state(e *env, name string) string { + c, _ := e.Docker.Container(name) + return c.State +} + +// A node Docker is about to restart is stopped too: left alone, Docker would bring +// it back after the drops are gone, past the app's Start guard. +func TestTurningIsolationOffStopsARestartingMysterium(t *testing.T) { + e := newEnv(t) + setting := "auto" + provider := gatedProvider(t, e, &setting) + e.Docker.SetState("cashpilot-mysterium", "restarting") + setting = "off" + provider.ReconcileFence(e.ctx()) + if got := state(e, "cashpilot-mysterium"); got != "exited" { + t.Fatalf("state = %q, want the restarting node stopped", got) + } +} + +// A node that cannot be stopped keeps the fence: the drops stay, and the status +// says why. +func TestTheFenceStaysWhenMysteriumCannotBeStopped(t *testing.T) { + e := newEnv(t) + setting := "auto" + provider := gatedProvider(t, e, &setting) + e.Docker.FailStop("cashpilot-mysterium") + setting = "off" + provider.ReconcileFence(e.ctx()) + st := provider.FenceStatus() + if st.State != "failed" || !strings.Contains(st.Error, "LAN fence was kept") { + t.Fatalf("status = %+v, want the fence kept and the failure named", st) + } +} + +// With the fence network already gone, a node that cannot be stopped is still +// named in the status, so the dashboard does not show a clean "n/a" while it runs. +func TestAStuckMysteriumIsReportedWhenTheFenceNetworkIsGone(t *testing.T) { + e := newEnv(t) + setting := "auto" + provider := gatedProvider(t, e, &setting) + e.Docker.FailStop("cashpilot-mysterium") + e.Docker.RemoveNetwork(runtime.FenceNetwork) + setting = "off" + provider.ReconcileFence(e.ctx()) + st := provider.FenceStatus() + if !strings.Contains(st.Error, "could not stop cashpilot-mysterium") { + t.Fatalf("status = %+v, want the failed stop named", st) + } +} + +// A container whose inspect fails cannot be judged, so Start and Restart refuse it. +func TestStartFailsClosedWhenTheContainerCannotBeInspected(t *testing.T) { + e := newEnv(t) + setting := "auto" + provider := gatedProvider(t, e, &setting) + if err := provider.Start(e.ctx(), "mysterium"); err != nil { + t.Fatalf("Start behind the fence: %v", err) + } + e.Docker.SetState("cashpilot-mysterium", "exited") + e.Docker.FailInspect("cashpilot-mysterium") + for name, err := range map[string]error{ + "Start": provider.Start(e.ctx(), "mysterium"), + "Restart": provider.Restart(e.ctx(), "mysterium", 5), + } { + if err == nil || !strings.Contains(err.Error(), "could not check") { + t.Errorf("%s: err = %v, want a refusal", name, err) + } + } + if got := state(e, "cashpilot-mysterium"); got == "running" { + t.Fatal("a container that could not be inspected was started") + } +} + +// A Mysterium made before the fence-gated label existed is found by its catalog +// entry: stopped when isolation goes off, and not started again. +func TestAnUnlabelledMysteriumIsGatedByItsCatalogEntry(t *testing.T) { + e := newEnv(t) + setting := "auto" + provider := gatedProvider(t, e, &setting) + e.Docker.SetLabel("cashpilot-mysterium", runtime.LabelFenceGated, "") + setting = "off" + provider.ReconcileFence(e.ctx()) + if got := state(e, "cashpilot-mysterium"); got != "exited" { + t.Fatalf("state = %q, want the unlabelled node stopped", got) + } + if err := provider.Start(e.ctx(), "mysterium"); err == nil || !strings.Contains(err.Error(), "only behind the LAN fence") { + t.Fatalf("Start: err = %v, want a refusal", err) + } +} + +// Behind a working fence, a node that is not on the fenced network alone is still +// refused: a container made on the host's network, or attached elsewhere too. +func TestStartRefusesAMysteriumOffTheFencedNetwork(t *testing.T) { + e := newEnv(t) + setting := "auto" + provider := gatedProvider(t, e, &setting) + e.Docker.SetState("cashpilot-mysterium", "exited") + e.Docker.AttachNetwork("cashpilot-mysterium", "bridge") + if err := provider.Start(e.ctx(), "mysterium"); err == nil || !strings.Contains(err.Error(), "also attached to bridge") { + t.Fatalf("Start: err = %v, want a refusal naming the other network", err) + } + if got := state(e, "cashpilot-mysterium"); got == "running" { + t.Fatal("a node attached outside the fence was started") + } +} diff --git a/e2e/harness/docker.go b/e2e/harness/docker.go index 97ca027..d0caf1b 100644 --- a/e2e/harness/docker.go +++ b/e2e/harness/docker.go @@ -14,6 +14,7 @@ import ( "log" "net/http" "net/http/httptest" + goruntime "runtime" "sort" "strings" "sync" @@ -46,6 +47,19 @@ type Container struct { Entrypoint []string StopTimeout *int Mounts []Mount + // Runtime, NetworkMode and PortBindings are the HostConfig fields of the same + // names, as the deploy sent them. + Runtime string + NetworkMode string + PortBindings map[string][]PortBinding + // ExtraNetworks are networks a test attached it to beside its network mode. + ExtraNetworks []string +} + +// PortBinding is one host binding of a published port. +type PortBinding struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` } // VolumeNames lists the names of the container's named volumes, in the order the @@ -81,6 +95,8 @@ type Docker struct { // networks are the user-defined networks, by name. The app creates one, its // LAN fence network, and runs a one-shot helper container to write the fence. networks map[string]map[string]any + // failInspect and failStop name containers whose inspect or stop fails. + failInspect, failStop map[string]bool } // NewDocker starts the fake daemon on a loopback TCP port. @@ -283,6 +299,14 @@ func (d *Docker) handle(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{ "Version": "28.0.0-fake", "ApiVersion": "1.51", "Os": "linux", "Arch": "amd64", }) + case path == "/info": + // What a real daemon reports and the deploy asks for: its CPU (this + // machine's, as the fallback assumed before /info was answered), its OS + // and its OCI runtimes. + writeJSON(w, http.StatusOK, map[string]any{ + "Architecture": goruntime.GOARCH, "OSType": "linux", + "Runtimes": map[string]any{"runc": map[string]string{"path": "runc"}}, + }) case path == "/images/create": d.pull(w, r) case strings.HasPrefix(path, "/images/") && strings.HasSuffix(path, "/json"): @@ -356,7 +380,10 @@ func (d *Docker) create(w http.ResponseWriter, r *http.Request) { StopTimeout *int `json:"StopTimeout"` Labels map[string]string `json:"Labels"` HostConfig struct { - Mounts []struct { + Runtime string `json:"Runtime"` + NetworkMode string `json:"NetworkMode"` + PortBindings map[string][]PortBinding `json:"PortBindings"` + Mounts []struct { Type string `json:"Type"` Source string `json:"Source"` Target string `json:"Target"` @@ -388,6 +415,10 @@ func (d *Docker) create(w http.ResponseWriter, r *http.Request) { Cmd: body.Cmd, Entrypoint: body.Entrypoint, StopTimeout: body.StopTimeout, + + Runtime: body.HostConfig.Runtime, + NetworkMode: body.HostConfig.NetworkMode, + PortBindings: body.HostConfig.PortBindings, } for _, m := range body.HostConfig.Mounts { if m.Type == "volume" && m.Source != "" { @@ -435,6 +466,14 @@ func (d *Docker) list(w http.ResponseWriter) { func (d *Docker) container(w http.ResponseWriter, r *http.Request, rest string) { id, action, _ := strings.Cut(rest, "/") + d.mu.Lock() + failed := (action == "json" && d.failing(d.failInspect, id)) || (action == "stop" && d.failing(d.failStop, id)) + d.mu.Unlock() + if failed { + writeJSON(w, http.StatusInternalServerError, map[string]string{"message": "fake daemon: " + action + " failed for " + id}) + return + } + switch { case action == "json": d.inspect(w, id) @@ -493,9 +532,87 @@ func (d *Docker) inspect(w http.ResponseWriter, id string) { "State": map[string]any{"Status": c.State, "Running": c.State == "running"}, "Config": map[string]any{"Image": c.Image, "Labels": c.Labels, "Env": c.Env, "Cmd": c.Cmd}, "Mounts": mounts, + "HostConfig": map[string]any{ + "NetworkMode": c.NetworkMode, "Runtime": c.Runtime, "PortBindings": c.PortBindings, + }, + "NetworkSettings": map[string]any{"Networks": c.networks()}, }) } +// networks is what inspect reports the container attached to: its network mode's +// network, plus any a test attached it to. +func (c *Container) networks() map[string]any { + out := map[string]any{} + if c.NetworkMode != "" { + out[c.NetworkMode] = map[string]any{} + } + for _, n := range c.ExtraNetworks { + out[n] = map[string]any{} + } + return out +} + +// SetState sets a container's state as Docker would report it ("restarting"). +func (d *Docker) SetState(name, state string) { + d.mu.Lock() + defer d.mu.Unlock() + if c := d.find(name); c != nil { + c.State = state + } +} + +// SetLabel sets (or, with value "", removes) one label of a container, the way a +// container made by an older version lacks a label a newer one sets. +func (d *Docker) SetLabel(name, key, value string) { + d.mu.Lock() + defer d.mu.Unlock() + if c := d.find(name); c != nil { + if value == "" { + delete(c.Labels, key) + } else { + c.Labels[key] = value + } + } +} + +// AttachNetwork reports the container attached to one more network. +func (d *Docker) AttachNetwork(name, network string) { + d.mu.Lock() + defer d.mu.Unlock() + if c := d.find(name); c != nil { + c.ExtraNetworks = append(c.ExtraNetworks, network) + } +} + +// RemoveNetwork deletes a network behind the app's back, the way a user's +// `docker network rm` or a daemon reset would. +func (d *Docker) RemoveNetwork(name string) { + d.mu.Lock() + defer d.mu.Unlock() + delete(d.networks, name) +} + +// FailInspect makes inspecting this container fail with a server error. +func (d *Docker) FailInspect(name string) { d.setFail(&d.failInspect, name) } + +// FailStop makes stopping this container fail with a server error. +func (d *Docker) FailStop(name string) { d.setFail(&d.failStop, name) } + +func (d *Docker) setFail(set *map[string]bool, name string) { + d.mu.Lock() + defer d.mu.Unlock() + if *set == nil { + *set = map[string]bool{} + } + (*set)[name] = true +} + +// failing reports a container whose name or id is in set. Called with d.mu held. +func (d *Docker) failing(set map[string]bool, id string) bool { + c := d.find(id) + return set[id] || (c != nil && (set[c.Name] || set[c.ID])) +} + func (d *Docker) setState(w http.ResponseWriter, id, state string) { d.mu.Lock() defer d.mu.Unlock() diff --git a/e2e/stop_grace_test.go b/e2e/stop_grace_test.go index a40c680..d9bce5f 100644 --- a/e2e/stop_grace_test.go +++ b/e2e/stop_grace_test.go @@ -2,7 +2,10 @@ package e2e import ( "slices" + "strings" "testing" + + "github.com/GeiserX/CashPilot-Desktop/internal/runtime" ) // A Storj node is created with the catalog's stop grace as the container's own stop @@ -62,9 +65,10 @@ func TestAnEarnerWithoutAWrapperKeepsTheImagesEntrypoint(t *testing.T) { } } -// Mysterium's command now carries ${UI_ADDRESS}. Left blank, the variable's default -// must fill it, or the node starts with an empty --ui.address. -func TestMysteriumsUIAddressFallsBackToItsDefault(t *testing.T) { +// Mysterium runs in its own namespace on the fenced network, under runc. UI_ADDRESS +// left blank publishes the WebUI on loopback only, and UDP_PORTS left blank +// publishes the default range 1:1 and hands it to the node. +func TestMysteriumDeploysBehindTheFenceWithItsDefaults(t *testing.T) { e := newEnv(t) if _, err := e.Manager.Deploy(e.ctx(), "mysterium", map[string]string{}); err != nil { t.Fatalf("Deploy: %v", err) @@ -73,15 +77,96 @@ func TestMysteriumsUIAddressFallsBackToItsDefault(t *testing.T) { if !ok { t.Fatal("the daemon has no cashpilot-mysterium container") } - if !slices.Contains(container.Cmd, "--ui.address=127.0.0.1") { - t.Errorf("command = %v, want --ui.address=127.0.0.1 from the default", container.Cmd) + if container.NetworkMode != runtime.FenceNetwork { + t.Errorf("network mode = %q, want %s", container.NetworkMode, runtime.FenceNetwork) + } + if container.Runtime != "runc" { + t.Errorf("runtime = %q, want runc", container.Runtime) + } + if ui := container.PortBindings["4449/tcp"]; len(ui) != 1 || ui[0].HostIP != "127.0.0.1" || ui[0].HostPort != "4449" { + t.Errorf("4449/tcp bindings = %+v, want 127.0.0.1:4449 only", ui) + } + udp := 0 + for port, b := range container.PortBindings { + if strings.HasSuffix(port, "/udp") { + udp++ + if len(b) != 1 || b[0].HostPort+"/udp" != port { + t.Errorf("%s bindings = %+v, want the same port on the host", port, b) + } + } + } + if udp != 30 { + t.Errorf("published %d UDP ports, want the 30 of 23000:23029", udp) + } + if !slices.Contains(container.Cmd, "--udp.ports=23000:23029") { + t.Errorf("command = %v, want --udp.ports=23000:23029 from the default", container.Cmd) + } +} + +// With LAN isolation off the node would sit on Docker's default bridge, where +// nothing keeps its NET_ADMIN namespace off the LAN. The deploy is refused and +// nothing is created. +func TestMysteriumIsRefusedWithIsolationOff(t *testing.T) { + e := newEnv(t) + provider := runtime.NewDockerProvider() + provider.SetIsolationSetting(func() string { return "off" }) + svc, _ := e.Catalog.Get("mysterium") + _, err := provider.Deploy(e.ctx(), runtime.DeploySpec{Slug: "mysterium", Service: svc}, nil) + if err == nil || !strings.Contains(err.Error(), "only behind the LAN fence") { + t.Fatalf("err = %v, want a refusal naming the fence", err) + } + if _, ok := e.Docker.Container("cashpilot-mysterium"); ok { + t.Fatal("a container was created for a refused deploy") } - if slices.ContainsFunc(container.Cmd, func(arg string) bool { return arg == "--ui.address=" }) { - t.Errorf("the node was given an empty WebUI address: %v", container.Cmd) +} + +// Turning isolation off takes the fence's drops away, so the node is stopped first, +// and it cannot be started again until isolation is back on. Ordinary earners keep +// running. +func TestTurningIsolationOffStopsMysteriumAndKeepsItStopped(t *testing.T) { + e := newEnv(t) + setting := "auto" + provider := runtime.NewDockerProvider() + provider.SetIsolationSetting(func() string { return setting }) + for _, slug := range []string{"mysterium", "honeygain"} { + svc, _ := e.Catalog.Get(slug) + creds := map[string]string{} + if slug == "honeygain" { + creds = honeygainCreds + } + if _, err := provider.Deploy(e.ctx(), runtime.DeploySpec{Slug: slug, Service: svc, Env: creds}, nil); err != nil { + t.Fatalf("deploy %s: %v", slug, err) + } + } + if c, _ := e.Docker.Container("cashpilot-mysterium"); c.Labels[runtime.LabelFenceGated] != "true" { + t.Fatalf("labels = %v, want the fence-gated label", c.Labels) + } + + setting = "off" + provider.ReconcileFence(e.ctx()) + if c, _ := e.Docker.Container("cashpilot-mysterium"); c.State == "running" { + t.Fatal("mysterium is still running with isolation off") + } + if c, _ := e.Docker.Container("cashpilot-honeygain"); c.State != "running" { + t.Fatalf("honeygain state = %q, want it left running", c.State) + } + for name, err := range map[string]error{ + "Start": provider.Start(e.ctx(), "mysterium"), + "Restart": provider.Restart(e.ctx(), "mysterium", 5), + } { + if err == nil || !strings.Contains(err.Error(), "only behind the LAN fence") { + t.Errorf("%s with isolation off: err = %v, want a refusal", name, err) + } + } + if c, _ := e.Docker.Container("cashpilot-mysterium"); c.State == "running" { + t.Fatal("mysterium was started with isolation off") + } + + setting = "on" + if err := provider.Start(e.ctx(), "mysterium"); err != nil { + t.Fatalf("Start with isolation on: %v", err) } - // The guide says to forward UDP 56000-56100; without the flag the node picks - // from 10000:60000 and the forward is never used. - if !slices.Contains(container.Cmd, "--udp.ports=56000:56100") { - t.Errorf("command = %v, want --udp.ports=56000:56100 from the default", container.Cmd) + if c, _ := e.Docker.Container("cashpilot-mysterium"); c.State != "running" { + t.Fatalf("state = %q, want running", c.State) } } diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 1b50445..6cffcba 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -259,6 +259,22 @@ type DockerConfig struct { // seconds after any stop, however long Docker is willing to wait. Entrypoint []string `json:"entrypoint" yaml:"entrypoint"` NetworkMode string `json:"networkMode" yaml:"network_mode"` + // Runtime is the one container runtime this service works under ("runc"), set on + // the container whatever the daemon's default is. Mysterium needs it: gVisor has + // no iptables-nft and no kernel WireGuard, so the node starts there and carries + // nothing. + Runtime string `json:"runtime" yaml:"runtime"` + // UIPort publishes a WebUI port on each address the named env var lists + // (comma separated), instead of a static ports entry. See UIBindings. + UIPort *UIPort `json:"uiPort,omitempty" yaml:"ui_port"` + // UDPPortRange publishes the start:end range the named env var holds, 1:1, at + // most MaxPorts ports. See ExpandUDPRange. + UDPPortRange *UDPPortRange `json:"udpPortRange,omitempty" yaml:"udp_port_range"` + // FenceGated services run only behind the LAN fence (internal/runtime/lanfence.go): + // a deploy anywhere else is refused, and they have no compose export. Mysterium + // holds NET_ADMIN in its own network namespace, and the fence is what keeps that + // namespace off the LAN, this machine and the tailnet. + FenceGated bool `json:"fenceGated" yaml:"fence_gated"` // NodeFirewall says where a node firewalls its own customers off private ranges // (the flag in Command carrying the ranges, the nat chain, the blackhole address). // The web worker reads it to check the node's isolation; Desktop parses it so the diff --git a/internal/catalog/netshape.go b/internal/catalog/netshape.go new file mode 100644 index 0000000..867e8ba --- /dev/null +++ b/internal/catalog/netshape.go @@ -0,0 +1,119 @@ +package catalog + +import ( + "fmt" + "net/netip" + "strconv" + "strings" +) + +// The ports a service publishes from its env rather than from static ports entries: +// docker.ui_port and docker.udp_port_range. The web catalog's rules (app/catalog.py +// ui_bindings, expand_udp_range, MAX_UDP_PORTS), so a value one app accepts the +// other publishes the same way. + +// MaxUDPPorts is the most UDP ports one udp_port_range may publish. Each published +// port is a rule and, with Docker's userland proxy, a process on the host. +const MaxUDPPorts = 1000 + +// UIPort is docker.ui_port: Port is published on each address the Env var lists. +type UIPort struct { + Env string `json:"env" yaml:"env"` + Port int `json:"port" yaml:"port"` +} + +// UDPPortRange is docker.udp_port_range: the start:end range the Env var holds is +// published 1:1, at most MaxPorts ports (0 means MaxUDPPorts). +type UDPPortRange struct { + Env string `json:"env" yaml:"env"` + MaxPorts int `json:"maxPorts,omitempty" yaml:"max_ports"` +} + +// Cap is the most ports this range may publish: MaxPorts within 1..MaxUDPPorts. +func (r UDPPortRange) Cap() int { + if r.MaxPorts < 1 || r.MaxPorts > MaxUDPPorts { + return MaxUDPPorts + } + return r.MaxPorts +} + +// PublishesPorts reports whether the container publishes any port on the host: +// static ports, a WebUI port or a UDP range. Two copies cannot share one, and a +// proxied copy cannot publish one. +func (d DockerConfig) PublishesPorts() bool { + return len(d.Ports) > 0 || d.UIPort != nil || d.UDPPortRange != nil +} + +// UIBindings is each IPv4 address in a comma-separated list, deduplicated, with +// "localhost" read as 127.0.0.1 (Docker binds addresses, not names). An empty list +// is 127.0.0.1, the catalog's default. Anything that is not an IPv4 address is an +// error: published as is, it would fail the container's start. +func UIBindings(value string) ([]netip.Addr, error) { + var found []netip.Addr + for _, raw := range strings.Split(value, ",") { + address := strings.TrimSpace(raw) + if address == "" { + continue + } + if address == "localhost" { + address = "127.0.0.1" + } + addr, err := netip.ParseAddr(address) + if err != nil || !addr.Is4() { + return nil, fmt.Errorf("%q is not an IPv4 address", address) + } + dup := false + for _, f := range found { + dup = dup || f == addr + } + if !dup { + found = append(found, addr) + } + } + if len(found) == 0 { + found = []netip.Addr{netip.MustParseAddr("127.0.0.1")} + } + return found, nil +} + +// ExpandUDPRange is the ports of "start:end", in order. A blank value publishes +// nothing. A value that is not a well-ordered range, or one wider than limit, is an +// error: a node can only use a forwarded range when each port reaches the same port +// in the container, so nothing short of the whole range is published. +func ExpandUDPRange(value string, limit int) ([]int, error) { + value = strings.TrimSpace(value) + if value == "" { + return nil, nil + } + if !IsPortRange(value) { + return nil, fmt.Errorf("%q is not a port range start:end with start <= end <= 65535", value) + } + start, end, _ := strings.Cut(value, ":") + first, _ := strconv.Atoi(start) + last, _ := strconv.Atoi(end) + if n := last - first + 1; n > limit { + return nil, fmt.Errorf("%q spans %d ports; at most %d can be published", value, n, limit) + } + ports := make([]int, 0, last-first+1) + for p := first; p <= last; p++ { + ports = append(ports, p) + } + return ports, nil +} + +// CheckPublishedValues reports whether env (the values the container would get) +// publishes cleanly through ui_port and udp_port_range: the deploy validation asks it +// so a bad address or an oversized range is refused before anything is touched. +func (d DockerConfig) CheckPublishedValues(env map[string]string) error { + if d.UIPort != nil { + if _, err := UIBindings(env[d.UIPort.Env]); err != nil { + return fmt.Errorf("%s: %w", d.UIPort.Env, err) + } + } + if d.UDPPortRange != nil { + if _, err := ExpandUDPRange(env[d.UDPPortRange.Env], d.UDPPortRange.Cap()); err != nil { + return fmt.Errorf("%s: %w", d.UDPPortRange.Env, err) + } + } + return nil +} diff --git a/internal/catalog/netshape_test.go b/internal/catalog/netshape_test.go new file mode 100644 index 0000000..4f354a4 --- /dev/null +++ b/internal/catalog/netshape_test.go @@ -0,0 +1,68 @@ +package catalog + +import ( + "slices" + "testing" +) + +func TestUIBindings(t *testing.T) { + got, err := UIBindings(" 127.0.0.1 ,172.18.0.1,localhost,127.0.0.1") + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].String() != "127.0.0.1" || got[1].String() != "172.18.0.1" { + t.Fatalf("UIBindings = %v, want 127.0.0.1 and 172.18.0.1 once each", got) + } + if got, _ := UIBindings(""); len(got) != 1 || got[0].String() != "127.0.0.1" { + t.Fatalf("an empty list = %v, want loopback", got) + } + for _, bad := range []string{"256.0.0.1", "example.com", "::1", "127.0.0.1 10.0.0.1"} { + if _, err := UIBindings(bad); err == nil { + t.Errorf("UIBindings(%q) accepted it", bad) + } + } +} + +func TestExpandUDPRange(t *testing.T) { + got, err := ExpandUDPRange("23000:23002", 1000) + if err != nil || !slices.Equal(got, []int{23000, 23001, 23002}) { + t.Fatalf("ExpandUDPRange = %v, %v", got, err) + } + if got, err := ExpandUDPRange(" ", 1000); err != nil || got != nil { + t.Fatalf("a blank value = %v, %v, want nothing published", got, err) + } + if _, err := ExpandUDPRange("1:1001", 1000); err == nil { + t.Error("1001 ports passed a cap of 1000") + } + if _, err := ExpandUDPRange("1:1000", 1000); err != nil { + t.Errorf("exactly the cap was refused: %v", err) + } + for _, bad := range []string{"23002:23000", "0:10", "60000:70000", "23000", "a:b"} { + if _, err := ExpandUDPRange(bad, 1000); err == nil { + t.Errorf("ExpandUDPRange(%q) accepted it", bad) + } + } +} + +func TestUDPCapStaysWithinTheCeiling(t *testing.T) { + for in, want := range map[int]int{0: 1000, -1: 1000, 5000: 1000, 30: 30} { + if got := (UDPPortRange{MaxPorts: in}).Cap(); got != want { + t.Errorf("Cap(%d) = %d, want %d", in, got, want) + } + } +} + +func TestPublishesPorts(t *testing.T) { + if (DockerConfig{}).PublishesPorts() { + t.Fatal("nothing declared, nothing published") + } + for _, d := range []DockerConfig{ + {Ports: []string{"1:1"}}, + {UIPort: &UIPort{Env: "A", Port: 1}}, + {UDPPortRange: &UDPPortRange{Env: "B"}}, + } { + if !d.PublishesPorts() { + t.Errorf("%+v publishes a port", d) + } + } +} diff --git a/internal/catalog/schema_parity_test.go b/internal/catalog/schema_parity_test.go index 2ba407b..f1c9300 100644 --- a/internal/catalog/schema_parity_test.go +++ b/internal/catalog/schema_parity_test.go @@ -96,9 +96,35 @@ func TestMysteriumCapabilitiesAndDeviceParse(t *testing.T) { if svc.Payout.Model != "external" || svc.Payout.Chain != "polygon" || svc.Payout.AddressSource != "manual" { t.Errorf("mysterium payout = %+v, want model=external chain=polygon address_source=manual", svc.Payout) } - // devices_per_ip: 0 is a verified "no per-IP limit", not an omission. - if svc.Requirements.DevicesPerIP == nil || *svc.Requirements.DevicesPerIP != 0 { - t.Errorf("mysterium devices_per_ip = %v, want an explicit 0", svc.Requirements.DevicesPerIP) + // MystNodes runs one node per public IP; the others show offline. The web + // catalog changed 0 to 1 on purpose, citing the MystNodes help centre. + if svc.Requirements.DevicesPerIP == nil || *svc.Requirements.DevicesPerIP != 1 { + t.Errorf("mysterium devices_per_ip = %v, want an explicit 1", svc.Requirements.DevicesPerIP) + } + + // The fenced-bridge model: its own namespace, runc only, the WebUI and the UDP + // range published from the env, and no deploy outside the fence. + if svc.Docker.NetworkMode == "host" { + t.Error("mysterium must no longer share the host's network: NET_ADMIN there reaches the host firewall") + } + if svc.Docker.Runtime != "runc" { + t.Errorf("mysterium docker.runtime = %q, want runc", svc.Docker.Runtime) + } + if ui := svc.Docker.UIPort; ui == nil || ui.Env != "UI_ADDRESS" || ui.Port != 4449 { + t.Errorf("mysterium docker.ui_port = %+v, want UI_ADDRESS on 4449", ui) + } + if rng := svc.Docker.UDPPortRange; rng == nil || rng.Env != "UDP_PORTS" || rng.Cap() != 1000 { + t.Errorf("mysterium docker.udp_port_range = %+v, want UDP_PORTS capped at 1000", rng) + } + if !svc.Docker.FenceGated { + t.Error("mysterium docker.fence_gated must parse as true") + } + env := map[string]string{} + for _, item := range svc.Docker.Env { + env[item.Key] = item.Default + } + if err := svc.Docker.CheckPublishedValues(env); err != nil { + t.Errorf("mysterium's own defaults do not publish: %v", err) } // Desktop-only: the native block must survive the catalog sync. if !svc.HasNative() { diff --git a/internal/compose/compose.go b/internal/compose/compose.go index d848cf2..03cfeb2 100644 --- a/internal/compose/compose.go +++ b/internal/compose/compose.go @@ -116,6 +116,17 @@ func Generate(src Source, slugs []string, opts Options) (string, error) { if catalog.IsRetired(svc.Status) { return "", fmt.Errorf("%s is no longer available (%s), so there is nothing to export", svc.Name, svc.Status) } + if svc.Docker.FenceGated { + // Compose would start it on Docker's default bridge, where nothing keeps its + // NET_ADMIN namespace off the LAN (web app/compose_generator.py refuses too). + return "", fmt.Errorf("%s runs only behind CashPilot's LAN fence, so it has no compose export. Deploy it from the dashboard", svc.Name) + } + if svc.Docker.UIPort != nil || svc.Docker.UDPPortRange != nil { + // The port it publishes follows a setting, and the file carries settings as + // .env placeholders: a published port written from today's value would part + // from the one the command uses as soon as the .env changes it. + return "", fmt.Errorf("%s publishes ports from its settings, which a compose file cannot follow, so it has no compose export. Deploy it from the dashboard", svc.Name) + } if svc.Custom || catalog.IsCustomSlug(slug) { // The export is built from catalog entries; a custom service's variables and // command are the user's own and stay encrypted in the app's database. @@ -168,6 +179,7 @@ type composeService struct { Volumes []string `yaml:"volumes,omitempty"` Devices []string `yaml:"devices,omitempty"` NetworkMode string `yaml:"network_mode,omitempty"` + Runtime string `yaml:"runtime,omitempty"` CapDrop []string `yaml:"cap_drop,omitempty"` CapAdd []string `yaml:"cap_add,omitempty"` SecurityOpt []string `yaml:"security_opt,omitempty"` @@ -247,6 +259,9 @@ func serviceBlock(svc catalog.Service, family, hostname string) (*composeService Ports: ports(svc.Docker.Ports), Devices: devices, NetworkMode: svc.Docker.NetworkMode, + // The runtime the entry requires, as the deploy sets it: the daemon's default + // (gVisor, say) may be one the service cannot run under. + Runtime: strings.TrimSpace(svc.Docker.Runtime), // Every capability dropped, then only the ones this entry declares added // back, and no path to new privileges. Same set the web CashPilot worker // runs these images under. diff --git a/internal/compose/compose_test.go b/internal/compose/compose_test.go index 5e1a5dd..5970d72 100644 --- a/internal/compose/compose_test.go +++ b/internal/compose/compose_test.go @@ -553,6 +553,13 @@ func TestEveryShippingServiceExports(t *testing.T) { if svc.Docker.Image == "" { continue } + if svc.Docker.FenceGated { + // Refused on purpose (TestAFenceGatedServiceHasNoComposeExport). + if _, err := Generate(cat, []string{svc.Slug}, Options{}); err == nil { + t.Errorf("%s is fence-gated and exported anyway", svc.Slug) + } + continue + } exported++ text, err := Generate(cat, []string{svc.Slug}, Options{Hostname: "mac-mini"}) if err != nil { @@ -602,7 +609,7 @@ func TestEveryExportedImageIsPinnedExceptTheKnownFloatingTags(t *testing.T) { seen := map[string]bool{} checked := 0 for _, svc := range cat.ListVisible() { - if svc.Docker.Image == "" { + if svc.Docker.Image == "" || svc.Docker.FenceGated { continue } for _, arch := range []string{"", "amd64", "arm64", "arm"} { @@ -722,3 +729,44 @@ func TestAnUnfilledHostPathIsARequiredVariable(t *testing.T) { t.Errorf("the environment block changed too: %v", env["IDENTITY_DIR"]) } } + +// A fence-gated service runs only behind the LAN fence; compose would start it on +// Docker's default bridge, so the export refuses it (web app/compose_generator.py). +func TestAFenceGatedServiceHasNoComposeExport(t *testing.T) { + gated := honeygain() + gated.Slug, gated.Name = "mysterium", "MystNodes" + gated.Docker.FenceGated = true + _, err := Generate(fakeCatalog{"mysterium": gated, "honeygain": honeygain()}, []string{"honeygain", "mysterium"}, Options{}) + if err == nil || !strings.Contains(err.Error(), "LAN fence") { + t.Fatalf("err = %v, want the export refused naming the fence", err) + } +} + +// A port published from a setting cannot be written into a file that carries the +// setting as a placeholder, so the export refuses it. +func TestAServicePublishingFromItsSettingsHasNoComposeExport(t *testing.T) { + for _, mutate := range []func(*catalog.Service){ + func(s *catalog.Service) { s.Docker.UIPort = &catalog.UIPort{Env: "UI", Port: 4449} }, + func(s *catalog.Service) { s.Docker.UDPPortRange = &catalog.UDPPortRange{Env: "UDP"} }, + } { + svc := honeygain() + mutate(&svc) + if _, err := Generate(fakeCatalog{"honeygain": svc}, []string{"honeygain"}, Options{}); err == nil || !strings.Contains(err.Error(), "from its settings") { + t.Errorf("err = %v, want the export refused", err) + } + } +} + +// The runtime the entry requires travels with the file. +func TestTheRequiredRuntimeIsExported(t *testing.T) { + svc := honeygain() + svc.Docker.Runtime = "runc" + block := serviceBlockOf(t, parse(t, generate(t, fakeCatalog{"honeygain": svc}, []string{"honeygain"}, Options{})), "cashpilot-honeygain") + if block["runtime"] != "runc" { + t.Fatalf("runtime = %v, want runc", block["runtime"]) + } + plain := parse(t, generate(t, fakeCatalog{"honeygain": honeygain()}, []string{"honeygain"}, Options{})) + if _, ok := serviceBlockOf(t, plain, "cashpilot-honeygain")["runtime"]; ok { + t.Fatal("an entry naming no runtime must leave the daemon's default") + } +} diff --git a/internal/proxies/proxies.go b/internal/proxies/proxies.go index f12296b..c06ff38 100644 --- a/internal/proxies/proxies.go +++ b/internal/proxies/proxies.go @@ -333,7 +333,10 @@ func Refusal(svc catalog.Service) string { if strings.HasPrefix(dc.NetworkMode, "container:") { return name + " joins another container's network, so it cannot be moved behind a proxy." } - if len(dc.Ports) > 0 { + if dc.FenceGated { + return name + " runs only behind the LAN fence, so it cannot run through a proxy." + } + if dc.PublishesPorts() { return name + " publishes a port on the host, which a proxied copy cannot do." } if len(dc.Volumes) > 0 { diff --git a/internal/proxies/proxies_test.go b/internal/proxies/proxies_test.go index a150150..48e36dd 100644 --- a/internal/proxies/proxies_test.go +++ b/internal/proxies/proxies_test.go @@ -90,6 +90,7 @@ func TestRefusal(t *testing.T) { "host's network": func(s *catalog.Service) { s.Docker.NetworkMode = "host" }, "another container": func(s *catalog.Service) { s.Docker.NetworkMode = "container:x" }, "publishes a port": func(s *catalog.Service) { s.Docker.Ports = []string{"1:1"} }, + "LAN fence": func(s *catalog.Service) { s.Docker.FenceGated = true }, "volume": func(s *catalog.Service) { s.Docker.Volumes = []string{"v:/d"} }, "device": func(s *catalog.Service) { s.Docker.Devices = []string{"/dev/net/tun"} }, "privileged": func(s *catalog.Service) { s.Docker.Privileged = true }, @@ -105,6 +106,17 @@ func TestRefusal(t *testing.T) { t.Errorf("Refusal = %q, want it to mention %q", r, want) } } + // Ports published from the env are published ports too. + for _, mutate := range []func(*catalog.Service){ + func(s *catalog.Service) { s.Docker.UIPort = &catalog.UIPort{Env: "A", Port: 1} }, + func(s *catalog.Service) { s.Docker.UDPPortRange = &catalog.UDPPortRange{Env: "B"} }, + } { + svc := base + mutate(&svc) + if r := Refusal(svc); !strings.Contains(r, "publishes a port") { + t.Errorf("Refusal = %q, want it to mention the published port", r) + } + } ok := base ok.Docker.CapAdd = []string{"SETUID"} if r := Refusal(ok); r != "" { diff --git a/internal/runtime/arch.go b/internal/runtime/arch.go index c5e6f54..96caf6e 100644 --- a/internal/runtime/arch.go +++ b/internal/runtime/arch.go @@ -243,6 +243,9 @@ type daemonInfo struct { // Windows containers, and that daemon REFUSES cap_add/cap_drop outright, so the // Linux-only hardening has to be skipped there rather than fail every deploy. OSType string + // Runtimes are the OCI runtimes the daemon offers ("runc", "runsc"), nil when it + // did not say. A service whose entry names a runtime is checked against it. + Runtimes map[string]bool } // infoClient is the one call daemonFacts needs, narrowed so the fallback behaviour is @@ -267,5 +270,11 @@ func daemonFacts(ctx context.Context, cli infoClient) daemonInfo { if osType := strings.TrimSpace(strings.ToLower(result.Info.OSType)); osType != "" { facts.OSType = osType } + if len(result.Info.Runtimes) > 0 { + facts.Runtimes = make(map[string]bool, len(result.Info.Runtimes)) + for name := range result.Info.Runtimes { + facts.Runtimes[name] = true + } + } return facts } diff --git a/internal/runtime/container_drift.go b/internal/runtime/container_drift.go index bfa9018..d29fee2 100644 --- a/internal/runtime/container_drift.go +++ b/internal/runtime/container_drift.go @@ -53,11 +53,13 @@ type ContainerShape struct { CapDrop []string Privileged bool NetworkMode string - Devices []string - Ports []string - Mounts []string - EnvKeys []string - Entrypoint []string + // Runtime is the OCI runtime the container runs under, as inspect reports it. + Runtime string + Devices []string + Ports []string + Mounts []string + EnvKeys []string + Entrypoint []string } // shapeClient is the part of the Docker client ContainerShapes needs, so the @@ -135,6 +137,7 @@ func shapeOf(ctr container.InspectResponse) ContainerShape { shape.CapDrop = capNames(host.CapDrop) shape.Privileged = host.Privileged shape.NetworkMode = string(host.NetworkMode) + shape.Runtime = host.Runtime devices := map[string]bool{} for _, d := range host.Devices { if path := strings.TrimRight(d.PathInContainer, "/"); path != "" { @@ -200,6 +203,11 @@ func ShapeDrift(svc catalog.Service, shape ContainerShape, customImage string) [ found = append(found, fmt.Sprintf("still has device %s, which the catalog no longer passes in", strings.Join(extra, ", "))) } + // A container that reports no runtime is not judged: nothing is known. + if want := strings.TrimSpace(dc.Runtime); want != "" && shape.Runtime != "" && shape.Runtime != want { + found = append(found, fmt.Sprintf("runs under the %s runtime; the catalog requires %s", shape.Runtime, want)) + } + if shape.Privileged && !dc.Privileged { found = append(found, "runs privileged, which CashPilot no longer allows") } @@ -217,9 +225,18 @@ func ShapeDrift(svc catalog.Service, shape ContainerShape, customImage string) [ case !wantHost && haveHost: found = append(found, "shares the host's network") case !wantHost: - if missing := minus(catalogPortKeys(dc.Ports), toSet(shape.Ports)); len(missing) > 0 { + want := catalogPortKeys(dc.Ports) + if dc.UIPort != nil { + want[fmt.Sprintf("%d/tcp", dc.UIPort.Port)] = true + } + if missing := minus(want, toSet(shape.Ports)); len(missing) > 0 { found = append(found, fmt.Sprintf("does not publish port %s", strings.Join(missing, ", "))) } + // Which range is the user's setting, which the shape does not carry, so only + // a container publishing none of it is judged. + if dc.UDPPortRange != nil && !slices.ContainsFunc(shape.Ports, func(p string) bool { return strings.HasSuffix(p, "/udp") }) { + found = append(found, "publishes none of its UDP ports") + } } wantMounts := map[string]bool{} @@ -293,7 +310,12 @@ func ShapeFingerprint(svc catalog.Service) string { Ports []string `json:"ports"` Volumes []string `json:"volumes"` Env []envField `json:"env"` - }{dc.Image, dc.ImageByArch, dc.CapAdd, dc.Devices, dc.Privileged, dc.Entrypoint, dc.NetworkMode, dc.Ports, dc.Volumes, env} + // Only when set, so an entry without them keeps the fingerprint (and the + // Ignore) it had before these fields existed. + UIPort *catalog.UIPort `json:"ui_port,omitempty"` + UDPPortRange *catalog.UDPPortRange `json:"udp_port_range,omitempty"` + Runtime string `json:"runtime,omitempty"` + }{dc.Image, dc.ImageByArch, dc.CapAdd, dc.Devices, dc.Privileged, dc.Entrypoint, dc.NetworkMode, dc.Ports, dc.Volumes, env, dc.UIPort, dc.UDPPortRange, strings.TrimSpace(dc.Runtime)} blob, _ := json.Marshal(judged) // map keys are sorted by encoding/json sum := sha256.Sum256(blob) return hex.EncodeToString(sum[:])[:16] diff --git a/internal/runtime/container_drift_test.go b/internal/runtime/container_drift_test.go index 80c11ce..538e467 100644 --- a/internal/runtime/container_drift_test.go +++ b/internal/runtime/container_drift_test.go @@ -38,9 +38,9 @@ func bitping(capAdd ...string) catalog.Service { func deployedShape(t *testing.T, svc catalog.Service, creds map[string]string) ContainerShape { t.Helper() env := buildEnv(svc, creds) - _, bindings, err := buildPorts(svc.Docker.Ports) + _, bindings, err := buildServicePorts(svc.Docker, env) if err != nil { - t.Fatalf("buildPorts: %v", err) + t.Fatalf("buildServicePorts: %v", err) } mounts := buildMounts(svc.Docker.Volumes, env) host, err := buildHostConfig(svc, bindings, mounts, "linux") diff --git a/internal/runtime/fence_gated_lock_test.go b/internal/runtime/fence_gated_lock_test.go new file mode 100644 index 0000000..f4b1212 --- /dev/null +++ b/internal/runtime/fence_gated_lock_test.go @@ -0,0 +1,161 @@ +package runtime + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/GeiserX/CashPilot-Desktop/e2e/harness" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/network" +) + +// A fence-gated start holds the reconcile lock from the fence check to the start, +// so a reconcile that turns isolation off cannot reduce the fence in between. +func TestAFenceGatedStartWaitsForAReconcile(t *testing.T) { + docker := harness.NewDocker() + t.Cleanup(docker.Close) + t.Setenv("DOCKER_HOST", docker.Host()) + t.Setenv("DOCKER_TLS_VERIFY", "") + t.Setenv("DOCKER_CERT_PATH", "") + t.Setenv("DOCKER_API_VERSION", "") + + p := NewDockerProvider() + svc := mysteriumShaped() + if _, err := p.Deploy(context.Background(), DeploySpec{Slug: "mysterium", Service: svc}, nil); err != nil { + t.Fatalf("Deploy: %v", err) + } + docker.SetState("cashpilot-mysterium", "exited") + + p.fence.reconcileMu.Lock() // a reconcile in progress + docker.ResetCalls() + done := make(chan error, 1) + go func() { done <- p.Start(context.Background(), "mysterium") }() + // The inspect just before the lock, so the check below cannot pass only + // because the goroutine had not got that far yet. + waitForCall(t, docker, "GET /containers/cashpilot-mysterium/json") + time.Sleep(200 * time.Millisecond) + if c, _ := docker.Container("cashpilot-mysterium"); c.State == "running" { + p.fence.reconcileMu.Unlock() + t.Fatal("the node started while a reconcile held the fence") + } + p.fence.reconcileMu.Unlock() + select { + case err := <-done: + if err != nil { + t.Fatalf("Start: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Start never finished after the reconcile") + } + if c, _ := docker.Container("cashpilot-mysterium"); c.State != "running" { + t.Fatalf("state = %q, want running", c.State) + } +} + +// An ordinary earner's start takes no lock, so it never waits on the fence helper. +func TestAnOrdinaryStartDoesNotWaitForAReconcile(t *testing.T) { + docker := harness.NewDocker() + t.Cleanup(docker.Close) + t.Setenv("DOCKER_HOST", docker.Host()) + t.Setenv("DOCKER_TLS_VERIFY", "") + t.Setenv("DOCKER_CERT_PATH", "") + t.Setenv("DOCKER_API_VERSION", "") + + p := NewDockerProvider() + svc := mysteriumShaped() + svc.Slug, svc.Docker.FenceGated, svc.Docker.Runtime = "plain", false, "" + svc.Docker.CapAdd, svc.Docker.Devices = nil, nil + if _, err := p.Deploy(context.Background(), DeploySpec{Slug: "plain", Service: svc}, nil); err != nil { + t.Fatalf("Deploy: %v", err) + } + docker.SetState("cashpilot-plain", "exited") + p.fence.reconcileMu.Lock() + defer p.fence.reconcileMu.Unlock() + done := make(chan error, 1) + go func() { done <- p.Start(context.Background(), "plain") }() + select { + case err := <-done: + if err != nil { + t.Fatalf("Start: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("an ordinary start waited on the reconcile lock") + } +} + +func TestOnlyOnFence(t *testing.T) { + on := func(mode string, nets ...string) container.InspectResponse { + ns := map[string]*network.EndpointSettings{} + for _, n := range nets { + ns[n] = &network.EndpointSettings{} + } + return container.InspectResponse{ + HostConfig: &container.HostConfig{NetworkMode: container.NetworkMode(mode)}, + NetworkSettings: &container.NetworkSettings{Networks: ns}, + } + } + if p := onlyOnFence(on(FenceNetwork, FenceNetwork)); p != "" { + t.Fatalf("on the fence alone: %q", p) + } + for name, c := range map[string]container.InspectResponse{ + "host networking": on("host", "host"), + "also elsewhere": on(FenceNetwork, FenceNetwork, "bridge"), + "not attached": on(FenceNetwork), + "no host config": {}, + "no networks": {HostConfig: &container.HostConfig{NetworkMode: FenceNetwork}}, + } { + if onlyOnFence(c) == "" { + t.Errorf("%s passed", name) + } + } +} + +// A fence-gated deploy holds the same lock from the fence check to the start. +func TestAFenceGatedDeployWaitsForAReconcile(t *testing.T) { + docker := harness.NewDocker() + t.Cleanup(docker.Close) + t.Setenv("DOCKER_HOST", docker.Host()) + t.Setenv("DOCKER_TLS_VERIFY", "") + t.Setenv("DOCKER_CERT_PATH", "") + t.Setenv("DOCKER_API_VERSION", "") + + p := NewDockerProvider() + p.fence.reconcileMu.Lock() + done := make(chan error, 1) + go func() { + _, err := p.Deploy(context.Background(), DeploySpec{Slug: "mysterium", Service: mysteriumShaped()}, nil) + done <- err + }() + // The image inspect comes just before the lock. + waitForCall(t, docker, "GET /images/") + time.Sleep(200 * time.Millisecond) + if _, ok := docker.Container("cashpilot-mysterium"); ok { + p.fence.reconcileMu.Unlock() + t.Fatal("the node was created while a reconcile held the fence") + } + p.fence.reconcileMu.Unlock() + select { + case err := <-done: + if err != nil { + t.Fatalf("Deploy: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Deploy never finished after the reconcile") + } +} + +// waitForCall waits until the fake daemon has received a call starting with prefix. +func waitForCall(t *testing.T, docker *harness.Docker, prefix string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if slices.ContainsFunc(docker.Calls(), func(c string) bool { return strings.HasPrefix(c, prefix) }) { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("the daemon never received %s; calls: %v", prefix, docker.Calls()) +} diff --git a/internal/runtime/fence_gated_test.go b/internal/runtime/fence_gated_test.go new file mode 100644 index 0000000..91abd95 --- /dev/null +++ b/internal/runtime/fence_gated_test.go @@ -0,0 +1,313 @@ +package runtime + +import ( + "context" + "net/netip" + "strconv" + "strings" + "testing" + + "github.com/GeiserX/CashPilot-Desktop/internal/catalog" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/network" + "github.com/moby/moby/api/types/system" + "github.com/moby/moby/client" +) + +// mysteriumShaped is the network shape of the real Mysterium entry since the web +// moved it off host networking: no static ports, the WebUI on each UI_ADDRESS +// address, the UDP range 1:1, runc only, and fence-gated. +func mysteriumShaped() catalog.Service { + return catalog.Service{ + Name: "MystNodes", + Slug: "mysterium", + Docker: catalog.DockerConfig{ + Image: "mysteriumnetwork/myst:1.39.9-alpine@sha256:f89347d244efeeaf29555e876404cf8aebcd41561c5cbed06c365ba64beb8564", + Env: []catalog.EnvVar{ + {Key: "UI_ADDRESS", Default: "127.0.0.1"}, + {Key: "UDP_PORTS", Default: "23000:23029", PortRange: true}, + }, + Runtime: "runc", + UIPort: &catalog.UIPort{Env: "UI_ADDRESS", Port: 4449}, + UDPPortRange: &catalog.UDPPortRange{Env: "UDP_PORTS", MaxPorts: 1000}, + FenceGated: true, + CapAdd: []string{"NET_ADMIN", "SETUID", "SETGID"}, + Devices: []string{"/dev/net/tun"}, + }, + } +} + +func TestTheWebUIIsPublishedOnEachAddressAndTheUDPRangeOneToOne(t *testing.T) { + svc := mysteriumShaped() + env := buildEnv(svc, map[string]string{"UI_ADDRESS": "127.0.0.1,172.18.0.1,localhost", "UDP_PORTS": "23000:23004"}) + ports, bindings, err := buildServicePorts(svc.Docker, env) + if err != nil { + t.Fatalf("buildServicePorts: %v", err) + } + ui := bindings[network.MustParsePort("4449/tcp")] + want := []network.PortBinding{ + {HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "4449"}, + {HostIP: netip.MustParseAddr("172.18.0.1"), HostPort: "4449"}, + } + if len(ui) != len(want) || ui[0] != want[0] || ui[1] != want[1] { + t.Fatalf("4449/tcp bindings = %+v, want %+v (localhost folded into 127.0.0.1)", ui, want) + } + for p := 23000; p <= 23004; p++ { + key := network.MustParsePort(strings.Join([]string{itoa(p), "udp"}, "/")) + if _, ok := ports[key]; !ok { + t.Errorf("%s is not exposed", key) + } + if b := bindings[key]; len(b) != 1 || b[0].HostPort != itoa(p) { + t.Errorf("%s bindings = %+v, want host port %d (1:1)", key, b, p) + } + } + if len(bindings) != 6 { + t.Errorf("published %d ports, want the WebUI and 5 UDP ports", len(bindings)) + } +} + +func TestTheCatalogDefaultsPublishLoopbackAndTheDefaultRange(t *testing.T) { + svc := mysteriumShaped() + _, bindings, err := buildServicePorts(svc.Docker, buildEnv(svc, nil)) + if err != nil { + t.Fatalf("buildServicePorts: %v", err) + } + if ui := bindings[network.MustParsePort("4449/tcp")]; len(ui) != 1 || ui[0].HostIP != netip.MustParseAddr("127.0.0.1") { + t.Fatalf("4449/tcp bindings = %+v, want loopback only", ui) + } + if len(bindings) != 31 { + t.Fatalf("published %d ports, want the WebUI and the 30 default UDP ports", len(bindings)) + } +} + +func TestABadAddressOrRangeIsRefusedBeforeAnythingIsPublished(t *testing.T) { + svc := mysteriumShaped() + svc.Docker.UDPPortRange.MaxPorts = 10 + for name, creds := range map[string]map[string]string{ + "not an address": {"UI_ADDRESS": "127.0.0.1,example.com"}, + "IPv6": {"UI_ADDRESS": "::1"}, + "reversed range": {"UDP_PORTS": "23010:23000"}, + "over the cap": {"UDP_PORTS": "23000:23010"}, + "not a range": {"UDP_PORTS": "23000"}, + } { + if _, _, err := buildServicePorts(svc.Docker, buildEnv(svc, creds)); err == nil { + t.Errorf("%s: %v was published, want a refusal", name, creds) + } + } + if _, _, err := buildServicePorts(svc.Docker, buildEnv(svc, map[string]string{"UDP_PORTS": "23000:23009"})); err != nil { + t.Fatalf("a range of exactly the cap was refused: %v", err) + } +} + +func TestTheRuntimeTheEntryNamesIsSetAndChecked(t *testing.T) { + svc := mysteriumShaped() + host, err := buildHostConfig(svc, nil, nil, "linux") + if err != nil { + t.Fatalf("buildHostConfig: %v", err) + } + if host.Runtime != "runc" { + t.Fatalf("Runtime = %q, want runc whatever the daemon's default", host.Runtime) + } + plain := svc + plain.Docker.Runtime = "" + if host, _ := buildHostConfig(plain, nil, nil, "linux"); host.Runtime != "" { + t.Fatalf("an entry naming no runtime got %q, want the daemon's default", host.Runtime) + } + + if err := checkRuntime(svc, map[string]bool{"runc": true, "runsc": true}); err != nil { + t.Fatalf("runc is available: %v", err) + } + if err := checkRuntime(svc, map[string]bool{"crun": true}); err == nil || !strings.Contains(err.Error(), "crun") { + t.Fatalf("a daemon without runc: err = %v, want a refusal naming what it has", err) + } + if err := checkRuntime(svc, nil); err == nil { + t.Fatal("a daemon that did not list its runtimes must be refused: the create would fail after the removal") + } + if err := checkRuntime(plain, nil); err != nil { + t.Fatalf("an entry naming no runtime needs no check: %v", err) + } +} + +type runtimesInfo struct { + runtimes map[string]system.RuntimeWithStatus +} + +func (r runtimesInfo) Info(context.Context, client.InfoOptions) (client.SystemInfoResult, error) { + return client.SystemInfoResult{Info: system.Info{Architecture: "x86_64", OSType: "linux", Runtimes: r.runtimes}}, nil +} + +func TestDaemonFactsListTheRuntimes(t *testing.T) { + facts := daemonFacts(context.Background(), runtimesInfo{map[string]system.RuntimeWithStatus{"runc": {}, "runsc": {}}}) + if !facts.Runtimes["runc"] || !facts.Runtimes["runsc"] || len(facts.Runtimes) != 2 { + t.Fatalf("Runtimes = %v, want runc and runsc", facts.Runtimes) + } + if facts := daemonFacts(context.Background(), runtimesInfo{}); facts.Runtimes != nil { + t.Fatalf("a daemon listing none gave %v, want nil (unknown)", facts.Runtimes) + } +} + +func TestAFenceGatedServiceDeploysOnlyOnTheFencedNetwork(t *testing.T) { + svc := mysteriumShaped() + on := &container.HostConfig{NetworkMode: FenceNetwork} + if p := fenceGateProblem(svc, on, FenceNetwork, "linux", FenceModeOn); p != "" { + t.Fatalf("on the fenced network: %q, want no problem", p) + } + bridge := &container.HostConfig{NetworkMode: "bridge"} + for mode, want := range map[string]string{ + FenceModeOff: "off in Settings", + FenceModeLegacy: "Turn on isolation", + FenceModeUnsupported: "Podman", + "": "off in Settings", + } { + p := fenceGateProblem(svc, bridge, "", "linux", mode) + if !strings.Contains(p, "only behind the LAN fence") || !strings.Contains(p, want) || !strings.Contains(p, "not touched") { + t.Errorf("mode %q: %q, want a refusal mentioning %q", mode, p, want) + } + } + if p := fenceGateProblem(svc, &container.HostConfig{NetworkMode: "host"}, "", "linux", FenceModeOn); !strings.Contains(p, "outside the fence") { + t.Errorf("host networking: %q, want a refusal", p) + } + if p := fenceGateProblem(svc, &container.HostConfig{NetworkMode: "nat"}, "", "windows", ""); !strings.Contains(p, "Windows containers") { + t.Errorf("Windows containers: %q, want a refusal", p) + } + + plain := svc + plain.Docker.FenceGated = false + if p := fenceGateProblem(plain, bridge, "", "linux", FenceModeOff); p != "" { + t.Fatalf("an ordinary earner on the default bridge was refused: %q", p) + } +} + +func TestAFencedMysteriumMissingItsWebUIPortIsDrift(t *testing.T) { + svc := mysteriumShaped() + shape := deployedShape(t, svc, nil) + if found := ShapeDrift(svc, shape, ""); len(found) != 0 { + t.Fatalf("a freshly deployed node shows drift: %v", found) + } + var kept []string + for _, p := range shape.Ports { + if p != "4449/tcp" && !strings.HasSuffix(p, "/udp") { + kept = append(kept, p) + } + } + shape.Ports = kept + got := strings.Join(ShapeDrift(svc, shape, ""), "; ") + if !strings.Contains(got, "4449/tcp") || !strings.Contains(got, "UDP") { + t.Fatalf("drift = %q, want the missing WebUI port and UDP range named", got) + } +} + +func itoa(n int) string { return strconv.Itoa(n) } + +// An entry that names the fenced network itself never went through the fence, so +// it is refused like any other network: the fence's rules may not be there. +func TestNamingTheFencedNetworkDoesNotSkipTheFence(t *testing.T) { + svc := mysteriumShaped() + named := &container.HostConfig{NetworkMode: FenceNetwork} + if p := fenceGateProblem(svc, named, "", "linux", FenceModeOff); !strings.Contains(p, "only behind the LAN fence") { + t.Fatalf("a fenced network the fence did not hand out: %q, want a refusal", p) + } +} + +// A blank (or whitespace-only) UI_ADDRESS or UDP_PORTS keeps the catalog default, as the deploy +// validation assumes; otherwise the node would get no UDP ports and an empty +// --udp.ports. +func TestABlankPublishedSettingKeepsItsDefault(t *testing.T) { + svc := mysteriumShaped() + for _, blank := range []string{"", " "} { + env := buildEnv(svc, map[string]string{"UI_ADDRESS": blank, "UDP_PORTS": blank}) + if env["UDP_PORTS"] != "23000:23029" || env["UI_ADDRESS"] != "127.0.0.1" { + t.Fatalf("blank %q: env = %v, want the defaults", blank, env) + } + _, bindings, err := buildServicePorts(svc.Docker, env) + if err != nil || len(bindings) != 31 { + t.Fatalf("blank %q: published %d ports (%v), want the WebUI and the 30 default UDP ports", blank, len(bindings), err) + } + } + // Any other variable may still be blanked on purpose. + other := svc + other.Docker.Env = append(other.Docker.Env, catalog.EnvVar{Key: "EXTRA", Default: "x"}) + if got := buildEnv(other, map[string]string{"EXTRA": ""})["EXTRA"]; got != "" { + t.Fatalf("EXTRA = %q, want the blank the user chose", got) + } +} + +func TestARuntimeOtherThanTheCatalogsIsDrift(t *testing.T) { + svc := mysteriumShaped() + shape := deployedShape(t, svc, nil) + if shape.Runtime != "runc" { + t.Fatalf("shape runtime = %q, want runc", shape.Runtime) + } + shape.Runtime = "runsc" + if got := strings.Join(ShapeDrift(svc, shape, ""), "; "); !strings.Contains(got, "runsc") || !strings.Contains(got, "requires runc") { + t.Fatalf("drift = %q, want the runtime named", got) + } + shape.Runtime = "" + if found := ShapeDrift(svc, shape, ""); len(found) != 0 { + t.Fatalf("a container reporting no runtime was judged: %v", found) + } + plain := svc + plain.Docker.Runtime = "" + if ShapeFingerprint(plain) == ShapeFingerprint(svc) { + t.Fatal("a runtime-only catalog change keeps the fingerprint, so an Ignore would hide it") + } +} + +func TestAFenceGatedContainerIsLabelled(t *testing.T) { + svc := mysteriumShaped() + if gatedLabels(containerLabels("mysterium", 1), svc)[LabelFenceGated] != "true" { + t.Fatal("a fence-gated container must carry the label the fence stops it by") + } + svc.Docker.FenceGated = false + if _, ok := gatedLabels(containerLabels("x", 1), svc)[LabelFenceGated]; ok { + t.Fatal("an ordinary earner must not carry the fence-gated label") + } +} + +// A fence that cannot be put back in on mode must not leave a fence-gated node +// running: the table may already be gone, and the fence is the only thing keeping it +// off the host's own addresses. Other earners keep running; a fence that applies +// leaves the node alone. +func TestAFailedFenceStopsTheFenceGatedNode(t *testing.T) { + earner := func(id string, labels map[string]string) map[string]any { + return map[string]any{ + "Id": id, "Names": []string{"/cashpilot-" + id}, "State": "running", "Labels": labels, + "HostConfig": map[string]any{"NetworkMode": FenceNetwork}, + "NetworkSettings": map[string]any{"Networks": map[string]any{FenceNetwork: map[string]any{"NetworkID": "fencenet"}}}, + } + } + gated := earner("mysterium", map[string]string{LabelManaged: "true", LabelService: "mysterium", LabelFenceGated: "true"}) + plain := earner("honeygain", map[string]string{LabelManaged: "true", LabelService: "honeygain"}) + stops := func(fake *fakeDocker) (gatedStopped, plainStopped bool) { + for _, call := range fake.recorded() { + gatedStopped = gatedStopped || call == "POST /containers/mysterium/stop" + plainStopped = plainStopped || call == "POST /containers/honeygain/stop" + } + return + } + + for _, exit := range []int{1, 0} { + fake := &fakeDocker{earners: []map[string]any{gated, plain}, fenceExit: exit} + fake.start(t) + p := &DockerProvider{} + p.SetIsolationSetting(func() string { return "on" }) + p.ReconcileFence(context.Background()) + st := p.FenceStatus() + gatedStopped, plainStopped := stops(fake) + if plainStopped { + t.Errorf("exit %d: an earner that is not fence-gated was stopped (calls %v)", exit, fake.recorded()) + } + if exit != 0 { + if st.Mode != FenceModeOn || st.State != "failed" { + t.Fatalf("exit %d: status %+v, want on and failed", exit, st) + } + if !gatedStopped { + t.Fatalf("the fence failed (%s) but the fence-gated node was not stopped (calls %v)", st.Error, fake.recorded()) + } + continue + } + if st.State != "applied" || gatedStopped { + t.Fatalf("a fence that applied: status %+v, node stopped %v", st, gatedStopped) + } + } +} diff --git a/internal/runtime/lanfence.go b/internal/runtime/lanfence.go index f93b5ea..f268d87 100644 --- a/internal/runtime/lanfence.go +++ b/internal/runtime/lanfence.go @@ -146,6 +146,7 @@ type fenceAPI interface { ContainerInspect(ctx context.Context, containerID string, options client.ContainerInspectOptions) (client.ContainerInspectResult, error) ContainerCreate(ctx context.Context, options client.ContainerCreateOptions) (client.ContainerCreateResult, error) ContainerStart(ctx context.Context, containerID string, options client.ContainerStartOptions) (client.ContainerStartResult, error) + ContainerStop(ctx context.Context, containerID string, options client.ContainerStopOptions) (client.ContainerStopResult, error) ContainerWait(ctx context.Context, containerID string, options client.ContainerWaitOptions) client.ContainerWaitResult ContainerLogs(ctx context.Context, containerID string, options client.ContainerLogsOptions) (client.ContainerLogsResult, error) ContainerRemove(ctx context.Context, containerID string, options client.ContainerRemoveOptions) (client.ContainerRemoveResult, error) @@ -190,12 +191,14 @@ type fence struct { // (NAT loopback), so the table drops that address too. egress and egressAt cache // it; mu guards them. lookupEgress func(ctx context.Context) string - egress string - egressAt time.Time - mu sync.Mutex // the helper and the apply sequence - reconcileMu sync.Mutex // whole reconciles, so one cannot overwrite a later one's status - stateMu sync.Mutex // status - status FenceStatus + // gatedSlug reports a service the catalog marks fence_gated (SetFenceGated). + gatedSlug func(slug string) bool + egress string + egressAt time.Time + mu sync.Mutex // the helper and the apply sequence + reconcileMu sync.Mutex // whole reconciles, so one cannot overwrite a later one's status + stateMu sync.Mutex // status + status FenceStatus } // FenceSetting normalises a Settings value: auto, on or off. Anything else is auto. @@ -954,18 +957,37 @@ func (f *fence) reconcile(ctx context.Context, cli fenceAPI) { f.setStatus(func(s *FenceStatus) { s.State, s.Error = "failed", err.Error() }) return } + var gatedErr error + if mode != FenceModeOn { + // Before the drops go: a fence-gated earner must not outlive the fence. + gatedErr = f.stopGated(ctx, cli) + } switch { case mode == FenceModeOn: - _ = f.apply(ctx, cli) + if err := f.apply(ctx, cli); err != nil { + // The table may be gone (a ruleset flush, a runtime restart that brought + // the node back), so a fence-gated earner must not keep running on it. + if gatedErr = f.stopGated(ctx, cli); gatedErr != nil { + f.setStatus(func(s *FenceStatus) { s.Error = withGatedErr(s.Error, gatedErr) }) + } + } if fresh, err := listEarners(ctx, cli); err == nil { earners = fresh } + case mode == FenceModeOff && net != nil && gatedErr != nil: + // The drops stay until every fence-gated earner is stopped; the next + // reconcile tries again. + f.setStatus(func(s *FenceStatus) { + s.State, s.Error = "failed", gatedErr.Error()+". The LAN fence was kept" + }) case mode == FenceModeOff && net != nil: f.reduce(ctx, cli, net) case mode == FenceModeUnsupported: - f.setStatus(func(s *FenceStatus) { s.State, s.Backend, s.Error = "unsupported", "", podmanUnsupported+"." }) + f.setStatus(func(s *FenceStatus) { + s.State, s.Backend, s.Error = "unsupported", "", withGatedErr(podmanUnsupported+".", gatedErr) + }) default: - f.setStatus(func(s *FenceStatus) { s.State, s.Backend, s.Error = "n/a", "", "" }) + f.setStatus(func(s *FenceStatus) { s.State, s.Backend, s.Error = "n/a", "", withGatedErr("", gatedErr) }) } // Settings changed while this ran: its mode is stale, and the reconcile the // change triggers publishes the right one. @@ -975,6 +997,64 @@ func (f *fence) reconcile(ctx context.Context, cli fenceAPI) { f.setOutside(mode, earners) } +// withGatedErr adds a failed stop of a fence-gated earner to a status message, so a +// node still running without the fence shows on the dashboard instead of an empty +// error. The next reconcile tries the stop again. +func withGatedErr(msg string, gatedErr error) string { + if gatedErr == nil { + return msg + } + if msg == "" { + return gatedErr.Error() + } + return msg + " " + gatedErr.Error() +} + +// isGated reports a fence-gated container: labelled so at deploy, or a service the +// catalog marks fence_gated, which covers containers made before the label. +func (f *fence) isGated(labels map[string]string) bool { + if labels[LabelFenceGated] == "true" { + return true + } + return f.gatedSlug != nil && labels[LabelService] != "" && f.gatedSlug(labels[LabelService]) +} + +// stopGated stops every fence-gated container that is running or that Docker is +// about to restart. An explicit stop, so a restart policy of unless-stopped does not +// bring it back; Start refuses it until isolation is on again. An error means one +// may still be running, and the caller must keep the fence. +func (f *fence) stopGated(ctx context.Context, cli fenceAPI) error { + res, err := cli.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: make(client.Filters).Add("label", LabelManaged+"=true")}) + if err != nil { + return fmt.Errorf("could not list the fence-gated earners: %w", err) + } + var failed []string + for _, c := range res.Items { + if !f.isGated(c.Labels) { + continue + } + // A restarting container is stopped too, or Docker would bring it back + // after the drops are gone, past the app's Start guard. + if c.State != container.StateRunning && c.State != container.StateRestarting { + continue + } + if _, err := cli.ContainerStop(ctx, c.ID, client.ContainerStopOptions{}); err != nil { + failed = append(failed, strings.TrimPrefix(firstName(c.Names), "/")) + } + } + if len(failed) > 0 { + return fmt.Errorf("could not stop %s, which runs only behind the LAN fence", strings.Join(failed, ", ")) + } + return nil +} + +func firstName(names []string) string { + if len(names) == 0 { + return "a fence-gated earner" + } + return names[0] +} + // refreshOutside recounts the earners outside the fence after a deploy or removal, // so the dashboard does not keep naming a moved earner until the next check. func (f *fence) refreshOutside(ctx context.Context, cli fenceAPI) { diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 078031c..ef6e27c 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -130,6 +130,13 @@ func (p *DockerProvider) SetIsolationSetting(read func() string) { p.fence.setting = read } +// SetFenceGated wires in which services the catalog marks fence_gated, so the fence +// also covers a container made before LabelFenceGated existed (a Mysterium deployed +// on the host's network by an older version). Call it before the first deploy. +func (p *DockerProvider) SetFenceGated(gated func(slug string) bool) { + p.fence.gatedSlug = gated +} + // SetIsolationMemory wires in where "isolation is on" is kept, so a worker stays // fenced after Docker loses every trace of it. Call it before the first deploy. func (p *DockerProvider) SetIsolationMemory(read func() bool, write func(on bool)) { @@ -226,7 +233,7 @@ func (p *DockerProvider) Deploy(ctx context.Context, spec DeploySpec, progress f } // Two copies would share the same host directories and fight over the same host // ports. The manager refuses this first; this is the backstop. - if spec.Instance > 1 && (len(svc.Docker.Volumes) > 0 || len(svc.Docker.Ports) > 0) { + if spec.Instance > 1 && (len(svc.Docker.Volumes) > 0 || svc.Docker.PublishesPorts()) { return ContainerInfo{}, fmt.Errorf("%s has volumes or published ports, so it runs once", svc.Name) } @@ -249,8 +256,11 @@ func (p *DockerProvider) Deploy(ctx context.Context, spec DeploySpec, progress f if err != nil { return ContainerInfo{}, fmt.Errorf("%s copy %d: %w", svc.Name, spec.Instance, err) } - ports, bindings, err := buildPorts(svc.Docker.Ports) + ports, bindings, err := buildServicePorts(svc.Docker, env) if err != nil { + return ContainerInfo{}, fmt.Errorf("%s was not deployed: %w. The running container was not touched", svc.Name, err) + } + if err := checkRuntime(svc, facts.Runtimes); err != nil { return ContainerInfo{}, err } mounts := buildMounts(svc.Docker.Volumes, env) @@ -264,7 +274,7 @@ func (p *DockerProvider) Deploy(ctx context.Context, spec DeploySpec, progress f // The grace period travels with the container, so stopping it later does not // need the catalog to find it again. StopTimeout: &stopTimeout, - Labels: containerLabels(spec.Slug, spec.Instance), + Labels: gatedLabels(containerLabels(spec.Slug, spec.Instance), svc), Hostname: name, } if svc.Docker.Command != "" { @@ -309,13 +319,23 @@ func (p *DockerProvider) Deploy(ctx context.Context, spec DeploySpec, progress f // Which network a bridge earner joins is settled on this side of the line too: if // the LAN fence cannot be applied, the deploy is refused and the running container // keeps earning. A Windows-containers daemon has no Linux bridge to fence. + if svc.Docker.FenceGated { + // From the fence check to the start: a reconcile that turns isolation off + // in between would reduce the fence under a node about to start. + p.fence.reconcileMu.Lock() + defer p.fence.reconcileMu.Unlock() + } + fenced := "" if hostConfig.NetworkMode == "bridge" && facts.OSType == "linux" { - fenced, err := p.fence.networkForDeploy(ctx, cli) + fenced, err = p.fence.networkForDeploy(ctx, cli) if err != nil { return ContainerInfo{}, err } isolate(hostConfig, fenced) } + if problem := fenceGateProblem(svc, hostConfig, fenced, facts.OSType, p.fence.Status().Mode); problem != "" { + return ContainerInfo{}, errors.New(problem) + } defer p.fence.refreshOutside(context.Background(), cli) // Past this point the deploy is destructive: the existing container goes. @@ -415,8 +435,86 @@ func (p *DockerProvider) Start(ctx context.Context, slug string) error { return err } defer cli.Close() - _, err = cli.ContainerStart(ctx, containerName(slug), client.ContainerStartOptions{}) - return err + name := containerName(slug) + return p.startGuarded(ctx, cli, name, func() error { + _, err := cli.ContainerStart(ctx, name, client.ContainerStartOptions{}) + return err + }) +} + +// startGuarded runs start for name. A fence-gated container is started only behind +// the fence (gatedStartProblem), with the reconcile lock held from the check to the +// start so a reconcile cannot reduce the fence between them. Other containers take +// no lock, so their starts never wait on the fence helper. +func (p *DockerProvider) startGuarded(ctx context.Context, cli *client.Client, name string, start func() error) error { + got, err := cli.ContainerInspect(ctx, name, client.ContainerInspectOptions{}) + if err != nil { + // Whether it is fence-gated cannot be known, so the start fails closed. + return fmt.Errorf("could not check %s before starting it: %w", name, err) + } + if got.Container.Config == nil || !p.fence.isGated(got.Container.Config.Labels) { + return start() + } + p.fence.reconcileMu.Lock() + defer p.fence.reconcileMu.Unlock() + if err := p.gatedStartProblem(ctx, cli, name); err != nil { + return err + } + return start() +} + +// gatedStartProblem refuses to start a fence-gated container unless the fence is on +// and has just been applied: a stop while isolation was off must stay a stop. +// The caller holds p.fence.reconcileMu. It inspects again under the lock, and a +// container that cannot be inspected is refused. +func (p *DockerProvider) gatedStartProblem(ctx context.Context, cli *client.Client, name string) error { + got, err := cli.ContainerInspect(ctx, name, client.ContainerInspectOptions{}) + if err != nil { + return fmt.Errorf("could not check %s before starting it: %w", name, err) + } + var labels map[string]string + if got.Container.Config != nil { + labels = got.Container.Config.Labels + } + if !p.fence.isGated(labels) { + return nil + } + fenced, err := p.fence.networkForDeploy(ctx, cli) + if err != nil { + return err + } + if fenced != FenceNetwork { + return fmt.Errorf("%s runs only behind the LAN fence, and LAN isolation is %s here. Set it to auto or on in Settings first", name, p.fence.Status().Mode) + } + if problem := onlyOnFence(got.Container); problem != "" { + return fmt.Errorf("%s runs only behind the LAN fence, but %s. Redeploy it from the dashboard, which puts it back on the fence", name, problem) + } + return nil +} + +// onlyOnFence is why a container is not on the fenced network alone, or "": a node +// made before the fence (host networking) or moved to another network would start +// with access the fence does not cover. +func onlyOnFence(c container.InspectResponse) string { + if c.HostConfig == nil || string(c.HostConfig.NetworkMode) != FenceNetwork { + mode := "" + if c.HostConfig != nil { + mode = string(c.HostConfig.NetworkMode) + } + return fmt.Sprintf("its network mode is %q", mode) + } + if c.NetworkSettings == nil { + return "Docker did not say which networks it is attached to" + } + for name := range c.NetworkSettings.Networks { + if name != FenceNetwork { + return "it is also attached to " + name + } + } + if _, ok := c.NetworkSettings.Networks[FenceNetwork]; !ok { + return "it is not attached to " + FenceNetwork + } + return "" } func (p *DockerProvider) Restart(ctx context.Context, slug string, stopTimeoutSeconds int) error { @@ -427,8 +525,10 @@ func (p *DockerProvider) Restart(ctx context.Context, slug string, stopTimeoutSe defer cli.Close() name := containerName(slug) timeout := stopTimeoutForContainer(ctx, cli, name, stopTimeoutSeconds) - _, err = cli.ContainerRestart(ctx, name, client.ContainerRestartOptions{Timeout: &timeout}) - return err + return p.startGuarded(ctx, cli, name, func() error { + _, err := cli.ContainerRestart(ctx, name, client.ContainerRestartOptions{Timeout: &timeout}) + return err + }) } // PlanRemoval reports what removing this service would delete. @@ -916,6 +1016,18 @@ func containerLabels(slug string, instance int) map[string]string { // catalog. The web worker uses the same label. const LabelCustom = "cashpilot.custom" +// LabelFenceGated marks a container whose catalog entry is fence_gated. The fence +// stops it whenever isolation is not on, and Start refuses it then (lanfence.go +// stopGated), without needing the catalog to find it again. +const LabelFenceGated = "cashpilot.fence-gated" + +func gatedLabels(labels map[string]string, svc catalog.Service) map[string]string { + if svc.Docker.FenceGated { + labels[LabelFenceGated] = "true" + } + return labels +} + // otherOwner reports, for the container already holding a deploy's name, whose it is // when that is not this copy of this service: "" when it is ours to replace. // @@ -960,8 +1072,18 @@ func buildEnv(svc catalog.Service, overrides map[string]string) map[string]strin // Only catalog-declared keys reach the container: an orphaned stored credential // (e.g. a pre-migration USER_ID/DEVICE_NAME left in the blob) must not leak into // the environment of a service whose schema no longer includes it. + // The variables ui_port and udp_port_range publish from keep their default when + // left blank or whitespace, as the deploy validation assumes: a blank UDP_PORTS would publish + // nothing and hand the node an empty --udp.ports. + published := map[string]bool{} + if ui := svc.Docker.UIPort; ui != nil { + published[ui.Env] = true + } + if rng := svc.Docker.UDPPortRange; rng != nil { + published[rng.Env] = true + } for key, value := range overrides { - if declared[key] { + if declared[key] && !(strings.TrimSpace(value) == "" && published[key] && env[key] != "") { env[key] = value } } @@ -1005,6 +1127,99 @@ func buildPorts(raw []string) (network.PortSet, network.PortMap, error) { return ports, bindings, nil } +// buildServicePorts is everything the container publishes: the static ports, the +// WebUI port (ui_port) on each address its env var lists, and the UDP range +// (udp_port_range) its env var holds, 1:1. env is what the container gets, defaults +// included. A bad address or a range that is reversed or wider than its cap is an +// error, raised before the running container is touched. +func buildServicePorts(dc catalog.DockerConfig, env map[string]string) (network.PortSet, network.PortMap, error) { + ports, bindings, err := buildPorts(dc.Ports) + if err != nil { + return nil, nil, err + } + if ui := dc.UIPort; ui != nil { + if ui.Port < 1 || ui.Port > 65535 { + return nil, nil, fmt.Errorf("ui_port %d is not a port", ui.Port) + } + addrs, err := catalog.UIBindings(env[ui.Env]) + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", ui.Env, err) + } + port, err := network.ParsePort(fmt.Sprintf("%d/tcp", ui.Port)) + if err != nil { + return nil, nil, err + } + ports[port] = struct{}{} + for _, addr := range addrs { + bindings[port] = append(bindings[port], network.PortBinding{HostIP: addr, HostPort: strconv.Itoa(ui.Port)}) + } + } + if rng := dc.UDPPortRange; rng != nil { + udp, err := catalog.ExpandUDPRange(env[rng.Env], rng.Cap()) + if err != nil { + return nil, nil, fmt.Errorf("%s: %w", rng.Env, err) + } + for _, p := range udp { + port, err := network.ParsePort(fmt.Sprintf("%d/udp", p)) + if err != nil { + return nil, nil, err + } + ports[port] = struct{}{} + bindings[port] = []network.PortBinding{{HostPort: strconv.Itoa(p)}} + } + } + return ports, bindings, nil +} + +// checkRuntime refuses a service whose catalog entry names a runtime the daemon does +// not provide, before the running container is touched. runtimes nil means the +// daemon did not say, which is refused too: the create would fail after the removal. +func checkRuntime(svc catalog.Service, runtimes map[string]bool) error { + want := strings.TrimSpace(svc.Docker.Runtime) + if want == "" { + return nil + } + if runtimes == nil { + return fmt.Errorf("%s runs only under the %s runtime, and the container runtime did not say which runtimes it has. The running container was not touched", svc.Name, want) + } + if !runtimes[want] { + have := make([]string, 0, len(runtimes)) + for name := range runtimes { + have = append(have, name) + } + slices.Sort(have) + return fmt.Errorf("%s runs only under the %s runtime, which this container runtime does not provide (it has %s). The running container was not touched", svc.Name, want, strings.Join(have, ", ")) + } + return nil +} + +// fenceGateProblem is why a fence-gated service cannot be deployed with this host +// config, or "". Such a service runs only on the fenced network: anywhere else its +// NET_ADMIN namespace sits on a bridge that reaches the LAN, this machine and the +// tailnet (web app/lan_fence.py mode_problem). fenced is the network the fence +// handed out after applying its rules ("" when it applied none), so an entry that +// names the fenced network itself does not skip the fence. mode is the fence mode +// the deploy saw. +func fenceGateProblem(svc catalog.Service, host *container.HostConfig, fenced, osType, mode string) string { + if !svc.Docker.FenceGated || (fenced == FenceNetwork && string(host.NetworkMode) == FenceNetwork) { + return "" + } + why := "" + switch { + case osType != "linux": + why = "the container runtime runs Windows containers, which have no LAN fence" + case string(host.NetworkMode) != "bridge" && string(host.NetworkMode) != FenceNetwork: + why = fmt.Sprintf("its network mode %q is outside the fence", host.NetworkMode) + case mode == FenceModeUnsupported: + why = podmanUnsupported + case mode == FenceModeLegacy: + why = "LAN isolation is not on yet on this machine. Press \"Turn on isolation\" in the dashboard's notice, which moves the other earners behind the fence, then deploy it again" + default: + why = "LAN isolation is off in Settings. Set it to auto or on, then deploy it again" + } + return fmt.Sprintf("%s runs only behind the LAN fence: %s. The running container was not touched", svc.Name, why) +} + func buildMounts(raw []string, env map[string]string) []mount.Mount { mounts := make([]mount.Mount, 0, len(raw)) for _, mapping := range raw { @@ -1053,6 +1268,9 @@ func buildHostConfig(svc catalog.Service, bindings network.PortMap, mounts []mou Name: "unless-stopped", }, NetworkMode: container.NetworkMode(svc.Docker.NetworkMode), + // The runtime the entry names wins over the daemon's default (Deploy has + // checked the daemon has it); none leaves the daemon's default. + Runtime: strings.TrimSpace(svc.Docker.Runtime), } applyHardening(hostConfig, svc, osType) // "" and "default" both name the default bridge on a Linux daemon, so normalise diff --git a/internal/services/fence_gated_test.go b/internal/services/fence_gated_test.go new file mode 100644 index 0000000..97801cb --- /dev/null +++ b/internal/services/fence_gated_test.go @@ -0,0 +1,127 @@ +package services + +import ( + "context" + "fmt" + goruntime "runtime" + "strings" + "testing" + "testing/fstest" + + "github.com/GeiserX/CashPilot-Desktop/internal/catalog" + "github.com/GeiserX/CashPilot-Desktop/internal/runtime" +) + +// gatedServiceYAML has Mysterium's network shape: no static ports, the WebUI and a +// UDP range published from the env, and fence-gated. +const gatedServiceYAML = `name: Gated Example +slug: gated-example +category: bandwidth +status: active +docker: + image: example/gated@sha256:` + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ` + env: + - key: UI_ADDRESS + default: "127.0.0.1" + - key: UDP_PORTS + default: "23000:23029" + port_range: true + ui_port: + env: UI_ADDRESS + port: 4449 + udp_port_range: + env: UDP_PORTS + max_ports: 100 + fence_gated: true +` + +func newGatedCatalog(t *testing.T) *catalog.Catalog { + t.Helper() + cat, err := catalog.LoadEmbedded(fstest.MapFS{"services/bandwidth/gated.yml": {Data: []byte(gatedServiceYAML)}}) + if err != nil { + t.Fatalf("LoadEmbedded error: %v", err) + } + return cat +} + +// It holds NET_ADMIN, so it runs only the image the catalog pins. +func TestAFenceGatedServiceRefusesAChosenImage(t *testing.T) { + fake := &fakeProvider{} + m := NewManager(fake, newGatedCatalog(t), newTestStore(t)) + _, err := m.DeployImage(context.Background(), "gated-example", nil, strPtr("example/other:1.0")) + if err == nil || !strings.Contains(err.Error(), "only the image the catalog pins") { + t.Fatalf("err = %v, want a refusal of the chosen image", err) + } + if fake.deployCalls != 0 { + t.Fatalf("the runtime was called %d times for a refused image", fake.deployCalls) + } + if _, err := m.DeployImage(context.Background(), "gated-example", nil, strPtr("")); err != nil { + t.Fatalf("the catalog image was refused: %v", err) + } +} + +// A range wider than the entry's cap passes the pattern, so the deploy validation +// must catch it before the runtime is reached. +func TestAPublishedValueThatCannotBePublishedIsRefusedBeforeTheRuntime(t *testing.T) { + fake := &fakeProvider{} + m := NewManager(fake, newGatedCatalog(t), newTestStore(t)) + for _, creds := range []map[string]string{ + {"UDP_PORTS": "23000:23100"}, + {"UI_ADDRESS": "999.0.0.1"}, + } { + if err := m.ValidateCredentials("gated-example", creds); err == nil { + t.Errorf("%v validated", creds) + } + if _, err := m.Deploy(context.Background(), "gated-example", creds); err == nil { + t.Errorf("%v deployed", creds) + } + } + if fake.deployCalls != 0 { + t.Fatalf("the runtime was called %d times for refused values", fake.deployCalls) + } + if err := m.ValidateCredentials("gated-example", map[string]string{"UDP_PORTS": "23000:23099"}); err != nil { + t.Fatalf("a range of exactly the cap was refused: %v", err) + } +} + +// Ports published from the env still belong to one container. +func TestAServicePublishingFromItsEnvRunsOnce(t *testing.T) { + m := NewManager(&fakeProvider{}, newGatedCatalog(t), newTestStore(t)) + svc, _ := m.catalog.Get("gated-example") + if limit := m.ScalingLimit(svc); limit.Max != 1 || !strings.Contains(limit.Reason, "port") { + t.Fatalf("ScalingLimit = %+v, want one copy because of its published ports", limit) + } +} + +// The native node publishes nothing and ignores the UDP range, so a range the Docker +// path would refuse does not stop a native deploy. +func TestTheNativeNodeIsNotHeldToWhatDockerPublishes(t *testing.T) { + yaml := gatedServiceYAML + fmt.Sprintf(`native: + binaries: + - os: %s + arch: %s + url: "https://example.test/gated" + sha256: "%s" + archive: none + bin: "gatedstub" + command: "" +`, goruntime.GOOS, goruntime.GOARCH, strings.Repeat("0", 64)) + cat, err := catalog.LoadEmbedded(fstest.MapFS{"services/bandwidth/gated.yml": {Data: []byte(yaml)}}) + if err != nil { + t.Fatalf("LoadEmbedded error: %v", err) + } + docker := &fakeProvider{} + native := &fakeProvider{deployResult: runtime.ContainerInfo{ContainerID: "n1", Status: "running"}} + m := NewManager(docker, cat, newTestStore(t)) + m.Register(nativeRuntimeKind, native) + wide := map[string]string{"UDP_PORTS": "20000:30000"} + if err := m.ValidateCredentials("gated-example", wide); err != nil { + t.Fatalf("ValidateCredentials refused a range the native node ignores: %v", err) + } + if _, err := m.Deploy(context.Background(), "gated-example", wide); err != nil { + t.Fatalf("native deploy refused: %v", err) + } + if native.deployCalls != 1 || docker.deployCalls != 0 { + t.Fatalf("native=%d docker=%d, want the native runtime", native.deployCalls, docker.deployCalls) + } +} diff --git a/internal/services/instances.go b/internal/services/instances.go index 4e0a2d1..1dce904 100644 --- a/internal/services/instances.go +++ b/internal/services/instances.go @@ -37,7 +37,7 @@ func (m *Manager) ScalingLimit(svc catalog.Service) ScalingLimit { if len(svc.Docker.Volumes) > 0 { return ScalingLimit{1, name + " keeps data in a host directory, which two copies would share."} } - if len(svc.Docker.Ports) > 0 { + if svc.Docker.PublishesPorts() { return ScalingLimit{1, name + " publishes a port on the host, which two copies cannot share."} } if link := svc.DeviceLink; link != nil && link.IDEnv != "" { diff --git a/internal/services/manager.go b/internal/services/manager.go index e1eedcd..c7cea0d 100644 --- a/internal/services/manager.go +++ b/internal/services/manager.go @@ -276,6 +276,11 @@ func (m *Manager) DeployWith(ctx context.Context, slug string, credentials map[s // runtime. Record whatever kind actually served the deploy so the persisted // Runtime is derived rather than a hardcoded literal. kind := m.kindForService(svc) + if custom != "" && svc.Docker.FenceGated { + // It holds NET_ADMIN in its own namespace, so it runs only the image the + // catalog pins (web app/main.py _fence_gated_spec). + return store.Deployment{}, fmt.Errorf("%s runs only the image the catalog pins, since it holds NET_ADMIN; clear the Image box to deploy it", svc.Name) + } if custom != "" { // The chosen image replaces the catalog's and every per-architecture // override with it: it is the user's answer for this machine. An image only @@ -285,6 +290,11 @@ func (m *Manager) DeployWith(ctx context.Context, slug string, credentials map[s kind = "" } provider, runtimeKind := m.resolveProvider(kind) + if runtimeKind != nativeRuntimeKind { + if err := checkPublished(svc, credentials); err != nil { + return store.Deployment{}, err + } + } // Copies are read only when copies are involved, so a plain deploy asks the // runtime nothing it did not ask before. copier, _ := provider.(runtime.CopyProvider) @@ -723,6 +733,24 @@ func validateRequired(svc catalog.Service, credentials map[string]string) error return nil } +// checkPublished refuses what ui_port and udp_port_range would publish from the env +// when it cannot be published (a range the pattern lets through but wider than the +// cap), before the deploy rather than in the runtime. Only a Docker deploy publishes +// anything, so the caller asks it only for one: the native node ignores the range. +func checkPublished(svc catalog.Service, credentials map[string]string) error { + env := map[string]string{} + for _, item := range svc.Docker.Env { + env[item.Key] = item.Default + if value := credentials[item.Key]; value != "" { + env[item.Key] = value + } + } + if err := svc.Docker.CheckPublishedValues(env); err != nil { + return fmt.Errorf("invalid value for field: %w", err) + } + return nil +} + // isPortRange reports whether value is "start:end" with 1 <= start <= end <= 65535. func isPortRange(value string) bool { return catalog.IsPortRange(value) } @@ -740,7 +768,13 @@ func (m *Manager) ValidateCredentials(slug string, credentials map[string]string if err := refuseRetired(svc); err != nil { return err } - return validateRequired(svc, credentials) + if err := validateRequired(svc, credentials); err != nil { + return err + } + if m.kindForService(svc) == nativeRuntimeKind { + return nil + } + return checkPublished(svc, credentials) } // RequiredCredentialsMet reports whether every required field for the service is present diff --git a/services/bandwidth/mysterium.yml b/services/bandwidth/mysterium.yml index 4c81993..0dd973b 100644 --- a/services/bandwidth/mysterium.yml +++ b/services/bandwidth/mysterium.yml @@ -7,8 +7,9 @@ description: > MystNodes (Mysterium Network) is a decentralized VPN and proxy network built on blockchain technology. You earn MYST tokens by running a node that provides VPN, proxy, and data scraping services to users. Requires NET_ADMIN, SETUID and SETGID capabilities and - host networking for full functionality. Includes a built-in web UI for node - management. Works on both residential and VPS connections. + a TUN device, which it uses inside its own network namespace on the fenced + earner bridge. Includes a built-in web UI for node management. Works on both + residential and VPS connections. short_description: Decentralized VPN node - blockchain-based, works on VPS referral: @@ -16,7 +17,12 @@ referral: code: "do7v7YOoBBpbOstKQovX2pUvZYKia4ZhH3QIdNtE" docker: - image: "mysteriumnetwork/myst@sha256:1b68b8336112567a2b4596eccebd53cb704bfa37630dd3e2aef63d8a0eca2b2a" + # Pinned by digest: the node holds NET_ADMIN, so the catalog refuses it any + # image it has not named exactly (app/catalog.py). Bump tag and digest together. + image: "mysteriumnetwork/myst:1.39.9-alpine@sha256:f89347d244efeeaf29555e876404cf8aebcd41561c5cbed06c365ba64beb8564" + # gVisor cannot run this node (no iptables-nft, no kernel WireGuard), so an + # operator default of CASHPILOT_CONTAINER_RUNTIME=runsc must not reach it. + runtime: runc platforms: [linux/amd64, linux/arm64, linux/arm/v7] resources: mem_limit: "768m" @@ -27,13 +33,13 @@ docker: required: false secret: false description: >- - Where the node's WebUI (port 4449) listens. 127.0.0.1 keeps it on this - machine. To reach it through a reverse proxy, add the address the proxy - connects to, comma separated with no spaces: 127.0.0.1,172.18.0.1. + The host addresses the node's WebUI (port 4449) is published on. + 127.0.0.1 keeps it on this machine. To reach it through a reverse proxy, + add the address the proxy connects to, comma separated with no spaces: + 127.0.0.1,172.18.0.1. default: "127.0.0.1" - # The value lands on the node's command line, so anything but addresses - # would become an argument: "127.0.0.1, 10.0.0.1" puts a stray word before - # `service`, the node refuses to start and stops earning. + # Each address becomes one host binding of port 4449 (ui_port below), so + # anything but addresses would fail the deploy. # Each octet 0 to 255, ASCII digits only ([0-9], not \d, which in Python also # matches other scripts' digits): 256.0.0.1 is not an address, and the WebUI # listener would fail to bind while the node kept running. @@ -44,15 +50,36 @@ docker: secret: false description: >- The UDP ports the node uses for VPN sessions, as start:end. Forward this - same range on your router. - default: "56000:56100" - # Without it the node picks from 10000:60000, so a forwarded 56000-56100 - # is never used and every session falls back to hole punching. + same range on your router. Each session takes about two ports, so the + default 30 carry about 15 sessions at once, more than a home node gets. + Every published port costs about 2.5 MB of memory in Docker's port + proxy, so a wider range only costs more. Keep it below 32768: ports + 32768-60999 are Linux's ephemeral range, where any program on the host + can hold one for a moment, and one taken port fails the node's start. + A node that already has a range keeps it. + default: "23000:23029" + # Without it the node picks from 10000:60000, so a forwarded range is + # never used and every session falls back to hole punching. 30 ports, + # below Linux's ephemeral range (32768-60999): Docker publishes the range + # 1:1 and fails the whole start on one port a host process holds, where a + # node on the host's network just skipped it. Through 2.1.0 the default + # was 56000:56029 (56000:56100 through 2.0.6); a range on record is kept. # ASCII digits only, as above; port_range checks start <= end <= 65535. pattern: '[0-9]{4,5}:[0-9]{4,5}' port_range: true - ports: - - "4449:4449" + # No static ports. The WebUI is published on each UI_ADDRESS address, and the + # VPN range 1:1 on every address, from the env (app/catalog.py). + ui_port: + env: UI_ADDRESS + port: 4449 + udp_port_range: + env: UDP_PORTS + max_ports: 1000 + # Docker never starts this node on its own (restart policy "no"). The worker + # starts it only after it has confirmed the LAN fence is in place, at deploy, + # after a reboot and after a crash, and stops it within 30 s if the fence or + # the container's shape changes (app/lan_fence.py guard). + fence_gated: true volumes: - "mysterium-data:/var/lib/mysterium-node" critical_volumes: @@ -60,19 +87,20 @@ docker: holds: >- Node identity keystore. Destroying it creates a brand-new identity: the node loses its reputation and any earnings still held against the old one. - # Loopback by default, on purpose. With host networking, 0.0.0.0 publishes - # the node's WebUI (4449) and Tequila API (4050) on every interface of the - # machine, and nothing off the machine needs them: the worker reads the - # node's earnings from inside the container (collector.node_read below), not - # over the network. Reach the WebUI from - # another machine over an SSH tunnel (ssh -L 4449:127.0.0.1:4449 ), or - # put a reverse proxy in front by adding the one address it connects to in - # UI_ADDRESS (the node binds every address in the comma-separated list). The - # Tequila API stays on loopback: the WebUI proxies to it from inside the node. + # The node runs in its own network namespace on the fenced earner bridge, so + # 0.0.0.0 inside it reaches only that namespace: the WebUI (4449) is + # reachable from the host only where ui_port publishes it, loopback by + # default. Nothing off the machine needs it: the worker reads the node's + # earnings from inside the container (collector.node_read below). Reach the + # WebUI from another machine over an SSH tunnel (ssh -L 4449:127.0.0.1:4449 + # ), or add the one address a reverse proxy connects to in UI_ADDRESS. + # The Tequila API stays on the node's own loopback: the WebUI proxies to it + # from inside the node. # --firewall.protected.networks: the node blackholes these for its customers. # Its default leaves out 100.64.0.0/10 (CGNAT and Tailscale) and - # 169.254.0.0/16 (cloud metadata), so a VPN customer could reach the tailnet. - command: "--ui.address=${UI_ADDRESS} --tequilapi.address=127.0.0.1 --firewall.protected.networks=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.0/8,100.64.0.0/10,169.254.0.0/16 --udp.ports=${UDP_PORTS} service --agreed-terms-and-conditions" + # 169.254.0.0/16 (cloud metadata). It stays as a second layer inside the + # node's namespace; the host's fence is the first. + command: "--ui.address=0.0.0.0 --tequilapi.address=127.0.0.1 --firewall.protected.networks=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.0/8,100.64.0.0/10,169.254.0.0/16 --udp.ports=${UDP_PORTS} service --agreed-terms-and-conditions" # The node's own firewall for its customers, read by the worker's isolation # check: the ranges in this flag of `command` go to the nat chain MYST, which # sends each one to the blackhole address. @@ -80,7 +108,6 @@ docker: flag: "--firewall.protected.networks=" chain: MYST blackhole: "240.0.0.1" - network_mode: "host" # SETUID and SETGID are for sudo. The node runs as root and still configures # its interface and firewall through `sudo ip ...` / `sudo iptables ...`, and # sudo switches uid and gid on the way (setresuid, setgroups). Every container @@ -88,6 +115,11 @@ docker: # "sudo: PERM_SUDOERS: setresuid(-1, 1, -1): Operation not permitted", every # session dies at setup, and the node keeps registering and looking healthy. # no-new-privileges stays on: the node is already root, so it blocks nothing here. + # NET_ADMIN reaches only the node's own network namespace: its tun and wg + # interfaces, its routes and its own NAT. It cannot touch the host's + # firewall, routes or interfaces, and the host's fence on the bridge drops + # what the node sends to the LAN, the host, the tailnet and cloud metadata + # whatever the node does to its own tables. cap_add: [NET_ADMIN, SETUID, SETGID] # Wireguard/dvpn need a TUN device. Without it the node starts, registers and # advertises itself to the network but carries no traffic - it looks healthy @@ -131,7 +163,9 @@ docker: requirements: residential_ip: false - devices_per_ip: 0 + # MystNodes help centre: one node per public IP; the others show offline + # (docs/research/per-ip-device-limits.md). + devices_per_ip: 1 min_bandwidth: "" gpu: false min_storage: "" @@ -147,7 +181,7 @@ earnings: monthly_high: 10 currency: USD per: "device" - notes: "Earnings in MYST tokens. Residential IPs earn significantly more. Node WebUI at port 4449 for management. VPS accepted. Port forwarding UDP 56000-56100 (the UDP_PORTS default) strongly recommended -- without it, nodes get Strict NAT status and many VPN sessions fail, severely reducing income. Important: after first run, set your beneficiary (settlement) wallet via the node WebUI or CLI to match your mystnodes.com account -- this links on-chain earnings to your cloud dashboard." + notes: "Earnings in MYST tokens. Residential IPs earn significantly more. Node WebUI at port 4449 (loopback by default) for management. VPS accepted. Port forwarding UDP 23000-23029 (the UDP_PORTS default) strongly recommended -- without it, nodes get Strict NAT status and many VPN sessions fail, severely reducing income. Important: after first run, set your beneficiary (settlement) wallet via the node WebUI or CLI to match your mystnodes.com account -- this links on-chain earnings to your cloud dashboard." cashout: method: auto @@ -315,13 +349,14 @@ rating: # Native (no-Docker) execution: run the official Mysterium headless node binary as a # supervised child process. Pinned to release 1.38.5; sha256 is the SHA-256 of each # published release artifact. There is no linux/arm64 tarball upstream (only .deb), so -# arm64 Linux hosts fall back to the docker: image above. +# arm64 Linux hosts fall back to the docker: image above, which deploys only behind +# the LAN fence (docker.fence_gated). # -# The launch args deliberately do NOT mirror docker.command any more. The container -# binds its UI and Tequila API to 0.0.0.0 because it runs inside a container with -# host networking on a worker box; a native process binds them to 127.0.0.1 because -# it runs directly on the user's own machine, where 0.0.0.0 would publish an -# unauthenticated node admin API to the local network. Port 4449 and the accepted +# The launch args deliberately do NOT mirror docker.command. The container binds its +# UI to 0.0.0.0 inside its own network namespace, which reaches the host only where +# docker.ui_port publishes it; a native process binds it to 127.0.0.1 because it runs +# directly on the user's own machine, where 0.0.0.0 would publish an unauthenticated +# node admin API to the local network. Port 4449 and the accepted # terms are unchanged, so the earnings collector at # http://localhost:4449/tequilapi still works. #