From 9559fdc287e5bc5222304106ccc8fdebfcfa002e Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Mon, 9 Nov 2026 22:52:46 +0100
Subject: [PATCH 1/3] fix(worker): stop tearing down a node Docker is still
starting
A Mysterium node publishing 1,000 UDP ports takes about 51 s to start. The
socket proxy (sockguard 2.4.6) gives up on a call whose answer has not begun
after 30 s (its default response_header_timeout; our policy only turns off
request_timeout) and answers 502, so the worker treated a slow start as a
refused one and removed the new container while Docker was still bringing it
up. The node went down.
- After a create or start that got no answer (a 502/504 from the proxy, or
docker-py's own read timeout), the worker now reads the container until it
runs, Docker reports the start failed, or 120 s from the create have
passed. Only then is the deploy rolled back. The UI's wait for the
worker's deploy answer grows by the same 120 s.
- The UDP range cap drops from 1,000 to 300 ports (catalog ceiling and the
Mysterium entry): 300 start in about 12 s, well inside the proxy's 30 s.
A node already running a wider range keeps running; a redeploy of it is
refused before the worker is asked, with the range to use instead.
- The guard's reason for holding a node whose last deploy failed now says
it also holds one put back or started by hand, and that only a deploy
from CashPilot lifts it. The worker reports the reason per container, so
the dashboard shows a "Held stopped" card with a Redeploy button and the
container-down alert carries it, not only the Fleet page.
---
app/catalog.py | 15 +-
app/constants.py | 7 +
app/drift.py | 4 +-
app/lan_fence.py | 11 +-
app/main.py | 27 ++-
app/orchestrator.py | 108 +++++++++++-
app/static/js/app.js | 16 ++
app/worker_api.py | 6 +-
docs/isolation.md | 5 +-
services/bandwidth/mysterium.yml | 6 +-
tests/test_main_deploy_routes.py | 5 +-
tests/test_mysterium_fenced.py | 8 +-
tests/test_slow_start_deploy.py | 293 +++++++++++++++++++++++++++++++
13 files changed, 483 insertions(+), 28 deletions(-)
create mode 100644 tests/test_slow_start_deploy.py
diff --git a/app/catalog.py b/app/catalog.py
index af540897..a578b4fc 100644
--- a/app/catalog.py
+++ b/app/catalog.py
@@ -81,8 +81,12 @@ def parse_port(mapping: object) -> tuple[str, int | list[str | int]]:
NETNS_CAPABILITIES = frozenset({"NET_ADMIN", "SYS_ADMIN"})
#: The most UDP ports one udp_port_range may publish. Each published port is a
-#: rule and, with Docker's userland proxy, a process on the host.
-MAX_UDP_PORTS = 1000
+#: rule and, with Docker's userland proxy, a process on the host, and each one
+#: slows the container's start. Measured on a live node (Nov 2026): 300 ports
+#: start in about 12 s, about 350 take 30 s, where the socket proxy stops
+#: waiting for Docker's answer, and 1,000 take about 51 s. 300 keeps a deploy's
+#: start well inside the proxy's 30 s on a slower host too.
+MAX_UDP_PORTS = 300
def is_port_range(value: str) -> bool:
@@ -114,7 +118,12 @@ def expand_udp_range(value: str, cap: int = MAX_UDP_PORTS) -> dict[str, int]:
start, _, end = value.partition(":")
first, last = int(start), int(end)
if last - first + 1 > cap:
- raise ValueError(f"{value!r} spans {last - first + 1} ports; at most {cap} can be published")
+ raise ValueError(
+ f"{value!r} spans {last - first + 1} ports; at most {cap} can be published, since every published "
+ f"port slows the container's start and a wider range runs past what a deploy can wait for. Choose a "
+ f"range of {cap} ports or fewer, such as {first}:{min(first + cap - 1, 65535)}, and forward the same "
+ "range on your router"
+ )
return {f"{port}/udp": port for port in range(first, last + 1)}
diff --git a/app/constants.py b/app/constants.py
index bd6c6de8..9089a3ad 100644
--- a/app/constants.py
+++ b/app/constants.py
@@ -58,6 +58,13 @@
#: then fails the whole start.
EPHEMERAL_PORTS = (32768, 60999)
+#: How long a deploy waits, counted from its create call, for Docker to finish
+#: starting a container when the call itself got no answer. The socket proxy
+#: gives up on a call after 30 s and docker-py after 60 s, while Docker carries
+#: on: a node publishing 1,000 UDP ports took about 51 s to start. The UI adds
+#: it to its own wait for the worker's deploy answer.
+START_SETTLE_SECONDS = 120
+
def drift_warnings_enabled() -> bool:
"""CASHPILOT_DRIFT_WARNINGS: off stops the worker reporting container settings
diff --git a/app/drift.py b/app/drift.py
index 7653611c..dcf705de 100644
--- a/app/drift.py
+++ b/app/drift.py
@@ -267,7 +267,9 @@ def container_drift(
(v.get("default") for v in docker_conf.get("env") or [] if v.get("key") == rng.get("env")), ""
)
with contextlib.suppress(ValueError):
- want_ports |= set(catalog.expand_udp_range(str(value or ""), int(rng.get("max_ports") or 1000)))
+ want_ports |= set(
+ catalog.expand_udp_range(str(value or ""), int(rng.get("max_ports") or catalog.MAX_UDP_PORTS))
+ )
if missing := want_ports - set(shape.get("ports") or []):
found.append(f"does not publish port {_names(missing)}")
if gated and shape.get("restart_policy") not in (None, "no"):
diff --git a/app/lan_fence.py b/app/lan_fence.py
index 39c24cd5..47331e8e 100644
--- a/app/lan_fence.py
+++ b/app/lan_fence.py
@@ -1268,8 +1268,15 @@ def guard() -> dict[str, str]:
return held
-#: Why the guard holds a container a failed deploy left behind.
-FAILED_DEPLOY = "its last deploy failed and left this container behind. Redeploy it from the dashboard"
+#: Why the guard holds a container under a name whose last deploy failed. The
+#: record is by name, so it also holds the old container put back by hand, or
+#: one started with docker start: the worker cannot tell it from the failed
+#: one, and the operator has to hear that only a deploy from CashPilot lifts it.
+FAILED_DEPLOY = (
+ "its last deploy from CashPilot failed, so the worker keeps any container under this name stopped, one put "
+ "back or started by hand included, until a deploy from CashPilot succeeds. Redeploy it from the CashPilot "
+ "dashboard; starting it with Docker does not lift this"
+)
#: A start the guard could not read back: stopped, but tried again next pass.
diff --git a/app/main.py b/app/main.py
index ef86f843..22c1426d 100644
--- a/app/main.py
+++ b/app/main.py
@@ -283,6 +283,9 @@ async def _get_all_worker_containers(workers: list[dict[str, Any]] | None = None
# When that refused run ended, to tell a refusal
# older than the last balance read (stale) apart.
"fatal_at": c.get("fatal_at"),
+ # Why the worker's fence guard keeps this copy stopped, and
+ # what brings it back (lan_fence.guard). Absent when it does not.
+ "held": c.get("held"),
# What the container was created with, for the drift
# check (app/drift.py). None from a worker too old to send it.
"shape": c.get("shape"),
@@ -586,6 +589,10 @@ def _container_down_message(slug: str, copies: list[dict[str, Any]]) -> str:
how = f" (exit code {', '.join(str(code) for code in codes)})" if codes else ""
message = f"{name} has not been running since {since} UTC on {nodes}{how}, so it is not earning."
if catalog.fence_gated(catalog.get_service(slug)):
+ # What the worker's guard says holds it; a reason only a redeploy clears
+ # must reach the operator here, not only on the Fleet page.
+ if held := sorted({str(c["held"]) for c in copies if c.get("held")}):
+ return message + f" The worker keeps it stopped: {'; '.join(held)}."
# No restart policy on purpose: the worker starts it once the LAN fence is confirmed.
return message + (
" The worker starts it as soon as the LAN fence is confirmed; while it stays down, the Fleet page "
@@ -2820,6 +2827,7 @@ async def api_services_deployed(request: Request) -> list[dict[str, Any]]:
# What a windowed health signal says is wrong with this copy.
"hints": _windowed_hints.get((slug, inst.get("_worker_id"), int(inst.get("instance") or 1)), []),
"fatal": inst.get("fatal"),
+ "held": inst.get("held"),
}
# Only containers CashPilot made (constants.CASHPILOT_DEPLOYED): a
# container the user runs by hand is theirs to shape, even when it
@@ -3658,7 +3666,9 @@ def fence_gated_ports(docker_conf: dict[str, Any], env: dict[str, Any]) -> dict[
ports[f"{int(ui['port'])}/tcp"] = bindings[0] if len(bindings) == 1 else bindings
rng = docker_conf.get("udp_port_range")
if isinstance(rng, dict):
- ports.update(catalog.expand_udp_range(str(env.get(rng["env"]) or ""), int(rng.get("max_ports") or 1000)))
+ ports.update(
+ catalog.expand_udp_range(str(env.get(rng["env"]) or ""), int(rng.get("max_ports") or catalog.MAX_UDP_PORTS))
+ )
return ports
@@ -3696,7 +3706,12 @@ async def _fence_gated_spec(
try:
ports = fence_gated_ports(docker_conf, env)
except ValueError as exc:
- raise HTTPException(status_code=400, detail=f"Invalid value for {slug}: {exc}") from None
+ # Refused before the worker is asked: a node already running keeps
+ # running as it is, whatever range it has.
+ raise HTTPException(
+ status_code=400,
+ detail=f"Invalid value for {slug}: {exc}. Nothing was deployed; a running {slug} was not touched.",
+ ) from None
merged = dict(spec)
# The record's network mode and ports do not win here, so saying they were kept would be false.
divergence[:] = [d for d in divergence if not d.startswith(("network_mode: keeping", "ports: keeping"))]
@@ -4293,8 +4308,14 @@ async def _proxy_worker_deploy(worker_id: int, slug: str, spec: dict[str, Any])
image = _image_for_arch(docker_conf, arch)
if image:
spec = {**spec, "image": image}
+ # A start Docker is slow to answer is waited out by the worker for up to
+ # START_SETTLE_SECONDS (orchestrator._await_start), so the wait here covers it.
return await _proxy_to_worker(
- worker_id, "POST", f"/api/containers/{slug}/deploy", json=spec, timeout=60 + stop_timeout
+ worker_id,
+ "POST",
+ f"/api/containers/{slug}/deploy",
+ json=spec,
+ timeout=60 + stop_timeout + constants.START_SETTLE_SECONDS,
)
diff --git a/app/orchestrator.py b/app/orchestrator.py
index 59f96064..8bc0e62c 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -22,6 +22,7 @@
from typing import Any
import docker
+import requests
from docker.errors import APIError, DockerException, NotFound
# docker-py's run() and create() refuse stop_timeout although the Engine API and
@@ -56,6 +57,7 @@
LABEL_SERVICE,
LABEL_STATE_CREATED,
LABEL_VERSION,
+ START_SETTLE_SECONDS,
drift_warnings_enabled,
)
@@ -415,6 +417,18 @@ def _fatal_entry(container: Any, slug: str) -> dict[str, Any]:
return {"fatal": kind, "fatal_at": finished} if kind else {}
+def _held_entry(container: Any) -> dict[str, Any]:
+ """``{"held": why}`` for a container the fence guard keeps stopped (lan_fence.guard), else nothing.
+
+ The guard's last pass, so the dashboard can say on the service itself why
+ the node is down and what brings it back, not only on the Fleet page.
+ """
+ from app import lan_fence
+
+ why = (lan_fence._state.get("guard") or {}).get(getattr(container, "name", None))
+ return {"held": str(why)} if why else {}
+
+
def _shape(container: Any) -> dict[str, Any]:
"""What a container was created with, on the settings that decide whether it works.
@@ -1090,17 +1104,28 @@ def deploy_raw(
**resource_kwargs,
)
created = None
+ began = time.monotonic()
try:
- container = (
- client.containers.create(**run_kwargs) if gated else client.containers.run(detach=True, **run_kwargs)
- )
- created = container
+ try:
+ container = (
+ client.containers.create(**run_kwargs) if gated else client.containers.run(detach=True, **run_kwargs)
+ )
+ created = container
+ if gated:
+ _mark_stopped(name, False)
+ # Left created, not started, when the fence is not confirmed: the guard
+ # starts it as soon as it is, and nothing runs before that.
+ _require_fence(slug)
+ container.start()
+ except ContainerNetworkError:
+ raise
+ except Exception as exc:
+ # No answer is not a refusal: Docker may still be starting it. A gated
+ # create that got none was never started, so there is nothing to wait for.
+ if not _outcome_unknown(exc) or (gated and created is None):
+ raise
+ container = created = _await_start(client, name, created, began, exc)
if gated:
- _mark_stopped(name, False)
- # Left created, not started, when the fence is not confirmed: the guard
- # starts it as soon as it is, and nothing runs before that.
- _require_fence(slug)
- container.start()
from app import lan_fence
if problem := lan_fence._started_unattached(container, gated_mode):
@@ -1292,6 +1317,69 @@ def _deploy_failure(exc: BaseException, restored: str, slug: str = "") -> Contai
return ContainerRuntimeError(f"The container could not be created: {exc}. {restored}", 500)
+#: What the socket proxy answers when it stopped waiting for Docker (sockguard
+#: cuts a call whose answer has not begun after 30 s): Docker may still be at it.
+_GATEWAY_STATUSES = frozenset({502, 504})
+#: Seconds between two looks at a container whose start got no answer.
+_START_POLL_SECONDS = 2.0
+
+
+def _outcome_unknown(exc: BaseException) -> bool:
+ """Whether a Docker call that raised may still have done its work: no answer, rather than a refusal.
+
+ docker-py's own read timeout or a dropped connection, or the socket proxy's
+ gateway answer. A refusal (a taken port, a proxy rule, a bad spec) is an
+ answer, and is not waited on.
+ """
+ # APIError first: docker-py derives it from requests' HTTPError, and it is Docker's answer.
+ if isinstance(exc, APIError):
+ return getattr(exc, "status_code", None) in _GATEWAY_STATUSES
+ return isinstance(exc, requests.exceptions.RequestException)
+
+
+def _await_start(client: Any, name: str, created: Any | None, began: float, exc: BaseException) -> Any:
+ """The container under ``name`` once Docker has started it; raises when it did not.
+
+ For a create or start whose call ended without Docker's answer: Docker goes
+ on starting the container after the proxy or docker-py stopped waiting, and
+ each published port slows the start (a node with 1,000 UDP ports took about
+ 51 s). Removing it then, as a refused start is, took down a node Docker was
+ still bringing up. Looks until it runs, Docker reports the start failed, or
+ START_SETTLE_SECONDS from the create call have passed.
+ """
+ logger.warning(
+ "Docker did not answer the start of %s (%s); waiting up to %d s for it to come up",
+ name,
+ exc,
+ START_SETTLE_SECONDS,
+ )
+ deadline = began + START_SETTLE_SECONDS
+ while True:
+ try:
+ current = created if created is not None else client.containers.get(name)
+ # Docker answers an inspect only once the start lets go of the container.
+ current.reload()
+ except NotFound:
+ raise ContainerRuntimeError(f"{name} is gone: Docker did not start it ({exc}).", 500) from exc
+ except Exception as poll_exc: # noqa: BLE001 - not known yet; looked at again below
+ logger.debug("Could not read %s while it starts: %s", name, poll_exc)
+ else:
+ attrs = current.attrs if isinstance(getattr(current, "attrs", None), dict) else {}
+ state = attrs.get("State") if isinstance(attrs.get("State"), dict) else {}
+ status = getattr(current, "status", "")
+ if status == "running":
+ logger.info("%s came up %.0f s after its create call", name, time.monotonic() - began)
+ return current
+ if status in ("exited", "dead") or (status == "created" and state.get("Error")):
+ why = state.get("Error") or f"it is {status}"
+ raise ContainerRuntimeError(f"{name} did not start: {why}.", 500) from exc
+ if time.monotonic() >= deadline:
+ raise ContainerRuntimeError(
+ f"{name} was not running {START_SETTLE_SECONDS} s after Docker was asked to start it ({exc}).", 504
+ ) from exc
+ time.sleep(_START_POLL_SECONDS)
+
+
from app.constants import EPHEMERAL_PORTS # noqa: E402 - Linux's default ephemeral range
@@ -3109,6 +3197,7 @@ def get_status() -> list[dict[str, Any]]:
**_stop_facts(c),
**_state_id_entry(c, slug, client),
**_fatal_entry(c, slug),
+ **_held_entry(c),
}
# Present only when the service declares a dial-back address var;
# the key's absence is "nothing to say", never "checked and fine".
@@ -3303,6 +3392,7 @@ def get_status_light() -> list[dict[str, Any]]:
**_shape_entry(c),
**_stop_facts(c),
**_fatal_entry(c, slug),
+ **_held_entry(c),
}
# Same contract as get_status: key present only when declared+set.
if advertised := _advertised_address(c, slug):
diff --git a/app/static/js/app.js b/app/static/js/app.js
index 06099ebf..69db02c1 100644
--- a/app/static/js/app.js
+++ b/app/static/js/app.js
@@ -1538,6 +1538,22 @@ const CP = (() => {
? ` data-a2="${escapeHtml(String(inst.worker_id))}"${inst.instance > 1 ? ` data-a3="${escapeHtml(String(inst.instance))}"` : ''}`
: '';
const controllable = _canWrite && !inst.unmanaged && !svc.unmanaged && inst.has_docker && !inst.is_android;
+ if (inst.held) {
+ // The worker's fence guard keeps it stopped (a fence-gated node): Start
+ // is refused for the same reason, so the card says why and offers what works.
+ items.push({
+ level: 'fail',
+ tag: 'Held stopped',
+ what: svc.name,
+ where,
+ why: `the worker keeps it stopped: ${escapeHtml(inst.held)}.`,
+ actions: _isOwner && !inst.unmanaged && inst.worker_id != null
+ ? `
+ `
+ : '',
+ });
+ continue;
+ }
items.push({
level: st,
tag: st === 'fail' ? 'Down' : 'Degraded',
diff --git a/app/worker_api.py b/app/worker_api.py
index c2434c89..82c0a838 100644
--- a/app/worker_api.py
+++ b/app/worker_api.py
@@ -1543,8 +1543,10 @@ def _validate_instance(instance: int) -> None:
raise HTTPException(status_code=400, detail=f"instance must be between 1 and {MAX_INSTANCES}")
-#: The most container ports one deploy may publish: a 1000-port UDP range
-#: (catalog.MAX_UDP_PORTS) plus a handful of TCP ones.
+#: The most container ports one deploy may publish: a 1000-port UDP range plus
+#: a handful of TCP ones. The catalog now caps a range at 300
+#: (catalog.MAX_UDP_PORTS), but a UI older than that may still send 1,000, and
+#: the worker waits out the slow start such a range brings (orchestrator._await_start).
MAX_PUBLISHED_PORTS = 1024
diff --git a/docs/isolation.md b/docs/isolation.md
index d1b2ea47..f3424795 100644
--- a/docs/isolation.md
+++ b/docs/isolation.md
@@ -171,7 +171,10 @@ every other earner:
the forward on your router is the same as on the host's network. Docker
fails the whole start when a host program holds one of its ports, so the
default sits below Linux's ephemeral range (32768-60999). Each port goes to the same port in
- the node, with the customer's own source address. At most 1000 ports.
+ the node, with the customer's own source address. At most 300 ports: each
+ published port slows the node's start, about 12 s for 300 and 51 s for
+ 1,000. A node already running with a wider range
+ keeps running; CashPilot refuses to redeploy it until the range is narrowed.
- Its WebUI listens on `0.0.0.0` inside its namespace and is published only
on the host addresses in `UI_ADDRESS`, `127.0.0.1` by default. The Tequila
API is never published.
diff --git a/services/bandwidth/mysterium.yml b/services/bandwidth/mysterium.yml
index 5ffded44..02955b02 100644
--- a/services/bandwidth/mysterium.yml
+++ b/services/bandwidth/mysterium.yml
@@ -74,7 +74,11 @@ docker:
port: 4449
udp_port_range:
env: UDP_PORTS
- max_ports: 1000
+ # Each published port slows the node's start: 300 start in about 12 s,
+ # 1,000 in about 51 s, past the 30 s the socket proxy waits (app/catalog.py
+ # MAX_UDP_PORTS). A node already running with a wider range keeps running;
+ # a redeploy of it is refused until the range is narrowed.
+ max_ports: 300
# Docker never starts this node on its own (restart policy "no"). The worker
# starts it only after it has confirmed the LAN fence is in place, at deploy,
# after a reboot and after a crash, and stops it within 30 s if the fence or
diff --git a/tests/test_main_deploy_routes.py b/tests/test_main_deploy_routes.py
index 79c997f5..83293d56 100644
--- a/tests/test_main_deploy_routes.py
+++ b/tests/test_main_deploy_routes.py
@@ -436,9 +436,10 @@ def test_trailing_whitespace_is_not_a_new_path(self, client):
assert sent["moved_mounts"] == []
assert list(sent["volumes"]) == ["/mnt/storj"]
- @pytest.mark.parametrize(("declared", "expected"), [(300, 360), (None, 90), ("junk", 90)])
+ @pytest.mark.parametrize(("declared", "expected"), [(300, 360 + 120), (None, 90 + 120), ("junk", 90 + 120)])
def test_the_deploy_wait_covers_the_old_containers_clean_stop(self, declared, expected):
- """The worker stops the old container with this timeout before replacing it."""
+ """The worker stops the old container with this timeout before replacing it, and may
+ then wait START_SETTLE_SECONDS (120) for a start Docker was slow to answer."""
import asyncio
from app import main
diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py
index ff98f662..68dc024a 100644
--- a/tests/test_mysterium_fenced.py
+++ b/tests/test_mysterium_fenced.py
@@ -131,9 +131,9 @@ def test_anything_but_a_range_is_refused(self, value):
catalog.expand_udp_range(value)
def test_a_range_over_the_cap_is_refused(self):
- with pytest.raises(ValueError, match="at most 1000"):
- catalog.expand_udp_range("10000:11000")
- assert len(catalog.expand_udp_range("10000:10999")) == 1000
+ with pytest.raises(ValueError, match="at most 300"):
+ catalog.expand_udp_range("10000:10300")
+ assert len(catalog.expand_udp_range("10000:10299")) == 300
def test_the_webui_addresses_become_bindings(self):
assert catalog.ui_bindings("localhost,172.18.0.1", 4449) == [["127.0.0.1", 4449], ["172.18.0.1", 4449]]
@@ -708,7 +708,7 @@ class TestStartRefusesAHeldNode:
def test_a_node_a_failed_deploy_left_is_not_started(self, markers, action):
orchestrator._mark_failed("cashpilot-mysterium", True)
node = _node(status="exited")
- with pytest.raises(orchestrator.ContainerNetworkError, match="last deploy failed"):
+ with pytest.raises(orchestrator.ContainerNetworkError, match="last deploy from CashPilot failed"):
_start(node, action)
node.start.assert_not_called()
node.restart.assert_not_called()
diff --git a/tests/test_slow_start_deploy.py b/tests/test_slow_start_deploy.py
new file mode 100644
index 00000000..84c45429
--- /dev/null
+++ b/tests/test_slow_start_deploy.py
@@ -0,0 +1,293 @@
+"""A slow start is not a failed deploy; the range cap fits a start; a held node says why.
+
+Measured on a live node (Nov 2026): a Mysterium container publishing 1,000 UDP
+ports took about 51 s to start. The worker reaches Docker through the socket
+proxy, which stops waiting for an answer after 30 s and answers 502. The worker
+took that as a refused start, removed the new container while Docker was still
+starting it, and the node was down. The operator put the old container back by
+hand, and the guard stopped it again as a leftover of the failed deploy, with
+nothing saying that only a redeploy from CashPilot brings it back.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+from unittest.mock import MagicMock, patch
+
+import pytest
+import requests
+
+from app import catalog, constants, drift, lan_fence, orchestrator
+from tests import test_mysterium_fenced as fenced
+from tests.test_mysterium_fenced import (
+ BIND_REFUSED,
+ MYST,
+ _api_error,
+ _FakeDocker,
+ _guard,
+ _node,
+ _raw_deploy,
+ _running_old,
+ _shape,
+ _ui_deploy,
+ _up_on,
+)
+
+# The fixtures the fenced tests use, shared rather than copied.
+markers = fenced.markers
+ui = fenced.ui
+
+pytestmark = pytest.mark.real_gated_mode
+
+
+def _gateway(status=502):
+ """What the socket proxy answers when it stopped waiting for Docker."""
+ response = MagicMock(status_code=status)
+ return orchestrator.APIError(
+ f"{status} Server Error", response=response, explanation="upstream Docker socket unreachable"
+ )
+
+
+def _starts_late(container, *, polls=1, ends="running", error=""):
+ """A start Docker finishes after the call gave up: ``polls`` reads show it created, then ``ends``."""
+ seen = {"n": 0}
+ container.status = "created"
+ container.attrs = {**_up_on(running=False), "State": {"Running": False, "Error": ""}}
+
+ def reload():
+ seen["n"] += 1
+ if seen["n"] > polls:
+ container.status = ends
+ container.attrs = {
+ **_up_on(running=ends == "running"),
+ "State": {"Running": ends == "running", "Error": error},
+ }
+
+ container.reload.side_effect = reload
+ return seen
+
+
+@pytest.fixture
+def fast_poll(monkeypatch):
+ monkeypatch.setattr(orchestrator, "_START_POLL_SECONDS", 0)
+
+
+# ---------------------------------------------------------------------------
+# 1. A start Docker is slow to answer is waited out, not rolled back
+# ---------------------------------------------------------------------------
+
+
+class TestASlowStartIsNotAFailedDeploy:
+ def test_a_start_the_proxy_gave_up_on_is_kept_once_it_runs(self, markers, fast_poll):
+ old = _running_old()
+ docker = _FakeDocker(old)
+ docker.created.start.side_effect = _gateway(502)
+ seen = _starts_late(docker.created, polls=2)
+ _raw_deploy(client=docker.client, old=old)
+ assert seen["n"] >= 3
+ assert not getattr(docker.created, "removed", False), "the node Docker was starting must not be removed"
+ assert docker.get("cashpilot-mysterium") is docker.created
+ assert old.removed, "the old node goes once the new one runs"
+ old.start.assert_not_called()
+ assert orchestrator.deploy_failed() == set()
+
+ def test_a_gateway_timeout_is_waited_out_too(self, markers, fast_poll):
+ old = _running_old()
+ docker = _FakeDocker(old)
+ docker.created.start.side_effect = _gateway(504)
+ _starts_late(docker.created)
+ _raw_deploy(client=docker.client, old=old)
+ assert docker.get("cashpilot-mysterium") is docker.created and old.removed
+
+ def test_docker_pys_own_read_timeout_on_an_ordinary_earner(self, markers, fast_poll):
+ labels = {"cashpilot.service": "honeygain", "cashpilot.managed": "true"}
+ old = _running_old("cashpilot-honeygain", labels)
+ docker = _FakeDocker(old)
+ # run() is create and start in one call: the container exists, the call timed out.
+ docker.create_error = requests.exceptions.ReadTimeout("Read timed out. (read timeout=60)")
+ _starts_late(docker.created)
+ _raw_deploy(client=docker.client, old=old, slug="honeygain")
+ assert docker.get("cashpilot-honeygain") is docker.created
+ assert not getattr(docker.created, "removed", False) and old.removed
+
+ def test_a_start_that_ends_exited_is_rolled_back(self, markers, fast_poll):
+ old = _running_old()
+ docker = _FakeDocker(old)
+ docker.created.start.side_effect = _gateway(502)
+ _starts_late(docker.created, ends="exited")
+ with pytest.raises(orchestrator.ContainerRuntimeError, match="did not start") as failed:
+ _raw_deploy(client=docker.client, old=old)
+ assert "put back and is running again" in str(failed.value)
+ assert docker.get("cashpilot-mysterium") is old and docker.created.removed
+
+ def test_a_start_docker_reports_failed_is_rolled_back(self, markers, fast_poll):
+ old = _running_old()
+ docker = _FakeDocker(old)
+ docker.created.start.side_effect = _gateway(502)
+ _starts_late(docker.created, ends="created", error=BIND_REFUSED)
+ with pytest.raises(orchestrator.ContainerRuntimeError, match="address already in use"):
+ _raw_deploy(client=docker.client, old=old)
+ assert docker.get("cashpilot-mysterium") is old
+
+ def test_a_start_still_not_running_at_the_bound_is_rolled_back(self, markers, fast_poll):
+ old = _running_old()
+ docker = _FakeDocker(old)
+ docker.created.start.side_effect = _gateway(502)
+ _starts_late(docker.created, polls=10**9)
+ clock = iter(range(0, 10**6, 30))
+ with (
+ patch.object(orchestrator.time, "monotonic", side_effect=lambda: next(clock)),
+ pytest.raises(orchestrator.ContainerRuntimeError, match=f"not running {constants.START_SETTLE_SECONDS} s"),
+ ):
+ _raw_deploy(client=docker.client, old=old)
+ assert docker.get("cashpilot-mysterium") is old
+
+ def test_control_a_refusal_is_not_waited_on(self, markers, fast_poll):
+ old = _running_old()
+ docker = _FakeDocker(old)
+ docker.created.start.side_effect = _api_error(BIND_REFUSED)
+ with pytest.raises(orchestrator.ContainerRuntimeError, match="put back"):
+ _raw_deploy(client=docker.client, old=old)
+ docker.created.reload.assert_not_called()
+
+ def test_control_a_gated_create_with_no_answer_is_not_waited_on(self, markers, fast_poll):
+ # The node was never started, so there is no start to wait for.
+ old = _running_old()
+ docker = _FakeDocker(old)
+ docker.create_error = _gateway(502)
+ with pytest.raises(orchestrator.ContainerRuntimeError, match="put back"):
+ _raw_deploy(client=docker.client, old=old)
+ docker.created.start.assert_not_called()
+ docker.created.reload.assert_not_called()
+
+ def test_the_ui_waits_as_long_as_the_worker_may(self):
+ import asyncio
+ from unittest.mock import AsyncMock
+
+ from app import main
+
+ proxy = AsyncMock(return_value={})
+ with patch("app.main._proxy_to_worker", proxy):
+ asyncio.run(main._proxy_worker_deploy(1, "mysterium", {}))
+ assert proxy.call_args.kwargs["timeout"] >= 60 + 30 + constants.START_SETTLE_SECONDS
+
+
+# ---------------------------------------------------------------------------
+# 2. The UDP range cap is one a deploy can start within
+# ---------------------------------------------------------------------------
+
+WIDE = {
+ "image": MYST["docker"]["image"],
+ "env": {"UI_ADDRESS": "127.0.0.1", "UDP_PORTS": "23000:23999"},
+ "volumes": {"mysterium-data": {"bind": "/var/lib/mysterium-node", "mode": "rw"}},
+}
+
+
+class TestTheRangeCap:
+ def test_the_cap_is_300(self):
+ assert catalog.MAX_UDP_PORTS == 300
+ assert MYST["docker"]["udp_port_range"]["max_ports"] == 300
+
+ def test_301_ports_are_refused_and_300_are_not(self):
+ with pytest.raises(ValueError, match="at most 300"):
+ catalog.expand_udp_range("23000:23300", 300)
+ assert len(catalog.expand_udp_range("23000:23299", 300)) == 300
+
+ def test_a_typed_range_over_the_cap_is_refused_with_what_to_do(self, ui):
+ resp, sent = _ui_deploy(ui, {"env": {"UDP_PORTS": "23000:23999"}})
+ assert resp.status_code == 400
+ detail = resp.json()["detail"]
+ assert "at most 300" in detail and "23000:23299" in detail and "router" in detail
+ assert not sent
+
+ def test_a_redeploy_of_a_wider_recorded_range_is_refused_not_cut(self, ui):
+ resp, sent = _ui_deploy(ui, {"env": {}}, recorded=WIDE)
+ assert resp.status_code == 400
+ detail = resp.json()["detail"]
+ assert "spans 1000 ports" in detail and "was not touched" in detail
+ assert not sent, "nothing reaches the worker, so the running node keeps running"
+
+ def test_control_a_narrower_range_typed_over_the_wide_record_deploys(self, ui):
+ resp, sent = _ui_deploy(ui, {"env": {"UDP_PORTS": "23000:23299"}}, recorded=WIDE)
+ assert resp.status_code == 200, resp.text
+ assert sum(k.endswith("/udp") for k in sent["ports"]) == 300
+
+ def test_the_guard_keeps_a_running_node_with_a_wider_range(self, markers):
+ node = _node(status="running")
+ node.attrs["Config"]["Env"] = ["UDP_PORTS=23000:23999"]
+ node.attrs["HostConfig"]["PortBindings"].update(
+ {f"{p}/udp": [{"HostIp": "", "HostPort": str(p)}] for p in range(23000, 24000)}
+ )
+ held, _ = _guard([node])
+ assert held == {}
+ node.stop.assert_not_called()
+
+ def test_drift_does_not_flag_a_node_running_its_wider_recorded_range(self):
+ shape = _shape()
+ shape["ports"] = sorted({*shape["ports"], *(f"{p}/udp" for p in range(23000, 24000))})
+ found = drift.container_drift(MYST, shape, {"env": {"UDP_PORTS": "23000:23999"}})
+ assert not any("/udp" in f for f in found)
+
+
+# ---------------------------------------------------------------------------
+# 3. A node the failed-deploy record holds says why, and what brings it back
+# ---------------------------------------------------------------------------
+
+
+class TestAHeldNodeSaysWhy:
+ def test_a_hand_restored_node_is_stopped_with_the_way_back_in_the_log(self, markers, caplog):
+ orchestrator._mark_failed("cashpilot-mysterium", True)
+ node = _node(status="running")
+ with caplog.at_level(logging.WARNING, logger="app.lan_fence"):
+ held, _ = _guard([node])
+ node.stop.assert_called_once()
+ why = held["cashpilot-mysterium"]
+ assert "put back or started by hand" in why and "Redeploy it from the CashPilot dashboard" in why
+ assert any(
+ "Stopping cashpilot-mysterium" in r.message and "Redeploy it from" in r.message for r in caplog.records
+ )
+
+ def test_the_heartbeat_carries_the_reason(self, markers):
+ orchestrator._mark_failed("cashpilot-mysterium", True)
+ node = _node(status="exited")
+ _guard([node])
+ assert orchestrator._held_entry(node) == {"held": lan_fence.FAILED_DEPLOY}
+
+ def test_control_a_node_the_guard_does_not_hold_carries_nothing(self, markers):
+ node = _node(status="running")
+ _guard([node])
+ assert orchestrator._held_entry(node) == {}
+
+ def test_the_container_down_alert_says_why_and_what_to_do(self):
+ from app import main
+
+ copies = [
+ {"_down_since": datetime.now(UTC), "_node": "w1", "restart_policy": "no", "held": lan_fence.FAILED_DEPLOY}
+ ]
+ message = main._container_down_message("mysterium", copies)
+ assert "The worker keeps it stopped: its last deploy from CashPilot failed" in message
+ assert "Redeploy it from the CashPilot dashboard" in message
+
+ def test_the_dashboard_offers_a_redeploy_for_a_held_copy(self):
+ from pathlib import Path
+
+ js = (Path(__file__).resolve().parent.parent / "app/static/js/app.js").read_text()
+ block = js[js.index("if (inst.held) {") : js.index("continue;", js.index("if (inst.held) {"))]
+ assert "Held stopped" in block and 'data-action="redeployOnWorker"' in block
+
+ def test_the_ui_keeps_the_reason_from_the_heartbeat(self):
+ import asyncio
+ import json
+
+ from app import main
+
+ worker = {
+ "id": 7,
+ "name": "alpha",
+ "status": "online",
+ "system_info": json.dumps({"docker_available": True}),
+ "containers": json.dumps([{"slug": "mysterium", "status": "exited", "held": lan_fence.FAILED_DEPLOY}]),
+ }
+ [entry] = asyncio.run(main._get_all_worker_containers([worker]))
+ assert entry["held"] == lan_fence.FAILED_DEPLOY
From 06aad6fa7625ff8ce05de1fc17b71477c28ef953 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Mon, 9 Nov 2026 23:16:40 +0100
Subject: [PATCH 2/3] fix(drift): tell a host-network node with a range over
the cap to narrow it, not reserve it
Reserving an oversized range on the host no longer makes the redeploy possible:
the 300-port cap refuses it. The move advice now says to narrow the range first,
and keeps the reservation option for ranges within the cap.
---
app/drift.py | 23 ++++++++++++++++-------
tests/test_slow_start_deploy.py | 15 +++++++++++++++
2 files changed, 31 insertions(+), 7 deletions(-)
diff --git a/app/drift.py b/app/drift.py
index dcf705de..67c835e7 100644
--- a/app/drift.py
+++ b/app/drift.py
@@ -166,13 +166,22 @@ def move_advice(
key = str(rng.get("env") or "") if isinstance(rng, dict) else ""
value = str(((recorded or {}).get("env") or {}).get(key) or "").strip() if key else ""
m = re.fullmatch(r"(\d{1,5}):(\d{1,5})", value)
- if m and int(m[1]) <= EPHEMERAL_PORTS[1] and int(m[2]) >= EPHEMERAL_PORTS[0]:
- low = EPHEMERAL_PORTS[0]
- default = next((str(v.get("default") or "") for v in docker_conf.get("env") or [] if v.get("key") == key), "")
- example = ""
- d = re.fullmatch(r"(\d{1,5}):(\d{1,5})", default)
- if d and int(d[2]) + 100 < low:
- example = f", such as {int(d[1]) + 100}:{int(d[2]) + 100}"
+ cap = int(rng.get("max_ports") or catalog.MAX_UDP_PORTS) if isinstance(rng, dict) else catalog.MAX_UDP_PORTS
+ low = EPHEMERAL_PORTS[0]
+ default = next((str(v.get("default") or "") for v in docker_conf.get("env") or [] if v.get("key") == key), "")
+ example = ""
+ d = re.fullmatch(r"(\d{1,5}):(\d{1,5})", default)
+ if d and int(d[2]) + 100 < low:
+ example = f", such as {int(d[1]) + 100}:{int(d[2]) + 100}"
+ if m and int(m[2]) - int(m[1]) + 1 > cap:
+ # Wider than a deploy publishes: the redeploy is refused, so reserving it is no way out.
+ advice["warning"] = (
+ f"Its recorded {key}, {value}, spans {int(m[2]) - int(m[1]) + 1} ports, more than the {cap} a deploy "
+ f"publishes, so the redeploy is refused until it is narrower. Before you redeploy, set {key} to a free "
+ f"range of at most {cap} ports below {low} other than the default{example} (a value equal to the default "
+ "keeps the recorded one), and change the router forward to match."
+ )
+ elif m and int(m[1]) <= EPHEMERAL_PORTS[1] and int(m[2]) >= EPHEMERAL_PORTS[0]:
advice["warning"] = (
f"Its recorded {key}, {value}, overlaps Linux's ephemeral range ({low}-{EPHEMERAL_PORTS[1]}), which the "
"redeploy keeps: on its own network every port in it is published, and one a host program holds fails "
diff --git a/tests/test_slow_start_deploy.py b/tests/test_slow_start_deploy.py
index 84c45429..e299f89d 100644
--- a/tests/test_slow_start_deploy.py
+++ b/tests/test_slow_start_deploy.py
@@ -291,3 +291,18 @@ def test_the_ui_keeps_the_reason_from_the_heartbeat(self):
}
[entry] = asyncio.run(main._get_all_worker_containers([worker]))
assert entry["held"] == lan_fence.FAILED_DEPLOY
+
+
+class TestTheMoveAdviceForAWideRange:
+ WIDE_HOST_RECORD = {"env": {"UI_ADDRESS": "127.0.0.1", "UDP_PORTS": "56000:56999"}, "network_mode": "host"}
+
+ def test_a_range_over_the_cap_is_told_to_narrow_not_to_reserve(self):
+ move = drift.move_advice(MYST, _shape(network_mode="host"), self.WIDE_HOST_RECORD, lan_fence.MODE_ON)
+ warning = move["warning"]
+ assert "spans 1000 ports" in warning and "at most 300 ports" in warning
+ assert "reserve it on the host" not in warning
+
+ def test_control_a_range_within_the_cap_keeps_the_reservation_option(self):
+ record = {**self.WIDE_HOST_RECORD, "env": {"UDP_PORTS": "56000:56100"}}
+ warning = drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"]
+ assert "reserve it on the host" in warning
From 76de548c52bffd9e2c93962d6a3221e27463a446 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Mon, 9 Nov 2026 23:41:21 +0100
Subject: [PATCH 3/3] fix(deploy): let a Stop outwait a redeploy's slow start,
and answer a timed-out start as final
A Stop of a fence-gated node waits on the lock a redeploy holds while it
waits out a slow start, so main now gives that Stop the deploy's own bound
on top of the stop timeout; cut off sooner, the stop went unrecorded and
the container-down alert fired for a node stopped on purpose.
A start still not running at the bound is rolled back before the worker
answers, so it answers 500, not 504: main reads 504 as still deploying and
kept the proxied-copy record of a copy no box runs.
---
app/main.py | 22 ++++++++++++++--------
app/orchestrator.py | 5 ++++-
tests/test_slow_start_deploy.py | 27 ++++++++++++++++++++++++++-
3 files changed, 44 insertions(+), 10 deletions(-)
diff --git a/app/main.py b/app/main.py
index 22c1426d..715840a4 100644
--- a/app/main.py
+++ b/app/main.py
@@ -4242,6 +4242,10 @@ async def _proxy_worker_command(
# for storj). Cut off sooner, a stop that went through answers 504 and is
# never recorded, so the copy raises a container-down alert.
timeout = 30 + _catalog_stop_timeout(slug) if command in ("stop", "restart", "remove") else 30
+ if command == "stop" and catalog.fence_gated(catalog.get_service(slug)):
+ # The worker stops a fence-gated node under the lock a redeploy holds
+ # (orchestrator.GATED_DEPLOY_LOCK), so a Stop sent during one waits for it first.
+ timeout += _deploy_timeout(slug)
if command == "remove":
return await _proxy_to_worker(worker_id, "DELETE", f"/api/containers/{slug}", params=params, timeout=timeout)
return await _proxy_to_worker(
@@ -4286,6 +4290,15 @@ def _catalog_stop_timeout(slug: str) -> int:
return timeout if timeout > 0 else 30
+def _deploy_timeout(slug: str) -> int:
+ """How long a deploy of ``slug`` may take on the worker.
+
+ It stops the old container first, and a start Docker is slow to answer is
+ waited out for up to START_SETTLE_SECONDS (orchestrator._await_start).
+ """
+ return 60 + _catalog_stop_timeout(slug) + constants.START_SETTLE_SECONDS
+
+
async def _proxy_worker_deploy(worker_id: int, slug: str, spec: dict[str, Any]) -> dict[str, Any]:
"""Forward a deploy command with full spec to a worker.
@@ -4295,7 +4308,6 @@ async def _proxy_worker_deploy(worker_id: int, slug: str, spec: dict[str, Any])
while the worker was still shutting the old node down cleanly.
"""
docker_conf = (catalog.get_service(slug) or {}).get("docker") or {}
- stop_timeout = _catalog_stop_timeout(slug)
# Which build THIS worker runs is decided here, at the edge, from the
# architecture in its heartbeat. The deployment record stays on the
# catalog's default image: it is one row for the whole fleet, and a
@@ -4308,14 +4320,8 @@ async def _proxy_worker_deploy(worker_id: int, slug: str, spec: dict[str, Any])
image = _image_for_arch(docker_conf, arch)
if image:
spec = {**spec, "image": image}
- # A start Docker is slow to answer is waited out by the worker for up to
- # START_SETTLE_SECONDS (orchestrator._await_start), so the wait here covers it.
return await _proxy_to_worker(
- worker_id,
- "POST",
- f"/api/containers/{slug}/deploy",
- json=spec,
- timeout=60 + stop_timeout + constants.START_SETTLE_SECONDS,
+ worker_id, "POST", f"/api/containers/{slug}/deploy", json=spec, timeout=_deploy_timeout(slug)
)
diff --git a/app/orchestrator.py b/app/orchestrator.py
index 8bc0e62c..1afc00dc 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -1374,8 +1374,11 @@ def _await_start(client: Any, name: str, created: Any | None, began: float, exc:
why = state.get("Error") or f"it is {status}"
raise ContainerRuntimeError(f"{name} did not start: {why}.", 500) from exc
if time.monotonic() >= deadline:
+ # 500, not 504: the deploy is rolled back by the time this is answered,
+ # and a 504 tells a caller the worker may still be at it (main keeps a
+ # proxied copy's record on one).
raise ContainerRuntimeError(
- f"{name} was not running {START_SETTLE_SECONDS} s after Docker was asked to start it ({exc}).", 504
+ f"{name} was not running {START_SETTLE_SECONDS} s after Docker was asked to start it ({exc}).", 500
) from exc
time.sleep(_START_POLL_SECONDS)
diff --git a/tests/test_slow_start_deploy.py b/tests/test_slow_start_deploy.py
index e299f89d..a37c7e2f 100644
--- a/tests/test_slow_start_deploy.py
+++ b/tests/test_slow_start_deploy.py
@@ -138,10 +138,15 @@ def test_a_start_still_not_running_at_the_bound_is_rolled_back(self, markers, fa
clock = iter(range(0, 10**6, 30))
with (
patch.object(orchestrator.time, "monotonic", side_effect=lambda: next(clock)),
- pytest.raises(orchestrator.ContainerRuntimeError, match=f"not running {constants.START_SETTLE_SECONDS} s"),
+ pytest.raises(
+ orchestrator.ContainerRuntimeError, match=f"not running {constants.START_SETTLE_SECONDS} s"
+ ) as failed,
):
_raw_deploy(client=docker.client, old=old)
assert docker.get("cashpilot-mysterium") is old
+ # Rolled back, so final: a 504 would tell main the worker may still be
+ # deploying, and main keeps a proxied copy's record for a copy no box runs.
+ assert failed.value.status_code != 504
def test_control_a_refusal_is_not_waited_on(self, markers, fast_poll):
old = _running_old()
@@ -172,6 +177,26 @@ def test_the_ui_waits_as_long_as_the_worker_may(self):
asyncio.run(main._proxy_worker_deploy(1, "mysterium", {}))
assert proxy.call_args.kwargs["timeout"] >= 60 + 30 + constants.START_SETTLE_SECONDS
+ @pytest.mark.parametrize(("slug", "waits_for_deploy"), [("mysterium", True), ("honeygain", False)])
+ def test_a_stop_sent_during_a_redeploy_waits_for_the_deploy_lock(self, slug, waits_for_deploy):
+ # The worker stops a fence-gated node under the lock deploy_raw holds
+ # while it waits out a slow start. Cut off sooner, a stop that goes
+ # through answers 504, is never recorded, and the container-down alert
+ # fires for a node stopped on purpose. A service without the fence
+ # takes no lock, and its wait is unchanged (the control).
+ import asyncio
+ from unittest.mock import AsyncMock
+
+ from app import main
+
+ proxy = AsyncMock(return_value={})
+ with patch("app.main._proxy_to_worker", proxy):
+ asyncio.run(main._proxy_worker_command(1, "stop", slug))
+ stop_alone = 30 + main._catalog_stop_timeout(slug)
+ deploy = main._deploy_timeout(slug)
+ expected = stop_alone + deploy if waits_for_deploy else stop_alone
+ assert proxy.call_args.kwargs["timeout"] == expected
+
# ---------------------------------------------------------------------------
# 2. The UDP range cap is one a deploy can start within