diff --git a/README.md b/README.md index c277810..0419992 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ TF edge defaults to a 180-second upstream I/O timeout, leaving room for the gene ### GPU memory ceiling on inference nodes -Set `services.omlx.gpu_memory_gb` in `tfconfig.yaml` for a shared ceiling, or override it with `nodes..runtime.gpu_memory_gb`. Use a positive integer in **GiB**. Omitted node values inherit the service setting; explicit `null` selects actual RAM minus 10 GiB. A 128 GiB node therefore defaults to 118 GiB (`iogpu.wired_limit_mb=120832`). RAM comes from `hw.memsize` on the inference node, not the controller or the inventory's `ram_gb`. +Set `services.omlx.gpu_memory_gb` in `tfconfig.yaml` for a shared ceiling, or override it with `nodes..runtime.gpu_memory_gb`. Use a positive integer in **GiB**. Omitted node values inherit the service setting; explicit `null` selects actual RAM minus 3 GiB. A 128 GiB node therefore defaults to 125 GiB (`iogpu.wired_limit_mb=128000`). RAM comes from `hw.memsize` on the inference node, not the controller or the inventory's `ram_gb`. `make bootstrap `, `make restart `, and sync with runtime restart install the setting in the oMLX system LaunchDaemon. At every start, including after reboot, a short root-owned plist command validates the limit, applies it only when changed, verifies it by reading sysctl back, then uses `sudo -n -H -u ` to start oMLX as the non-root runtime user. No extra sysctl sudoers permission is granted. An unsupported host, invalid limit, or failed write/readback prevents oMLX startup and reports the error in its stderr log. Dry-run only renders the plist. The legacy user LaunchAgent and process managers do not apply GPU memory settings; use the system `daemon` manager. @@ -220,7 +220,7 @@ Thunder Forge service ports live in `tfconfig.yaml` under `services:`: | `services.edge.host` | `0.0.0.0` | TF edge bind address; use `0.0.0.0` for LAN clients, keep raw Olla private | | `services.edge.port` | `40116` | Local TF edge OpenAI-compatible proxy | | `services.omlx.port` | `8018` | Default oMLX node runtime port when a node runtime omits `port` | -| `services.omlx.gpu_memory_gb` | `null` | Host-wide GPU wired-memory ceiling in GiB; null/omitted means actual node RAM minus 10 GiB | +| `services.omlx.gpu_memory_gb` | `null` | Host-wide GPU wired-memory ceiling in GiB; null/omitted means actual node RAM minus 3 GiB | | `services.edge.access_log` | `logs/tf-edge-access.jsonl` | TF edge JSONL accounting log | | `services.frontend.admin_user` | empty | Admin account used for frontend system-daemon sudo operations | diff --git a/src/thunder_forge/cluster/config.py b/src/thunder_forge/cluster/config.py index a1ab859..0e9ee43 100644 --- a/src/thunder_forge/cluster/config.py +++ b/src/thunder_forge/cluster/config.py @@ -382,7 +382,7 @@ def _parse_gpu_memory_gb(raw: object, *, name: str) -> int | None: if raw is None: return None if type(raw) is not int or raw <= 0: - raise ValueError(f"{name} must be a positive integer in GiB or null (RAM minus 10 GiB)") + raise ValueError(f"{name} must be a positive integer in GiB or null (RAM minus 3 GiB)") return raw diff --git a/src/thunder_forge/cluster/omlx.py b/src/thunder_forge/cluster/omlx.py index 1134f0e..aa42f16 100644 --- a/src/thunder_forge/cluster/omlx.py +++ b/src/thunder_forge/cluster/omlx.py @@ -803,7 +803,7 @@ def _gpu_memory_start_script(node: Node, program_arguments: list[str]) -> str: if limit is not None and limit * 1024 > 2**63 - 1: raise ValueError("gpu_memory_gb exceeds the supported system integer range") # GB in tfconfig follows the binary units of hw.memsize and iogpu.wired_limit_mb. - target = str(limit * 1024) if limit is not None else "$((total_mb - 10240))" + target = str(limit * 1024) if limit is not None else "$((total_mb - 3072))" command = shlex.join([ "/usr/bin/sudo", "-n", "-H", "-u", node.user, "--", "/usr/bin/env", f"HOME={node.home_dir}", diff --git a/tests/test_runtime_install.py b/tests/test_runtime_install.py index 407cc41..04cceb7 100644 --- a/tests/test_runtime_install.py +++ b/tests/test_runtime_install.py @@ -443,11 +443,14 @@ def fake_ssh_run(user, ip, cmd, *, timeout): @pytest.mark.parametrize( "configured,total_gib,current_mb,write_failure,readback_bad,expected_mb,success", [ - (None, 128, 0, False, False, 120832, True), - (None, 64, 0, False, False, 55296, True), + (None, 128, 0, False, False, 128000, True), + (None, 128, 128000, False, False, 128000, True), + (None, 64, 0, False, False, 62464, True), (100, 128, 0, False, False, 102400, True), (118, 128, 120832, False, False, 120832, True), - (None, 8, 0, False, False, None, False), + (None, 8, 0, False, False, 5120, True), + (None, 3, 0, False, False, None, False), + (None, 2, 0, False, False, None, False), (128, 128, 0, False, False, None, False), (129, 128, 0, False, False, None, False), (118, 128, 0, True, False, 120832, False), diff --git a/tfconfig.example.yaml b/tfconfig.example.yaml index 53cd99e..0023c97 100644 --- a/tfconfig.example.yaml +++ b/tfconfig.example.yaml @@ -27,7 +27,7 @@ services: omlx: port: 8018 # Host-wide Metal wired-memory ceiling, in GiB (1 GiB = 1024 MiB). - # null/omitted: actual node RAM minus 10 GiB, computed at each daemon start. + # null/omitted: actual node RAM minus 3 GiB, computed at each daemon start. # Applied before oMLX starts by the system LaunchDaemon, including after reboot. gpu_memory_gb: null @@ -176,8 +176,8 @@ nodes: # Optional oMLX memory guard; omit to preserve saved settings/defaults. # memory_guard: balanced # safe, balanced, aggressive, or quoted "off" # memory_guard_gb: 100.5 # custom GB ceiling; enables the guard, overrides the tier - # Optional override of services.omlx.gpu_memory_gb; null restores RAM minus 10 GiB. - # gpu_memory_gb: 118 + # Optional override of services.omlx.gpu_memory_gb; null restores RAM minus 3 GiB. + # gpu_memory_gb: 125 trusted_network: true models: - memory