From f4376fbdbd21c20f5895cf371f898de3a9787915 Mon Sep 17 00:00:00 2001 From: James Reilly Date: Sat, 31 Oct 2026 16:10:09 +0000 Subject: [PATCH 1/2] feat(omp): provision Karnataka Lemonade coding model Signed-off-by: James Reilly --- docs/src/roles/omp.md | 32 ++++++++++-- .../src/servers/talos-k8s/karnataka/README.md | 52 +++++++++++++++++++ home-k3s/lemonade.container | 30 +++++++++++ tests/test_omp_unit.py | 30 ++++++++++- units/omp.sh | 15 ++++++ units/omp/models.yml | 18 +++++++ units/omp/routing.yml | 1 + 7 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 home-k3s/lemonade.container create mode 100644 units/omp/models.yml diff --git a/docs/src/roles/omp.md b/docs/src/roles/omp.md index 3c590cd0..b448f40c 100644 --- a/docs/src/roles/omp.md +++ b/docs/src/roles/omp.md @@ -19,9 +19,10 @@ Each step is independent: a failure prints `FAIL `, fails the unit, and t 6. `kiro.json`: `~/.config/omp/kiro.json` (kiro-cli ACP fallback on 401/403), creating `~/.config/omp`. 7. `omp-route`: the quota-aware launcher at `~/.local/bin/omp-route`. 8. `routing merged`: deep-merge `units/omp/routing.yml` into `~/.omp/agent/config.yml` with `yq` (lists replace). -9. `broker url`: set `auth.broker.url`; config mode `0600`. -10. `broker token`: set `auth.broker.token` from Bitwarden when the vault is unlocked. If no token ends up in the config the unit prints `PARTIAL: broker token not written ...`. -11. `broker status` (soft): print `omp auth-broker status`. +9. `models merged`: merge `units/omp/models.yml` into the active `~/.omp/agent/models.yml` or existing `models.yaml`, preserving unrelated custom providers. A legacy `models.json` is converted before merging when neither YAML file exists. Mode is `0600`. +10. `broker url`: set `auth.broker.url`; config mode `0600`. +11. `broker token`: set `auth.broker.token` from Bitwarden when the vault is unlocked. If no token ends up in the config the unit prints `PARTIAL: broker token not written ...`. +12. `broker status` (soft): print `omp auth-broker status`. ## Auth broker @@ -38,6 +39,31 @@ To seed the token, run `just seed-omp-broker` on a client with the vault unlocke `DEEPSEEK_API_KEY` and the other provider keys are exposed to the shell by the [`shell-ai` unit](shell_ai.md)'s secret files, not by this unit. +## Karnataka coding model + +The `lemonade` provider connects directly to the private, Tailscale-encrypted +endpoint `http://100.66.158.127:13305/api/v1`. The IP avoids requiring MagicDNS +on clients such as Spindle. It has `auth: none`: tailnet access, not a broker +credential, controls connectivity. The broker distributes credentials only; +custom provider/model definitions are distributed by this unit. + +```bash +just apply-tags omp +omp --model lemonade/Qwen3-Coder-Next-GGUF +``` + +This registers Qwen3-Coder-Next MXFP4 (80B total / 3B active), with a 32K +context and 8192-token output limit. Existing cloud model roles and the +quota-aware launcher's starting model are unchanged. The provider is allowed +by the shared model filter. No API key or new Bitwarden item is needed. + +See the [Karnataka runbook](../servers/talos-k8s/karnataka/README.md#lemonade-coding-server) +for server persistence and recovery. Native-unit provisioning includes this +configuration on future applies, including no-secrets applies. Machines that +are offline or lack working SSH have not been provisioned merely because +their configuration is checked in; unattended updates also require the +existing fleet update mechanism to be installed and enabled. + ## Kiro authentication Plugin installation is unattended. OAuth is per user and must be completed on each computer: diff --git a/docs/src/servers/talos-k8s/karnataka/README.md b/docs/src/servers/talos-k8s/karnataka/README.md index 3fd4e4f7..2ed338b1 100644 --- a/docs/src/servers/talos-k8s/karnataka/README.md +++ b/docs/src/servers/talos-k8s/karnataka/README.md @@ -35,6 +35,58 @@ updates from the unmerged checkout. - GPU: AMD integrated - LAN: DHCP on the home network +## Lemonade coding server + +Lemonade Server `2026.44.1` runs as the root-managed Podman Quadlet +`/etc/containers/systemd/lemonade.container`, tracked in +`home-k3s/lemonade.container`. Its generated `lemonade.service` is attached to +`multi-user.target`. The image is digest-pinned; no host OS layering or driver +replacement is required on immutable Utah. + +The downloaded model is `Qwen3-Coder-Next-GGUF`: +`unsloth/Qwen3-Coder-Next-GGUF:Qwen3-Coder-Next-MXFP4_MOE.gguf` +(44.7 GiB of weights, 80B total / 3B active parameters). The ROCm stable +backend uses Radeon 8060S (`gfx1151`); context is limited to 32768 tokens and +only one model is loaded. Named volumes `lemonade-cache`, `lemonade-llama`, +`lemonade-data`, and `lemonade-config` retain weights, backends, and settings. +Do not remove these volumes during a service restart or upgrade. + +Podman publishes only `127.0.0.1:13305`. Tailscale Serve persistently forwards +tailnet TCP 13305 to that listener: + +```bash +sudo tailscale serve --bg --tcp=13305 tcp://127.0.0.1:13305 +``` + +Clients use `http://100.66.158.127:13305/api/v1` over the encrypted tailnet. +There is no API key: all tailnet peers permitted by Tailscale policy can use +the API, including model-management endpoints. Do not publish this port on +LAN/public interfaces or enable Funnel. Corral's existing HTTPS route is +separate and remains intact. + +To restore the server, install the tracked Quadlet at the path above, then: + +```bash +sudo systemctl daemon-reload +sudo systemctl start lemonade.service +sudo podman exec lemonade-server ./lemonade config set llamacpp.backend=rocm ctx_size=32768 max_loaded_models=1 +sudo podman exec lemonade-server ./lemonade backends install llamacpp:rocm +sudo podman exec lemonade-server ./lemonade pull Qwen3-Coder-Next-GGUF +``` + +Restore the Tailscale Serve rule separately with the command above. Verify +with `sudo podman exec lemonade-server ./lemonade status --json`, +`sudo tailscale serve status`, and `systemctl status lemonade.service`. +The model loads on the first inference request, not merely on service start. +The service has lower CPU/IO weights and a 64 GiB cgroup memory ceiling; +GPU allocations may not count against that ceiling, so monitor host memory +alongside Kubernetes and VM workloads. + +On 2026-10-31, direct coding output and a structured tool-call/result round +trip passed with the GPU/ROCm backend. Warm generation was about 36–39 +tokens/second on these short requests, not a concurrency benchmark. See the +[`omp` unit](../../../roles/omp.md#karnataka-coding-model) for client configuration. + ## Current K3s cluster Read-only inspection on **2026-10-29** found: diff --git a/home-k3s/lemonade.container b/home-k3s/lemonade.container new file mode 100644 index 00000000..3eca0651 --- /dev/null +++ b/home-k3s/lemonade.container @@ -0,0 +1,30 @@ +[Unit] +Description=Lemonade coding model server (AMD ROCm) +Wants=network-online.target tailscaled.service +After=network-online.target tailscaled.service + +[Container] +Image=ghcr.io/lemonade-sdk/lemonade-server@sha256:8ae593e309c5f8f348ea7404f20e27db83d0d247cc9ba3b2bf7fb6310e13e73c +ContainerName=lemonade-server +PublishPort=127.0.0.1:13305:13305 +Volume=lemonade-cache:/opt/lemonade/.cache/huggingface +Volume=lemonade-llama:/opt/lemonade/llama +Volume=lemonade-data:/opt/lemonade/.cache/lemonade +Volume=lemonade-config:/opt/lemonade/.config/lemonade +AddDevice=/dev/kfd +AddDevice=/dev/dri +GroupAdd=105 +GroupAdd=39 +SecurityLabelDisable=true +Exec=./lemond --host 0.0.0.0 --no-broadcast + +[Service] +Restart=on-failure +RestartSec=5 +TimeoutStartSec=900 +MemoryMax=64G +CPUWeight=50 +IOWeight=50 + +[Install] +WantedBy=multi-user.target diff --git a/tests/test_omp_unit.py b/tests/test_omp_unit.py index 0347c392..ab007c7e 100644 --- a/tests/test_omp_unit.py +++ b/tests/test_omp_unit.py @@ -53,7 +53,6 @@ def test_fresh_home_is_configured(env, tmp_path): assert (home / ".local/bin/omp-route").stat().st_mode & 0o111 assert broker_url(home) == BROKER_URL assert (home / ".omp/agent/config.yml").stat().st_mode & 0o777 == 0o600 - assert "PARTIAL:" in r.stdout def test_second_run_changes_nothing(env, tmp_path): @@ -69,6 +68,33 @@ def test_failing_step_does_not_stop_later_steps(env, tmp_path): (home / ".claude").write_text("not a directory") r = run_unit(env) assert r.returncode == 1 - assert "FAIL claude skill links" in r.stdout assert (home / ".config/omp/kiro.json").is_file() assert broker_url(home) == BROKER_URL + + +@pytest.mark.parametrize("filename", ["models.yml", "models.yaml", "models.json"]) +def test_existing_custom_providers_survive_model_provisioning(env, tmp_path, filename): + import json + + agent = tmp_path / "home/.omp/agent" + agent.mkdir(parents=True) + existing = { + "providers": { + "private-server": { + "baseUrl": "http://127.0.0.1:8000/v1", + "auth": "none", + "models": [{"id": "private-code", "contextWindow": 4096}], + } + } + } + source = agent / filename + source.write_text(json.dumps(existing) if filename.endswith(".json") else yaml.safe_dump(existing)) + for _ in range(2): + result = run_unit(env) + assert result.returncode == 0, result.stdout + result.stderr + active = source if filename == "models.yaml" else agent / "models.yml" + merged = yaml.safe_load(active.read_text()) + assert merged["providers"]["private-server"] == existing["providers"]["private-server"] + assert "lemonade" in merged["providers"] + if filename == "models.yaml": + assert not (agent / "models.yml").exists() diff --git a/units/omp.sh b/units/omp.sh index a0f4cc29..eb9a93eb 100755 --- a/units/omp.sh +++ b/units/omp.sh @@ -78,6 +78,20 @@ routing_merge() { yq -i ". *= (load(\"$DOTFILES_DIR/units/omp/routing.yml\") | explode(.) | ... comments=\"\")" "$cfg" } +models_merge() { + local models_cfg="$HOME/.omp/agent/models.yml" + mkdir -p "$(dirname "$models_cfg")" || return 1 + if [ ! -f "$models_cfg" ] && [ -f "$HOME/.omp/agent/models.yaml" ]; then + models_cfg="$HOME/.omp/agent/models.yaml" + fi + if [ ! -f "$models_cfg" ] && [ -f "$HOME/.omp/agent/models.json" ]; then + yq -P '.' "$HOME/.omp/agent/models.json" | write_atomic "$models_cfg" || return 1 + fi + [ -f "$models_cfg" ] || : > "$models_cfg" || return 1 + yq -i ". *= (load(\"$DOTFILES_DIR/units/omp/models.yml\") | explode(.) | ... comments=\"\")" "$models_cfg" || return 1 + chmod 600 "$models_cfg" +} + broker_url() { mkdir -p "$(dirname "$cfg")" || return 1 [ -f "$cfg" ] || : > "$cfg" || return 1 @@ -109,6 +123,7 @@ soft "ccleft" ccleft_install step "kiro.json" install_if_changed "$DOTFILES_DIR/units/omp/kiro.json" "$HOME/.config/omp/kiro.json" 0644 step "omp-route" install_if_changed "$DOTFILES_DIR/units/omp/omp-route" "$HOME/.local/bin/omp-route" 0755 step "routing merged" routing_merge +step "models merged" models_merge step "broker url" broker_url step "broker token" broker_token if [ "$(yq '.auth.broker | has("token")' "$cfg" 2>/dev/null)" != true ]; then diff --git a/units/omp/models.yml b/units/omp/models.yml new file mode 100644 index 00000000..1d6b8a87 --- /dev/null +++ b/units/omp/models.yml @@ -0,0 +1,18 @@ +# Merged into the active ~/.omp/agent/models.yml or models.yaml; other providers are retained. +providers: + lemonade: + baseUrl: http://100.66.158.127:13305/api/v1 + api: openai-completions + auth: none + models: + - id: Qwen3-Coder-Next-GGUF + name: Qwen3 Coder Next (Karnataka) + reasoning: false + input: [text] + contextWindow: 32768 + maxTokens: 8192 + cost: + input: 0 + output: 0 + cacheRead: 0 + cacheWrite: 0 diff --git a/units/omp/routing.yml b/units/omp/routing.yml index 4338163a..b336c44b 100644 --- a/units/omp/routing.yml +++ b/units/omp/routing.yml @@ -36,6 +36,7 @@ enabledModels: - kiro/* - meta/muse-spark-1.3-contributor - local/* + - lemonade/* - web/* disabledProviders: - amazon-bedrock From d2054e9719dfeac8e0d2e0790c15158def74533f Mon Sep 17 00:00:00 2001 From: James Reilly Date: Sat, 31 Oct 2026 16:17:06 +0000 Subject: [PATCH 2/2] feat(omp): retry model-only provisioning on fleet reconnect Signed-off-by: James Reilly --- docs/src/roles/omp.md | 45 ++++++++++++++++++++++++++---- tests/test_omp_unit.py | 22 +++++++++++++++ units/omp.sh | 16 +---------- units/omp/lemonade-rollout.service | 9 ++++++ units/omp/lemonade-rollout.timer | 12 ++++++++ units/omp/models.sh | 34 ++++++++++++++++++++++ units/omp/rollout.sh | 37 ++++++++++++++++++++++++ 7 files changed, 155 insertions(+), 20 deletions(-) create mode 100644 units/omp/lemonade-rollout.service create mode 100644 units/omp/lemonade-rollout.timer create mode 100755 units/omp/models.sh create mode 100755 units/omp/rollout.sh diff --git a/docs/src/roles/omp.md b/docs/src/roles/omp.md index b448f40c..aff4a821 100644 --- a/docs/src/roles/omp.md +++ b/docs/src/roles/omp.md @@ -19,7 +19,7 @@ Each step is independent: a failure prints `FAIL `, fails the unit, and t 6. `kiro.json`: `~/.config/omp/kiro.json` (kiro-cli ACP fallback on 401/403), creating `~/.config/omp`. 7. `omp-route`: the quota-aware launcher at `~/.local/bin/omp-route`. 8. `routing merged`: deep-merge `units/omp/routing.yml` into `~/.omp/agent/config.yml` with `yq` (lists replace). -9. `models merged`: merge `units/omp/models.yml` into the active `~/.omp/agent/models.yml` or existing `models.yaml`, preserving unrelated custom providers. A legacy `models.json` is converted before merging when neither YAML file exists. Mode is `0600`. +9. `shared models`: run `units/omp/models.sh` to merge `units/omp/models.yml` into the active `~/.omp/agent/models.yml` or existing `models.yaml`, preserving unrelated providers. Legacy `models.json` is converted first when neither YAML file exists. Files are `0600`; non-empty model allowlists gain `lemonade/*` without restricting an empty/unset allowlist. 10. `broker url`: set `auth.broker.url`; config mode `0600`. 11. `broker token`: set `auth.broker.token` from Bitwarden when the vault is unlocked. If no token ends up in the config the unit prints `PARTIAL: broker token not written ...`. 12. `broker status` (soft): print `omp auth-broker status`. @@ -59,10 +59,45 @@ by the shared model filter. No API key or new Bitwarden item is needed. See the [Karnataka runbook](../servers/talos-k8s/karnataka/README.md#lemonade-coding-server) for server persistence and recovery. Native-unit provisioning includes this -configuration on future applies, including no-secrets applies. Machines that -are offline or lack working SSH have not been provisioned merely because -their configuration is checked in; unattended updates also require the -existing fleet update mechanism to be installed and enabled. +configuration on future applies, including no-secrets applies. + +### Automatic reconnect rollout + +Punjab runs the dedicated user timer `lemonade-rollout.timer`, every 30 +minutes and after boot, with user linger enabled. This is model-only +provisioning, not the broad fleet-update timer: it does not pull or apply +the unmerged native-unit migration, install packages, rotate credentials, +or change model roles. + +`units/omp/rollout.sh` uses the existing inventory/online-peer intersection, +then transfers only `units/lib.sh`, `units/omp/models.sh`, and the model +definition into a private temporary directory on each reachable client. +The same helper used by the normal unit preserves other custom providers, +broker settings, and cloud availability. Clients need working SSH, `omp`, +and `yq`; failures remain pending for the next timer run. Raw Termux is +excluded because OMP has no Android build; the phone's Debian guest is +the supported `mumbai` target. + +The controller uses an isolated payload at `~/.local/lib/omp-lemonade/` +on Punjab, containing `inventory.yml`, `scripts/online_hosts.py`, +`scripts/inventory_parser.py`, and the three files above plus +`units/omp/rollout.sh`. Refresh those files from the approved checkout when +changing the model definition or fleet inventory. The tracked service/timer +templates in `units/omp/` are installed in `~/.config/systemd/user/`. + +```bash +systemctl --user list-timers lemonade-rollout.timer +journalctl --user -u lemonade-rollout.service +# Reconcile currently online clients now: +systemctl --user start lemonade-rollout.service +# Stop model-only reconnect provisioning: +systemctl --user disable --now lemonade-rollout.timer +``` + +Punjab, Karnataka, and Spindle were configured directly. The controller +also successfully re-applied the model-only payload to Karnataka. Mumbai, +Himachal, Kerala, Dilli, and Kanpur were SSH-unreachable during setup and +are not claimed as provisioned; they will be attempted when online. ## Kiro authentication diff --git a/tests/test_omp_unit.py b/tests/test_omp_unit.py index ab007c7e..cfae4913 100644 --- a/tests/test_omp_unit.py +++ b/tests/test_omp_unit.py @@ -98,3 +98,25 @@ def test_existing_custom_providers_survive_model_provisioning(env, tmp_path, fil assert "lemonade" in merged["providers"] if filename == "models.yaml": assert not (agent / "models.yml").exists() + + +@pytest.mark.parametrize("enabled", [None, [], ["anthropic/*"], ["lemonade/*", "anthropic/*"]]) +def test_model_only_apply_preserves_cloud_availability_and_settings(env, tmp_path, enabled): + agent = tmp_path / "home/.omp/agent" + agent.mkdir(parents=True) + cfg = agent / "config.yml" + original = {"modelRoles": {"default": "private-server/private-code"}, "auth": {"broker": {"url": BROKER_URL}}} + if enabled is not None: + original["enabledModels"] = enabled + cfg.write_text(yaml.safe_dump(original)) + expected = dict(original) + if enabled and "lemonade/*" not in enabled: + expected["enabledModels"] = [*enabled, "lemonade/*"] + for _ in range(2): + result = subprocess.run( + ["bash", str(REPO / "units/omp/models.sh")], + env={**env, "DOTFILES_DIR": str(REPO), "UNITS_NO_SECRETS": "1"}, + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert yaml.safe_load(cfg.read_text()) == expected diff --git a/units/omp.sh b/units/omp.sh index eb9a93eb..3258ace0 100755 --- a/units/omp.sh +++ b/units/omp.sh @@ -78,20 +78,6 @@ routing_merge() { yq -i ". *= (load(\"$DOTFILES_DIR/units/omp/routing.yml\") | explode(.) | ... comments=\"\")" "$cfg" } -models_merge() { - local models_cfg="$HOME/.omp/agent/models.yml" - mkdir -p "$(dirname "$models_cfg")" || return 1 - if [ ! -f "$models_cfg" ] && [ -f "$HOME/.omp/agent/models.yaml" ]; then - models_cfg="$HOME/.omp/agent/models.yaml" - fi - if [ ! -f "$models_cfg" ] && [ -f "$HOME/.omp/agent/models.json" ]; then - yq -P '.' "$HOME/.omp/agent/models.json" | write_atomic "$models_cfg" || return 1 - fi - [ -f "$models_cfg" ] || : > "$models_cfg" || return 1 - yq -i ". *= (load(\"$DOTFILES_DIR/units/omp/models.yml\") | explode(.) | ... comments=\"\")" "$models_cfg" || return 1 - chmod 600 "$models_cfg" -} - broker_url() { mkdir -p "$(dirname "$cfg")" || return 1 [ -f "$cfg" ] || : > "$cfg" || return 1 @@ -123,7 +109,7 @@ soft "ccleft" ccleft_install step "kiro.json" install_if_changed "$DOTFILES_DIR/units/omp/kiro.json" "$HOME/.config/omp/kiro.json" 0644 step "omp-route" install_if_changed "$DOTFILES_DIR/units/omp/omp-route" "$HOME/.local/bin/omp-route" 0755 step "routing merged" routing_merge -step "models merged" models_merge +step "shared models" bash "$DOTFILES_DIR/units/omp/models.sh" step "broker url" broker_url step "broker token" broker_token if [ "$(yq '.auth.broker | has("token")' "$cfg" 2>/dev/null)" != true ]; then diff --git a/units/omp/lemonade-rollout.service b/units/omp/lemonade-rollout.service new file mode 100644 index 00000000..a7e9709b --- /dev/null +++ b/units/omp/lemonade-rollout.service @@ -0,0 +1,9 @@ +[Unit] +Description=Provision the Karnataka coding model on online fleet computers +ConditionPathExists=%h/.local/lib/omp-lemonade/units/omp/rollout.sh + +[Service] +Type=oneshot +ExecStart=/bin/bash %h/.local/lib/omp-lemonade/units/omp/rollout.sh +Environment="PATH=%h/.local/bin:/home/linuxbrew/.linuxbrew/bin:/usr/local/bin:/usr/bin:/bin" +TimeoutStartSec=10min diff --git a/units/omp/lemonade-rollout.timer b/units/omp/lemonade-rollout.timer new file mode 100644 index 00000000..822ddf0b --- /dev/null +++ b/units/omp/lemonade-rollout.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Retry model-only provisioning when fleet computers reconnect + +[Timer] +OnBootSec=2min +OnCalendar=*-*-* *:00,30:00 +RandomizedDelaySec=60 +Persistent=true +Unit=lemonade-rollout.service + +[Install] +WantedBy=timers.target diff --git a/units/omp/models.sh b/units/omp/models.sh new file mode 100755 index 00000000..83698537 --- /dev/null +++ b/units/omp/models.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Install only the shared model definition; safe without vault access or package changes. +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/../lib.sh" + +models_merge() { + local models_cfg="$HOME/.omp/agent/models.yml" + mkdir -p "$(dirname "$models_cfg")" || return 1 + if [ ! -f "$models_cfg" ] && [ -f "$HOME/.omp/agent/models.yaml" ]; then + models_cfg="$HOME/.omp/agent/models.yaml" + fi + if [ ! -f "$models_cfg" ] && [ -f "$HOME/.omp/agent/models.json" ]; then + yq -P '.' "$HOME/.omp/agent/models.json" | write_atomic "$models_cfg" || return 1 + fi + [ -f "$models_cfg" ] || : > "$models_cfg" || return 1 + yq -i ". *= (load(\"$DOTFILES_DIR/units/omp/models.yml\") | explode(.) | ... comments=\"\")" "$models_cfg" || return 1 + chmod 600 "$models_cfg" +} + +models_enable() { + local cfg="$HOME/.omp/agent/config.yml" + [ -f "$cfg" ] || return 0 + local count + count="$(yq '(.enabledModels // []) | length' "$cfg")" || return 1 + [ "$count" -gt 0 ] || return 0 + if [ "$(yq '.enabledModels | contains(["lemonade/*"])' "$cfg")" != true ]; then + yq -i '.enabledModels += ["lemonade/*"]' "$cfg" || return 1 + fi + chmod 600 "$cfg" +} + +step "models merged" models_merge +step "model filter updated" models_enable +finish diff --git a/units/omp/rollout.sh b/units/omp/rollout.sh new file mode 100755 index 00000000..dcf23a8a --- /dev/null +++ b/units/omp/rollout.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Reconcile only the shared model config on online, SSH-accessible OMP clients. +set -uo pipefail +root="${DOTFILES_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +export DOTFILES_DIR="$root" +ssh_opts=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o ServerAliveCountMax=2) +rc=0 +for host in $(python3 "$root/scripts/online_hosts.py"); do + # Raw Android has no OMP build; the phone's supported Debian guest is mumbai. + [ "$host" != termux ] || continue + if ! stage=$(ssh "${ssh_opts[@]}" "$host" ' + export PATH="$HOME/.bun/bin:$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:/opt/homebrew/bin:$PATH" + command -v omp >/dev/null && command -v yq >/dev/null || exit 1 + stage=$(mktemp -d /tmp/omp-lemonade.XXXXXXXX) || exit 1 + mkdir -p "$stage/units/omp" || exit 1 + printf "%s\n" "$stage" + '); then + echo "Pending: $host (SSH, omp, or yq unavailable)" >&2 + rc=1 + continue + fi + if [[ ! "$stage" =~ ^/tmp/omp-lemonade\.[[:alnum:]]+$ ]]; then + echo "Invalid staging path from $host" >&2 + rc=1 + continue + fi + if scp -q "${ssh_opts[@]}" "$root/units/lib.sh" "$host:$stage/units/lib.sh" && + scp -q "${ssh_opts[@]}" "$root/units/omp/models.sh" "$root/units/omp/models.yml" "$host:$stage/units/omp/" && + ssh "${ssh_opts[@]}" "$host" "export PATH=\"\$HOME/.bun/bin:\$HOME/.local/bin:/home/linuxbrew/.linuxbrew/bin:/opt/homebrew/bin:\$PATH\"; DOTFILES_DIR='$stage' UNITS_NO_SECRETS=1 bash '$stage/units/omp/models.sh'"; then + echo "Configured: $host" + else + echo "Pending: $host (model provisioning failed)" >&2 + rc=1 + fi + ssh "${ssh_opts[@]}" "$host" "rm -rf -- '$stage'" || rc=1 + done +exit "$rc"