From 0c7f8871505d0ebb9b4d5fbf56b105e243ab3c8e Mon Sep 17 00:00:00 2001 From: pi Date: Thu, 19 Nov 2026 18:23:21 -0500 Subject: [PATCH 1/2] feat(contribute): run two isolated review-backed workers Keep one Ornith worker/reviewer lane per node at the native 262144-token window. Pin the verified OMP 19 Lab image and put reviewer evidence on per-lane claims. Capture unapproved worker lessons separately from the shared startup bundle. Assisted-by: github-copilot/gpt-6.2-sol via omp --- argocd/infra-application.yaml | 2 +- docs/skills/hive-contribute/SKILL.md | 179 ++++++----- docs/skills/hive-contribute/contribute.just | 326 +++++++++++++++----- manifests/contribute.yaml | 198 ++++++++++-- tests/unit/test_contribute_placement.py | 154 +++++++++ tests/unit/test_contribute_startup.py | 289 +++++++++++++++-- 6 files changed, 921 insertions(+), 227 deletions(-) create mode 100644 tests/unit/test_contribute_placement.py diff --git a/argocd/infra-application.yaml b/argocd/infra-application.yaml index b397ba59b0..38c24fda89 100644 --- a/argocd/infra-application.yaml +++ b/argocd/infra-application.yaml @@ -56,7 +56,7 @@ spec: kind: StatefulSet jsonPointers: - /spec/replicas - name: contribute + name: contribute-reviewed namespace: contribute project: default source: diff --git a/docs/skills/hive-contribute/SKILL.md b/docs/skills/hive-contribute/SKILL.md index a124c53f03..5870999103 100644 --- a/docs/skills/hive-contribute/SKILL.md +++ b/docs/skills/hive-contribute/SKILL.md @@ -1,24 +1,24 @@ --- name: hive-contribute description: > - Run the projectbluefin/contribute Hive contributor (OMP in tmux) on the lab, - driven by local llm-d Ornith models on exo-0 (worker) and ghost (advisor). - Use when turning contribution on/off, attaching to watch it, changing the - models or their flags, or debugging why the contributor is idle. + Use when launching, attaching to, draining, or troubleshooting Hive + contributors on the lab, changing their Ornith configuration, or verifying + that a dedicated review gates publication. metadata: context7-sources: - /can1357/oh-my-pi - /ggml-org/llama.cpp - /websites/cli_github_manual - /git/htmldocs + - /casey/just --- # hive-contribute ## When to Use -Use when launching, observing, stopping, or changing the lab contributor and -its local worker/advisor models or deterministic tooling. +Use when launching, observing, or stopping the lab contributors or changing +their local Ornith model, publication gate, or deterministic tooling. ## When NOT to Use @@ -35,68 +35,73 @@ General cluster administration belongs to the relevant cluster operations skill. | Piece | Where | Manifest | |---|---|---| -| llama-server (`ornith-ai/Ornith-1.5-35B-A3B-GGUF` `Ornith-1.5-35B-Q6_K.gguf`, 262,144 tokens); flags in ConfigMap `llm-d-presets` (`presets.ini`) | `llm-d/llm-d-exo-0`, exo-0, NodePort 30800 (worker) | `manifests/llm-d.yaml` | -| Same Ornith model as dedicated advisor | `llm-d/llm-d-ghost`, ghost, NodePort 30801 (advisor) | `manifests/llm-d.yaml` | -| Contributor: `ghcr.io/projectbluefin/contribute:stable` | StatefulSet `contribute/contribute`, 1 replica when on: `contribute-0` on exo-0 | `manifests/contribute.yaml` | +| `llm-d/llm-d-exo-0`, exo-0, NodePort 30800 | Ornith worker and dedicated reviewer on exo-0 | `manifests/llm-d.yaml` | +| `llm-d/llm-d-ghost`, ghost, NodePort 30801 | Ornith worker and dedicated reviewer on ghost | `manifests/llm-d.yaml` | +| StatefulSet `contribute/contribute-reviewed` | One contributor per node with hard pod anti-affinity; `spec.nodeName` seeds that node's model URL | `manifests/contribute.yaml` | -exo-0 runs the Ornith worker model and ghost runs the Ornith advisor model. -Ornith implements; the advisor reviews concurrently per turn via OMP `advisor.reviewMode: turn` with `advisor.syncBacklog: "off"`. -OMP roles are configured by `contribute-on`: `default` and `smol` → `ornith` on exo-0, -`advisor` → `ornith` on ghost (or optionally Luna via `advisor="luna"`). - -Both pods share one Hive registration: `HIVE_SESSION` is the pod name, and -relays with distinct sessions hold independent tasks under one account -(upstream hivecommons/hive v5 `contributor-relay.md`, "Running multiple -backends under one account"). +Both lanes use `lab-worker/ornith`, served with one slot per node. A lane +reviews only after its worker has handed off a commit; no per-turn advisor or +cross-node requests. Both pods share one Hive registration but set distinct +`HIVE_SESSION` values from their pod names. Separate sessions allow separate +leases; the hub still decides whether each session is eligible for work. ## Use -The recipes live in `~/.justfile`, so `just contribute-on` works from any directory. This repo's Justfile has `set fallback`, which reaches them too. If chezmoi manages that file, merge updates there: downloading a replacement overwrites machine-specific registration defaults. Otherwise install or refresh it: +The canonical recipes are in `docs/skills/hive-contribute/contribute.just`. +The workstation's chezmoi `dot_justfile` delegates there, keeping its Reef +registration default (`~/.config/hive/contributor.tunaos-lab.env`). The Lab +Justfile's `set fallback := true` lets Just find that home file when invoked +from the repo. Point `lab_contribute` in the wrapper at the checkout containing +the dual-review launcher. Do not replace the home Justfile with the raw Lab recipe. ```bash -curl -fsSL https://raw.githubusercontent.com/projectbluefin/lab/main/docs/skills/hive-contribute/contribute.just -o ~/.justfile +just contribute on # warm both nodes and launch the pair, or attach to a running pair +just contribute models # warm both native-window servers without Hive registration or leases +just contribute attach # attach without making changes on the cluster +just contribute status # node/pod state and Ornith slots, no inference +just contribute off # release both GPUs once contributors are drained and stopped +just contribute-learning-save ``` -```bash -just contribute-on # write Secret, scale models to 1, warm up, scale contributor to 1 and attach -just contribute # alias of contribute-on -just contribute-attach # attach to contribute-0 (detach: C-b d) -just contribute-status -just contribute-off # scale everything to 0; frees both GPUs for video jobs -just contribute-learning-save # save learned skills to the isolated Ghost profile -``` - -`contribute-on` runs a warm-up gate before starting the contributor: it ensures -`ornith` is loaded and responsive on both nodes. -Context and safe-tool assets are validated before the registration token is -reissued, so an invalid startup bundle does not invalidate the active lease. - -The k8s MCP `resources_scale` tool can toggle the same Deployments and StatefulSet. -Scaling up through MCP reuses the last Secret that `contribute-on` wrote and skips the warm-up. - -`contribute-on` sends, every time, with no login step: - -- `gh auth token` → `GH_TOKEN`. Hive contributors fork and open PRs as themselves (`gh-wrapper.sh`). -- The registration file (`HIVE_CONTRIBUTE_REGISTRATION`; the repo default is `~/.config/hive/contributor.bluefin.env`, the owner's chezmoi default is `contributor.tunaos-lab.env`). Its `HIVE_HUB` picks the hive, so it decides whose work the lab takes. Its token is reissued with your gh identity on every `contribute-on` (`POST /api/contribute/reissue-token`, what upstream's `contribute-move` does) and written back to the file. That invalidates any other client using this registration. The recipe reissues only the first hub, so keep one hub per file. -- `~/.omp/profiles/ghost/agent/config.yml`: isolated Ghost OMP settings, shipped with `modelRoles` rewritten (`yq`) to the lab models. The default workstation OMP configuration remains unchanged. -- `skills/`, `managed-skills/` and `APPEND_SYSTEM.md` from that profile: bundled into the existing Secret, then copied into the container's native OMP agent directory by `seed-home`. The bundle ships the `ponytail` skill (profile `skills/ponytail` symlinks the omp ponytail plugin's skill; tar dereferences it), and `APPEND_SYSTEM.md` makes it always-on. - -Override the paths with `HIVE_CONTRIBUTE_REGISTRATION` and `HIVE_CONTRIBUTE_OMP_CONFIG`; `KUBECONFIG` defaults to `~/.kube/bluespeed.yaml`. +`contribute-on`, `contribute-attach`, `contribute-status`, and `contribute-off` +are still accepted. The `hive-lab` session puts exo-0 on the left and ghost on +the right, independent of pod ordinal. Input is not synchronized. Detach +with `Ctrl-b d`; it does not stop either worker. `HIVE_CONTRIBUTE_NO_ATTACH=1` +waits for both lanes without opening tmux. Run `attach` to reconnect a dead pane. + +A clean `on` requires a StatefulSet explicitly marked `dual-review-v1`. Do +not add that annotation to bypass the gate: it certifies that the deployed +image enforces review and credential isolation. `on` validates the context +bundle and checks a single 262,144-token MTP slot and one generated token on +each node before reissuing one registration token. No long prompt runs at startup. +If both workers are already running, `on` just attaches without rotating +credentials. Supply one hub per file; override with `HIVE_CONTRIBUTE_REGISTRATION`. + +`off` must not race a new assignment. An idle status sample is not a drain +acknowledgment. `off` checks every relay, requests a drain, waits for current +leases to settle, saves skills, then scales down contributors followed by +the models. If a relay lacks drain support, `off` refuses to scale while any +contributor replica or pod remains. Export transcripts before shutdown. Every `contribute-on` launch reads `mcp.json` from the `contribute-omp` ConfigMap and includes it in the existing context bundle. `seed-home` extracts it into `~/.omp/agent/mcp.json`. It enables the Bluefin knowledge server (`https://mcp.projectbluefin.io/mcp`) and Context7 -(`https://mcp.context7.com/mcp`) over HTTP, independently of workstation or -Ghost-profile MCP settings. No workstation API keys are copied. The canonical -configuration lives in `manifests/contribute.yaml`; both local and Luna -launches use it. Changing this ConfigMap does not restart the contributor. +from the ConfigMap, without copying workstation provider credentials into the +OMP bundle. `seed-home` restores curated skills and the system appendix from +the Ghost profile, and shares only the lessons in its approved `managed-skills`. + +The worker and reviewer run in separate containers without GitHub or Hive +credentials. Only the relay mounts the registration and holds the lease's +publication token; it shares Git bundles with the worker and talks to the +reviewer over a separate private Unix socket. Disable pod process sharing +and service-account token mounts. Never put the workstation token in the +startup Secret: it authenticates the reissue request only. The ConfigMap also ships `safe-tools.js`, `checkout.py`, and `verify.py` through the context bundle into `~/.omp/agent/hooks/pre/`. New OMP sessions discover -the extension there; no workstation profile or contributor pod template is -changed. The remaining `tooling.md` is appended to the profile's -`APPEND_SYSTEM.md` without editing the profile. +the extension there. The recipe appends `tooling.md` to the Ghost profile's +`APPEND_SYSTEM.md` in the bundle without editing the profile itself. - `contribute_checkout` accepts a Git URL, a new absolute destination, an optional ref and positive depth (default 1). It fetches that ref into a @@ -189,22 +194,26 @@ edits `~/.config/hive/contributor.env`. ## Failure modes -- `/advisor status` shows `[no model]` although `modelRoles.advisor` is set: - compare the selected role with `kubectl -n contribute exec contribute-0 -c contribute -- omp models lab-advisor --json`. - Local mode requires `lab-advisor/ornith`; a catalog that still lists only - `lab-advisor/qwen-advisor` cannot resolve it. The catalog in - `manifests/contribute.yaml` must register `ornith` at - `http://llm-d-ghost.llm-d.svc.cluster.local:8000/v1`. - Wait until the active contributor finishes or is idle before pushing the - manifest changes to git: ArgoCD auto-syncs `testing-lab-infra` on push, and - the pod template diff (removing `POD_NAME` and updating `seed-home`) triggers - an immediate rolling restart under Kubernetes default `RollingUpdate`, while - `manifests/llm-d.yaml` removes init containers from the model servers. - Push to `main` only when the active task completes, then run - `just contribute-on` to seed the refreshed `models.yml` and attach. - `seed-home` copies `models.yml` at pod initialization: syncing the ConfigMap - alone does not update the running pod's `~/.omp/agent/models.yml`. - Do not overwrite workstation settings or start another Hive relay to debug it. +- `on` refuses to launch against the legacy runtime: do not patch the + capability annotation or directly scale up. Wait for a pinned, verified + review-capable image and a drained rollout. Kubernetes cannot add + `volumeClaimTemplates` to the existing StatefulSet. Sync the replacement + under `contribute-reviewed` only after the old replica is stopped; do not + force recreation of a StatefulSet with active work. +- Review silence, invalid output, a mismatched commit, timeout, lost lease, + or a second negative review must never publish. A passing result binds + publication to the reviewed SHA; it does not trust a pane verdict or a + branch name that the worker can still move. +- Protected publishing requires a Hub-authorized direct-push GitHub issue + or a single fix to an existing same-repository PR. A fork route, unsupported + forge, or unverified base branch fails closed. Check the hub's actual + assignment routes before enabling it; a connected relay is not evidence + that the hub can issue a publishable lease. +- A disposable OMP catalog with unknown reasoning support probes multiple + effort levels on first use. Set the reviewer-only model's catalog + `reasoning: false` to avoid those extra requests; the llama.cpp preset + controls Ornith's native reasoning. Confirm the adapter sends exactly one + inference request using a disposable endpoint, not a busy GPU. - `400 model name=ornith is not running` in a contributor pane: the router evicted a model as idle between admitting a request and proxying it, because requests for the other model arrived at the same time. It persists in b11151 @@ -212,10 +221,9 @@ edits `~/.config/hive/contributor.env`. Unstick it with `kubectl -n contribute exec contribute- -c contribute -- tmux send-keys -t contributor '' Enter`. Avoid overlapping requests for the two models: never send manual requests to a router while its contributor works. -- `contribute-on` needs `jq`, `yq`, `gh` and `kubectl` as real binaries on PATH. - A shell function or alias works interactively and fails in a fresh shell. - If it fails after the reissue POST, just rerun it: the reissue authenticates - with gh, so the discarded token is replaced. +- The recipe uses `jq`, `gh`, `kubectl`, `flock`, and `tmux` executables on + `PATH`. If startup fails after the reissue, check that there are no active + pods before retrying; reissue invalidates the previous registration token. - Applying `argocd/infra-application.yaml` with new Deployment names before the manifests reach `main` removes the old names from the ignore list, so self-heal scales the old Deployments back up until the sync prunes them. @@ -226,23 +234,29 @@ edits `~/.config/hive/contributor.env`. The owner maintains the Ghost profile and curated library in chezmoi. Native skill paths are `~/.omp/agent/skills//SKILL.md` inside the container. `APPEND_SYSTEM.md` adds guidance without replacing OMP's built-in prompt. -User-level `AGENTS.md` links to Hive's `~/agent.md` export, keeping that knowledge -visible when a task moves into a cloned repository. Project instructions and +User-level `AGENTS.md` points to the relay's read-only Hive knowledge export +so it stays visible after switching repositories. Project instructions and skills remain authoritative over generic or learned guidance. -`contribute-learning-save` copies managed skills back to the separate Ghost -profile before normal on/off operations; startup restores them from the same -Secret bundle. Failed saves stop the operation instead of discarding lessons. -Review lessons for accuracy and secrets before `chezmoi add`. Capture is -additive, so prune obsolete lessons during review. Unexpected pod loss can -still lose unsaved lessons; no persistent volume is introduced. +`contribute-learning-save` captures lessons in +`~/.local/state/hive/lab/review-pending-skills`, not in the startup bundle. +Review them for accuracy and secrets, then copy selected skill directories into +the profile's `managed-skills` to share them on the next launch. Differing +versions of the same lesson stop capture; select a pod explicitly and reconcile. + +Worker and reviewer transcripts are ephemeral; export them before shutdown +(`~/.omp/agent/sessions` in `contribute`, `~/.omp/review-sessions` in +`reviewer`). The relay's evidence volume is for compact redacted review records only. +Auto-Learn is enabled for workers only. A dedicated reviewer starts with +no Auto-Learn or worker skills. Auto-Learn writes managed skills, never +Ornith weights; review captured lessons before restoring them on both nodes. ## Verification Check the actual prompt with a disposable OMP `--mode rpc --no-session` instance and `get_state`, not a second process sharing an active session's state. Confirm skill names, a distinctive Hive knowledge phrase, and matching -model context limits (`ornith` 262144), and that +worker model context limits (`ornith` 262144), and that `curl :3080x/v1/models` lists the Ornith id. No inference call is needed to verify context construction. Never use a copied Hive entrypoint to lease additional work merely for a smoke check. @@ -260,6 +274,9 @@ OMP prompt; remove the marker from both the profile and Secret afterward. - [ ] Required regression skips and missing tests fail the gate. - [ ] Startup bundle errors do not reissue registration. - [ ] Existing owner context and credentials remain unchanged. +- [ ] Distinct Ready sessions have overlapping leases and each uses only its local GPU. +- [ ] Reject, timeout, revoke, stale SHA and failed publication cannot produce a success verdict. +- [ ] Review evidence survives pod replacement; model processes cannot read publication credentials. ## Common Rationalizations diff --git a/docs/skills/hive-contribute/contribute.just b/docs/skills/hive-contribute/contribute.just index 61e3c0da86..41cd4aeead 100644 --- a/docs/skills/hive-contribute/contribute.just +++ b/docs/skills/hive-contribute/contribute.just @@ -3,101 +3,261 @@ export KUBECONFIG := env("KUBECONFIG", home_directory() / ".kube/bluespeed.yaml") -alias contribute := contribute-on - -# Start models and Hive contributor with local Ornith advisor (default), or advisor="luna" -contribute-on advisor="local": contribute-learning-save +# Start or view the pair; detach from tmux without stopping either worker. +contribute $action="on": #!/usr/bin/env bash set -euo pipefail - reg="${HIVE_CONTRIBUTE_REGISTRATION:-$HOME/.config/hive/contributor.bluefin.env}" - profile="$HOME/.omp/profiles/ghost/agent" - omp="${HIVE_CONTRIBUTE_OMP_CONFIG:-$profile/config.yml}" - tok="$(mktemp)"; trap 'rm -f "$tok"' EXIT - printf %s "$(gh auth token)" > "$tok" - # Ship standalone contributor config with Ornith worker and Ornith advisor - cfg="$(mktemp)"; bundle="$(mktemp)"; context="$(mktemp -d)" - trap 'rm -f "$tok" "$cfg" "$bundle"; rm -rf "$context"' EXIT - mkdir -p "$profile/managed-skills" - kubectl -n contribute get configmap contribute-omp -o json > "$context/config.json" - jq -er '.data["mcp.json"]' "$context/config.json" > "$context/mcp.json" - { cat "$profile/APPEND_SYSTEM.md"; printf '\n'; jq -er '.data["tooling.md"]' "$context/config.json"; } > "$context/APPEND_SYSTEM.md" - mkdir -p "$context/hooks/pre" - for asset in safe-tools.js checkout.py verify.py; do - jq -er --arg name "$asset" '.data[$name]' "$context/config.json" > "$context/hooks/pre/$asset" - done - tar -czhf "$bundle" -C "$profile" skills managed-skills -C "$context" APPEND_SYSTEM.md mcp.json hooks - advisor_mode="{{advisor}}" - if [ "$advisor_mode" = "luna" ]; then - cat > "$cfg" <<'EOF' + die() { printf '%s\n' "$*" >&2; exit 1; } + k() { timeout --kill-after=2s 20s kubectl --request-timeout=15s "$@"; } + contributor_pods() { k -n contribute get pods -l app.kubernetes.io/name=contribute -o json; } + workload=sts/contribute-reviewed + + start_models() { + k -n llm-d scale deploy/llm-d-exo-0 deploy/llm-d-ghost --replicas=1 >/dev/null + echo 'Starting both Ornith servers; checking the native slot and one generated token...' + warm() { + local node="$1" deployment="deploy/llm-d-$1" + timeout --kill-after=2s 310s kubectl -n llm-d rollout status "$deployment" --timeout=5m || return + k -n llm-d exec "$deployment" -- curl -fsS --max-time 10 'http://localhost:8000/slots?model=ornith' | + jq -e 'length == 1 and .[0].n_ctx == 262144 and .[0].speculative == true' >/dev/null || + die "$node: expected one 262144-token Ornith slot with MTP. Check the synced Lab preset and model rollout." + timeout --kill-after=2s 40s kubectl -n llm-d exec "$deployment" -- curl -fsS --max-time 30 http://localhost:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"ornith","messages":[{"role":"user","content":"ok"}],"chat_template_kwargs":{"enable_thinking":false},"max_tokens":1}' | + jq -e '.usage.completion_tokens == 1' >/dev/null || die "$node: Ornith inference check failed." + echo "$node: Ornith ready (262144 tokens, MTP active)." + } + local a b failed=0 + warm exo-0 & a=$! + warm ghost & b=$! + wait "$a" || failed=1 + wait "$b" || failed=1 + [ "$failed" = 0 ] || die 'Ornith readiness failed; no contributor was started.' + } + + case "$action" in + status) exec just --justfile {{quote(justfile())}} contribute-status ;; + on|models|attach|off) ;; + *) die "Usage: just contribute [on|models|attach|status|off]" ;; + esac + + # Serialize local launches and pane reconciliation; don't hold the lock while viewing. + umask 077 + lock_dir="${XDG_RUNTIME_DIR:-$HOME/.cache}" + mkdir -p "$lock_dir" + exec 9>"$lock_dir/hive-lab.lock" + flock -n 9 || die "Another contributor command is running; retry when it finishes." + + if [ "$action" != attach ]; then + sts="$(k -n contribute get "$workload" -o json)" + replicas="$(jq -er '.spec.replicas | numbers' <<< "$sts")" + if [ "$action" = models ]; then + [ "$replicas" = 0 ] && contributor_pods | jq -e '.items == []' >/dev/null || + die 'Refusing to warm models while a Hive contributor is running or stopping.' + start_models + exit + fi + if [ "$action" = off ]; then + pods="$(contributor_pods)" + if [ "$replicas" != 0 ]; then + # A legacy idle snapshot is not a drain acknowledgment. Check every + # replica before touching any marker, and keep the lock through scale-down. + jq -e --argjson count "$replicas" '.items | length == $count and all(.[]; .metadata.deletionTimestamp == null and .status.phase == "Running")' <<< "$pods" >/dev/null || + die "Refusing to stop: cannot drain while a contributor replica is missing or starting." + membership() { jq -c '[.items[] | [.metadata.name, .metadata.uid]] | sort'; } + original="$(membership <<< "$pods")" + mapfile -t workers < <(jq -r '.items[].metadata.name' <<< "$pods") + drain_state() { + k -n contribute exec "$1" -c relay -- cat /home/hive/.config/hive/relay-status.json | + jq -er '.operator_drain | select(. == "open" or . == "draining" or . == "drained")' + } + for pod in "${workers[@]}"; do + drain_state "$pod" >/dev/null || + die "Refusing to stop: $pod does not report a supported drain capability." + done + for pod in "${workers[@]}"; do + k -n contribute exec "$pod" -c relay -- touch /tmp/hive-operator-drain + done + echo 'Waiting for contributor leases to drain...' + deadline=$((SECONDS + 600)) + while :; do + current="$(contributor_pods)" + [ "$(membership <<< "$current")" = "$original" ] || + die "Contributor pods changed during drain; rerun just contribute off." + all_drained=1 + for pod in "${workers[@]}"; do + [ "$(drain_state "$pod")" = drained ] || all_drained=0 + done + [ "$all_drained" = 0 ] || break + [ "$SECONDS" -lt "$deadline" ] || + die "Drain still in progress; existing leases are untouched. Rerun just contribute off once they finish." + sleep 2 + done + just --justfile {{quote(justfile())}} contribute-learning-save + current="$(contributor_pods)" + [ "$(membership <<< "$current")" = "$original" ] || + die "Contributor pods changed while saving skills; rerun just contribute off." + k -n contribute scale "$workload" --current-replicas="$replicas" --replicas=0 + deadline=$((SECONDS + 120)) + while :; do + pods="$(contributor_pods)" + if jq -e '.items == []' <<< "$pods" >/dev/null; then break; fi + [ "$SECONDS" -lt "$deadline" ] || die "Contributor pods are still stopping; GPUs left on." + sleep 2 + done + fi + jq -e '.items == []' <<< "$pods" >/dev/null || + die "Refusing to stop GPUs: contributor pods are still present after drain." + k -n llm-d scale deploy/llm-d-exo-0 deploy/llm-d-ghost --replicas=0 + exit + fi + jq -e '.spec.template.metadata.annotations["bluefin.io/contributor-mode"] == "dual-review-v1"' <<< "$sts" >/dev/null || + die "Refusing launch: the live StatefulSet lacks the verified dedicated reviewer runtime (dual-review-v1). Use 'just contribute models' for models only." + case "$replicas" in + 0) ;; + 2) action=attach ;; + *) die "Refusing launch: unexpected desired replica count ($replicas); no in-place takeover." ;; + esac + fi + + command -v tmux >/dev/null || die "Install tmux on the workstation first." + if [ "$action" = on ]; then + pods="$(contributor_pods)" + jq -e '.items == []' <<< "$pods" >/dev/null || + die "Refusing launch: old contributor pods are still present. Wait for them to exit." + reg="${HIVE_CONTRIBUTE_REGISTRATION:-$HOME/.config/hive/contributor.bluefin.env}" + profile="$HOME/.omp/profiles/ghost/agent" + context="$(mktemp -d)" + trap 'rm -rf "$context"' EXIT + mkdir -p "$profile/managed-skills" "$context/hooks/pre" + k -n contribute get configmap contribute-omp -o json > "$context/config.json" + jq -er '.data["mcp.json"] | strings | select(length > 0)' "$context/config.json" > "$context/mcp.json" + jq -e 'type == "object"' "$context/mcp.json" >/dev/null + { cat "$profile/APPEND_SYSTEM.md"; printf '\n'; jq -er '.data["tooling.md"] | strings | select(length > 0)' "$context/config.json"; } > "$context/APPEND_SYSTEM.md" + for asset in safe-tools.js checkout.py verify.py; do + jq -er --arg name "$asset" '.data[$name] | strings | select(length > 0)' "$context/config.json" > "$context/hooks/pre/$asset" + done + tar -czhf "$context/ghost-context.tgz" -C "$profile" skills managed-skills -C "$context" APPEND_SYSTEM.md mcp.json hooks + cat > "$context/config.yml" <<'EOF' modelRoles: default: lab-worker/ornith smol: lab-worker/ornith - advisor: github-copilot/gpt-6-luna:max advisor: - enabled: true - reviewMode: turn - syncBacklog: "off" - async: - enabled: false - task: - batch: false - autolearn: enabled: false - EOF - else - cat > "$cfg" <<'EOF' - modelRoles: - default: lab-worker/ornith - smol: lab-worker/ornith - advisor: lab-advisor/ornith - advisor: - enabled: true - reviewMode: turn - syncBacklog: "off" async: enabled: false task: batch: false autolearn: - enabled: false + enabled: true EOF + hub="$(awk '/^HIVE_HUB=/{sub(/^HIVE_HUB=/, ""); print; exit}' "$reg")" + [[ "$hub" == wss://* ]] || die "Missing or invalid HIVE_HUB in registration." + grep -q '^HIVE_REGISTRATION_TOKEN=' "$reg" || die "Registration token field is missing." + + start_models + + # Recheck after the potentially long warm-up before touching the one registration. + sts="$(k -n contribute get "$workload" -o json)" + jq -e '.spec.replicas == 0 and .spec.template.metadata.annotations["bluefin.io/contributor-mode"] == "dual-review-v1"' <<< "$sts" >/dev/null || + die "StatefulSet changed during warm-up; refusing to reissue registration." + pods="$(contributor_pods)" + jq -e '.items == []' <<< "$pods" >/dev/null || die "Contributor pods appeared during warm-up." + token="$(timeout --kill-after=2s 20s gh auth token)" + [ -n "$token" ] || die "gh returned an empty token." + http="https://${hub#wss://}" + new="$(curl -fsS --max-time 20 -X POST "${http%/contribute}/api/contribute/reissue-token" -H "Authorization: Bearer $token" | + jq -er '.registration_token | strings | select(length > 0 and (test("[[:space:]]") | not))')" + HIVE_NEW_TOKEN="$new" awk '/^HIVE_REGISTRATION_TOKEN=/{ $0="HIVE_REGISTRATION_TOKEN=" ENVIRON["HIVE_NEW_TOKEN"] } 1' "$reg" > "$context/contributor.env" + cat "$context/contributor.env" > "$reg" + k -n contribute create secret generic contribute \ + --from-file=contributor.env="$context/contributor.env" \ + --from-file=config.yml="$context/config.yml" --from-file=ghost-context.tgz="$context/ghost-context.tgz" \ + --dry-run=client -o yaml | k apply -f - >/dev/null + rm -rf "$context" + trap - EXIT + k -n contribute scale "$workload" --current-replicas=0 --replicas=2 >/dev/null + # Check remote session readiness directly, even if only one pod can schedule. + deadline=$((SECONDS + 600)) + else + deadline=$SECONDS fi - # Refresh the Hive registration token with the gh identity (upstream's contribute-move call), - # so a stale or reissued-elsewhere token never needs a manual repair. The lab is the one contributor. - hub="$(sed -nE 's/^HIVE_HUB=//p' "$reg" | head -1)"; http="https://${hub#wss://}"; http="${http%/contribute}" - new="$(curl -fsS --max-time 20 -X POST "${http}/api/contribute/reissue-token" -H "Authorization: Bearer $(cat "$tok")" | jq -er .registration_token)" - sed -i "s|^HIVE_REGISTRATION_TOKEN=.*|HIVE_REGISTRATION_TOKEN=${new}|" "$reg" - kubectl -n contribute create secret generic contribute \ - --from-file=GH_TOKEN="$tok" --from-file=contributor.env="$reg" --from-file=config.yml="$cfg" \ - --from-file=ghost-context.tgz="$bundle" \ - --dry-run=client -o yaml | kubectl apply -f - >/dev/null - kubectl -n llm-d scale deploy/llm-d-exo-0 deploy/llm-d-ghost --replicas=1 >/dev/null - echo "Warming Ornith on both nodes before starting contributor..." - kubectl -n llm-d rollout status deploy/llm-d-exo-0 --timeout=60m - kubectl -n llm-d rollout status deploy/llm-d-ghost --timeout=60m - # Warm-up gate: load Ornith on each node (exo-0 worker, ghost advisor). - warm() { - kubectl -n llm-d exec "deploy/llm-d-$1" -- curl -fsS --max-time 3600 localhost:8000/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -d '{"model":"ornith","messages":[{"role":"user","content":"ok"}],"max_tokens":1}' >/dev/null + + find_lanes() { + local pods listing node pod + exo= ghost= + pods="$(contributor_pods)" || return + listing="$(jq -r --arg action "$action" '.items[] + | select(.metadata.deletionTimestamp == null and .status.phase == "Running") + | select($action != "on" or any(.status.conditions[]?; .type == "Ready" and .status == "True")) + | select(.spec.nodeName == "exo-0" or .spec.nodeName == "ghost") + | [.spec.nodeName, .metadata.name] | @tsv' <<< "$pods")" || return + while IFS=$'\t' read -r node pod; do + case "$node" in + exo-0) [ -z "$exo" ] || continue ;; + ghost) [ -z "$ghost" ] || continue ;; + *) continue ;; + esac + if k -n contribute exec "$pod" -c contribute --pod-running-timeout=5s -- tmux has-session -t =contributor 2>/dev/null; then + if [ "$node" = exo-0 ]; then exo="$pod"; else ghost="$pod"; fi + fi + done <<< "$listing" } - warm exo-0 & a=$!; warm ghost & b=$! - wait "$a"; wait "$b" - kubectl -n contribute scale sts/contribute --replicas=1 >/dev/null - kubectl -n contribute rollout restart sts/contribute >/dev/null - kubectl -n contribute rollout status sts/contribute --timeout=10m - until kubectl -n contribute exec contribute-0 -- tmux has-session -t contributor 2>/dev/null; do sleep 2; done - just contribute-attach - -# Attach to the contributor (contribute-0 on exo-0; ghost runs advisor). -# Detach: C-b d; contributor keeps running. -contribute-attach: - kubectl -n contribute exec -it contribute-0 -- env TERM=xterm-256color COLORTERM=truecolor tmux attach -t contributor - -# Stop both contributors and free both GPUs. -contribute-off: contribute-learning-save - kubectl -n contribute scale sts/contribute --replicas=0 - kubectl -n llm-d scale deploy/llm-d-exo-0 deploy/llm-d-ghost --replicas=0 + + until find_lanes && [ -n "$exo" ] && [ -n "$ghost" ]; do + [ "$SECONDS" -lt "$deadline" ] || break + sleep 2 + done + if [ "$action" = on ] || [ "${HIVE_CONTRIBUTE_NO_ATTACH:-0}" = 1 ]; then + [ -n "$exo" ] && [ -n "$ghost" ] || die 'Both reviewer-backed workers did not become ready; run just contribute status.' + printf 'Dedicated-review workers ready: exo-0=%s ghost=%s\n' "$exo" "$ghost" + fi + [ "${HIVE_CONTRIBUTE_NO_ATTACH:-0}" != 1 ] || exit 0 + session="=hive-lab" + window="hive-lab:lanes" + if ! tmux has-session -t "$session" 2>/dev/null; then + left="$(tmux new-session -d -P -F '#{pane_id}' -s hive-lab -n lanes -x 120 -y 32 sleep infinity 9>&-)" + tmux set-window-option -t "$window" remain-on-exit on + right="$(tmux split-window -d -h -P -F '#{pane_id}' -t "$left" sleep infinity 9>&-)" + tmux set-option -p -t "$left" @hive_lane exo-0 + tmux set-option -p -t "$right" @hive_lane ghost + tmux set-window-option -t "$window" pane-border-status top + tmux set-window-option -t "$window" pane-border-format ' #{@hive_lane} ' + fi + panes="$(tmux list-panes -t "$window" -F '#{pane_id} #{@hive_lane} #{pane_dead} #{@hive_pod}')" + tmux set-window-option -t "$window" synchronize-panes off + while read -r pane node dead attached_pod; do + case "$node" in + exo-0) pod="$exo" ;; + ghost) pod="$ghost" ;; + *) continue ;; + esac + if [ -z "$pod" ]; then + printf '%s: remote lane unavailable; rerun just contribute attach to reconnect.\n' "$node" >&2 + if [ -z "$attached_pod" ] && [ "$dead" = 0 ]; then + tmux respawn-pane -k -t "$pane" bash -c 'printf "%s: lane unavailable. Reconnect with just contribute attach.\n" "$1"' bash "$node" 9>&- + fi + elif [ "$dead" = 1 ] || [ -z "$attached_pod" ]; then + tmux respawn-pane -k -t "$pane" bash -c ' + KUBECONFIG="$1" kubectl -n contribute exec -it "$2" -c contribute --pod-running-timeout=5s \ + -- env TERM=xterm-256color COLORTERM=truecolor tmux attach -t =contributor + printf "\n%s: disconnected. Reconnect with just contribute attach.\n" "$3" + ' bash "$KUBECONFIG" "$pod" "$node" 9>&- + tmux set-option -p -t "$pane" @hive_pod "$pod" + fi + done <<< "$panes" + exec 9>&- + if [ -n "${TMUX:-}" ]; then + exec tmux switch-client -t "$session" + else + exec tmux attach-session -t "$session" + fi + +contribute-on: (contribute "on") + +contribute-attach: (contribute "attach") + +contribute-off: (contribute "off") # Read-only node resources, pod state and Ornith inference snapshots (no model requests). contribute-status: @@ -157,12 +317,12 @@ contribute-status: contribute-learning-save pod="": #!/usr/bin/env bash set -euo pipefail - pods="{{pod}}" + pods={{quote(pod)}} if [ -z "$pods" ]; then pods="$(kubectl -n contribute get pods -l app.kubernetes.io/name=contribute -o json | jq -r '.items[] | select(any(.status.containerStatuses[]?; .name == "contribute" and .state.running != null)) | .metadata.name')" fi [ -n "$pods" ] || exit 0 - dest="$HOME/.omp/profiles/ghost/agent/managed-skills" + dest="$HOME/.local/state/hive/lab/review-pending-skills" mkdir -p "$dest" # ponytail: additive capture from every contributor; prune obsolete lessons during review. # Two pods changing the same skill would overwrite each other: stop instead of losing a lesson. @@ -180,4 +340,4 @@ contribute-learning-save pod="": done fi for pod in $pods; do cp -r "$stage/$pod/." "$dest"; done - printf 'Saved learned skills to %s; review before chezmoi add.\n' "$dest" + printf 'Saved lessons to %s; review and promote selected skills to the profile managed-skills directory before sharing.\n' "$dest" diff --git a/manifests/contribute.yaml b/manifests/contribute.yaml index a256e20a57..1dffab20aa 100644 --- a/manifests/contribute.yaml +++ b/manifests/contribute.yaml @@ -1,7 +1,8 @@ # hive-contribute: projectbluefin/contribute (OMP in tmux) on the local llm-d models. -# StatefulSet, replicas: 0 in git; `just contribute-on` (in ~/.justfile) writes the Secret (your `gh auth token`, -# Hive registration, OMP config.yml) and scales it to 1. Ornith on exo-0 implements; -# Ornith on ghost advises. See docs/skills/hive-contribute/SKILL.md. +# StatefulSet, replicas: 0 in Git. Each contributor uses the Ornith server on +# its own node for both implementation and the separate review session. +# Rollout requires the pinned review runtime and a drained StatefulSet recreate: +# Kubernetes cannot add volumeClaimTemplates to an existing StatefulSet. # Files are copied into an emptyDir home by seed-home: subPath mounts would make # ~/.omp root-owned, and OMP must write ~/.omp/natives. apiVersion: v1 @@ -20,15 +21,7 @@ data: models.yml: | providers: lab-worker: - baseUrl: http://llm-d-exo-0.llm-d.svc.cluster.local:8000/v1 - api: openai-completions - apiKey: local - models: - - id: ornith - name: Ornith 1.5 35B-A3B Q6_K - contextWindow: 262144 - lab-advisor: - baseUrl: http://llm-d-ghost.llm-d.svc.cluster.local:8000/v1 + baseUrl: http://llm-d-@NODE_NAME@.llm-d.svc.cluster.local:8000/v1 api: openai-completions apiKey: local models: @@ -446,17 +439,19 @@ data: configured Bluefin knowledge and Context7 tools for platform/API facts; separate verified facts from hypotheses. roles.yml: | - # Layered last: pinned local models for worker roles and priority advisor settings + # Layered last: the relay runs a separate review, not OMP's advisor. modelRoles: default: lab-worker/ornith smol: lab-worker/ornith advisor: - syncBacklog: "off" + enabled: false + autolearn: + enabled: true --- apiVersion: apps/v1 kind: StatefulSet metadata: - name: contribute + name: contribute-reviewed namespace: contribute labels: app.kubernetes.io/name: contribute @@ -469,17 +464,43 @@ spec: app.kubernetes.io/name: contribute template: metadata: + annotations: + bluefin.io/contributor-mode: dual-review-v1 labels: app.kubernetes.io/name: contribute spec: + automountServiceAccountToken: false + shareProcessNamespace: false + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: [exo-0, ghost] + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: contribute + topologyKey: kubernetes.io/hostname securityContext: runAsUser: 65532 runAsGroup: 65532 fsGroup: 65532 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - name: contribute - image: ghcr.io/projectbluefin/contribute:stable + image: &review_image 192.168.1.102:30500/lab-contribute@sha256:30f5a7de16abde578864172ff5a58a911b4e013a1b6f13730c54188874feb96f imagePullPolicy: Always + command: ["/usr/local/bin/hive-review-runtime", "worker"] + securityContext: &unprivileged + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] env: - name: AGENT_BACKEND value: omp @@ -491,15 +512,10 @@ spec: value: truecolor - name: PI_CONFIG_FILES value: /usr/share/hive/contribute/omp-config.yml:/etc/contribute/roles.yml - - name: HIVE_SESSION - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: GH_TOKEN - valueFrom: - secretKeyRef: - name: contribute - key: GH_TOKEN + readinessProbe: + exec: + command: ["/usr/bin/tmux", "has-session", "-t", "=contributor"] + periodSeconds: 5 volumeMounts: - name: home mountPath: /home/hive @@ -508,6 +524,13 @@ spec: - name: omp mountPath: /etc/contribute readOnly: true + - name: handoff + mountPath: /handoff + - name: worker-socket + mountPath: /run/hive-worker + - name: hive-knowledge + mountPath: /run/hive-knowledge + readOnly: true resources: requests: cpu: '1' @@ -515,40 +538,146 @@ spec: limits: cpu: '8' memory: 16Gi + - name: relay + image: *review_image + imagePullPolicy: Always + command: ["/usr/local/bin/hive-review-runtime", "relay"] + securityContext: *unprivileged + env: + - name: HIVE_SESSION + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: HIVE_OPERATOR_DRAIN_FILE + value: /tmp/hive-operator-drain + - name: HIVE_PREPUBLISH_REVIEW_DIR + value: /handoff + - name: HIVE_REVIEW_EVIDENCE_DIR + value: /var/lib/hive-review + readinessProbe: + exec: + command: ["/usr/bin/bash", "-c", "test -s /home/hive/.config/hive/relay-status.json"] + periodSeconds: 5 + volumeMounts: + - name: registration + mountPath: /registration + readOnly: true + - name: relay-tmp + mountPath: /tmp + - name: worker-socket + mountPath: /run/hive-worker + - name: reviewer-socket + mountPath: /run/hive-reviewer + - name: handoff + mountPath: /handoff + - name: hive-knowledge + mountPath: /run/hive-knowledge + - name: review-evidence + mountPath: /var/lib/hive-review + subPath: evidence + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '2' + memory: 2Gi + - name: reviewer + image: *review_image + imagePullPolicy: Always + command: ["/usr/local/bin/hive-review-runtime", "reviewer"] + securityContext: *unprivileged + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + readinessProbe: + exec: + command: ["/usr/bin/bash", "-c", "test -S /run/hive-reviewer/review.sock"] + periodSeconds: 5 + volumeMounts: + - name: reviewer-tmp + mountPath: /tmp + - name: reviewer-socket + mountPath: /run/hive-reviewer + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '2' + memory: 2Gi volumes: - name: home emptyDir: sizeLimit: 50Gi - name: tmp emptyDir: {} + - name: relay-tmp + emptyDir: {} + - name: reviewer-tmp + emptyDir: {} + - name: worker-socket + emptyDir: {} + - name: reviewer-socket + emptyDir: {} + - name: hive-knowledge + emptyDir: {} + - name: handoff + emptyDir: + sizeLimit: 512Mi + - name: registration + secret: + secretName: contribute + defaultMode: 288 + items: + - key: contributor.env + path: contributor.env - name: secret secret: secretName: contribute defaultMode: 288 + items: + - key: config.yml + path: config.yml + - key: ghost-context.tgz + path: ghost-context.tgz - name: omp configMap: name: contribute-omp initContainers: - name: seed-home - image: ghcr.io/projectbluefin/contribute:stable + image: *review_image imagePullPolicy: Always + securityContext: *unprivileged command: - bash - -c - | set -euo pipefail + case "${NODE_NAME:-}" in + exo-0|ghost) ;; + *) printf 'Unsupported contributor node: %s\n' "${NODE_NAME:-}" >&2; exit 1 ;; + esac + umask 077 mkdir -p ~/.omp/agent ~/.config/hive [ -d ~/workspace ] && [ ! -O ~/workspace ] && rm -rf ~/workspace || true mkdir -p ~/workspace chmod 755 ~/workspace install -m600 /seed/secret/config.yml ~/.omp/agent/config.yml - install -m600 /seed/omp/models.yml ~/.omp/agent/models.yml - install -m600 /seed/secret/contributor.env ~/.config/hive/contributor.env + sed "s/@NODE_NAME@/$NODE_NAME/" "/seed/omp/models.yml" > ~/.omp/agent/models.yml tar -xzf /seed/secret/ghost-context.tgz -C ~/.omp/agent - ln -sfn "$HOME/agent.md" ~/.omp/agent/AGENTS.md + ln -sfn /run/hive-knowledge/agent.md ~/.omp/agent/AGENTS.md + # local-path's volume root may be world writable; expose only this private directory. + install -d -m0700 "/seed/review/evidence" env: - name: HOME value: /home/hive + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName volumeMounts: - name: home mountPath: /home/hive @@ -558,3 +687,14 @@ spec: - name: omp mountPath: /seed/omp readOnly: true + - name: review-evidence + mountPath: /seed/review + volumeClaimTemplates: + - metadata: + name: review-evidence + spec: + accessModes: [ReadWriteOnce] + storageClassName: local-path + resources: + requests: + storage: 1Gi diff --git a/tests/unit/test_contribute_placement.py b/tests/unit/test_contribute_placement.py new file mode 100644 index 0000000000..b752d08545 --- /dev/null +++ b/tests/unit/test_contribute_placement.py @@ -0,0 +1,154 @@ +import os +import stat +import subprocess +import tarfile +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture(scope="module") +def contributor(): + with (ROOT / "manifests/contribute.yaml").open() as manifest: + resources = {doc["kind"]: doc for doc in yaml.safe_load_all(manifest)} + return resources["StatefulSet"], resources["ConfigMap"]["data"] + + +def test_contributor_placement_and_session_identity(contributor): + stateful_set, _ = contributor + spec = stateful_set["spec"] + pod = spec["template"] + labels = {"app.kubernetes.io/name": "contribute"} + app = yaml.safe_load((ROOT / "argocd/infra-application.yaml").read_text()) + assert any(rule.get("kind") == "StatefulSet" + and rule.get("name") == stateful_set["metadata"]["name"] + and rule.get("namespace") == stateful_set["metadata"]["namespace"] + and "/spec/replicas" in rule.get("jsonPointers", []) + for rule in app["spec"]["ignoreDifferences"]) + assert spec["replicas"] == 0 + assert spec["selector"]["matchLabels"] == labels + assert pod["metadata"]["labels"].items() >= labels.items() + affinity = pod["spec"]["affinity"] + required_nodes = affinity["nodeAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + (term,) = required_nodes["nodeSelectorTerms"] + (node_selector,) = term["matchExpressions"] + assert node_selector["key"] == "kubernetes.io/hostname" + assert node_selector["operator"] == "In" + assert set(node_selector["values"]) == {"exo-0", "ghost"} + (anti_affinity,) = affinity["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + assert anti_affinity["topologyKey"] == "kubernetes.io/hostname" + assert anti_affinity["labelSelector"]["matchLabels"] == labels + relay = next(container for container in pod["spec"]["containers"] if container["name"] == "relay") + env = {item["name"]: item for item in relay["env"]} + assert env["HIVE_SESSION"]["valueFrom"]["fieldRef"]["fieldPath"] == "metadata.name" + + +def test_review_evidence_uses_a_private_subdirectory_of_a_per_pod_claim(contributor): + stateful_set, _ = contributor + spec = stateful_set["spec"] + (claim,) = spec["volumeClaimTemplates"] + assert claim["spec"]["accessModes"] == ["ReadWriteOnce"] + assert claim["spec"]["storageClassName"] == "local-path" + assert claim["spec"]["resources"]["requests"]["storage"] == "1Gi" + (mount,) = ( + mount for container in spec["template"]["spec"]["containers"] if container["name"] == "relay" + for mount in container["volumeMounts"] + if mount["mountPath"] == "/var/lib/hive-review" + ) + assert mount["name"] == claim["metadata"]["name"] + assert mount["subPath"] == "evidence" + + +def test_model_containers_cannot_mount_relay_credentials_or_each_others_control_sockets(contributor): + stateful_set, _ = contributor + pod = stateful_set["spec"]["template"]["spec"] + assert pod["automountServiceAccountToken"] is False + assert pod["shareProcessNamespace"] is False + mounts = {container["name"]: {mount["name"] for mount in container["volumeMounts"]} + for container in pod["containers"]} + assert "registration" in mounts["relay"] + assert not mounts["contribute"] & {"registration", "reviewer-socket", "relay-tmp", "review-evidence"} + assert not mounts["reviewer"] & {"registration", "worker-socket", "relay-tmp", "review-evidence", "handoff"} + init = next(container for container in pod["initContainers"] if container["name"] == "seed-home") + assert not any(mount["name"] == "registration" for mount in init["volumeMounts"]) + projected = next(volume["secret"]["items"] for volume in pod["volumes"] if volume["name"] == "secret") + assert {item["key"] for item in projected} == {"config.yml", "ghost-context.tgz"} + + +@pytest.fixture +def seed_home(contributor, tmp_path): + stateful_set, data = contributor + pod = stateful_set["spec"]["template"]["spec"] + (init,) = (container for container in pod["initContainers"] if container["name"] == "seed-home") + (node_env,) = ( + item for item in init["env"] + if item.get("valueFrom", {}).get("fieldRef", {}).get("fieldPath") == "spec.nodeName" + ) + home = tmp_path / "home" + home.mkdir() + mounts = {mount["name"]: mount["mountPath"] for mount in init["volumeMounts"]} + sources = {name: tmp_path / mount.lstrip("/") for name, mount in mounts.items() if name != "home"} + for directory in sources.values(): + directory.mkdir(parents=True) + (sources["omp"] / "models.yml").write_text(data["models.yml"]) + (sources["secret"] / "config.yml").write_text("advisor:\n enabled: true\n") + (sources["secret"] / "contributor.env").write_text("HIVE_REGISTRATION_TOKEN=never-seed-this") + with tarfile.open(sources["secret"] / "ghost-context.tgz", "w:gz"): + pass + + # Simulate the mounts without rewriting the actual seed logic. + command = [arg.replace("/seed/", f"{tmp_path}/seed/") for arg in init["command"]] + evidence = next(mount for container in pod["containers"] if container["name"] == "relay" + for mount in container["volumeMounts"] if mount["mountPath"] == "/var/lib/hive-review") + + def run(node): + return subprocess.run( + command, + env={"HOME": str(home), "PATH": os.environ["PATH"], node_env["name"]: node}, + capture_output=True, + text=True, + ) + + return run, home / ".omp/agent/models.yml", sources[evidence["name"]] / evidence["subPath"] + + +@pytest.mark.parametrize(("node", "endpoint"), [ + ("exo-0", "http://llm-d-exo-0.llm-d.svc.cluster.local:8000/v1"), + ("ghost", "http://llm-d-ghost.llm-d.svc.cluster.local:8000/v1"), +]) +def test_seed_writes_only_the_assigned_nodes_model(seed_home, node, endpoint): + run, catalog, evidence = seed_home + result = run(node) + assert result.returncode == 0, result.stderr + providers = yaml.safe_load(catalog.read_text())["providers"] + assert set(providers) == {"lab-worker"} + assert providers["lab-worker"]["baseUrl"] == endpoint + assert [model["id"] for model in providers["lab-worker"]["models"]] == ["ornith"] + assert evidence.is_dir() + assert stat.S_IMODE(evidence.stat().st_mode) == 0o700 + assert not (catalog.parents[2] / ".config/hive/contributor.env").exists() + + +@pytest.mark.parametrize("node", ["other-node", ""]) +def test_seed_rejects_an_unmapped_node_before_writing_a_catalog(seed_home, node): + run, catalog, evidence = seed_home + result = run(node) + assert result.returncode != 0 + assert "Unsupported contributor node" in result.stderr + assert not catalog.exists() + assert not evidence.exists() + + +def test_last_worker_overlay_disables_the_per_turn_advisor(contributor): + stateful_set, config_map = contributor + container = stateful_set["spec"]["template"]["spec"]["containers"][0] + env = {item["name"]: item.get("value") for item in container["env"]} + overlay = env["PI_CONFIG_FILES"].split(":")[-1] + (config_mount,) = (mount for mount in container["volumeMounts"] if mount["name"] == "omp") + roles = yaml.safe_load(config_map[str(Path(overlay).relative_to(config_mount["mountPath"]))]) + assert roles["modelRoles"] == {"default": "lab-worker/ornith", "smol": "lab-worker/ornith"} + assert roles["advisor"] == {"enabled": False} + assert roles["autolearn"]["enabled"] is True diff --git a/tests/unit/test_contribute_startup.py b/tests/unit/test_contribute_startup.py index 9825c30d38..e0a7502799 100644 --- a/tests/unit/test_contribute_startup.py +++ b/tests/unit/test_contribute_startup.py @@ -1,52 +1,275 @@ import json import os +import shutil import subprocess from pathlib import Path -ROOT = Path(__file__).resolve().parents[2] +import pytest +ROOT = Path(__file__).resolve().parents[2] -def test_missing_safe_tool_assets_do_not_rotate_registration(tmp_path): - """An invalid startup bundle must not invalidate the existing contributor lease.""" - lines = (ROOT / "docs/skills/hive-contribute/contribute.just").read_text().splitlines() - start = next(i for i, line in enumerate(lines) if line.startswith("contribute-on advisor=")) - body = [] - for line in lines[start + 1:]: - if line and not line.startswith(" "): - break - body.append(line[4:] if line else "") - script = "\n".join(body).replace("{{advisor}}", "local") +@pytest.fixture +def launcher(tmp_path): + recipe = ROOT / "docs/skills/hive-contribute/contribute.just" home = tmp_path / "home" profile = home / ".omp/profiles/ghost/agent" - for name in ("skills", "managed-skills"): - (profile / name).mkdir(parents=True, exist_ok=True) + (profile / "skills").mkdir(parents=True) (profile / "APPEND_SYSTEM.md").write_text("Existing owner instructions.\n") registration = tmp_path / "registration.env" - original = "HIVE_HUB=wss://example.invalid/contribute\nHIVE_REGISTRATION_TOKEN=existing-token\n" - registration.write_text(original) + registration.write_text("HIVE_HUB=wss://example.invalid/contribute\nHIVE_REGISTRATION_TOKEN=existing-token\n") config = tmp_path / "config.json" config.write_text(json.dumps({"data": {"mcp.json": "{}", "tooling.md": "guidance\n"}})) - binaries = tmp_path / "bin" binaries.mkdir() - gh = binaries / "gh" - gh.write_text("#!/bin/sh\nprintf '%s' fixture-token\n") - kubectl = binaries / "kubectl" - kubectl.write_text( - f"#!/bin/sh\nprintf queried > '{tmp_path / 'config_queried'}'\n" - f"cat '{config}'\n" - ) - curl = binaries / "curl" - curl.write_text( - f"#!/bin/sh\nprintf rotated > '{tmp_path / 'registration_rotated'}'\n" - "printf '%s' '{\"registration_token\":\"replacement-token\"}'\n" - ) - for executable in (gh, kubectl, curl): + calls = tmp_path / "calls" + calls.touch() + scripts = { + "kubectl": """case "$*" in + *'get sts/contribute-reviewed'*) printf '%s' "$STATEFULSET" ;; + *'get configmap contribute-omp'*) cat "$BUNDLE_CONFIG" ;; + *'get pods'*) printf '%s' "$PODS" ;; + *'has-session'*) exit 0 ;; + *) exit 99 ;; +esac""", + "gh": "exit 97", + "curl": "exit 98", + "tmux": """case "$*" in + 'has-session -t =hive-lab') exit 0 ;; + *list-panes*) printf '%s\\n' '%0 exo-0 0 worker-1' '%1 ghost 0 worker-0' ;; +esac""", + } + + def stub(name, script): + executable = binaries / name + executable.write_text(f'#!/bin/sh\nprintf "%s %s\\n" "{name}" "$*" >> "$CALLS"\n{script}\n') executable.chmod(0o755) - environment = {**os.environ, "HOME": str(home), "HIVE_CONTRIBUTE_REGISTRATION": str(registration), "PATH": f"{binaries}:{os.environ['PATH']}"} - result = subprocess.run(["bash", "-c", script], env=environment, capture_output=True, text=True) + return executable + + for name, script in scripts.items(): + stub(name, script) + + def run(action="on", replicas=0, mode="dual-review-v1", **overrides): + pods = [ + {"metadata": {"name": name}, "spec": {"nodeName": node}, "status": {"phase": "Running"}} + for name, node in (("worker-0", "ghost"), ("worker-1", "exo-0")) + ] + sts = {"spec": {"replicas": replicas, "template": {"metadata": { + "annotations": {"bluefin.io/contributor-mode": mode} + }}}} + environment = {**os.environ, "HOME": str(home), "XDG_RUNTIME_DIR": str(tmp_path), + "HIVE_CONTRIBUTE_REGISTRATION": str(registration), "BUNDLE_CONFIG": str(config), + "PATH": f"{binaries}:{os.environ['PATH']}", "CALLS": str(calls), + "STATEFULSET": json.dumps(sts), "PODS": json.dumps({"items": pods if replicas else []}), + **overrides} + if "TMUX" not in overrides: + environment.pop("TMUX", None) + calls.write_text("") + result = subprocess.run( + ["just", "--justfile", str(recipe), "contribute", *([action] if action else [])], + env=environment, capture_output=True, text=True, timeout=20, + ) + return result, calls.read_text() + + return run, registration, config, stub + + +def test_missing_safe_tool_assets_do_not_rotate_registration(launcher): + run, registration, _, _ = launcher + original = registration.read_text() + result, calls = run() assert result.returncode != 0 - assert (tmp_path / "config_queried").exists(), result.stderr - assert not (tmp_path / "registration_rotated").exists() + assert "get configmap contribute-omp" in calls, result.stderr + assert not any(command in calls for command in ("gh auth token", "curl ", "scale ")) assert registration.read_text() == original + + +@pytest.mark.parametrize(("replicas", "mode"), [(0, None), (1, None), (1, "dual-review-v1"), (3, "dual-review-v1")]) +def test_on_refuses_legacy_or_unexpected_replica_state(launcher, replicas, mode): + run, _, _, _ = launcher + result, calls = run(replicas=replicas, mode=mode) + assert result.returncode != 0 + assert "get sts/contribute-reviewed" in calls + assert not any(command in calls for command in ("gh auth token", "curl ", "scale ", "rollout ", "configmap ")) + + +@pytest.mark.parametrize("action", ["on", ""]) +def test_already_on_attaches_without_startup_side_effects(launcher, action): + run, _, _, _ = launcher + result, calls = run(action=action, replicas=2) + assert result.returncode == 0, result.stderr + assert "tmux attach-session -t =hive-lab" in calls + assert not any(command in calls for command in ("gh auth token", "curl ", "scale ", "rollout ", "configmap ")) + + +def test_cold_start_warms_both_models_before_reissuing_once(launcher, tmp_path): + run, registration, config, stub = launcher + config.write_text(json.dumps({"data": { + "mcp.json": "{}", "tooling.md": "guidance\n", "safe-tools.js": "// hook\n", + "checkout.py": "# hook\n", "verify.py": "# hook\n", + }})) + stub("gh", "printf fixture-token") + stub("curl", """printf '%s' '{"registration_token":"replacement-token"}'""") + stub("kubectl", """case "$*" in + *'get sts/contribute-reviewed'*) printf '%s' "$STATEFULSET" ;; + *'get configmap contribute-omp'*) cat "$BUNDLE_CONFIG" ;; + *'get pods'*) + if [ -e "$STARTED" ]; then printf '%s' "$READY_PODS"; else printf '{"items":[]}'; fi ;; + *'has-session'*) exit 0 ;; + *'scale sts/contribute-reviewed --current-replicas=0 --replicas=2'*) touch "$STARTED" ;; + *'scale deploy/'*|*'rollout status'*) exit 0 ;; + *'/slots?model=ornith'*) printf '[{"n_ctx":%s,"speculative":true}]' "${SLOT_CONTEXT:-262144}" ;; + *'v1/chat/completions'*) printf '{"usage":{"completion_tokens":1}}' ;; + *'create secret'*) + for arg do + case "$arg" in --from-file=ghost-context.tgz=*) tar -tzf "${arg#--from-file=ghost-context.tgz=}" > "$BUNDLE_CONTENTS" ;; esac + done + printf 'secret YAML' ;; + *'apply -f -'*) cat >/dev/null ;; + *) exit 99 ;; +esac""") + ready = {"items": [ + {"metadata": {"name": pod}, "spec": {"nodeName": node}, + "status": {"phase": "Running", "conditions": [{"type": "Ready", "status": "True"}]}} + for node, pod in (("ghost", "worker-0"), ("exo-0", "worker-1")) + ]} + home = tmp_path / "home" + approved = home / ".omp/profiles/ghost/agent/managed-skills/approved/SKILL.md" + pending = home / ".local/state/hive/lab/review-pending-skills/unapproved/SKILL.md" + for skill in (approved, pending): + skill.parent.mkdir(parents=True) + skill.write_text("Test lesson\n") + contents = tmp_path / "bundle-contents" + options = dict(READY_PODS=json.dumps(ready), BUNDLE_CONTENTS=str(contents)) + result, calls = run(STARTED=str(tmp_path / "started"), **options) + assert result.returncode == 0, result.stderr + assert calls.count("/api/contribute/reissue-token") == 1 + rotation = calls.index("gh auth token") + for node in ("exo-0", "ghost"): + assert calls.index(f"exec deploy/llm-d-{node}") < rotation + assert rotation < calls.index("scale sts/contribute-reviewed --current-replicas=0 --replicas=2") + assert "rollout restart" not in calls + assert "HIVE_REGISTRATION_TOKEN=replacement-token\n" in registration.read_text() + assert "managed-skills/approved/SKILL.md" in contents.read_text() + assert "unapproved" not in contents.read_text() + result, calls = run(SLOT_CONTEXT="102400", STARTED=str(tmp_path / "must-not-start"), **options) + assert result.returncode != 0 + assert "262144" in result.stderr + assert "reissue-token" not in calls + assert not (tmp_path / "must-not-start").exists() + assert registration.read_text().endswith("HIVE_REGISTRATION_TOKEN=replacement-token\n") + + +def test_models_starts_native_context_without_hive_credentials(launcher): + run, registration, _, stub = launcher + previous = registration.read_text() + stub("kubectl", """case "$*" in + *'get sts/contribute-reviewed'*) printf '%s' "$STATEFULSET" ;; + *'get pods'*) printf '%s' "$PODS" ;; + *'scale deploy/'*|*'rollout status'*) exit 0 ;; + *'/slots?model=ornith'*) printf '[{"n_ctx":262144,"speculative":true}]' ;; + *'v1/chat/completions'*) printf '{"usage":{"completion_tokens":1}}' ;; + *) exit 99 ;; +esac""") + result, calls = run("models", mode=None) + assert result.returncode == 0, result.stderr + assert set(line.split("exec deploy/")[1].split()[0] for line in calls.splitlines() + if "v1/chat/completions" in line) == {"llm-d-exo-0", "llm-d-ghost"} + assert "gh auth token" not in calls and "reissue-token" not in calls + assert "scale sts/contribute" not in calls + assert registration.read_text() == previous + + +def test_off_refuses_shutdown_without_a_safe_drain(launcher): + run, _, _, _ = launcher + result, calls = run(action="off", replicas=2) + assert result.returncode != 0 + assert "drain" in result.stderr + assert "scale " not in calls + + +def test_off_saves_both_workers_before_releasing_gpus(launcher, tmp_path): + run, _, _, stub = launcher + stub("kubectl", """case "$*" in + *'get sts/contribute-reviewed'*) printf '%s' "$STATEFULSET" ;; + *'scale sts/contribute-reviewed --current-replicas=2 --replicas=0'*) touch "$STOPPED" ;; + *'get pods'*) + if [ -e "$STOPPED" ]; then printf '{"items":[]}'; else printf '%s' "$PODS"; fi ;; + *'cat /home/hive/.config/hive/relay-status.json'*) + printf '{"operator_drain":"%s"}' "$(if [ -e "$STOPPED.$5" ]; then echo drained; else echo open; fi)" ;; + *'touch /tmp/hive-operator-drain'*) touch "$STOPPED.$5" ;; + *' cp '*) + for arg do dest=$arg; done + mkdir -p "$dest/new-lesson" && printf 'Captured lesson\\n' > "$dest/new-lesson/SKILL.md" ;; + *'mkdir -p'*|*'scale deploy/'*) exit 0 ;; + *) exit 99 ;; +esac""") + pods = {"items": [{ + "metadata": {"name": name}, "spec": {"nodeName": node}, + "status": {"phase": "Running", "containerStatuses": [{"name": "contribute", "state": {"running": {}}}]}, + } for name, node in (("worker-0", "ghost"), ("worker-1", "exo-0"))]} + result, calls = run("off", replicas=2, STOPPED=str(tmp_path / "stopped"), PODS=json.dumps(pods)) + assert result.returncode == 0, result.stderr + assert calls.count("touch /tmp/hive-operator-drain") == 2 + contributor_stop = calls.index("scale sts/contribute-reviewed --current-replicas=2 --replicas=0") + assert calls.index("worker-0:/home/hive/.omp/agent/managed-skills/.") < contributor_stop + assert calls.index("worker-1:/home/hive/.omp/agent/managed-skills/.") < contributor_stop + assert contributor_stop < calls.index("scale deploy/llm-d-exo-0 deploy/llm-d-ghost --replicas=0") + home = tmp_path / "home" + assert (home / ".local/state/hive/lab/review-pending-skills/new-lesson/SKILL.md").is_file() + assert not (home / ".omp/profiles/ghost/agent/managed-skills/new-lesson").exists() + + +def test_attach_switches_an_existing_tmux_client_and_reports_missing_lane(launcher): + run, _, _, _ = launcher + result, calls = run(action="attach", TMUX="existing", PODS=json.dumps({"items": [{ + "metadata": {"name": "worker-1"}, "spec": {"nodeName": "exo-0"}, "status": {"phase": "Running"} + }]})) + assert result.returncode == 0, result.stderr + assert "ghost: remote lane unavailable" in result.stderr + assert "tmux switch-client -t =hive-lab" in calls + assert "respawn-pane" not in calls + + +@pytest.mark.skipif(shutil.which("tmux") is None, reason="requires tmux") +def test_real_tmux_reconnect_preserves_healthy_pane(launcher, tmp_path): + run, _, _, stub = launcher + socket = tmp_path / "tmux.sock" + tmux = shutil.which("tmux") + stub("kubectl", """case "$*" in + *'get pods'*) printf '%s' "$PODS" ;; + *'has-session'*) exit 0 ;; + *'attach -t'*) printf 'attached %s\\n' "$*"; exec sleep infinity ;; + *) exit 99 ;; +esac""") + # Suppress only the final client attachment (pytest has no controlling TTY). + stub("tmux", f"""[ "$1" != attach-session ] || exit 0 +exec '{tmux}' -S '{socket}' -f /dev/null "$@" """) + + def command(*args): + return subprocess.check_output([tmux, "-S", str(socket), *args], text=True, timeout=5) + + def pane_rows(): + return [row.split() for row in command( + "list-panes", "-t", "hive-lab:lanes", "-F", + "#{pane_left} #{@hive_lane} #{pane_pid} #{@hive_pod}" + ).splitlines()] + + try: + ghost_only = {"items": [{"metadata": {"name": "worker-0"}, "spec": {"nodeName": "ghost"}, + "status": {"phase": "Running"}}]} + result, _ = run(action="attach", PODS=json.dumps(ghost_only)) + assert result.returncode == 0, result.stderr + rows = pane_rows() + assert rows[0][:2] == ["0", "exo-0"] + assert rows[1][1::2] == ["ghost", "worker-0"] + assert "exo-0: remote lane unavailable" in result.stderr + result, _ = run(action="attach", replicas=2) + assert result.returncode == 0, result.stderr + recovered = pane_rows() + assert recovered[0][-1] == "worker-1" + assert recovered[1] == rows[1] + assert command("show-options", "-wv", "-t", "hive-lab:lanes", "synchronize-panes").strip() == "off" + assert command("list-sessions", "-F", "#{session_name}").strip() == "hive-lab" + finally: + subprocess.run([tmux, "-S", str(socket), "kill-server"], capture_output=True, timeout=5) From 1ff7b379dd7a14ca390e840e7b48311654ca7627 Mon Sep 17 00:00:00 2001 From: pi Date: Thu, 19 Nov 2026 18:25:50 -0500 Subject: [PATCH 2/2] ci: install Just for contributor launcher tests --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 726316d830..684578138f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install Just for launcher tests + uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + - name: Install dependencies run: | python -m pip install --upgrade pip