From 5418e9933d569f787a469a99d6bd0b9fcf5c7331 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:11:20 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20dashboard:=20stream=20Job=20pod?= =?UTF-8?q?=20logs=20in=20the=20terminal=20for=20sandbox.runtime=20job=20a?= =?UTF-8?q?gents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Co-authored-by: clubanderson <407614+clubanderson@users.noreply.github.com> --- .github/workflows/v2-ci.yml | 9 ++ .../fixed-14798-ttyd-job-runtime-logs.md | 1 + src/deploy/test_ttyd_kubejob_logs.sh | 142 ++++++++++++++++++ src/deploy/ttyd-tmux.sh | 111 ++++++++++++++ 4 files changed, 263 insertions(+) create mode 100644 changelog.d/fixed-14798-ttyd-job-runtime-logs.md create mode 100755 src/deploy/test_ttyd_kubejob_logs.sh diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index 458052a823..b1de654fc2 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -1009,6 +1009,15 @@ jobs: working-directory: . run: bash src/deploy/test_ttyd_default_socket_uid.sh + # #14798: sandbox.runtime: job agents run each kick as a Kubernetes Job + # in its own pod, so no hive- tmux session ever exists and the + # dashboard terminal waited for one until it timed out. The helper now + # streams the newest Job pod's log instead. This stubs curl as a fake + # API server and asserts the log, exit status and "no run yet" message. + - name: ttyd streams Job pod logs for sandbox.runtime job agents (#14798) + working-directory: . + run: bash src/deploy/test_ttyd_kubejob_logs.sh + # #5435: hive-discord.service ran `node bot.js` with # WorkingDirectory=/tmp/hive/discord, resolving the executed code out of a # world-writable parent that is cleared on every reboot — a local user who diff --git a/changelog.d/fixed-14798-ttyd-job-runtime-logs.md b/changelog.d/fixed-14798-ttyd-job-runtime-logs.md new file mode 100644 index 0000000000..c2f0588d6e --- /dev/null +++ b/changelog.d/fixed-14798-ttyd-job-runtime-logs.md @@ -0,0 +1 @@ +- **The dashboard's ▶ terminal now shows what a `sandbox.runtime: job` agent is doing** (#14798). Job-runtime agents run each kick as a Kubernetes Job in their own pod, so there is never a `hive-` tmux session in the hive pod and the terminal sat on "tmux session not created yet" until its wait expired. `ttyd-tmux.sh` now looks up the agent's Job pods by the launcher's `app.kubernetes.io/managed-by=hive-kubejob` / `hive.kubestellar.io/agent` labels with the pod's existing ServiceAccount and streams the newest pod's log read-only — following it while it runs and printing the exit status when it finishes — picks up a Job that starts while the terminal is waiting, and says plainly when the agent has had no run yet. Tmux agents and non-Kubernetes installs are unchanged. diff --git a/src/deploy/test_ttyd_kubejob_logs.sh b/src/deploy/test_ttyd_kubejob_logs.sh new file mode 100755 index 0000000000..644c0dbc9f --- /dev/null +++ b/src/deploy/test_ttyd_kubejob_logs.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# The browser terminal must stream a sandbox.runtime: job agent's Job pod log +# instead of waiting for a tmux session that never appears (#14798). +# Run: bash src/deploy/test_ttyd_kubejob_logs.sh +# +# THE BUG. A job-runtime agent runs every kick as a Kubernetes Job in its own +# pod (src/crux/kubejob), so there is never a hive- tmux session in the +# hive pod. ttyd-tmux.sh only knew how to wait for one, so the dashboard's +# terminal showed "tmux session not created yet" for the whole wait window and +# then "no tmux socket found" — never what the sandboxed agent was doing. +# +# HOW THIS TESTS IT. `curl` is stubbed on PATH with a fake API server that +# answers the pod list, pod and pod log endpoints from fixture files, and the +# ServiceAccount directory is a scratch one. TMUX_TMPDIR points at an empty +# directory, so no tmux session can be found and the helper must take the Job +# path. HIVE_TTYD_WAIT_SEC=0 keeps the pre-fix behaviour (exit 1, no log) fast. +set -uo pipefail + +PASS=0 +FAIL=0 +pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } +fail() { echo " FAIL: $1"; [ $# -gt 1 ] && echo " $2"; FAIL=$((FAIL + 1)); } +finish() { + echo "" + echo "=== Results: $PASS passed, $FAIL failed ===" + [ "$FAIL" -eq 0 ] +} + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +ATTACH="${ROOT}/src/deploy/ttyd-tmux.sh" + +echo "=== ttyd-tmux.sh streams Job pod logs for sandbox.runtime: job agents (#14798) ===" + +if [ ! -f "$ATTACH" ]; then + fail "locate $ATTACH" "the layout moved — this test cannot verify anything" + finish; exit $? +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +mkdir -p "$WORK/bin" "$WORK/sa" "$WORK/tmux" "$WORK/api" +echo "test-token" > "$WORK/sa/token" +echo "hive-ns" > "$WORK/sa/namespace" + +# Fake API server. Every URL is logged; responses come from $WORK/api. When +# pods.after exists, the pod list is empty until that many list calls have +# been made (a Job that starts while the terminal is waiting). +cat > "$WORK/bin/curl" <<'EOF' +#!/usr/bin/env bash +for url; do :; done +echo "$url" >> "$FAKE_API/urls" +case "$url" in + *"/pods?labelSelector="*) + n=$(( $(cat "$FAKE_API/list-calls" 2>/dev/null || echo 0) + 1 )) + echo "$n" > "$FAKE_API/list-calls" + if [ -f "$FAKE_API/pods.json" ] && [ "$n" -gt "$(cat "$FAKE_API/pods.after" 2>/dev/null || echo 0)" ]; then + cat "$FAKE_API/pods.json" + else + echo '{"kind":"PodList","metadata":{"resourceVersion":"1"},"items":[]}' + fi ;; + *"/log?"*) cat "$FAKE_API/log.txt" ;; + *"/pods/"*) cat "$FAKE_API/pod.json" ;; + *) exit 22 ;; +esac +EOF +chmod +x "$WORK/bin/curl" + +# Two runs of agent My_Agent: the older one first in the list, ownerReferences +# naming the Jobs (which must not be mistaken for pods). +POD_OLD="hive-kick-hive-my-agent-1700000000-abcde" +POD_NEW="hive-kick-hive-my-agent-1700009999-x7k2p" +write_pods() { + printf '{"kind":"PodList","metadata":{"resourceVersion":"9"},"items":[{"metadata":{"name":"%s","ownerReferences":[{"kind":"Job","name":"hive-kick-hive-my-agent-1700000000"}]}},{"metadata":{"name":"%s","ownerReferences":[{"kind":"Job","name":"hive-kick-hive-my-agent-1700009999"}]}}]}\n' "$POD_OLD" "$POD_NEW" > "$WORK/api/pods.json" + echo '{"metadata":{"name":"'"$POD_NEW"'"},"status":{"phase":"Failed","containerStatuses":[{"name":"kick","state":{"terminated":{"exitCode":3,"reason":"Error"}}}]}}' > "$WORK/api/pod.json" + printf 'kick: cloning workspace\nkick: running tests\n' > "$WORK/api/log.txt" +} + +reset_api() { rm -f "$WORK/api/"*; } + +run_attach() { + local wait="$1" + OUT=$(env PATH="$WORK/bin:$PATH" FAKE_API="$WORK/api" TMUX_TMPDIR="$WORK/tmux" \ + HIVE_TTYD_KUBE_SA_DIR="$WORK/sa" HIVE_TTYD_KUBE_API="https://api.test" \ + HIVE_TTYD_WAIT_SEC="$wait" HIVE_TTYD_POLL_SEC=1 \ + timeout 30 bash "$ATTACH" "hive-My_Agent" 2>&1) + RC=$? +} + +# --- 1. a finished Job pod: its log and exit status are shown --------------- +reset_api; write_pods +run_attach 0 +if [ "$RC" -eq 0 ]; then pass "exits 0 after streaming the Job pod log"; else fail "exits 0 after streaming the Job pod log" "rc=$RC: $OUT"; fi +if grep -q "kick: running tests" <<<"$OUT"; then pass "the Job pod's log is shown"; else fail "the Job pod's log is shown" "$OUT"; fi +if grep -q "exit code 3" <<<"$OUT"; then pass "the run's exit status is shown"; else fail "the run's exit status is shown" "$OUT"; fi +if grep -q "/pods/${POD_NEW}/log?container=kick&follow=true" "$WORK/api/urls" 2>/dev/null; then + pass "follows the NEWEST run's kick container log" +else + fail "follows the NEWEST run's kick container log" "$(cat "$WORK/api/urls" 2>/dev/null)" +fi +if grep -q "/namespaces/hive-ns/pods?labelSelector=app.kubernetes.io%2Fmanaged-by%3Dhive-kubejob%2Chive.kubestellar.io%2Fagent%3Dhive-my-agent-sandbox" "$WORK/api/urls" 2>/dev/null; then + pass "selects pods by the launcher's managed-by and sanitized agent labels" +else + fail "selects pods by the launcher's managed-by and sanitized agent labels" "$(cat "$WORK/api/urls" 2>/dev/null)" +fi +if grep -q "tmux session not created yet" <<<"$OUT"; then + fail "does not claim to wait for a tmux session when a Job pod exists" "$OUT" +else + pass "does not claim to wait for a tmux session when a Job pod exists" +fi + +# --- 2. no run yet: said clearly --------------------------------------------- +reset_api +run_attach 0 +if [ "$RC" -ne 0 ] && grep -q "has had no run yet" <<<"$OUT"; then + pass "says clearly that a job-runtime agent has had no run yet" +else + fail "says clearly that a job-runtime agent has had no run yet" "rc=$RC: $OUT" +fi + +# --- 3. a Job that starts while the terminal waits is picked up ------------- +reset_api; write_pods; echo 2 > "$WORK/api/pods.after" +run_attach 10 +if [ "$RC" -eq 0 ] && grep -q "kick: cloning workspace" <<<"$OUT"; then + pass "streams a Job pod that appears during the wait" +else + fail "streams a Job pod that appears during the wait" "rc=$RC: $OUT" +fi + +# --- 4. no ServiceAccount: the API is never contacted ----------------------- +reset_api; write_pods +OUT=$(env PATH="$WORK/bin:$PATH" FAKE_API="$WORK/api" TMUX_TMPDIR="$WORK/tmux" \ + HIVE_TTYD_KUBE_SA_DIR="$WORK/no-such-sa" HIVE_TTYD_WAIT_SEC=0 HIVE_TTYD_POLL_SEC=1 \ + timeout 30 bash "$ATTACH" "hive-My_Agent" 2>&1) +RC=$? +if [ "$RC" -ne 0 ] && [ ! -s "$WORK/api/urls" ] && grep -q "tmux session not created yet" <<<"$OUT"; then + pass "outside Kubernetes the tmux wait is unchanged and no API call is made" +else + fail "outside Kubernetes the tmux wait is unchanged and no API call is made" "rc=$RC: $OUT" +fi + +finish diff --git a/src/deploy/ttyd-tmux.sh b/src/deploy/ttyd-tmux.sh index ac0c3abd6d..3ad4d5a046 100644 --- a/src/deploy/ttyd-tmux.sh +++ b/src/deploy/ttyd-tmux.sh @@ -148,8 +148,112 @@ if [ "$SESSION_FROM_ARG" -eq 0 ]; then exit 1 fi +# sandbox.runtime: job agents (#14798). Such an agent runs each kick as a +# Kubernetes Job in its own pod (src/crux/kubejob), so no hive- tmux +# session ever appears here and the wait below could only ever time out. When +# the hive pod has its ServiceAccount mounted, look for that agent's Job pods +# by the labels the launcher sets and stream the newest one's log read-only +# instead. The RBAC (pods, pods/log get+list) is the launcher's own, from +# src/deploy/k8s/sandbox-job-rbac.yaml. The image has no jq, so the few fields +# needed are read with grep/sed from the API server's compact JSON. +KUBEJOB_SA_DIR="${HIVE_TTYD_KUBE_SA_DIR:-/var/run/secrets/kubernetes.io/serviceaccount}" +KUBEJOB_API="${HIVE_TTYD_KUBE_API:-https://kubernetes.default.svc}" +KUBEJOB_API="${KUBEJOB_API%/}" + +kubejob_enabled() { + [ -r "${KUBEJOB_SA_DIR}/token" ] && [ -s "${KUBEJOB_SA_DIR}/namespace" ] && command -v curl >/dev/null 2>&1 +} + +kubejob_namespace() { + tr -d '[:space:]' < "${KUBEJOB_SA_DIR}/namespace" +} + +# kubejob_agent_label mirrors the launcher: LaunchSpec.Name is +# "hive--sandbox" and the label is its +# sanitizeLabel — lowercase, every run of other characters one dash, trimmed. +kubejob_agent_label() { + printf '%s-sandbox' "$SESSION" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//' +} + +kube_curl() { + local args=(-sS --fail -H "Authorization: Bearer $(cat "${KUBEJOB_SA_DIR}/token" 2>/dev/null)") + if [ -r "${KUBEJOB_SA_DIR}/ca.crt" ]; then + args+=(--cacert "${KUBEJOB_SA_DIR}/ca.crt") + fi + curl "${args[@]}" "$@" +} + +# kubejob_latest_pod prints the newest Job pod for this agent, or nothing. Pod +# names are - and the job is hive-kick--, so the +# second-to-last dash field orders the runs. +kubejob_latest_pod() { + local ns label raw + ns=$(kubejob_namespace) + label=$(kubejob_agent_label) + raw=$(kube_curl --max-time 10 "${KUBEJOB_API}/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io%2Fmanaged-by%3Dhive-kubejob%2Chive.kubestellar.io%2Fagent%3D${label}" 2>/dev/null) || return 0 + printf '%s' "$raw" \ + | grep -oE '"metadata": *\{ *"name": *"[a-z0-9.-]+"' \ + | sed -E 's/.*"([a-z0-9.-]+)"$/\1/' \ + | awk -F- 'NF > 2 { print $(NF-1), $0 }' \ + | sort -n | tail -n 1 | cut -d' ' -f2 || true +} + +kubejob_pod_json() { + kube_curl --max-time 10 "${KUBEJOB_API}/api/v1/namespaces/$(kubejob_namespace)/pods/$1" 2>/dev/null || true +} + +kubejob_field() { + # $1 JSON, $2 field name; prints the first string or integer value. + printf '%s' "$1" | grep -oE "\"$2\": *(\"[^\"]*\"|-?[0-9]+)" | head -n 1 | sed -E 's/^[^:]*: *"?([^"]*)"?$/\1/' || true +} + +kubejob_stream() { + local pod="$1" ns phase pod_json code waited=0 + ns=$(kubejob_namespace) + echo "agent '${SESSION#hive-}' runs as a Kubernetes Job (sandbox.runtime: job); there is no tmux session to attach to." >&2 + # A Pending pod has no log yet (the API answers 400 "waiting to start"). + phase=$(kubejob_field "$(kubejob_pod_json "$pod")" phase) + while [ "$phase" = "Pending" ] && [ "$waited" -lt "$TTYD_WAIT_SEC" ]; do + echo "Job pod '${pod}' is Pending; waiting for it to start..." >&2 + sleep "$TTYD_POLL_SEC" + waited=$((waited + TTYD_POLL_SEC)) + phase=$(kubejob_field "$(kubejob_pod_json "$pod")" phase) + done + echo "streaming the log of Job pod '${pod}' (read-only; follows while the pod runs)..." >&2 + echo "----------------------------------------------------------------------" >&2 + if ! kube_curl -N "${KUBEJOB_API}/api/v1/namespaces/${ns}/pods/${pod}/log?container=kick&follow=true"; then + echo "warning: could not read the log of Job pod '${pod}' (it may have been cleaned up)" >&2 + fi + echo "----------------------------------------------------------------------" >&2 + pod_json=$(kubejob_pod_json "$pod") + phase=$(kubejob_field "$pod_json" phase) + code=$(kubejob_field "$pod_json" exitCode) + if [ -n "$code" ]; then + echo "Job pod '${pod}' finished: phase ${phase:-unknown}, exit code ${code}." >&2 + else + echo "log stream for Job pod '${pod}' ended; pod phase ${phase:-unknown (pod gone)}." >&2 + fi + echo "reopen this terminal to follow the agent's next run." >&2 +} + +KUBEJOB_POD="" +kubejob_check() { + KUBEJOB_POD="" + kubejob_enabled || return 1 + KUBEJOB_POD=$(kubejob_latest_pod) + [ -n "$KUBEJOB_POD" ] +} + if ! find_tmux_socket; then + if kubejob_check; then + kubejob_stream "$KUBEJOB_POD" + exit 0 + fi echo "waiting for agent '${SESSION#hive-}' to start (tmux session not created yet); polling up to ${TTYD_WAIT_SEC}s..." >&2 + if kubejob_enabled; then + echo "no sandbox Job pod for agent '${SESSION#hive-}' either: if it runs with sandbox.runtime: job, it has had no run yet" >&2 + echo "(or its last run succeeded and the Job was cleaned up); its next Job pod's log will be streamed here when it starts." >&2 + fi WAITED=0 while [ "$WAITED" -lt "$TTYD_WAIT_SEC" ]; do sleep "$TTYD_POLL_SEC" @@ -158,10 +262,17 @@ if ! find_tmux_socket; then echo "agent '${SESSION#hive-}' tmux session is ready after ${WAITED}s; attaching..." >&2 break fi + if kubejob_check; then + kubejob_stream "$KUBEJOB_POD" + exit 0 + fi done if [ -z "$TMUX_SOCKET" ]; then echo "error: no tmux socket found for session '${SESSION}' after waiting ${TTYD_WAIT_SEC}s" >&2 echo "hint: '${SESSION}' was requested, but no matching tmux session was created before the wait expired." >&2 + if kubejob_enabled; then + echo "hint: no sandbox Job pod (sandbox.runtime: job) ran for it in that time either." >&2 + fi list_available_sessions exit 1 fi