Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/v2-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1009,6 +1009,15 @@ jobs:
working-directory: .
run: bash src/deploy/test_ttyd_default_socket_uid.sh

# #14798: sandbox.runtime: job agents run each kick as a Kubernetes Job
# in its own pod, so no hive-<agent> tmux session ever exists and the
# dashboard terminal waited for one until it timed out. The helper now
# streams the newest Job pod's log instead. This stubs curl as a fake
# API server and asserts the log, exit status and "no run yet" message.
- name: ttyd streams Job pod logs for sandbox.runtime job agents (#14798)
working-directory: .
run: bash src/deploy/test_ttyd_kubejob_logs.sh

# #5435: hive-discord.service ran `node bot.js` with
# WorkingDirectory=/tmp/hive/discord, resolving the executed code out of a
# world-writable parent that is cleared on every reboot — a local user who
Expand Down
1 change: 1 addition & 0 deletions changelog.d/fixed-14798-ttyd-job-runtime-logs.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **The dashboard's ▶ terminal now shows what a `sandbox.runtime: job` agent is doing** (#14798). Job-runtime agents run each kick as a Kubernetes Job in their own pod, so there is never a `hive-<agent>` tmux session in the hive pod and the terminal sat on "tmux session not created yet" until its wait expired. `ttyd-tmux.sh` now looks up the agent's Job pods by the launcher's `app.kubernetes.io/managed-by=hive-kubejob` / `hive.kubestellar.io/agent` labels with the pod's existing ServiceAccount and streams the newest pod's log read-only — following it while it runs and printing the exit status when it finishes — picks up a Job that starts while the terminal is waiting, and says plainly when the agent has had no run yet. Tmux agents and non-Kubernetes installs are unchanged.
142 changes: 142 additions & 0 deletions src/deploy/test_ttyd_kubejob_logs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
#!/usr/bin/env bash
# The browser terminal must stream a sandbox.runtime: job agent's Job pod log
# instead of waiting for a tmux session that never appears (#14798).
# Run: bash src/deploy/test_ttyd_kubejob_logs.sh
#
# THE BUG. A job-runtime agent runs every kick as a Kubernetes Job in its own
# pod (src/crux/kubejob), so there is never a hive-<agent> tmux session in the
# hive pod. ttyd-tmux.sh only knew how to wait for one, so the dashboard's
# terminal showed "tmux session not created yet" for the whole wait window and
# then "no tmux socket found" — never what the sandboxed agent was doing.
#
# HOW THIS TESTS IT. `curl` is stubbed on PATH with a fake API server that
# answers the pod list, pod and pod log endpoints from fixture files, and the
# ServiceAccount directory is a scratch one. TMUX_TMPDIR points at an empty
# directory, so no tmux session can be found and the helper must take the Job
# path. HIVE_TTYD_WAIT_SEC=0 keeps the pre-fix behaviour (exit 1, no log) fast.
set -uo pipefail

PASS=0
FAIL=0
pass() { echo " PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo " FAIL: $1"; [ $# -gt 1 ] && echo " $2"; FAIL=$((FAIL + 1)); }
finish() {
echo ""
echo "=== Results: $PASS passed, $FAIL failed ==="
[ "$FAIL" -eq 0 ]
}

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
ATTACH="${ROOT}/src/deploy/ttyd-tmux.sh"

echo "=== ttyd-tmux.sh streams Job pod logs for sandbox.runtime: job agents (#14798) ==="

if [ ! -f "$ATTACH" ]; then
fail "locate $ATTACH" "the layout moved — this test cannot verify anything"
finish; exit $?
fi

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

mkdir -p "$WORK/bin" "$WORK/sa" "$WORK/tmux" "$WORK/api"
echo "test-token" > "$WORK/sa/token"
echo "hive-ns" > "$WORK/sa/namespace"

# Fake API server. Every URL is logged; responses come from $WORK/api. When
# pods.after exists, the pod list is empty until that many list calls have
# been made (a Job that starts while the terminal is waiting).
cat > "$WORK/bin/curl" <<'EOF'
#!/usr/bin/env bash
for url; do :; done
echo "$url" >> "$FAKE_API/urls"
case "$url" in
*"/pods?labelSelector="*)
n=$(( $(cat "$FAKE_API/list-calls" 2>/dev/null || echo 0) + 1 ))
echo "$n" > "$FAKE_API/list-calls"
if [ -f "$FAKE_API/pods.json" ] && [ "$n" -gt "$(cat "$FAKE_API/pods.after" 2>/dev/null || echo 0)" ]; then
cat "$FAKE_API/pods.json"
else
echo '{"kind":"PodList","metadata":{"resourceVersion":"1"},"items":[]}'
fi ;;
*"/log?"*) cat "$FAKE_API/log.txt" ;;
*"/pods/"*) cat "$FAKE_API/pod.json" ;;
*) exit 22 ;;
esac
EOF
chmod +x "$WORK/bin/curl"

# Two runs of agent My_Agent: the older one first in the list, ownerReferences
# naming the Jobs (which must not be mistaken for pods).
POD_OLD="hive-kick-hive-my-agent-1700000000-abcde"
POD_NEW="hive-kick-hive-my-agent-1700009999-x7k2p"
write_pods() {
printf '{"kind":"PodList","metadata":{"resourceVersion":"9"},"items":[{"metadata":{"name":"%s","ownerReferences":[{"kind":"Job","name":"hive-kick-hive-my-agent-1700000000"}]}},{"metadata":{"name":"%s","ownerReferences":[{"kind":"Job","name":"hive-kick-hive-my-agent-1700009999"}]}}]}\n' "$POD_OLD" "$POD_NEW" > "$WORK/api/pods.json"
echo '{"metadata":{"name":"'"$POD_NEW"'"},"status":{"phase":"Failed","containerStatuses":[{"name":"kick","state":{"terminated":{"exitCode":3,"reason":"Error"}}}]}}' > "$WORK/api/pod.json"
printf 'kick: cloning workspace\nkick: running tests\n' > "$WORK/api/log.txt"
}

reset_api() { rm -f "$WORK/api/"*; }

run_attach() {
local wait="$1"
OUT=$(env PATH="$WORK/bin:$PATH" FAKE_API="$WORK/api" TMUX_TMPDIR="$WORK/tmux" \
HIVE_TTYD_KUBE_SA_DIR="$WORK/sa" HIVE_TTYD_KUBE_API="https://api.test" \
HIVE_TTYD_WAIT_SEC="$wait" HIVE_TTYD_POLL_SEC=1 \
timeout 30 bash "$ATTACH" "hive-My_Agent" 2>&1)
RC=$?
}

# --- 1. a finished Job pod: its log and exit status are shown ---------------
reset_api; write_pods
run_attach 0
if [ "$RC" -eq 0 ]; then pass "exits 0 after streaming the Job pod log"; else fail "exits 0 after streaming the Job pod log" "rc=$RC: $OUT"; fi
if grep -q "kick: running tests" <<<"$OUT"; then pass "the Job pod's log is shown"; else fail "the Job pod's log is shown" "$OUT"; fi
if grep -q "exit code 3" <<<"$OUT"; then pass "the run's exit status is shown"; else fail "the run's exit status is shown" "$OUT"; fi
if grep -q "/pods/${POD_NEW}/log?container=kick&follow=true" "$WORK/api/urls" 2>/dev/null; then
pass "follows the NEWEST run's kick container log"
else
fail "follows the NEWEST run's kick container log" "$(cat "$WORK/api/urls" 2>/dev/null)"
fi
if grep -q "/namespaces/hive-ns/pods?labelSelector=app.kubernetes.io%2Fmanaged-by%3Dhive-kubejob%2Chive.kubestellar.io%2Fagent%3Dhive-my-agent-sandbox" "$WORK/api/urls" 2>/dev/null; then
pass "selects pods by the launcher's managed-by and sanitized agent labels"
else
fail "selects pods by the launcher's managed-by and sanitized agent labels" "$(cat "$WORK/api/urls" 2>/dev/null)"
fi
if grep -q "tmux session not created yet" <<<"$OUT"; then
fail "does not claim to wait for a tmux session when a Job pod exists" "$OUT"
else
pass "does not claim to wait for a tmux session when a Job pod exists"
fi

# --- 2. no run yet: said clearly ---------------------------------------------
reset_api
run_attach 0
if [ "$RC" -ne 0 ] && grep -q "has had no run yet" <<<"$OUT"; then
pass "says clearly that a job-runtime agent has had no run yet"
else
fail "says clearly that a job-runtime agent has had no run yet" "rc=$RC: $OUT"
fi

# --- 3. a Job that starts while the terminal waits is picked up -------------
reset_api; write_pods; echo 2 > "$WORK/api/pods.after"
run_attach 10
if [ "$RC" -eq 0 ] && grep -q "kick: cloning workspace" <<<"$OUT"; then
pass "streams a Job pod that appears during the wait"
else
fail "streams a Job pod that appears during the wait" "rc=$RC: $OUT"
fi

# --- 4. no ServiceAccount: the API is never contacted -----------------------
reset_api; write_pods
OUT=$(env PATH="$WORK/bin:$PATH" FAKE_API="$WORK/api" TMUX_TMPDIR="$WORK/tmux" \
HIVE_TTYD_KUBE_SA_DIR="$WORK/no-such-sa" HIVE_TTYD_WAIT_SEC=0 HIVE_TTYD_POLL_SEC=1 \
timeout 30 bash "$ATTACH" "hive-My_Agent" 2>&1)
RC=$?
if [ "$RC" -ne 0 ] && [ ! -s "$WORK/api/urls" ] && grep -q "tmux session not created yet" <<<"$OUT"; then
pass "outside Kubernetes the tmux wait is unchanged and no API call is made"
else
fail "outside Kubernetes the tmux wait is unchanged and no API call is made" "rc=$RC: $OUT"
fi

finish
111 changes: 111 additions & 0 deletions src/deploy/ttyd-tmux.sh
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,112 @@ if [ "$SESSION_FROM_ARG" -eq 0 ]; then
exit 1
fi

# sandbox.runtime: job agents (#14798). Such an agent runs each kick as a
# Kubernetes Job in its own pod (src/crux/kubejob), so no hive-<agent> tmux
# session ever appears here and the wait below could only ever time out. When
# the hive pod has its ServiceAccount mounted, look for that agent's Job pods
# by the labels the launcher sets and stream the newest one's log read-only
# instead. The RBAC (pods, pods/log get+list) is the launcher's own, from
# src/deploy/k8s/sandbox-job-rbac.yaml. The image has no jq, so the few fields
# needed are read with grep/sed from the API server's compact JSON.
KUBEJOB_SA_DIR="${HIVE_TTYD_KUBE_SA_DIR:-/var/run/secrets/kubernetes.io/serviceaccount}"
KUBEJOB_API="${HIVE_TTYD_KUBE_API:-https://kubernetes.default.svc}"
KUBEJOB_API="${KUBEJOB_API%/}"

kubejob_enabled() {
[ -r "${KUBEJOB_SA_DIR}/token" ] && [ -s "${KUBEJOB_SA_DIR}/namespace" ] && command -v curl >/dev/null 2>&1
}

kubejob_namespace() {
tr -d '[:space:]' < "${KUBEJOB_SA_DIR}/namespace"
}

# kubejob_agent_label mirrors the launcher: LaunchSpec.Name is
# "hive-<sanitizeBranchPart(agent)>-sandbox" and the label is its
# sanitizeLabel — lowercase, every run of other characters one dash, trimmed.
kubejob_agent_label() {
printf '%s-sandbox' "$SESSION" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//'
}

kube_curl() {
local args=(-sS --fail -H "Authorization: Bearer $(cat "${KUBEJOB_SA_DIR}/token" 2>/dev/null)")
if [ -r "${KUBEJOB_SA_DIR}/ca.crt" ]; then
args+=(--cacert "${KUBEJOB_SA_DIR}/ca.crt")
fi
curl "${args[@]}" "$@"
}

# kubejob_latest_pod prints the newest Job pod for this agent, or nothing. Pod
# names are <job>-<random> and the job is hive-kick-<agent>-<unix>, so the
# second-to-last dash field orders the runs.
kubejob_latest_pod() {
local ns label raw
ns=$(kubejob_namespace)
label=$(kubejob_agent_label)
raw=$(kube_curl --max-time 10 "${KUBEJOB_API}/api/v1/namespaces/${ns}/pods?labelSelector=app.kubernetes.io%2Fmanaged-by%3Dhive-kubejob%2Chive.kubestellar.io%2Fagent%3D${label}" 2>/dev/null) || return 0
printf '%s' "$raw" \
| grep -oE '"metadata": *\{ *"name": *"[a-z0-9.-]+"' \
| sed -E 's/.*"([a-z0-9.-]+)"$/\1/' \
| awk -F- 'NF > 2 { print $(NF-1), $0 }' \
| sort -n | tail -n 1 | cut -d' ' -f2 || true
}

kubejob_pod_json() {
kube_curl --max-time 10 "${KUBEJOB_API}/api/v1/namespaces/$(kubejob_namespace)/pods/$1" 2>/dev/null || true
}

kubejob_field() {
# $1 JSON, $2 field name; prints the first string or integer value.
printf '%s' "$1" | grep -oE "\"$2\": *(\"[^\"]*\"|-?[0-9]+)" | head -n 1 | sed -E 's/^[^:]*: *"?([^"]*)"?$/\1/' || true
}

kubejob_stream() {
local pod="$1" ns phase pod_json code waited=0
ns=$(kubejob_namespace)
echo "agent '${SESSION#hive-}' runs as a Kubernetes Job (sandbox.runtime: job); there is no tmux session to attach to." >&2
# A Pending pod has no log yet (the API answers 400 "waiting to start").
phase=$(kubejob_field "$(kubejob_pod_json "$pod")" phase)
while [ "$phase" = "Pending" ] && [ "$waited" -lt "$TTYD_WAIT_SEC" ]; do
echo "Job pod '${pod}' is Pending; waiting for it to start..." >&2
sleep "$TTYD_POLL_SEC"
waited=$((waited + TTYD_POLL_SEC))
phase=$(kubejob_field "$(kubejob_pod_json "$pod")" phase)
done
echo "streaming the log of Job pod '${pod}' (read-only; follows while the pod runs)..." >&2
echo "----------------------------------------------------------------------" >&2
if ! kube_curl -N "${KUBEJOB_API}/api/v1/namespaces/${ns}/pods/${pod}/log?container=kick&follow=true"; then
echo "warning: could not read the log of Job pod '${pod}' (it may have been cleaned up)" >&2
fi
echo "----------------------------------------------------------------------" >&2
pod_json=$(kubejob_pod_json "$pod")
phase=$(kubejob_field "$pod_json" phase)
code=$(kubejob_field "$pod_json" exitCode)
if [ -n "$code" ]; then
echo "Job pod '${pod}' finished: phase ${phase:-unknown}, exit code ${code}." >&2
else
echo "log stream for Job pod '${pod}' ended; pod phase ${phase:-unknown (pod gone)}." >&2
fi
echo "reopen this terminal to follow the agent's next run." >&2
}

KUBEJOB_POD=""
kubejob_check() {
KUBEJOB_POD=""
kubejob_enabled || return 1
KUBEJOB_POD=$(kubejob_latest_pod)
[ -n "$KUBEJOB_POD" ]
}

if ! find_tmux_socket; then
if kubejob_check; then
kubejob_stream "$KUBEJOB_POD"
exit 0
fi
echo "waiting for agent '${SESSION#hive-}' to start (tmux session not created yet); polling up to ${TTYD_WAIT_SEC}s..." >&2
if kubejob_enabled; then
echo "no sandbox Job pod for agent '${SESSION#hive-}' either: if it runs with sandbox.runtime: job, it has had no run yet" >&2
echo "(or its last run succeeded and the Job was cleaned up); its next Job pod's log will be streamed here when it starts." >&2
fi
WAITED=0
while [ "$WAITED" -lt "$TTYD_WAIT_SEC" ]; do
sleep "$TTYD_POLL_SEC"
Expand All @@ -158,10 +262,17 @@ if ! find_tmux_socket; then
echo "agent '${SESSION#hive-}' tmux session is ready after ${WAITED}s; attaching..." >&2
break
fi
if kubejob_check; then
kubejob_stream "$KUBEJOB_POD"
exit 0
fi
done
if [ -z "$TMUX_SOCKET" ]; then
echo "error: no tmux socket found for session '${SESSION}' after waiting ${TTYD_WAIT_SEC}s" >&2
echo "hint: '${SESSION}' was requested, but no matching tmux session was created before the wait expired." >&2
if kubejob_enabled; then
echo "hint: no sandbox Job pod (sandbox.runtime: job) ran for it in that time either." >&2
fi
list_available_sessions
exit 1
fi
Expand Down
Loading