diff --git a/exgentic_a2a_runner/authbridge/wait-for-reload.sh b/exgentic_a2a_runner/authbridge/wait-for-reload.sh index 3860dc2..fd44bff 100755 --- a/exgentic_a2a_runner/authbridge/wait-for-reload.sh +++ b/exgentic_a2a_runner/authbridge/wait-for-reload.sh @@ -26,7 +26,9 @@ # - `authbridge-proxy` post-binary-split (proxy-sidecar mode); # the operator picks this name when injecting # the proxy-sidecar binary -# This script auto-detects which one is in the pod. +# This script auto-detects which one is in the pod. During a rollout it +# probes every sidecar-bearing pod each poll and succeeds as soon as any +# reports the target SHA, so it doesn't matter which pod is "first". set -euo pipefail @@ -35,57 +37,89 @@ AGENT_NAME=${2:?agent name required} WANT_SHA=${3:?want-sha required} TIMEOUT=${4:-180} -# Auto-detect the authbridge container name. Sidecars are injected by -# the AuthBridge webhook at pod-admission, so the Deployment spec only -# lists the agent — we have to look at a running pod. Try the legacy -# name first (`envoy-proxy`) since most clusters today still run the -# older operator layout; fall back to the post-#411 combined name -# (`authbridge`). -POD_CONTAINERS=$(kubectl -n "$NAMESPACE" get pods \ - -l app.kubernetes.io/name="$AGENT_NAME" \ - -o jsonpath='{.items[0].spec.containers[*].name}' 2>/dev/null || true) +# The operator-injected sidecar container is one of these names, +# depending on operator version (see header comment). AUTHBRIDGE_CANDIDATES=(authbridge-proxy authbridge envoy-proxy) -AUTHBRIDGE_CONTAINER="" -for candidate in "${AUTHBRIDGE_CANDIDATES[@]}"; do - if echo "$POD_CONTAINERS" | tr ' ' '\n' | grep -qx "$candidate"; then - AUTHBRIDGE_CONTAINER="$candidate" - break - fi -done -if [[ -z "$AUTHBRIDGE_CONTAINER" ]]; then - echo "ERROR: could not find an authbridge sidecar container in pods of $AGENT_NAME." >&2 - echo " Looked for: ${AUTHBRIDGE_CANDIDATES[*]}." >&2 - echo " Containers found: ${POD_CONTAINERS:-}" >&2 - exit 1 -fi -echo "[*] Authbridge container detected: $AUTHBRIDGE_CONTAINER" + +# Enumerate the agent's pods and, for each, the authbridge sidecar +# container it carries (if any). Emits one "podcontainer" line per +# sidecar-bearing pod. We re-run this every poll iteration rather than +# picking a single pod up front: +# +# During a rollout the pod list contains, in arbitrary order, the old +# pod (agent-only if it predates AuthBridge, or a sidecar still serving +# the stale config) alongside the new pod (mounting the patched +# ConfigMap). The old `.items[0]` assumption picked whichever came first +# — often the old/agent-only pod — and then either failed to find a +# sidecar at all or waited forever on a pod whose config will never +# advance (it's being terminated). Because success is defined purely by +# "some sidecar reports the target SHA", we just probe every sidecar pod +# each round; the converged pod wins regardless of list order, and +# old/terminating pods that never match are harmless. +list_sidecar_pods() { + kubectl -n "$NAMESPACE" get pods \ + -l app.kubernetes.io/name="$AGENT_NAME" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{range .spec.containers[*]}{.name}{","}{end}{"\n"}{end}' \ + 2>/dev/null | while IFS='|' read -r pod containers; do + [[ -z "$pod" ]] && continue + for candidate in "${AUTHBRIDGE_CANDIDATES[@]}"; do + if echo "$containers" | tr ',' '\n' | grep -qx "$candidate"; then + printf '%s\t%s\n' "$pod" "$candidate" + break + fi + done + done +} DEADLINE=$(( $(date +%s) + TIMEOUT )) echo "[*] Waiting for authbridge to load the patched config (timeout ${TIMEOUT}s)" echo " target SHA: $WANT_SHA" -ACTIVE_SHA="" +SAW_SIDECAR=false # did we ever find a sidecar-bearing pod? +LAST_POD="" # for the failure diagnostics +LAST_CONTAINER="" +LAST_SHA="" while [[ $(date +%s) -lt $DEADLINE ]]; do - ACTIVE_SHA=$(kubectl -n "$NAMESPACE" exec deploy/"$AGENT_NAME" -c "$AUTHBRIDGE_CONTAINER" -- \ - wget -q -O - http://localhost:9093/reload/status 2>/dev/null | \ - python3 -c 'import json, sys + SIDECAR_PODS=$(list_sidecar_pods || true) + if [[ -n "$SIDECAR_PODS" ]]; then + SAW_SIDECAR=true + while IFS=$'\t' read -r pod container; do + [[ -z "$pod" ]] && continue + LAST_POD="$pod"; LAST_CONTAINER="$container" + active=$(kubectl -n "$NAMESPACE" exec "$pod" -c "$container" -- \ + wget -q -O - http://localhost:9093/reload/status 2>/dev/null | \ + python3 -c 'import json, sys try: print(json.load(sys.stdin).get("active_config_sha256", "")) except Exception: pass' 2>/dev/null || true) - if [[ "$ACTIVE_SHA" == "$WANT_SHA" ]]; then - echo "[*] Active config SHA matches — patch is live." - exit 0 + [[ -n "$active" ]] && LAST_SHA="$active" + if [[ "$active" == "$WANT_SHA" ]]; then + echo "[*] Active config SHA matches on pod $pod ($container) — patch is live." + exit 0 + fi + done <<< "$SIDECAR_PODS" fi sleep 3 done +if ! $SAW_SIDECAR; then + echo "ERROR: could not find an authbridge sidecar container in pods of $AGENT_NAME." >&2 + echo " Looked for: ${AUTHBRIDGE_CANDIDATES[*]}." >&2 + echo " Pods and their containers:" >&2 + kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/name="$AGENT_NAME" \ + -o jsonpath='{range .items[*]}{" "}{.metadata.name}{": "}{range .spec.containers[*]}{.name}{" "}{end}{"\n"}{end}' >&2 2>/dev/null || true + exit 1 +fi + echo "ERROR: authbridge active config did not match patched SHA within ${TIMEOUT}s." >&2 echo " want: $WANT_SHA" >&2 -echo " last active: ${ACTIVE_SHA:-}" >&2 -echo " Last 20 lines of the $AUTHBRIDGE_CONTAINER container:" >&2 -kubectl -n "$NAMESPACE" logs deploy/"$AGENT_NAME" -c "$AUTHBRIDGE_CONTAINER" --tail=20 >&2 || true +echo " last active: ${LAST_SHA:-}" >&2 +if [[ -n "$LAST_POD" ]]; then + echo " Last 20 lines of the $LAST_CONTAINER container (pod $LAST_POD):" >&2 + kubectl -n "$NAMESPACE" logs "$LAST_POD" -c "$LAST_CONTAINER" --tail=20 >&2 || true +fi echo >&2 echo " Likely causes:" >&2 echo " - ConfigMap parse error (look for 'reload failed' above)" >&2 diff --git a/exgentic_a2a_runner/deploy-agent.sh b/exgentic_a2a_runner/deploy-agent.sh index 86310ed..8846d47 100755 --- a/exgentic_a2a_runner/deploy-agent.sh +++ b/exgentic_a2a_runner/deploy-agent.sh @@ -629,6 +629,98 @@ PYEOF ) || exit 1 fi +# Step 7.5: Bake OTEL-collector skip_hosts into the NAMESPACE authbridge base +# BEFORE the agent is created, so the sidecar has it from first pod boot — no +# post-deploy patch of the running agent, no reload race. +# +# Why the namespace base and not the per-agent ConfigMap: the rossoctl operator +# webhook force-overwrites `authbridge-config-` (server-side apply, +# ForceOwnership) on every pod admission, so a per-agent pre-seed can't survive. +# The one thing it does honor is the namespace-level `authbridge-runtime-config` +# `config.yaml`, which it reads as the base and MERGES into each per-agent CM — +# it only injects reverse_proxy_addr / reverse_proxy_backend / forward_proxy_addr +# into `listener`, so a `listener.skip_hosts` we add here rides through untouched. +# +# NOTE: `skip_hosts` is an AuthBridge-*binary* listener key; it is not defined in +# rossoctl/operator source and has NOT been verified against the sidecar's config +# schema. If the sidecar rejects the merged config, look for a "reload failed" / +# parse error in the sidecar container logs +# (kubectl -n "$NAMESPACE" logs deploy/ -c authbridge # or authbridge-proxy / envoy-proxy) +# and correct the key name/shape below. This edit is namespace-wide and persists +# for every agent in $NAMESPACE. +if [ "$AUTHBRIDGE_ENABLED" = "true" ] && [ "$CLUSTER_MODE" != "in-cluster" ]; then + echo "Step 7.5: Adding OTEL-collector skip_hosts to namespace authbridge base..." + NS_AB_CM="authbridge-runtime-config" + + if ! command -v python3 >/dev/null 2>&1 || ! python3 -c 'import yaml' 2>/dev/null; then + echo "Error: python3 with PyYAML is required to merge skip_hosts into $NS_AB_CM." >&2 + echo " Install with one of:" >&2 + echo " pip3 install --user pyyaml" >&2 + echo " brew install libyaml && pip3 install pyyaml # macOS" >&2 + echo " sudo apt install python3-yaml # Debian/Ubuntu" >&2 + exit 1 + elif ! kubectl -n "$NAMESPACE" get configmap "$NS_AB_CM" >/dev/null 2>&1; then + echo "Error: ConfigMap $NAMESPACE/$NS_AB_CM not found." >&2 + echo " The operator/Helm owns this base; it must exist before deploying an" >&2 + echo " AuthBridge-enabled agent. We do not fabricate one here (that would drop" >&2 + echo " the pipeline: block and break auth). Ensure the namespace is provisioned." >&2 + exit 1 + else + CURRENT_NS_YAML=$( + kubectl -n "$NAMESPACE" get configmap "$NS_AB_CM" \ + -o jsonpath='{.data.config\.yaml}' + ) + MERGED_NS_YAML=$(printf '%s' "$CURRENT_NS_YAML" | python3 - <<'PYEOF' +import sys +import yaml + +# Hosts the sidecar must NOT intercept (OTEL collector, rossoctl-system). +SKIP_HOSTS = [ + "otel-collector.rossoctl-system.svc.cluster.local", + "otel-collector", +] + +cfg = yaml.safe_load(sys.stdin.read()) or {} +listener = cfg.get("listener") +if not isinstance(listener, dict): + listener = {} +existing = listener.get("skip_hosts") +if not isinstance(existing, list): + existing = [] +# Union + sort + de-dupe so re-runs are byte-stable and we don't clobber +# hosts someone else added. +listener["skip_hosts"] = sorted(set(existing) | set(SKIP_HOSTS)) +cfg["listener"] = listener +# sort_keys=True keeps output deterministic across runs for the no-op check. +sys.stdout.write(yaml.safe_dump(cfg, default_flow_style=False, sort_keys=True)) +PYEOF + ) || { echo "Error: skip_hosts merge failed for $NAMESPACE/$NS_AB_CM (bad YAML in config.yaml?)" >&2; exit 1; } + + if [ -z "$MERGED_NS_YAML" ]; then + echo "Error: skip_hosts merge produced empty output for $NAMESPACE/$NS_AB_CM" >&2 + exit 1 + elif [ "$CURRENT_NS_YAML" = "$MERGED_NS_YAML" ]; then + echo "✓ Namespace authbridge base already has skip_hosts — nothing to patch" + else + TMP_NS_CONFIG=$(mktemp) + printf '%s' "$MERGED_NS_YAML" >"$TMP_NS_CONFIG" + # Conflict-free create --dry-run | apply (same pattern as apply-pipeline.sh). + if kubectl -n "$NAMESPACE" create configmap "$NS_AB_CM" \ + --from-file=config.yaml="$TMP_NS_CONFIG" \ + --dry-run=client -o yaml \ + | kubectl -n "$NAMESPACE" apply -f - >/dev/null; then + echo "✓ skip_hosts added to $NAMESPACE/$NS_AB_CM" + rm -f "$TMP_NS_CONFIG" + else + echo "Error: could not apply skip_hosts to $NAMESPACE/$NS_AB_CM" >&2 + rm -f "$TMP_NS_CONFIG" + exit 1 + fi + fi + fi + echo "" +fi + AGENT_JSON=$(cat <