Skip to content

Route OTEL collector around authbridge sidecar via skip_hosts #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yoavkatz wants to merge 1 commit into
base: main
Choose a base branch
from
+160 −34
Open

Route OTEL collector around authbridge sidecar via skip_hosts #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 68 additions & 34 deletions exgentic_a2a_runner/authbridge/wait-for-reload.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
# - `authbridge-proxy` post-binary-split (proxy-sidecar mode);
# the operator picks this name when injecting
# the proxy-sidecar binary
# This script auto-detects which one is in the pod.
# This script auto-detects which one is in the pod. During a rollout it
# probes every sidecar-bearing pod each poll and succeeds as soon as any
# reports the target SHA, so it doesn't matter which pod is "first".

set -euo pipefail

Expand All @@ -35,57 +37,89 @@ AGENT_NAME=${2:?agent name required}
WANT_SHA=${3:?want-sha required}
TIMEOUT=${4:-180}

# Auto-detect the authbridge container name. Sidecars are injected by
# the AuthBridge webhook at pod-admission, so the Deployment spec only
# lists the agent — we have to look at a running pod. Try the legacy
# name first (`envoy-proxy`) since most clusters today still run the
# older operator layout; fall back to the post-#411 combined name
# (`authbridge`).
POD_CONTAINERS=$(kubectl -n "$NAMESPACE" get pods \
-l app.kubernetes.io/name="$AGENT_NAME" \
-o jsonpath='{.items[0].spec.containers[*].name}' 2>/dev/null || true)
# The operator-injected sidecar container is one of these names,
# depending on operator version (see header comment).
AUTHBRIDGE_CANDIDATES=(authbridge-proxy authbridge envoy-proxy)
AUTHBRIDGE_CONTAINER=""
for candidate in "${AUTHBRIDGE_CANDIDATES[@]}"; do
if echo "$POD_CONTAINERS" | tr ' ' '\n' | grep -qx "$candidate"; then
AUTHBRIDGE_CONTAINER="$candidate"
break
fi
done
if [[ -z "$AUTHBRIDGE_CONTAINER" ]]; then
echo "ERROR: could not find an authbridge sidecar container in pods of $AGENT_NAME." >&2
echo " Looked for: ${AUTHBRIDGE_CANDIDATES[*]}." >&2
echo " Containers found: ${POD_CONTAINERS:-<none>}" >&2
exit 1
fi
echo "[*] Authbridge container detected: $AUTHBRIDGE_CONTAINER"

# Enumerate the agent's pods and, for each, the authbridge sidecar
# container it carries (if any). Emits one "pod<TAB>container" line per
# sidecar-bearing pod. We re-run this every poll iteration rather than
# picking a single pod up front:
#
# During a rollout the pod list contains, in arbitrary order, the old
# pod (agent-only if it predates AuthBridge, or a sidecar still serving
# the stale config) alongside the new pod (mounting the patched
# ConfigMap). The old `.items[0]` assumption picked whichever came first
# — often the old/agent-only pod — and then either failed to find a
# sidecar at all or waited forever on a pod whose config will never
# advance (it's being terminated). Because success is defined purely by
# "some sidecar reports the target SHA", we just probe every sidecar pod
# each round; the converged pod wins regardless of list order, and
# old/terminating pods that never match are harmless.
list_sidecar_pods() {
kubectl -n "$NAMESPACE" get pods \
-l app.kubernetes.io/name="$AGENT_NAME" \
-o jsonpath='{range .items[*]}{.metadata.name}{"|"}{range .spec.containers[*]}{.name}{","}{end}{"\n"}{end}' \
2>/dev/null | while IFS='|' read -r pod containers; do
[[ -z "$pod" ]] && continue
for candidate in "${AUTHBRIDGE_CANDIDATES[@]}"; do
if echo "$containers" | tr ',' '\n' | grep -qx "$candidate"; then
printf '%s\t%s\n' "$pod" "$candidate"
break
fi
done
done
}

DEADLINE=$(( $(date +%s) + TIMEOUT ))

echo "[*] Waiting for authbridge to load the patched config (timeout ${TIMEOUT}s)"
echo " target SHA: $WANT_SHA"

ACTIVE_SHA=""
SAW_SIDECAR=false # did we ever find a sidecar-bearing pod?
LAST_POD="" # for the failure diagnostics
LAST_CONTAINER=""
LAST_SHA=""
while [[ $(date +%s) -lt $DEADLINE ]]; do
ACTIVE_SHA=$(kubectl -n "$NAMESPACE" exec deploy/"$AGENT_NAME" -c "$AUTHBRIDGE_CONTAINER" -- \
wget -q -O - http://localhost:9093/reload/status 2>/dev/null | \
python3 -c 'import json, sys
SIDECAR_PODS=$(list_sidecar_pods || true)
if [[ -n "$SIDECAR_PODS" ]]; then
SAW_SIDECAR=true
while IFS=$'\t' read -r pod container; do
[[ -z "$pod" ]] && continue
LAST_POD="$pod"; LAST_CONTAINER="$container"
active=$(kubectl -n "$NAMESPACE" exec "$pod" -c "$container" -- \
wget -q -O - http://localhost:9093/reload/status 2>/dev/null | \
python3 -c 'import json, sys
try:
print(json.load(sys.stdin).get("active_config_sha256", ""))
except Exception:
pass' 2>/dev/null || true)
if [[ "$ACTIVE_SHA" == "$WANT_SHA" ]]; then
echo "[*] Active config SHA matches — patch is live."
exit 0
[[ -n "$active" ]] && LAST_SHA="$active"
if [[ "$active" == "$WANT_SHA" ]]; then
echo "[*] Active config SHA matches on pod $pod ($container) — patch is live."
exit 0
fi
done <<< "$SIDECAR_PODS"
fi
sleep 3
done

if ! $SAW_SIDECAR; then
echo "ERROR: could not find an authbridge sidecar container in pods of $AGENT_NAME." >&2
echo " Looked for: ${AUTHBRIDGE_CANDIDATES[*]}." >&2
echo " Pods and their containers:" >&2
kubectl -n "$NAMESPACE" get pods -l app.kubernetes.io/name="$AGENT_NAME" \
-o jsonpath='{range .items[*]}{" "}{.metadata.name}{": "}{range .spec.containers[*]}{.name}{" "}{end}{"\n"}{end}' >&2 2>/dev/null || true
exit 1
fi

echo "ERROR: authbridge active config did not match patched SHA within ${TIMEOUT}s." >&2
echo " want: $WANT_SHA" >&2
echo " last active: ${ACTIVE_SHA:-<none>}" >&2
echo " Last 20 lines of the $AUTHBRIDGE_CONTAINER container:" >&2
kubectl -n "$NAMESPACE" logs deploy/"$AGENT_NAME" -c "$AUTHBRIDGE_CONTAINER" --tail=20 >&2 || true
echo " last active: ${LAST_SHA:-<none>}" >&2
if [[ -n "$LAST_POD" ]]; then
echo " Last 20 lines of the $LAST_CONTAINER container (pod $LAST_POD):" >&2
kubectl -n "$NAMESPACE" logs "$LAST_POD" -c "$LAST_CONTAINER" --tail=20 >&2 || true
fi
echo >&2
echo " Likely causes:" >&2
echo " - ConfigMap parse error (look for 'reload failed' above)" >&2
Expand Down
92 changes: 92 additions & 0 deletions exgentic_a2a_runner/deploy-agent.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,98 @@ PYEOF
) || exit 1
fi

# Step 7.5: Bake OTEL-collector skip_hosts into the NAMESPACE authbridge base
# BEFORE the agent is created, so the sidecar has it from first pod boot — no
# post-deploy patch of the running agent, no reload race.
#
# Why the namespace base and not the per-agent ConfigMap: the rossoctl operator
# webhook force-overwrites `authbridge-config-<agent>` (server-side apply,
# ForceOwnership) on every pod admission, so a per-agent pre-seed can't survive.
# The one thing it does honor is the namespace-level `authbridge-runtime-config`
# `config.yaml`, which it reads as the base and MERGES into each per-agent CM —
# it only injects reverse_proxy_addr / reverse_proxy_backend / forward_proxy_addr
# into `listener`, so a `listener.skip_hosts` we add here rides through untouched.
#
# NOTE: `skip_hosts` is an AuthBridge-*binary* listener key; it is not defined in
# rossoctl/operator source and has NOT been verified against the sidecar's config
# schema. If the sidecar rejects the merged config, look for a "reload failed" /
# parse error in the sidecar container logs
# (kubectl -n "$NAMESPACE" logs deploy/<agent> -c authbridge # or authbridge-proxy / envoy-proxy)
# and correct the key name/shape below. This edit is namespace-wide and persists
# for every agent in $NAMESPACE.
if [ "$AUTHBRIDGE_ENABLED" = "true" ] && [ "$CLUSTER_MODE" != "in-cluster" ]; then
echo "Step 7.5: Adding OTEL-collector skip_hosts to namespace authbridge base..."
NS_AB_CM="authbridge-runtime-config"

if ! command -v python3 >/dev/null 2>&1 || ! python3 -c 'import yaml' 2>/dev/null; then
echo "Error: python3 with PyYAML is required to merge skip_hosts into $NS_AB_CM." >&2
echo " Install with one of:" >&2
echo " pip3 install --user pyyaml" >&2
echo " brew install libyaml && pip3 install pyyaml # macOS" >&2
echo " sudo apt install python3-yaml # Debian/Ubuntu" >&2
exit 1
elif ! kubectl -n "$NAMESPACE" get configmap "$NS_AB_CM" >/dev/null 2>&1; then
echo "Error: ConfigMap $NAMESPACE/$NS_AB_CM not found." >&2
echo " The operator/Helm owns this base; it must exist before deploying an" >&2
echo " AuthBridge-enabled agent. We do not fabricate one here (that would drop" >&2
echo " the pipeline: block and break auth). Ensure the namespace is provisioned." >&2
exit 1
else
CURRENT_NS_YAML=$(
kubectl -n "$NAMESPACE" get configmap "$NS_AB_CM" \
-o jsonpath='{.data.config\.yaml}'
)
MERGED_NS_YAML=$(printf '%s' "$CURRENT_NS_YAML" | python3 - <<'PYEOF'
import sys
import yaml

# Hosts the sidecar must NOT intercept (OTEL collector, rossoctl-system).
SKIP_HOSTS = [
"otel-collector.rossoctl-system.svc.cluster.local",
"otel-collector",
]

cfg = yaml.safe_load(sys.stdin.read()) or {}
listener = cfg.get("listener")
if not isinstance(listener, dict):
listener = {}
existing = listener.get("skip_hosts")
if not isinstance(existing, list):
existing = []
# Union + sort + de-dupe so re-runs are byte-stable and we don't clobber
# hosts someone else added.
listener["skip_hosts"] = sorted(set(existing) | set(SKIP_HOSTS))
cfg["listener"] = listener
# sort_keys=True keeps output deterministic across runs for the no-op check.
sys.stdout.write(yaml.safe_dump(cfg, default_flow_style=False, sort_keys=True))
PYEOF
) || { echo "Error: skip_hosts merge failed for $NAMESPACE/$NS_AB_CM (bad YAML in config.yaml?)" >&2; exit 1; }

if [ -z "$MERGED_NS_YAML" ]; then
echo "Error: skip_hosts merge produced empty output for $NAMESPACE/$NS_AB_CM" >&2
exit 1
elif [ "$CURRENT_NS_YAML" = "$MERGED_NS_YAML" ]; then
echo "✓ Namespace authbridge base already has skip_hosts — nothing to patch"
else
TMP_NS_CONFIG=$(mktemp)
printf '%s' "$MERGED_NS_YAML" >"$TMP_NS_CONFIG"
# Conflict-free create --dry-run | apply (same pattern as apply-pipeline.sh).
if kubectl -n "$NAMESPACE" create configmap "$NS_AB_CM" \
--from-file=config.yaml="$TMP_NS_CONFIG" \
--dry-run=client -o yaml \
| kubectl -n "$NAMESPACE" apply -f - >/dev/null; then
echo "✓ skip_hosts added to $NAMESPACE/$NS_AB_CM"
rm -f "$TMP_NS_CONFIG"
else
echo "Error: could not apply skip_hosts to $NAMESPACE/$NS_AB_CM" >&2
rm -f "$TMP_NS_CONFIG"
exit 1
fi
fi
fi
echo ""
fi

AGENT_JSON=$(cat <<EOF
{
"name": "$AGENT_NAME",
Expand Down