privateip · Oct 7, 2026 · Oct 7, 2026 · Oct 7, 2026
@@ -10,12 +10,11 @@ source "${SCRIPT_DIR}/lib.sh"
 
 # config/bmp-collector/ (shared with production) is copied onto the node
 # nested under the lab overlay's own root, so its "collector" resource
-# reference resolves. rm -rf first: docker cp nests SRC inside an existing
-# DEST instead of replacing it (see deploy-fabric.sh's copy_fabric_config).
+# reference resolves. copy_to replaces resources/bmp-collector/ first, so
+# the docker cp below never lands in an existing DEST (see lib.sh).
 COLLECTOR_DIR=$(cd "${SCRIPT_DIR}/../../../config/bmp-collector" && pwd)
 
 node=$(control_plane iad)
-docker exec "${node}" rm -rf /galactic/resources/bmp-collector
 copy_to "${node}" bmp-collector
 docker cp "${COLLECTOR_DIR}" "${node}:/galactic/resources/bmp-collector/iad/collector"
 apply_k "${node}" /galactic/resources/bmp-collector/iad/

@@ -89,16 +89,13 @@ for site in dfw sjc iad; do
 
   # docker cp copies SRC *into* an already-existing DEST directory
   # (nesting SRC's basename underneath it) instead of overwriting it,
-  # which would break kustomize's "../base" resource reference — so
-  # rm -rf first and only docker cp into paths that don't yet exist:
-  # copy_to lands resources/galactic-cni/ fresh, then the config/galactic-cni/
+  # which would break kustomize's "../base" resource reference — so only
+  # docker cp into paths that don't yet exist: copy_to replaces
+  # resources/galactic-cni/ (see lib.sh), then the config/galactic-cni/
   # copy targets "base", a leaf copy_to didn't create. Each site then applies
   # its own overlay directory (galactic-cni/<site>/), which layers onto the
   # shared one -- dfw needs a different GALACTIC_CNI_EBPF_INTERFACES than the
-  # single-homed sites. A rerun without
-  # the rm -rf would otherwise silently keep serving the prior copy from
-  # underneath the new nested directory.
-  docker exec "${node}" rm -rf /galactic/resources/galactic-cni
+  # single-homed sites.
   copy_to "${node}" galactic-cni
   docker cp "${GALACTIC_CNI_DIR}" "${node}:/galactic/resources/galactic-cni/base"
 

@@ -35,11 +35,10 @@ copy_fabric_config() {
   docker cp "${FABRIC_DIR}" "${node}:/galactic/resources/fabric-router/base/fabric"
 }
 
-# rm -rf first -- see copy_fabric_config's comment; copy_to (lib.sh) doesn't
-# overwrite an already-provisioned node's copy on its own either.
+# copy_to (lib.sh) replaces an already-provisioned node's copy, and
+# copy_fabric_config then fills in base/fabric -- see its comment.
 for site in dfw iad sjc; do
   node=$(control_plane "${site}")
-  docker exec "${node}" rm -rf /galactic/resources/fabric-router
   copy_to "${node}" fabric-router
   copy_fabric_config "${node}"
   apply_k "${node}" "/galactic/resources/fabric-router/${site}/"
@@ -50,7 +49,6 @@ done
 # land in one pass.
 node=$(control_plane iad)
 echo "Copying resources to ${node}..."
-docker exec "${node}" rm -rf /galactic/resources/galactic-router /galactic/resources/galactic-control
 copy_to "${node}" galactic-router
 copy_to "${node}" galactic-control
 

@@ -71,7 +71,6 @@ for site in dfw sjc iad; do
   echo "Applying galactic-nat/${site} to ${node}..."
   delete_gateway "${node}"
   delete_stale_shards "${node}"
-  docker exec "${node}" rm -rf /galactic/resources/galactic-nat
   copy_to "${node}" galactic-nat
   copy_nat_config "${node}"
   apply_k "${node}" "/galactic/resources/galactic-nat/${site}/"

@@ -71,7 +71,6 @@ apply_galactic_router() {
 for site in dfw sjc; do
   node=$(control_plane "${site}")
   echo "Applying galactic-router/${site} to ${node}..."
-  docker exec "${node}" rm -rf /galactic/resources/galactic-router
   copy_to "${node}" galactic-router
   copy_router_config "${node}"
   apply_galactic_router "${node}" "${site}"

@@ -11,12 +11,26 @@ control_plane() {
 
 # copy_to NODE SRC [DEST]
 # SRC is relative to RESOURCES_DIR; DEST defaults to the same path under
-# /galactic/resources/ on the node. docker cp requires DEST's parent to
-# already exist (it won't create intermediate directories), which matters
-# once SRC nests more than one level deep (e.g. "tenants/ns10") — so create
-# it first.
+# /galactic/resources/ on the node. DEST is replaced, not merged into: docker
+# cp copies SRC *into* a DEST that already exists, nesting it one level down,
+# so a rerun against a provisioned node would leave the old copy where
+# kustomize reads it and `kubectl apply` would report it unchanged. docker cp
+# also requires DEST's parent to exist, which matters once SRC nests more
+# than one level deep (e.g. "tenants/ns10"), so that is created first.
 copy_to() {
   local node="$1" src="$2" dest="${3:-/galactic/resources/${2}/}"
+  case "${dest}" in
+    *..*)
+      echo "copy_to: refusing to replace ${dest}: it contains .." >&2
+      return 1
+      ;;
+    /galactic/?*) ;;
+    *)
+      echo "copy_to: refusing to replace ${dest}: not under /galactic/" >&2
+      return 1
+      ;;
+  esac
+  docker exec "${node}" rm -rf "${dest}"
   docker exec "${node}" mkdir -p "$(dirname "${dest}")"
   docker cp "${RESOURCES_DIR}/${src}" "${node}:${dest}"
 }

@@ -40,12 +40,57 @@ fi
 # session's address pair is unique to the router holding it. The local address
 # is GoBMP's router_ip, which both Peer Up and Peer Down carry; local_ip is
 # only on a Peer Up.
-states=$(docker exec "$(control_plane iad)" kubectl logs -n galactic-system deploy/bmp-collector |
+#
+# A router sends its Peer Ups once, when its BMP session opens, and GoBMP logs
+# every route-monitoring message after them, so on a lab that has run for a
+# while the kubelet has rotated them out of the current log file. kubectl logs
+# reads only that file, so this reads every file the collector's pod has
+# written on its node instead, rotated and compressed ones included, oldest
+# first. The kubelet names them <restart>.log for the live file and
+# <restart>.log.<YYYYMMDD-HHMMSS>[.gz] for each rotation, so sorting by restart
+# count, then rotation time, with the live file last, puts them in order. Each
+# line carries the CRI prefix "<time> <stream> <P|F> ", and a line the runtime
+# split is a run of P parts ending in an F.
+# The running pod that is not being deleted: the collector's Deployment uses
+# Recreate, so a terminating pod can still be listed while its replacement
+# starts.
+collector=$(docker exec "$(control_plane iad)" kubectl get pods -n galactic-system \
+  -l app.kubernetes.io/name=bmp-collector --field-selector=status.phase=Running \
+  -o jsonpath='{range .items[*]}{.spec.nodeName} {.metadata.name} {.metadata.uid} {.metadata.deletionTimestamp}{"\n"}{end}' |
+  awk 'NF == 3' | head -1)
+read -r collector_node collector_pod collector_uid <<<"${collector}"
+if [[ -z "${collector_uid:-}" ]]; then
+  echo "FAIL bmp-collector: no running collector pod in iad"
+  exit 1
+fi
+log_dir="/var/log/pods/galactic-system_${collector_pod}_${collector_uid}"
+if ! docker exec "${collector_node}" test -d "${log_dir}"; then
+  echo "FAIL bmp-collector: ${collector_node} has no log directory ${log_dir}"
+  exit 1
+fi
+# Listed and read in one exec, so a file the kubelet compresses in between is
+# read under its new .gz name. The kubelet keeps a bounded number of rotated
+# files, so a Peer Up older than all of them is still lost.
+states=$(docker exec -i "${collector_node}" sh -s "${log_dir}" <<'SCRIPT' |
+cd "$1" || exit 1
+ls */* |
+  sed -E 's#^(.*/)([0-9]+)\.log(\.([0-9-]+)(\.gz)?)?$#\2 \4 &#; s#^([0-9]+)  #\1 99999999-999999 #' |
+  sort -k1,1n -k2,2 | awk '{print $3}' |
+  while read -r f; do zcat -f "${f}" 2>/dev/null || zcat -f "${f}.gz"; done
+SCRIPT
   PEER_STATE_MSG="${PEER_STATE_MSG}" python3 -c '
 import json, os, sys
 want = int(os.environ["PEER_STATE_MSG"])
 last = {}
-for line in sys.stdin:
+partial = ""
+for raw in sys.stdin:
+    parts = raw.rstrip("\n").split(" ", 3)
+    if len(parts) < 4 or parts[2] not in ("P", "F"):
+        continue
+    if parts[2] == "P":
+        partial += parts[3]
+        continue
+    line, partial = partial + parts[3], ""
     if not line.startswith("{"):
         continue
     try: