From c18711b848d9362d40a7b9be94977ebc31057fe6 Mon Sep 17 00:00:00 2001
From: Peter Sprygada <psprygada@datum.net>
Date: Tue, 6 Oct 2026 20:19:12 -0400
Subject: [PATCH 1/2] fix(containerlab): Read the BMP collector's rotated logs
 in verify:bmp

verify:bmp failed on any lab whose collector had run long enough for the kubelet to rotate its log. A router sends its Peer Ups once, when its BMP session opens, and the route-monitoring messages that follow push them out of the live file. kubectl logs reads only that file, so every session looked like it had never come up.

The check now reads every log file the collector's pod has written on its node, compressed rotations included. It orders them by the kubelet's own file names (restart count, then rotation time, live file last) rather than by modification time, and joins lines the container runtime split into partial records.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 deploy/containerlab/scripts/verify-bmp.sh | 49 ++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/deploy/containerlab/scripts/verify-bmp.sh b/deploy/containerlab/scripts/verify-bmp.sh
index 10ff992b..11e2b31a 100755
--- a/deploy/containerlab/scripts/verify-bmp.sh
+++ b/deploy/containerlab/scripts/verify-bmp.sh
@@ -40,12 +40,57 @@ fi
 # session's address pair is unique to the router holding it. The local address
 # is GoBMP's router_ip, which both Peer Up and Peer Down carry; local_ip is
 # only on a Peer Up.
-states=$(docker exec "$(control_plane iad)" kubectl logs -n galactic-system deploy/bmp-collector |
+#
+# A router sends its Peer Ups once, when its BMP session opens, and GoBMP logs
+# every route-monitoring message after them, so on a lab that has run for a
+# while the kubelet has rotated them out of the current log file. kubectl logs
+# reads only that file, so this reads every file the collector's pod has
+# written on its node instead, rotated and compressed ones included, oldest
+# first. The kubelet names them <restart>.log for the live file and
+# <restart>.log.<YYYYMMDD-HHMMSS>[.gz] for each rotation, so sorting by restart
+# count, then rotation time, with the live file last, puts them in order. Each
+# line carries the CRI prefix "<time> <stream> <P|F> ", and a line the runtime
+# split is a run of P parts ending in an F.
+# The running pod that is not being deleted: the collector's Deployment uses
+# Recreate, so a terminating pod can still be listed while its replacement
+# starts.
+collector=$(docker exec "$(control_plane iad)" kubectl get pods -n galactic-system \
+  -l app.kubernetes.io/name=bmp-collector --field-selector=status.phase=Running \
+  -o jsonpath='{range .items[*]}{.spec.nodeName} {.metadata.name} {.metadata.uid} {.metadata.deletionTimestamp}{"\n"}{end}' |
+  awk 'NF == 3' | head -1)
+read -r collector_node collector_pod collector_uid <<<"${collector}"
+if [[ -z "${collector_uid:-}" ]]; then
+  echo "FAIL bmp-collector: no running collector pod in iad"
+  exit 1
+fi
+log_dir="/var/log/pods/galactic-system_${collector_pod}_${collector_uid}"
+if ! docker exec "${collector_node}" test -d "${log_dir}"; then
+  echo "FAIL bmp-collector: ${collector_node} has no log directory ${log_dir}"
+  exit 1
+fi
+# Listed and read in one exec, so a file the kubelet compresses in between is
+# read under its new .gz name. The kubelet keeps a bounded number of rotated
+# files, so a Peer Up older than all of them is still lost.
+states=$(docker exec -i "${collector_node}" sh -s "${log_dir}" <<'SCRIPT' |
+cd "$1" || exit 1
+ls */* |
+  sed -E 's#^(.*/)([0-9]+)\.log(\.([0-9-]+)(\.gz)?)?$#\2 \4 &#; s#^([0-9]+)  #\1 99999999-999999 #' |
+  sort -k1,1n -k2,2 | awk '{print $3}' |
+  while read -r f; do zcat -f "${f}" 2>/dev/null || zcat -f "${f}.gz"; done
+SCRIPT
   PEER_STATE_MSG="${PEER_STATE_MSG}" python3 -c '
 import json, os, sys
 want = int(os.environ["PEER_STATE_MSG"])
 last = {}
-for line in sys.stdin:
+partial = ""
+for raw in sys.stdin:
+    parts = raw.rstrip("\n").split(" ", 3)
+    if len(parts) < 4 or parts[2] not in ("P", "F"):
+        continue
+    if parts[2] == "P":
+        partial += parts[3]
+        continue
+    line, partial = partial + parts[3], ""
     if not line.startswith("{"):
         continue
     try:

From 71c4d91a8132265938ac0e009728c901bec9a5cb Mon Sep 17 00:00:00 2001
From: Peter Sprygada <psprygada@datum.net>
Date: Tue, 6 Oct 2026 20:19:12 -0400
Subject: [PATCH 2/2] fix(containerlab): Replace a node's resource copy on
 every deploy

copy_to used docker cp into a destination that might already exist, and docker cp then nests the new copy one level down instead of replacing the old one. Kustomize kept reading the old files, so a changed manifest applied as unchanged. Most deploy scripts worked around it by deleting the destination first, but deploy-ns.sh did not, so an edited tenant never reached a running lab.

copy_to now replaces its destination itself and refuses any path outside /galactic/. The per-script deletes it makes redundant are gone, and the comments that described the old behavior say what copy_to does now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../scripts/deploy-bmp-collector.sh           |  5 ++---
 deploy/containerlab/scripts/deploy-cni.sh     | 11 ++++------
 deploy/containerlab/scripts/deploy-fabric.sh  |  6 ++---
 .../scripts/deploy-galactic-nat.sh            |  1 -
 .../scripts/deploy-galactic-router.sh         |  1 -
 deploy/containerlab/scripts/lib.sh            | 22 +++++++++++++++----
 6 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/deploy/containerlab/scripts/deploy-bmp-collector.sh b/deploy/containerlab/scripts/deploy-bmp-collector.sh
index fe9e6118..98877227 100755
--- a/deploy/containerlab/scripts/deploy-bmp-collector.sh
+++ b/deploy/containerlab/scripts/deploy-bmp-collector.sh
@@ -10,12 +10,11 @@ source "${SCRIPT_DIR}/lib.sh"
 
 # config/bmp-collector/ (shared with production) is copied onto the node
 # nested under the lab overlay's own root, so its "collector" resource
-# reference resolves. rm -rf first: docker cp nests SRC inside an existing
-# DEST instead of replacing it (see deploy-fabric.sh's copy_fabric_config).
+# reference resolves. copy_to replaces resources/bmp-collector/ first, so
+# the docker cp below never lands in an existing DEST (see lib.sh).
 COLLECTOR_DIR=$(cd "${SCRIPT_DIR}/../../../config/bmp-collector" && pwd)
 
 node=$(control_plane iad)
-docker exec "${node}" rm -rf /galactic/resources/bmp-collector
 copy_to "${node}" bmp-collector
 docker cp "${COLLECTOR_DIR}" "${node}:/galactic/resources/bmp-collector/iad/collector"
 apply_k "${node}" /galactic/resources/bmp-collector/iad/
diff --git a/deploy/containerlab/scripts/deploy-cni.sh b/deploy/containerlab/scripts/deploy-cni.sh
index 49186131..a991626c 100755
--- a/deploy/containerlab/scripts/deploy-cni.sh
+++ b/deploy/containerlab/scripts/deploy-cni.sh
@@ -89,16 +89,13 @@ for site in dfw sjc iad; do
 
   # docker cp copies SRC *into* an already-existing DEST directory
   # (nesting SRC's basename underneath it) instead of overwriting it,
-  # which would break kustomize's "../base" resource reference — so
-  # rm -rf first and only docker cp into paths that don't yet exist:
-  # copy_to lands resources/galactic-cni/ fresh, then the config/galactic-cni/
+  # which would break kustomize's "../base" resource reference — so only
+  # docker cp into paths that don't yet exist: copy_to replaces
+  # resources/galactic-cni/ (see lib.sh), then the config/galactic-cni/
   # copy targets "base", a leaf copy_to didn't create. Each site then applies
   # its own overlay directory (galactic-cni/<site>/), which layers onto the
   # shared one -- dfw needs a different GALACTIC_CNI_EBPF_INTERFACES than the
-  # single-homed sites. A rerun without
-  # the rm -rf would otherwise silently keep serving the prior copy from
-  # underneath the new nested directory.
-  docker exec "${node}" rm -rf /galactic/resources/galactic-cni
+  # single-homed sites.
   copy_to "${node}" galactic-cni
   docker cp "${GALACTIC_CNI_DIR}" "${node}:/galactic/resources/galactic-cni/base"
 
diff --git a/deploy/containerlab/scripts/deploy-fabric.sh b/deploy/containerlab/scripts/deploy-fabric.sh
index cdaf8971..48ea31d6 100755
--- a/deploy/containerlab/scripts/deploy-fabric.sh
+++ b/deploy/containerlab/scripts/deploy-fabric.sh
@@ -35,11 +35,10 @@ copy_fabric_config() {
   docker cp "${FABRIC_DIR}" "${node}:/galactic/resources/fabric-router/base/fabric"
 }
 
-# rm -rf first -- see copy_fabric_config's comment; copy_to (lib.sh) doesn't
-# overwrite an already-provisioned node's copy on its own either.
+# copy_to (lib.sh) replaces an already-provisioned node's copy, and
+# copy_fabric_config then fills in base/fabric -- see its comment.
 for site in dfw iad sjc; do
   node=$(control_plane "${site}")
-  docker exec "${node}" rm -rf /galactic/resources/fabric-router
   copy_to "${node}" fabric-router
   copy_fabric_config "${node}"
   apply_k "${node}" "/galactic/resources/fabric-router/${site}/"
@@ -50,7 +49,6 @@ done
 # land in one pass.
 node=$(control_plane iad)
 echo "Copying resources to ${node}..."
-docker exec "${node}" rm -rf /galactic/resources/galactic-router /galactic/resources/galactic-control
 copy_to "${node}" galactic-router
 copy_to "${node}" galactic-control
 
diff --git a/deploy/containerlab/scripts/deploy-galactic-nat.sh b/deploy/containerlab/scripts/deploy-galactic-nat.sh
index 9d31fb49..cce53f0a 100755
--- a/deploy/containerlab/scripts/deploy-galactic-nat.sh
+++ b/deploy/containerlab/scripts/deploy-galactic-nat.sh
@@ -71,7 +71,6 @@ for site in dfw sjc iad; do
   echo "Applying galactic-nat/${site} to ${node}..."
   delete_gateway "${node}"
   delete_stale_shards "${node}"
-  docker exec "${node}" rm -rf /galactic/resources/galactic-nat
   copy_to "${node}" galactic-nat
   copy_nat_config "${node}"
   apply_k "${node}" "/galactic/resources/galactic-nat/${site}/"
diff --git a/deploy/containerlab/scripts/deploy-galactic-router.sh b/deploy/containerlab/scripts/deploy-galactic-router.sh
index 3f606b31..953b7c97 100755
--- a/deploy/containerlab/scripts/deploy-galactic-router.sh
+++ b/deploy/containerlab/scripts/deploy-galactic-router.sh
@@ -71,7 +71,6 @@ apply_galactic_router() {
 for site in dfw sjc; do
   node=$(control_plane "${site}")
   echo "Applying galactic-router/${site} to ${node}..."
-  docker exec "${node}" rm -rf /galactic/resources/galactic-router
   copy_to "${node}" galactic-router
   copy_router_config "${node}"
   apply_galactic_router "${node}" "${site}"
diff --git a/deploy/containerlab/scripts/lib.sh b/deploy/containerlab/scripts/lib.sh
index 80e9439f..a7646458 100755
--- a/deploy/containerlab/scripts/lib.sh
+++ b/deploy/containerlab/scripts/lib.sh
@@ -11,12 +11,26 @@ control_plane() {
 
 # copy_to NODE SRC [DEST]
 # SRC is relative to RESOURCES_DIR; DEST defaults to the same path under
-# /galactic/resources/ on the node. docker cp requires DEST's parent to
-# already exist (it won't create intermediate directories), which matters
-# once SRC nests more than one level deep (e.g. "tenants/ns10") — so create
-# it first.
+# /galactic/resources/ on the node. DEST is replaced, not merged into: docker
+# cp copies SRC *into* a DEST that already exists, nesting it one level down,
+# so a rerun against a provisioned node would leave the old copy where
+# kustomize reads it and `kubectl apply` would report it unchanged. docker cp
+# also requires DEST's parent to exist, which matters once SRC nests more
+# than one level deep (e.g. "tenants/ns10"), so that is created first.
 copy_to() {
   local node="$1" src="$2" dest="${3:-/galactic/resources/${2}/}"
+  case "${dest}" in
+    *..*)
+      echo "copy_to: refusing to replace ${dest}: it contains .." >&2
+      return 1
+      ;;
+    /galactic/?*) ;;
+    *)
+      echo "copy_to: refusing to replace ${dest}: not under /galactic/" >&2
+      return 1
+      ;;
+  esac
+  docker exec "${node}" rm -rf "${dest}"
   docker exec "${node}" mkdir -p "$(dirname "${dest}")"
   docker cp "${RESOURCES_DIR}/${src}" "${node}:${dest}"
 }