privateip · Oct 27, 2026 · Oct 27, 2026 · Oct 27, 2026 · Oct 27, 2026
@@ -52,8 +52,10 @@ tasks:
     deps: [golangci-lint, yamlfmt, yaml-ext, 'build:ebpf']
     cmds:
       - '{{.GOLANGCI_LINT}} run'
-      # examples/edge-provider is its own module, which the root run skips.
+      # examples/edge-provider and examples/looking-glass-client are their
+      # own modules, which the root run skips.
       - cd examples/edge-provider && {{.GOLANGCI_LINT}} run --config ../../.golangci.yaml
+      - cd examples/looking-glass-client && {{.GOLANGCI_LINT}} run --config ../../.golangci.yaml
       - '{{.YAMLFMT}} -lint'
   lint-fix:
     desc: Lint and apply fixes
@@ -181,13 +183,27 @@ tasks:
     cmds:
       - bash scripts/ci.sh unittest
       - task: test:edge-provider
+      - task: test:looking-glass-client
   test:edge-provider:
     desc: Build, vet and unit-test the reference edge provider (examples/edge-provider, its own module)
     dir: examples/edge-provider
     cmds:
       - go build ./...
       - go vet ./...
       - go test -race ./...
+  test:looking-glass-client:
+    desc: Build, vet and unit-test the independent Looking Glass client (examples/looking-glass-client, its own module)
+    dir: examples/looking-glass-client
+    cmds:
+      - go build ./...
+      - go vet ./...
+      - go test -race ./...
+      # The client must build from the published service API alone.
+      - |
+        if go list -deps ./... | grep -E '^go\.datum\.net/galactic/' | grep -vxE 'go\.datum\.net/galactic/api/fabric/(service/)?v1'; then
+          echo "the client imports Galactic packages beyond the published service API" >&2
+          exit 1
+        fi
   test:unit-root:
     desc: Re-run just the requireRoot(t)-gated unit tests, as root
     deps: ['build:ebpf']

@@ -260,6 +260,13 @@ its behalf.
 | iad-worker2 – tr3:bond1   | 2001:db8:1:32::/64   | 2001:db8:1:32::1   | 2001:db8:1:32::2   | 10.1.32.0/24   | 10.1.32.1    | 10.1.32.2      |
 | remote-host – tr4:eth4    | 2001:db8:1:40::/64   | 2001:db8:1:40::1   | 2001:db8:1:40::2   | 11.1.40.0/24   | 11.1.40.1    | 11.1.40.2      |
 
+`remote-host` also takes `2c0f:40::2/64` on the same link. fabric-api's public probe
+policy refuses `2001:db8::/32`, so a Looking Glass ping or traceroute over IPv6 needs a
+global destination, and this is it. `tr4` also originates `11.1.42.0/24` and `2c0f:42::/64`
+and drops them without an ICMP error: a probe there is lost and a traceroute ends in silent
+hops, keeping its routers busy for about 20 s (`verify:looking-glass` cancels, expires and
+fails over such probes). None of these prefixes leave the lab.
+
 ### Site-internal links (numbered, iBGP to the site's edge node)
 
 Same convention with the edge node in the router's seat: it takes `::1`/`.1`, the worker
@@ -552,6 +559,9 @@ task deploy
 | `verify:ns30`                    | Verify ns30 ping (dfw only, 2 pods)                                                                                                         |
 | `verify:ns40`                    | Verify ns40 ping (iad only, 2 pods)                                                                                                         |
 | `verify:traffic-policy`          | Every binding installed; each path forwards what it grants and drops, and counts, what it does not                                          |
+| `deploy:looking-glass`           | Enable the Looking Glass service on dfw in the documented order (restarts dfw's FRR twice); issue its test certificates                     |
+| `verify:looking-glass`           | Qualify the Looking Glass service on dfw with the independent client, with faults (about 45 min; not in `verify`)                           |
+| `undeploy:looking-glass`         | Switch the Looking Glass service off on dfw and roll its enabling steps back (restarts dfw's FRR twice)                                     |
 | `destroy`                        | Destroy the lab and remove all Kind clusters                                                                                                |
 | `restart`                        | Full rebuild — destroy then redeploy                                                                                                        |
 | `rebuild`                        | Full rebuild — clean (destroy + delete images/artifacts) then redeploy                                                                      |

@@ -43,8 +43,18 @@ tasks:
 
   "build:fabric-api":
     desc: Build the fabric-api image (node sidecar, cell gateway, janitor)
+    # The commit becomes the gateway's Looking Glass contract_revision, which
+    # verify:looking-glass records as the build under test.
+    vars:
+      GIT_COMMIT:
+        sh: git rev-parse HEAD
+      GIT_TREE_STATE:
+        sh: git diff --quiet HEAD && echo clean || echo dirty
     cmds:
-      - docker build --network=host -t fabric-api:latest -f ../../containers/fabric-api/Dockerfile ../..
+      - >-
+        docker build --network=host -t fabric-api:latest
+        --build-arg GIT_COMMIT={{.GIT_COMMIT}} --build-arg GIT_TREE_STATE={{.GIT_TREE_STATE}}
+        -f ../../containers/fabric-api/Dockerfile ../..
 
   "build:galactic-router":
     desc: Build the galactic-router container image
@@ -524,6 +534,45 @@ tasks:
     cmds:
       - ./scripts/verify-fabric-metrics.sh
 
+  # Looking Glass service qualification on dfw (tests/lgqualify): the
+  # independent client in examples/looking-glass-client against real fabric
+  # routers, with faults. deploy:looking-glass enables the service in the
+  # documented order (restarting dfw's FRR twice) and undeploy:looking-glass
+  # rolls it back. The run takes about 45 minutes, restarts and pauses the
+  # gateway leader, resets one of dfw-worker's BGP sessions 80 times, and
+  # writes its evidence to bin/lg-qualify/<run>/ at the repository root.
+  "deploy:looking-glass":
+    desc: Enable the Looking Glass service on dfw and issue the qualification's service certificates
+    deps: ["build:fabric-api"]
+    cmds:
+      - |
+        set -e
+        for node in dfw-worker dfw-worker2 dfw-worker3; do
+          docker save fabric-api:latest | docker exec -i "${node}" ctr --namespace k8s.io images import -
+        done
+      - ./scripts/deploy-looking-glass.sh
+
+  "undeploy:looking-glass":
+    desc: Switch the Looking Glass service off on dfw and roll its enabling steps back
+    cmds:
+      - ./scripts/undeploy-looking-glass.sh
+
+  "verify:looking-glass":
+    desc: Qualify the Looking Glass service on dfw with the independent client (about 45 minutes)
+    dir: ../..
+    vars:
+      OUT: '{{.ROOT_DIR}}/../../bin/lg-qualify'
+    cmds:
+      - mkdir -p {{.OUT}}
+      - cd examples/looking-glass-client && CGO_ENABLED=0 go build -o {{.OUT}}/looking-glass-client .
+      - CGO_ENABLED=0 go build -o {{.OUT}}/fabric-api ./cmd/fabric-api
+      - kind get kubeconfig --name dfw > {{.OUT}}/dfw.kubeconfig
+      - >-
+        LG_QUALIFY_KUBECONFIG={{.OUT}}/dfw.kubeconfig
+        LG_QUALIFY_CLIENT={{.OUT}}/looking-glass-client
+        LG_QUALIFY_FABRIC_API={{.OUT}}/fabric-api
+        go test -count=1 -v -timeout 120m -run TestQualify ./tests/lgqualify/
+
   "verify:fabric-api":
     desc: Verify the fabric-api looking glass on every fabric node, directly and through FabricQuery
     deps: ["verify:fabric-api-config"]

@@ -25,6 +25,8 @@ ip link set eth1 up
 # on the link, the same TR-.1/node-.2 convention every worker uplink uses.
 ip -6 addr add 2001:db8:1:40::2/64 dev eth1 || true
 ip -4 addr add 11.1.40.2/24 dev eth1 || true
+# The Looking Glass probe destination; see node_files/tr4/frr.conf.
+ip -6 addr add 2c0f:40::2/64 dev eth1 || true
 
 # Everything this host talks to is somewhere behind tr4: the whole fabric,
 # every site's nodes, and every tenant's egress source address. A default

@@ -25,8 +25,15 @@ interface eth4
  description remote-host-facing
  ip address 11.1.40.1/24
  ipv6 address 2001:db8:1:40::1/64
+ ipv6 address 2c0f:40::1/64
 !
 
+! A probe sink for verify:looking-glass: global prefixes tr4 originates and
+! drops without an ICMP error, so a probe toward them is lost and a
+! traceroute ends in silent hops, keeping its routers busy for about 20 s.
+ip route 11.1.42.0/24 blackhole
+ipv6 route 2c0f:42::/64 blackhole
+
 ipv6 prefix-list MIN48 seq 5 permit ::/0 le 47
 route-map FILTER_PREFIX deny 10
  match ipv6 address prefix-list MIN48
@@ -68,6 +75,14 @@ router bgp 65100
   ! makes nginx on 2001:db8:1:40::2 reachable from a tenant pod's egress
   ! path through the NAT66 shard tier, and from every other site's nodes.
   network 2001:db8:1:40::/64
+  ! The host's probe destination (2c0f:40::2), outside 2001:db8::/32: the
+  ! fabric-api public probe policy refuses documentation and other
+  ! special-use space, so only a global address can be pinged or traced
+  ! through the Looking Glass service (verify:looking-glass). Like
+  ! 11.1.40.0/24, it is never announced beyond the lab.
+  network 2c0f:40::/64
+  ! The probe sink, dropped here (see the blackhole routes above).
+  network 2c0f:42::/64
  exit-address-family
  !
  ! One numbered session per address family -- see node_files/tr1/frr.conf.
@@ -85,6 +100,7 @@ router bgp 65100
   ! The off-fabric host's IPv4 subnet -- see the IPv6 network statement
   ! above. A /24 passes FILTER_PREFIX_V4 on every receiving transit router.
   network 11.1.40.0/24
+  network 11.1.42.0/24
  exit-address-family
 !
 

@@ -0,0 +1,23 @@
+# Looking Glass service on dfw
+
+The overlays `deploy:looking-glass` applies to qualify the Looking Glass
+service (`verify:looking-glass`), one per step of the enabling order in
+[docs/fabric-api/operations.md](../../../../docs/fabric-api/operations.md#enabling-aggregate-probe-budgets).
+Each builds on the step before it:
+
+| Step | Overlay                      | Applied from        | Adds                                                     |
+| ---- | ---------------------------- | ------------------- | -------------------------------------------------------- |
+| 1    | `fabric-router/dfw/permits/` | `dfw-control-plane` | `components/fabric-api-permits`; restarts FRR            |
+| 2    | `gateway/dfw/budgets/`       | host                | `components/aggregate-budgets`                           |
+| 3    | `fabric-router/dfw/require/` | `dfw-control-plane` | `components/fabric-api-require-permits`; restarts FRR    |
+| 4    | `gateway/dfw/service/`       | host                | `components/service`, switched off until the deploy ends |
+
+`pki/services.yaml` holds the service certificates the suite authenticates
+with. `undeploy:looking-glass` switches the service off and walks the steps
+back in reverse, ending on the lab's ordinary `fabric-router` and gateway
+overlays.
+
+The `fabric-router` overlays are applied on `dfw-control-plane`, where
+`deploy-fabric.sh` copies `resources/fabric-router/` and nests
+`config/fabric-router/` under it as `fabric-router/base/fabric`; their
+relative paths resolve there, not in this checkout.
@@ -0,0 +1,27 @@
+# Step 1: dfw's fabric-router sidecars verify probe permits. The permit
+# Components set the sidecar's arguments whole, so the patch keeps the
+# lab's own (resources/fabric-router/base/fabric-lab-patch.yaml) beside them.
+resources:
+  - ../../../../fabric-router/dfw
+components:
+  - ../../../../fabric-router/base/fabric/components/fabric-api-permits
+patches:
+  - target:
+      kind: DaemonSet
+      name: fabric-router
+    patch: |-
+      apiVersion: apps/v1
+      kind: DaemonSet
+      metadata:
+        name: fabric-router
+      spec:
+        template:
+          spec:
+            containers:
+              - name: fabric-api
+                args:
+                  - node
+                  - --probe-permits
+                  - --operator-allow-prefixes=10.255.255.0/24,fc00:0::/32
+                  - --enable-expensive-queries
+                  - --log-level=debug
@@ -0,0 +1,29 @@
+# Step 3: dfw's fabric-router sidecars refuse any probe without a permit,
+# direct operator probes included, so the gateway's budgets charge every
+# probe in the cell. The lab's own sidecar arguments are kept as in step 1.
+resources:
+  - ../permits
+components:
+  - ../../../../fabric-router/base/fabric/components/fabric-api-require-permits
+patches:
+  - target:
+      kind: DaemonSet
+      name: fabric-router
+    patch: |-
+      apiVersion: apps/v1
+      kind: DaemonSet
+      metadata:
+        name: fabric-router
+      spec:
+        template:
+          spec:
+            containers:
+              - name: fabric-api
+                args:
+                  - node
+                  - --probe-permits
+                  - --require-probe-permits
+                  - --gateway-address=fabric-api-gateway.galactic-system.svc:9346
+                  - --operator-allow-prefixes=10.255.255.0/24,fc00:0::/32
+                  - --enable-expensive-queries
+                  - --log-level=debug
@@ -0,0 +1,6 @@
+# Step 2: dfw's gateway charges every probe to the cell's aggregate budgets,
+# at the contract's default limits.
+resources:
+  - ../../../../fabric-api/gateway/dfw
+components:
+  - ../../../../../../../config/fabric-api/components/aggregate-budgets
@@ -0,0 +1,7 @@
+# Step 4: dfw's gateway serves the Looking Glass service to the
+# looking-glass identity. The switch (ConfigMap fabric-api-service) is
+# created off; deploy-looking-glass.sh turns it on last.
+resources:
+  - ../budgets
+components:
+  - ../../../../../../../config/fabric-api/components/service
@@ -0,0 +1,87 @@
+# Looking Glass service certificates for verify:looking-glass, issued by the
+# lab's fabric-api Issuer (resources/fabric-api/pki). CELL_NAME is replaced
+# with the site name by deploy-looking-glass.sh.
+#
+# - looking-glass: the service identity the gateway allows
+#   (config/fabric-api/components/service).
+# - looking-glass-unlisted: a valid service identity of this cell that the
+#   gateway does not list.
+# - looking-glass-other-cell: the allowed service name in another cell,
+#   signed by this cell's CA.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: lg-qualify-looking-glass
+  namespace: galactic-system
+  labels:
+    app.kubernetes.io/part-of: lg-qualify
+spec:
+  secretName: lg-qualify-looking-glass
+  secretTemplate:
+    labels:
+      app.kubernetes.io/part-of: lg-qualify
+  duration: 24h
+  renewBefore: 8h
+  uris:
+    - spiffe://fabric-api.datumapis.com/cell/CELL_NAME/service/looking-glass
+  usages:
+    - digital signature
+    - client auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: fabric-api
+    kind: Issuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: lg-qualify-unlisted
+  namespace: galactic-system
+  labels:
+    app.kubernetes.io/part-of: lg-qualify
+spec:
+  secretName: lg-qualify-unlisted
+  secretTemplate:
+    labels:
+      app.kubernetes.io/part-of: lg-qualify
+  duration: 24h
+  renewBefore: 8h
+  uris:
+    - spiffe://fabric-api.datumapis.com/cell/CELL_NAME/service/looking-glass-unlisted
+  usages:
+    - digital signature
+    - client auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: fabric-api
+    kind: Issuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: lg-qualify-other-cell
+  namespace: galactic-system
+  labels:
+    app.kubernetes.io/part-of: lg-qualify
+spec:
+  secretName: lg-qualify-other-cell
+  secretTemplate:
+    labels:
+      app.kubernetes.io/part-of: lg-qualify
+  duration: 24h
+  renewBefore: 8h
+  uris:
+    - spiffe://fabric-api.datumapis.com/cell/not-CELL_NAME/service/looking-glass
+  usages:
+    - digital signature
+    - client auth
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: fabric-api
+    kind: Issuer
@@ -15,28 +15,11 @@ source "${SCRIPT_DIR}/lib.sh"
 # ConfigMap via NODE_NAME and each site's kustomization generates one such
 # ConfigMap per matching node. Re-running this after editing one node's
 # frr.conf changes only that node's ConfigMap, which its config-agent
-# sidecar reloads in place -- no pod restarts. Copied onto the node at deploy time nested under
-# the overlay's own root so its "fabric" resource reference resolves
-# (kustomize requires resources in or below the overlay root).
-FABRIC_DIR=$(cd "${SCRIPT_DIR}/../../../config/fabric-router" && pwd)
-
-# copy_fabric_config NODE copies config/fabric-router/ onto NODE, nested under
-# resources/fabric-router/base/ so the base overlay's "fabric" resource
-# reference resolves. rm -rf first: like deploy-galactic-router.sh's
-# copy_router_config, docker cp nests SRC inside an already-existing DEST
-# dir instead of overwriting it, so a rerun against an already-provisioned
-# node would silently keep serving the prior copy from underneath the new
-# one -- kubectl would then report the DaemonSet "unchanged" even after a
-# real manifest edit (found live: a fabric-lab-patch.yaml affinity fix
-# never took effect on a redeploy until this guard was added).
-copy_fabric_config() {
-  local node="$1"
-  docker exec "${node}" rm -rf /galactic/resources/fabric-router/base/fabric
-  docker cp "${FABRIC_DIR}" "${node}:/galactic/resources/fabric-router/base/fabric"
-}
-
-# copy_to (lib.sh) replaces an already-provisioned node's copy, and
-# copy_fabric_config then fills in base/fabric -- see its comment.
+# sidecar reloads in place -- no pod restarts. copy_to (lib.sh) copies
+# resources/fabric-router/ onto the control-plane node, replacing an
+# already-provisioned copy, and copy_fabric_config (lib.sh) nests
+# config/fabric-router/ under it as base/fabric, inside the overlay's own
+# root, since kustomize requires resources in or below that root.
 for site in dfw iad sjc; do
   node=$(control_plane "${site}")
   copy_to "${node}" fabric-router