diff --git a/Taskfile.yaml b/Taskfile.yaml index 21c77d3d8..89512cb5f 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -52,8 +52,10 @@ tasks: deps: [golangci-lint, yamlfmt, yaml-ext, 'build:ebpf'] cmds: - '{{.GOLANGCI_LINT}} run' - # examples/edge-provider is its own module, which the root run skips. + # examples/edge-provider and examples/looking-glass-client are their + # own modules, which the root run skips. - cd examples/edge-provider && {{.GOLANGCI_LINT}} run --config ../../.golangci.yaml + - cd examples/looking-glass-client && {{.GOLANGCI_LINT}} run --config ../../.golangci.yaml - '{{.YAMLFMT}} -lint' lint-fix: desc: Lint and apply fixes @@ -181,6 +183,7 @@ tasks: cmds: - bash scripts/ci.sh unittest - task: test:edge-provider + - task: test:looking-glass-client test:edge-provider: desc: Build, vet and unit-test the reference edge provider (examples/edge-provider, its own module) dir: examples/edge-provider @@ -188,6 +191,19 @@ tasks: - go build ./... - go vet ./... - go test -race ./... + test:looking-glass-client: + desc: Build, vet and unit-test the independent Looking Glass client (examples/looking-glass-client, its own module) + dir: examples/looking-glass-client + cmds: + - go build ./... + - go vet ./... + - go test -race ./... + # The client must build from the published service API alone. + - | + if go list -deps ./... | grep -E '^go\.datum\.net/galactic/' | grep -vxE 'go\.datum\.net/galactic/api/fabric/(service/)?v1'; then + echo "the client imports Galactic packages beyond the published service API" >&2 + exit 1 + fi test:unit-root: desc: Re-run just the requireRoot(t)-gated unit tests, as root deps: ['build:ebpf'] diff --git a/deploy/containerlab/README.md b/deploy/containerlab/README.md index 4b9cf645c..09c50758c 100644 --- a/deploy/containerlab/README.md +++ b/deploy/containerlab/README.md @@ -260,6 +260,13 @@ its behalf. | iad-worker2 – tr3:bond1 | 2001:db8:1:32::/64 | 2001:db8:1:32::1 | 2001:db8:1:32::2 | 10.1.32.0/24 | 10.1.32.1 | 10.1.32.2 | | remote-host – tr4:eth4 | 2001:db8:1:40::/64 | 2001:db8:1:40::1 | 2001:db8:1:40::2 | 11.1.40.0/24 | 11.1.40.1 | 11.1.40.2 | +`remote-host` also takes `2c0f:40::2/64` on the same link. fabric-api's public probe +policy refuses `2001:db8::/32`, so a Looking Glass ping or traceroute over IPv6 needs a +global destination, and this is it. `tr4` also originates `11.1.42.0/24` and `2c0f:42::/64` +and drops them without an ICMP error: a probe there is lost and a traceroute ends in silent +hops, keeping its routers busy for about 20 s (`verify:looking-glass` cancels, expires and +fails over such probes). None of these prefixes leave the lab. + ### Site-internal links (numbered, iBGP to the site's edge node) Same convention with the edge node in the router's seat: it takes `::1`/`.1`, the worker @@ -552,6 +559,9 @@ task deploy | `verify:ns30` | Verify ns30 ping (dfw only, 2 pods) | | `verify:ns40` | Verify ns40 ping (iad only, 2 pods) | | `verify:traffic-policy` | Every binding installed; each path forwards what it grants and drops, and counts, what it does not | +| `deploy:looking-glass` | Enable the Looking Glass service on dfw in the documented order (restarts dfw's FRR twice); issue its test certificates | +| `verify:looking-glass` | Qualify the Looking Glass service on dfw with the independent client, with faults (about 45 min; not in `verify`) | +| `undeploy:looking-glass` | Switch the Looking Glass service off on dfw and roll its enabling steps back (restarts dfw's FRR twice) | | `destroy` | Destroy the lab and remove all Kind clusters | | `restart` | Full rebuild — destroy then redeploy | | `rebuild` | Full rebuild — clean (destroy + delete images/artifacts) then redeploy | diff --git a/deploy/containerlab/Taskfile.yaml b/deploy/containerlab/Taskfile.yaml index fe8073d74..2af36b74c 100644 --- a/deploy/containerlab/Taskfile.yaml +++ b/deploy/containerlab/Taskfile.yaml @@ -43,8 +43,18 @@ tasks: "build:fabric-api": desc: Build the fabric-api image (node sidecar, cell gateway, janitor) + # The commit becomes the gateway's Looking Glass contract_revision, which + # verify:looking-glass records as the build under test. + vars: + GIT_COMMIT: + sh: git rev-parse HEAD + GIT_TREE_STATE: + sh: git diff --quiet HEAD && echo clean || echo dirty cmds: - - docker build --network=host -t fabric-api:latest -f ../../containers/fabric-api/Dockerfile ../.. + - >- + docker build --network=host -t fabric-api:latest + --build-arg GIT_COMMIT={{.GIT_COMMIT}} --build-arg GIT_TREE_STATE={{.GIT_TREE_STATE}} + -f ../../containers/fabric-api/Dockerfile ../.. "build:galactic-router": desc: Build the galactic-router container image @@ -524,6 +534,45 @@ tasks: cmds: - ./scripts/verify-fabric-metrics.sh + # Looking Glass service qualification on dfw (tests/lgqualify): the + # independent client in examples/looking-glass-client against real fabric + # routers, with faults. deploy:looking-glass enables the service in the + # documented order (restarting dfw's FRR twice) and undeploy:looking-glass + # rolls it back. The run takes about 45 minutes, restarts and pauses the + # gateway leader, resets one of dfw-worker's BGP sessions 80 times, and + # writes its evidence to bin/lg-qualify// at the repository root. + "deploy:looking-glass": + desc: Enable the Looking Glass service on dfw and issue the qualification's service certificates + deps: ["build:fabric-api"] + cmds: + - | + set -e + for node in dfw-worker dfw-worker2 dfw-worker3; do + docker save fabric-api:latest | docker exec -i "${node}" ctr --namespace k8s.io images import - + done + - ./scripts/deploy-looking-glass.sh + + "undeploy:looking-glass": + desc: Switch the Looking Glass service off on dfw and roll its enabling steps back + cmds: + - ./scripts/undeploy-looking-glass.sh + + "verify:looking-glass": + desc: Qualify the Looking Glass service on dfw with the independent client (about 45 minutes) + dir: ../.. + vars: + OUT: '{{.ROOT_DIR}}/../../bin/lg-qualify' + cmds: + - mkdir -p {{.OUT}} + - cd examples/looking-glass-client && CGO_ENABLED=0 go build -o {{.OUT}}/looking-glass-client . + - CGO_ENABLED=0 go build -o {{.OUT}}/fabric-api ./cmd/fabric-api + - kind get kubeconfig --name dfw > {{.OUT}}/dfw.kubeconfig + - >- + LG_QUALIFY_KUBECONFIG={{.OUT}}/dfw.kubeconfig + LG_QUALIFY_CLIENT={{.OUT}}/looking-glass-client + LG_QUALIFY_FABRIC_API={{.OUT}}/fabric-api + go test -count=1 -v -timeout 120m -run TestQualify ./tests/lgqualify/ + "verify:fabric-api": desc: Verify the fabric-api looking glass on every fabric node, directly and through FabricQuery deps: ["verify:fabric-api-config"] diff --git a/deploy/containerlab/node_files/remote-host/startup.sh b/deploy/containerlab/node_files/remote-host/startup.sh index f28c5f3c6..2a3eeb295 100755 --- a/deploy/containerlab/node_files/remote-host/startup.sh +++ b/deploy/containerlab/node_files/remote-host/startup.sh @@ -25,6 +25,8 @@ ip link set eth1 up # on the link, the same TR-.1/node-.2 convention every worker uplink uses. ip -6 addr add 2001:db8:1:40::2/64 dev eth1 || true ip -4 addr add 11.1.40.2/24 dev eth1 || true +# The Looking Glass probe destination; see node_files/tr4/frr.conf. +ip -6 addr add 2c0f:40::2/64 dev eth1 || true # Everything this host talks to is somewhere behind tr4: the whole fabric, # every site's nodes, and every tenant's egress source address. A default diff --git a/deploy/containerlab/node_files/tr4/frr.conf b/deploy/containerlab/node_files/tr4/frr.conf index 11764b896..0451720e8 100644 --- a/deploy/containerlab/node_files/tr4/frr.conf +++ b/deploy/containerlab/node_files/tr4/frr.conf @@ -25,8 +25,15 @@ interface eth4 description remote-host-facing ip address 11.1.40.1/24 ipv6 address 2001:db8:1:40::1/64 + ipv6 address 2c0f:40::1/64 ! +! A probe sink for verify:looking-glass: global prefixes tr4 originates and +! drops without an ICMP error, so a probe toward them is lost and a +! traceroute ends in silent hops, keeping its routers busy for about 20 s. +ip route 11.1.42.0/24 blackhole +ipv6 route 2c0f:42::/64 blackhole + ipv6 prefix-list MIN48 seq 5 permit ::/0 le 47 route-map FILTER_PREFIX deny 10 match ipv6 address prefix-list MIN48 @@ -68,6 +75,14 @@ router bgp 65100 ! makes nginx on 2001:db8:1:40::2 reachable from a tenant pod's egress ! path through the NAT66 shard tier, and from every other site's nodes. network 2001:db8:1:40::/64 + ! The host's probe destination (2c0f:40::2), outside 2001:db8::/32: the + ! fabric-api public probe policy refuses documentation and other + ! special-use space, so only a global address can be pinged or traced + ! through the Looking Glass service (verify:looking-glass). Like + ! 11.1.40.0/24, it is never announced beyond the lab. + network 2c0f:40::/64 + ! The probe sink, dropped here (see the blackhole routes above). + network 2c0f:42::/64 exit-address-family ! ! One numbered session per address family -- see node_files/tr1/frr.conf. @@ -85,6 +100,7 @@ router bgp 65100 ! The off-fabric host's IPv4 subnet -- see the IPv6 network statement ! above. A /24 passes FILTER_PREFIX_V4 on every receiving transit router. network 11.1.40.0/24 + network 11.1.42.0/24 exit-address-family ! diff --git a/deploy/containerlab/resources/looking-glass/README.md b/deploy/containerlab/resources/looking-glass/README.md new file mode 100644 index 000000000..27e12c604 --- /dev/null +++ b/deploy/containerlab/resources/looking-glass/README.md @@ -0,0 +1,23 @@ +# Looking Glass service on dfw + +The overlays `deploy:looking-glass` applies to qualify the Looking Glass +service (`verify:looking-glass`), one per step of the enabling order in +[docs/fabric-api/operations.md](../../../../docs/fabric-api/operations.md#enabling-aggregate-probe-budgets). +Each builds on the step before it: + +| Step | Overlay | Applied from | Adds | +| ---- | ---------------------------- | ------------------- | -------------------------------------------------------- | +| 1 | `fabric-router/dfw/permits/` | `dfw-control-plane` | `components/fabric-api-permits`; restarts FRR | +| 2 | `gateway/dfw/budgets/` | host | `components/aggregate-budgets` | +| 3 | `fabric-router/dfw/require/` | `dfw-control-plane` | `components/fabric-api-require-permits`; restarts FRR | +| 4 | `gateway/dfw/service/` | host | `components/service`, switched off until the deploy ends | + +`pki/services.yaml` holds the service certificates the suite authenticates +with. `undeploy:looking-glass` switches the service off and walks the steps +back in reverse, ending on the lab's ordinary `fabric-router` and gateway +overlays. + +The `fabric-router` overlays are applied on `dfw-control-plane`, where +`deploy-fabric.sh` copies `resources/fabric-router/` and nests +`config/fabric-router/` under it as `fabric-router/base/fabric`; their +relative paths resolve there, not in this checkout. diff --git a/deploy/containerlab/resources/looking-glass/fabric-router/dfw/permits/kustomization.yaml b/deploy/containerlab/resources/looking-glass/fabric-router/dfw/permits/kustomization.yaml new file mode 100644 index 000000000..d187bed49 --- /dev/null +++ b/deploy/containerlab/resources/looking-glass/fabric-router/dfw/permits/kustomization.yaml @@ -0,0 +1,27 @@ +# Step 1: dfw's fabric-router sidecars verify probe permits. The permit +# Components set the sidecar's arguments whole, so the patch keeps the +# lab's own (resources/fabric-router/base/fabric-lab-patch.yaml) beside them. +resources: + - ../../../../fabric-router/dfw +components: + - ../../../../fabric-router/base/fabric/components/fabric-api-permits +patches: + - target: + kind: DaemonSet + name: fabric-router + patch: |- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: fabric-router + spec: + template: + spec: + containers: + - name: fabric-api + args: + - node + - --probe-permits + - --operator-allow-prefixes=10.255.255.0/24,fc00:0::/32 + - --enable-expensive-queries + - --log-level=debug diff --git a/deploy/containerlab/resources/looking-glass/fabric-router/dfw/require/kustomization.yaml b/deploy/containerlab/resources/looking-glass/fabric-router/dfw/require/kustomization.yaml new file mode 100644 index 000000000..32bda5dd7 --- /dev/null +++ b/deploy/containerlab/resources/looking-glass/fabric-router/dfw/require/kustomization.yaml @@ -0,0 +1,29 @@ +# Step 3: dfw's fabric-router sidecars refuse any probe without a permit, +# direct operator probes included, so the gateway's budgets charge every +# probe in the cell. The lab's own sidecar arguments are kept as in step 1. +resources: + - ../permits +components: + - ../../../../fabric-router/base/fabric/components/fabric-api-require-permits +patches: + - target: + kind: DaemonSet + name: fabric-router + patch: |- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: fabric-router + spec: + template: + spec: + containers: + - name: fabric-api + args: + - node + - --probe-permits + - --require-probe-permits + - --gateway-address=fabric-api-gateway.galactic-system.svc:9346 + - --operator-allow-prefixes=10.255.255.0/24,fc00:0::/32 + - --enable-expensive-queries + - --log-level=debug diff --git a/deploy/containerlab/resources/looking-glass/gateway/dfw/budgets/kustomization.yaml b/deploy/containerlab/resources/looking-glass/gateway/dfw/budgets/kustomization.yaml new file mode 100644 index 000000000..955241d0e --- /dev/null +++ b/deploy/containerlab/resources/looking-glass/gateway/dfw/budgets/kustomization.yaml @@ -0,0 +1,6 @@ +# Step 2: dfw's gateway charges every probe to the cell's aggregate budgets, +# at the contract's default limits. +resources: + - ../../../../fabric-api/gateway/dfw +components: + - ../../../../../../../config/fabric-api/components/aggregate-budgets diff --git a/deploy/containerlab/resources/looking-glass/gateway/dfw/service/kustomization.yaml b/deploy/containerlab/resources/looking-glass/gateway/dfw/service/kustomization.yaml new file mode 100644 index 000000000..57dee3ac6 --- /dev/null +++ b/deploy/containerlab/resources/looking-glass/gateway/dfw/service/kustomization.yaml @@ -0,0 +1,7 @@ +# Step 4: dfw's gateway serves the Looking Glass service to the +# looking-glass identity. The switch (ConfigMap fabric-api-service) is +# created off; deploy-looking-glass.sh turns it on last. +resources: + - ../budgets +components: + - ../../../../../../../config/fabric-api/components/service diff --git a/deploy/containerlab/resources/looking-glass/pki/services.yaml b/deploy/containerlab/resources/looking-glass/pki/services.yaml new file mode 100644 index 000000000..ec9ca39ea --- /dev/null +++ b/deploy/containerlab/resources/looking-glass/pki/services.yaml @@ -0,0 +1,87 @@ +# Looking Glass service certificates for verify:looking-glass, issued by the +# lab's fabric-api Issuer (resources/fabric-api/pki). CELL_NAME is replaced +# with the site name by deploy-looking-glass.sh. +# +# - looking-glass: the service identity the gateway allows +# (config/fabric-api/components/service). +# - looking-glass-unlisted: a valid service identity of this cell that the +# gateway does not list. +# - looking-glass-other-cell: the allowed service name in another cell, +# signed by this cell's CA. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: lg-qualify-looking-glass + namespace: galactic-system + labels: + app.kubernetes.io/part-of: lg-qualify +spec: + secretName: lg-qualify-looking-glass + secretTemplate: + labels: + app.kubernetes.io/part-of: lg-qualify + duration: 24h + renewBefore: 8h + uris: + - spiffe://fabric-api.datumapis.com/cell/CELL_NAME/service/looking-glass + usages: + - digital signature + - client auth + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: fabric-api + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: lg-qualify-unlisted + namespace: galactic-system + labels: + app.kubernetes.io/part-of: lg-qualify +spec: + secretName: lg-qualify-unlisted + secretTemplate: + labels: + app.kubernetes.io/part-of: lg-qualify + duration: 24h + renewBefore: 8h + uris: + - spiffe://fabric-api.datumapis.com/cell/CELL_NAME/service/looking-glass-unlisted + usages: + - digital signature + - client auth + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: fabric-api + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: lg-qualify-other-cell + namespace: galactic-system + labels: + app.kubernetes.io/part-of: lg-qualify +spec: + secretName: lg-qualify-other-cell + secretTemplate: + labels: + app.kubernetes.io/part-of: lg-qualify + duration: 24h + renewBefore: 8h + uris: + - spiffe://fabric-api.datumapis.com/cell/not-CELL_NAME/service/looking-glass + usages: + - digital signature + - client auth + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: fabric-api + kind: Issuer diff --git a/deploy/containerlab/scripts/deploy-fabric.sh b/deploy/containerlab/scripts/deploy-fabric.sh index 48ea31d66..d191cc1a2 100755 --- a/deploy/containerlab/scripts/deploy-fabric.sh +++ b/deploy/containerlab/scripts/deploy-fabric.sh @@ -15,28 +15,11 @@ source "${SCRIPT_DIR}/lib.sh" # ConfigMap via NODE_NAME and each site's kustomization generates one such # ConfigMap per matching node. Re-running this after editing one node's # frr.conf changes only that node's ConfigMap, which its config-agent -# sidecar reloads in place -- no pod restarts. Copied onto the node at deploy time nested under -# the overlay's own root so its "fabric" resource reference resolves -# (kustomize requires resources in or below the overlay root). -FABRIC_DIR=$(cd "${SCRIPT_DIR}/../../../config/fabric-router" && pwd) - -# copy_fabric_config NODE copies config/fabric-router/ onto NODE, nested under -# resources/fabric-router/base/ so the base overlay's "fabric" resource -# reference resolves. rm -rf first: like deploy-galactic-router.sh's -# copy_router_config, docker cp nests SRC inside an already-existing DEST -# dir instead of overwriting it, so a rerun against an already-provisioned -# node would silently keep serving the prior copy from underneath the new -# one -- kubectl would then report the DaemonSet "unchanged" even after a -# real manifest edit (found live: a fabric-lab-patch.yaml affinity fix -# never took effect on a redeploy until this guard was added). -copy_fabric_config() { - local node="$1" - docker exec "${node}" rm -rf /galactic/resources/fabric-router/base/fabric - docker cp "${FABRIC_DIR}" "${node}:/galactic/resources/fabric-router/base/fabric" -} - -# copy_to (lib.sh) replaces an already-provisioned node's copy, and -# copy_fabric_config then fills in base/fabric -- see its comment. +# sidecar reloads in place -- no pod restarts. copy_to (lib.sh) copies +# resources/fabric-router/ onto the control-plane node, replacing an +# already-provisioned copy, and copy_fabric_config (lib.sh) nests +# config/fabric-router/ under it as base/fabric, inside the overlay's own +# root, since kustomize requires resources in or below that root. for site in dfw iad sjc; do node=$(control_plane "${site}") copy_to "${node}" fabric-router diff --git a/deploy/containerlab/scripts/deploy-looking-glass.sh b/deploy/containerlab/scripts/deploy-looking-glass.sh new file mode 100755 index 000000000..bed8fdf5d --- /dev/null +++ b/deploy/containerlab/scripts/deploy-looking-glass.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# deploy-looking-glass.sh — Enable the Looking Glass service on dfw for +# verify:looking-glass, in the enabling order of +# docs/fabric-api/operations.md#enabling-aggregate-probe-budgets (resources/looking-glass/): +# +# 1. fabric-router sidecars verify probe permits (restarts FRR) +# 2. the gateway enforces the cell's aggregate budgets +# 3. fabric-router sidecars require permits (restarts FRR) +# 4. the gateway serves the service, then the switch turns it on +# +# and issues the service certificates the suite authenticates with +# (resources/looking-glass/pki/services.yaml). Needs deploy:fabric-api. +# undeploy-looking-glass.sh reverses it. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +site=dfw +node=$(control_plane "${site}") +kc=$(site_kubeconfig "${site}") +trap 'rm -f "${kc}"' EXIT +k() { kubectl --kubeconfig "${kc}" "$@"; } + +if ! k -n galactic-system get deployment fabric-api-gateway >/dev/null; then + echo "${site}: no fabric-api gateway; run deploy:fabric-api first" >&2 + exit 1 +fi + +# fabric_router STEP applies resources/looking-glass/fabric-router/dfw/STEP +# on the control-plane node and waits for every fabric-router pod and its +# underlay sessions. +fabric_router() { + copy_to "${node}" fabric-router + copy_fabric_config "${node}" + copy_to "${node}" looking-glass + apply_k "${node}" "/galactic/resources/looking-glass/fabric-router/${site}/$1/" + k -n galactic-system rollout status daemonset/fabric-router --timeout=300s +} + +gateway() { + k apply -k "${RESOURCES_DIR}/looking-glass/gateway/${site}/$1/" + k -n galactic-system rollout status deployment/fabric-api-gateway --timeout=180s +} + +echo "${site}: 1/4 nodes verify probe permits" +fabric_router permits +echo "${site}: 2/4 gateway enforces aggregate budgets" +gateway budgets +echo "${site}: 3/4 nodes require probe permits" +fabric_router require +echo "${site}: 4/4 gateway serves the Looking Glass service" +gateway service + +sed "s/CELL_NAME/${site}/g" "${RESOURCES_DIR}/looking-glass/pki/services.yaml" | k apply -f - +k -n galactic-system wait --for condition=Ready --timeout=120s \ + certificate/lg-qualify-looking-glass certificate/lg-qualify-unlisted certificate/lg-qualify-other-cell + +k -n galactic-system patch configmap fabric-api-service --type merge -p '{"data":{"service":"enabled"}}' +echo "Done. The new gateway leader admits no probe for its 125-second recovery hold." diff --git a/deploy/containerlab/scripts/lib.sh b/deploy/containerlab/scripts/lib.sh index 3b980fc0d..6faf912f1 100755 --- a/deploy/containerlab/scripts/lib.sh +++ b/deploy/containerlab/scripts/lib.sh @@ -48,6 +48,21 @@ copy_config() { docker cp "${CONFIG_DIR}" "${node}:/galactic/config" } +# copy_fabric_config NODE copies config/fabric-router/ onto NODE, nested under +# resources/fabric-router/base/ so the base overlay's "fabric" resource +# reference resolves. rm -rf first: like deploy-galactic-router.sh's +# copy_router_config, docker cp nests SRC inside an already-existing DEST +# dir instead of overwriting it, so a rerun against an already-provisioned +# node would silently keep serving the prior copy from underneath the new +# one -- kubectl would then report the DaemonSet "unchanged" even after a +# real manifest edit (found live: a fabric-lab-patch.yaml affinity fix +# never took effect on a redeploy until this guard was added). +copy_fabric_config() { + local node="$1" + docker exec "${node}" rm -rf /galactic/resources/fabric-router/base/fabric + docker cp "${CONFIG_DIR}/fabric-router" "${node}:/galactic/resources/fabric-router/base/fabric" +} + apply_k() { local node="$1" path="$2" docker exec "${node}" kubectl apply -k "${path}" diff --git a/deploy/containerlab/scripts/undeploy-looking-glass.sh b/deploy/containerlab/scripts/undeploy-looking-glass.sh new file mode 100755 index 000000000..0891f0bf8 --- /dev/null +++ b/deploy/containerlab/scripts/undeploy-looking-glass.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# undeploy-looking-glass.sh — Switch the Looking Glass service off on dfw +# and roll deploy-looking-glass.sh back in the documented reverse order +# (docs/fabric-api/operations.md#upgrade-and-rollback): +# +# 1. switch the service off +# 2. remove components/service (its ConfigMap and Role go with it) +# 3. nodes stop requiring permits (restarts FRR) +# 4. remove the aggregate budgets +# 5. nodes stop verifying permits (restarts FRR) +# +# and deletes the suite's service certificates. It ends on the lab's +# ordinary fabric-router and gateway overlays. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +site=dfw +node=$(control_plane "${site}") +kc=$(site_kubeconfig "${site}") +trap 'rm -f "${kc}"' EXIT +k() { kubectl --kubeconfig "${kc}" "$@"; } + +echo "${site}: 1/5 switch the service off" +k -n galactic-system patch configmap fabric-api-service --type merge \ + -p '{"data":{"service":"disabled"}}' 2>/dev/null || true + +echo "${site}: 2/5 remove the service" +k apply -k "${RESOURCES_DIR}/looking-glass/gateway/${site}/budgets/" +k -n galactic-system delete configmap/fabric-api-service \ + role/fabric-api-gateway-service rolebinding/fabric-api-gateway-service --ignore-not-found +k -n galactic-system rollout status deployment/fabric-api-gateway --timeout=180s + +copy_to "${node}" fabric-router +copy_fabric_config "${node}" +copy_to "${node}" looking-glass +echo "${site}: 3/5 nodes stop requiring permits" +apply_k "${node}" "/galactic/resources/looking-glass/fabric-router/${site}/permits/" +k -n galactic-system rollout status daemonset/fabric-router --timeout=300s + +echo "${site}: 4/5 remove the aggregate budgets" +k apply -k "${RESOURCES_DIR}/fabric-api/gateway/${site}/" +k -n galactic-system rollout status deployment/fabric-api-gateway --timeout=180s + +echo "${site}: 5/5 nodes stop verifying permits" +apply_k "${node}" "/galactic/resources/fabric-router/${site}/" +k -n galactic-system delete role/fabric-api-node-lease rolebinding/fabric-api-node-lease --ignore-not-found +k -n galactic-system rollout status daemonset/fabric-router --timeout=300s + +k -n galactic-system delete certificate,secret -l app.kubernetes.io/part-of=lg-qualify --ignore-not-found +echo "Done." diff --git a/docs/archive/README.md b/docs/archive/README.md index b9acb8a75..a012f93e9 100644 --- a/docs/archive/README.md +++ b/docs/archive/README.md @@ -13,5 +13,6 @@ Use the [current egress rollout checklist](../nat/active-active-rollout.md). In ## Measurements - [fabric-api full-table measurements, October 7](benchmarks/fabric-api-2026-10-07.md). +- [Looking Glass qualification on the lab, October 27](benchmarks/looking-glass-qualification-2026-10-27.md). -Use the [maintained load-test procedure](../fabric-api/load-test.md) to qualify a new release. Record its revision, environment and result instead of replacing the older evidence. +Use the [maintained load-test procedure](../fabric-api/load-test.md) and the [Looking Glass qualification procedure](../fabric-api/qualification.md) to qualify a new release. Record its revision, environment and result instead of replacing the older evidence. diff --git a/docs/archive/benchmarks/looking-glass-qualification-2026-10-27.md b/docs/archive/benchmarks/looking-glass-qualification-2026-10-27.md new file mode 100644 index 000000000..be0f589bc --- /dev/null +++ b/docs/archive/benchmarks/looking-glass-qualification-2026-10-27.md @@ -0,0 +1,56 @@ +# Looking Glass qualification, 2026-10-27 + +One run of the [qualification procedure](../../fabric-api/qualification.md) on the containerlab lab. It qualifies the code at the commit below on the lab's routers. It is not qualification of a production or staging cell, and it is not a current capacity claim. + +## Run + +| Item | Value | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Code under test | Commit `980cc91c` of the branch that added the suite, before its final rebase. The fabric-api image was built from a clean tree at it. The published commits differ from it only in documentation, one gateway unit test, the lab task's output directory and a comment in the lab's fabric deploy script | +| Window | 2026-10-27, 17:59 to 18:23 UTC (1427 s) | +| Cell | dfw: three fabric routers running FRR 10.7.1, two gateway replicas | +| Limits | The contract's defaults, as the gateway reported them: cell requests 2/s burst 8, project 0.2/s burst 2, destination host 20 packets/s burst 640, 8 node slots per cell and 2 per project, 10 s queue wait | +| Result | 21 of 21 scenarios passed, none skipped | + +After the run, `undeploy:looking-glass` rolled the cell back and `verify:fabric-api` checked the existing interfaces on every site. + +## Results by acceptance row + +| Row | Scenarios | Measured | +| --- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A1 | `authorization` | Only the listed same-cell service identity was served, on new and reused connections and after its certificate rotated | +| A2 | `authorization` | Forged forwarding metadata was ignored, and the request was attributed to the authenticated service | +| A3 | `diagnostics`, `destinations` | All four operations in both families decoded in the independent client. Every special-use and malformed target was refused before the snapshot | +| A4 | `coverage` | Reachable routers' outcomes arrived within 21 ms. The unreachable router's failure arrived at 10.5 s | +| A5 | `coverage` | Duplicate, unknown, omitted and explicitly unselected routers were counted exactly. Selecting 33 routers was refused | +| A6 | `diagnostics` | Loss, silent hops, no route and a queue timeout after the snapshot stayed distinguishable from backend failures | +| A7 | `compatibility` | An unsupported API major, query enum and required feature were each refused, never downgraded | +| A8 | `bursts`, `fairness` | 8 of 12 concurrent projects and 2 of 5 same-project requests were admitted. The slow project was never refused while the fast one was refused 47 times in 60 | +| A9 | `destination-budget` | 193 packets were emitted to one host. The worst window was 1280 packets under the `640 + 20(T + 32)` bound | +| A10 | `destination-budget` | Once the service drained the host's bucket, operator and FabricQuery traceroutes to it were throttled by the same bucket | +| A11 | `leader-crash`, `api-isolation` | The new leader first admitted 132.3 s after acquiring, inside the 125 to 134 s window, and sent no probe during the hold | +| A12 | `cancellation`, `deadline`, `service-switch`, `blackhole` | Packets stopped before the trigger, or within 0.24 s of the switch and 4.2 s of the blackhole. Bounds 2 s, 3 s and 12 s | +| A13 | `resources` | Peak RSS was 57 MiB of 512 MiB for the gateway and 49 MiB of 768 MiB for a sidecar. Goroutines peaked 56 above baseline | +| A14 | `leader-pause`, `api-isolation` | A paused leader sent no packet after resuming. An isolated leader detected the loss in 1.9 s and its running work ended `OwnershipLost` | +| A15 | `service-switch`, `confidentiality` | No target or project appeared in a routine log line or a metric label | +| A16 | `legacy`, rollback | Operator and FabricQuery queries worked with the service on and after rollback. A direct probe worked 9.1 s after its node's certificate pod was replaced | +| A17 | `preflight`, `routing-impact`, `resources`, `cleanup` | Loaded p95 convergence was 3307 ms against a control p95 of 3309 ms (ratio 1.00, control half-width 2.4 ms), with no other session reset. No fault outlived the run | + +The per-scenario JSON and raw request records were kept outside the repository with the run. + +## Defects found + +An earlier run against the previous build found four defects in the service and one in the suite's cleanup. Each was fixed before this run: + +- The gateway admitted and charged probes to forbidden destinations, leaving the nodes to refuse them. +- A refused request's log line quoted the caller's probe target. +- With permits required, node certificates lacked client auth, so every direct operator probe failed. +- A node kept requesting permits under a replaced certificate pod's identity. +- The suite's API-isolation fault was left in place when the isolated gateway restarted, and that replica could not reach the API server afterwards. + +## Not established by this run + +- The target platform. Staging and production cells were not run. +- Direct operator probes from edge nodes. The lab's edge nodes cannot reach the gateway Service, so only one node's direct probes were judged. +- An inventory over 256 routers, CA rotation with an overlapping bundle, and cancellation timed during discovery. Unit tests cover these. +- Emission at its limit. With three routers, node slots kept emission 1280 packets below the bound. diff --git a/docs/fabric-api/README.md b/docs/fabric-api/README.md index b062961a3..c0fcb11ae 100644 --- a/docs/fabric-api/README.md +++ b/docs/fabric-api/README.md @@ -9,6 +9,7 @@ Use fabric-api to query FRR state and run bounded ICMP diagnostics from fabric r - [Operations](operations.md): observability, rollout, verification and troubleshooting. - [FabricQuery API](api.md): query and result semantics, conditions and errors. - [Load-test procedure](load-test.md): repeat the full-table diagnostic qualification. +- [Looking Glass qualification](qualification.md): run the service's acceptance suite on real fabric routers. - [Metric reference](../observability/fabric-api.md): exported series and alerts. The [Looking Glass service contract](../designs/looking-glass-service.md) specifies proposed service behavior. It does not announce a customer API or qualified aggregate limits. diff --git a/docs/fabric-api/architecture.md b/docs/fabric-api/architecture.md index 92abada9f..d0eaca8ed 100644 --- a/docs/fabric-api/architecture.md +++ b/docs/fabric-api/architecture.md @@ -87,4 +87,4 @@ The node runs with the socket group and only the raw-socket capability it needs. | BMP station and indexed searches | `internal/fabric/index` | Stream, synchronization and search tests; full-table load harness | | Schema | `internal/fabric/api/v1alpha1` | Envtest admission tests, requiring KUBEBUILDER_ASSETS | -The lab's `verify:fabric-api` task exercises node and gateway queries and FabricQuery completion. `verify:fabric-api-bootstrap` deliberately breaks certificate issuance to test the routing-isolation boundary. These require the lab; a unit-test run alone does not establish those outcomes. [Operational verification](operations.md#verifying) lists the commands and expected observations. +The lab's `verify:fabric-api` task exercises node and gateway queries and FabricQuery completion. `verify:looking-glass` runs the Looking Glass service's [qualification suite](qualification.md) (`tests/lgqualify`) through the independent client in `examples/looking-glass-client`. `verify:fabric-api-bootstrap` deliberately breaks certificate issuance to test the routing-isolation boundary. These require the lab; a unit-test run alone does not establish those outcomes. [Operational verification](operations.md#verifying) lists the commands and expected observations. diff --git a/docs/fabric-api/operations.md b/docs/fabric-api/operations.md index f787be4b3..244826000 100644 --- a/docs/fabric-api/operations.md +++ b/docs/fabric-api/operations.md @@ -274,6 +274,9 @@ cd deploy/containerlab && task verify:fabric-api cd deploy/containerlab && task verify:fabric-api-bootstrap ``` +Before switching the Looking Glass service on in a cell, qualify the build +with the [qualification suite](qualification.md). + In a real cell, read the same facts from the cluster: ```sh diff --git a/docs/fabric-api/qualification.md b/docs/fabric-api/qualification.md new file mode 100644 index 000000000..1dca91c7b --- /dev/null +++ b/docs/fabric-api/qualification.md @@ -0,0 +1,106 @@ +# Qualify the Looking Glass service + +The qualification suite (`tests/lgqualify`) runs the Looking Glass service on +real fabric routers through an independent client and checks every row of the +service contract's [acceptance matrix](../designs/looking-glass-service.md#acceptance-matrix-and-evidence) +against thresholds fixed before the run. Run it against a candidate build +before the service is switched on in a cell. Record each run's revision, +environment and results in the [archive](../archive/README.md#measurements), +next to earlier runs rather than over them. + +A pass on the containerlab lab qualifies the code on the lab's routers. It is +not qualification of a production cell, whose routers, kernels, address plan +and load differ (see [Limits of a lab run](#limits-of-a-lab-run)). + +## What it exercises + +The client in `examples/looking-glass-client` is its own Go module and imports +only the published service API (`api/fabric/service/v1`, `api/fabric/v1`); +`task test:looking-glass-client` fails if it imports anything else. Every +service request in the suite goes through it, authenticated with a service +certificate, against the cell gateway's Service. Operator and `FabricQuery` +traffic goes through `fabric-api query` and the Kubernetes API, as those +callers use them. + +| Scenario | Rows | Checks | +| -------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `preflight` | A17 | Records the gateway's build, contract revision, limits and capabilities, every router, and the client's build | +| `authorization` | A1, A2 | Missing, expired, unlisted, other-cell and operator credentials; a request naming another cell; every non-service method closed to the service identity; forged forwarding metadata ignored | +| `compatibility` | A7 | Unsupported API major, feature and query enum are refused, never downgraded | +| `diagnostics` | A3, A6 | `RouteLookup`, `BGPSummary`, `Ping` and `Traceroute` in both families decode; loss, silent hops and no-route results stay observations | +| `coverage` | A4, A5 | A fast router's outcome arrives while another is blocked; unreachable and omitted routers; explicit subsets keep `NotRequested` omissions | +| `destinations` | A3 | Special-use, operator-allowed and malformed targets are refused with `INVALID_ARGUMENT` before the snapshot | +| `legacy` | A16 | Operator gateway queries, direct operator probes (which now need a permit) and `FabricQuery` still work; a direct probe works again within 30 s of its node's `fabric-api-certs` pod being replaced | +| `bursts` | A8 | Cell and project bursts admit no more than their bursts allow; refusals are `Throttled` with scope and retry delay | +| `fairness` | A8 | A slow project is never refused while a fast one saturates its own rate; project slot caps hold | +| `destination-budget` | A9, A10 | Emitted packets per destination host, counted on every node, stay within `B + r(T + 32)` in every window; operator and `FabricQuery` probes are throttled by the same buckets | +| `cancellation` | A12 | Probe packets stop within 2 s of a client cancel | +| `deadline` | A12 | A request ends `DeadlineExceeded` and its packets stop within 2 s of `expires_at` | +| `service-switch` | A12, A15 | Switching the service off ends running streams `Cancelled` and stops packets within 3 s | +| `blackhole` | A12 | A client whose connection silently stops passing packets releases its streams within 12 s | +| `routing-impact` | A17 | 40 control and 40 loaded convergence trials of one hard-reset BGP session; no other session resets under load | +| `leader-crash` | A11, A14 | After the leader is killed, the new leader admits nothing for 125 s and resumes within 134 s; the old leader's work emits nothing after its dispatch plus 32 s | +| `leader-pause` | A14 | The same bounds when the leader is paused (SIGSTOP) and later resumed | +| `api-isolation` | A11, A14 | A leader cut off from the API server stops dispatch within 2 s of detecting it | +| `resources` | A13, A17 | Peak gateway and sidecar memory and goroutines stay within their container limits and 512 goroutines above baseline | +| `confidentiality` | A15, A17 | No service target or project in routine gateway or sidecar log lines, nor in any metric label | +| `cleanup` | A17 | No injected `DROP` rule remains on a node or in a gateway pod, and every gateway and `fabric-router` pod is Ready within 6 minutes | + +The thresholds are constants in `tests/lgqualify/thresholds_test.go`, taken +from the contract's [defaults and bounds](../designs/looking-glass-service.md#initial-defaults-and-hard-bounds). +A failure against one is a qualification failure. Change a threshold only with +a contract revision, never to make a run pass. The routing-impact gate passes +when the loaded p95 convergence time is at most 1.10 times the control p95. It +is inconclusive, which is not a pass, when the control p95's bootstrap 95 % +confidence half-width exceeds 10 % of it. + +## Running it on the lab + +Prerequisites: the containerlab lab deployed with `task deploy` (which includes +`deploy:cert-manager` and `deploy:fabric-api`), and an unchanged working tree, +since the image records its commit as the gateway's contract revision. + +From `deploy/containerlab/`: + +```sh +# Builds fabric-api from the checkout, loads it on dfw's nodes, and enables the +# service on dfw in the documented order. Restarts dfw's FRR twice. +task deploy:looking-glass + +# About 45 minutes. Kills, pauses and isolates the gateway leader, and resets +# one of dfw-worker's BGP sessions 80 times. +task verify:looking-glass + +# Switches the service off and walks the enabling steps back. Restarts dfw's +# FRR twice. Then check the existing interfaces after the rollback. +task undeploy:looking-glass +task verify:fabric-api +``` + +`verify:looking-glass` writes one JSON file per scenario, with its rows, +thresholds, observations and outcome, and the raw per-request records, to +`bin/lg-qualify//` at the repository root. The test log names +the directory. The run ends with `cleanup`; if it is interrupted before then, +run `task undeploy:looking-glass` and check that no gateway pod keeps a `DROP` +rule for ports 443 or 6443 in its network namespace. + +The overlays `deploy:looking-glass` applies are in +`deploy/containerlab/resources/looking-glass/`. tr4 announces `2c0f:40::/64` +for the off-fabric host and drops `11.1.42.0/24` and `2c0f:42::/64` silently, +since the public probe policy refuses documentation addresses; see +[the lab README](../../deploy/containerlab/README.md). + +## Limits of a lab run + +- Direct operator probes are judged on dfw-worker only. The lab's edge nodes' + host network reaches neither the cluster DNS server nor the gateway Service, + so their sidecars cannot ask the gateway for permits. A production edge node + must reach the gateway Service before `components/fabric-api-require-permits` + is applied. +- The lab's routers carry a few hundred routes, not full tables, and every node + shares one host's CPU. The [full-table load test](load-test.md) covers FRR + under real table sizes. +- With the default limits, a three-router traceroute toward a silent + destination from one project times out one router as `QueueTimeout` after + 10 s, since a project holds at most two node slots. The contract requires + qualification to report this default. diff --git a/examples/looking-glass-client/README.md b/examples/looking-glass-client/README.md new file mode 100644 index 000000000..37aa4cede --- /dev/null +++ b/examples/looking-glass-client/README.md @@ -0,0 +1,107 @@ +# Looking Glass service client + +An independent client of the Looking Glass service, the API a cell's +fabric-api gateway serves to an authorized customer-facing service +([docs/designs/looking-glass-service.md](../../docs/designs/looking-glass-service.md)). +It is built from the published service API alone: its module imports only +`go.datum.net/galactic/api/fabric/service/v1` and `api/fabric/v1`, and its +module path lies outside `go.datum.net/galactic`, so the compiler refuses any +import of Galactic's internal packages. `task test:looking-glass-client` +builds and tests it and fails if it imports any other Galactic package. + +The Looking Glass qualification (`task verify:looking-glass` in +`deploy/containerlab`, [docs/fabric-api/qualification.md](../../docs/fabric-api/qualification.md)) +sends every request through this client. + +## Identity + +The client authenticates with a service certificate whose URI SAN is +`spiffe://fabric-api.datumapis.com/cell//service/`, issued by the +cell's fabric-api CA, for client auth. The gateway accepts it only when +`` is in its `--service-identities`. The client accepts only a server +whose single URI SAN is `spiffe://fabric-api.datumapis.com/cell//gateway`, +verified against `--ca`. + +## Build + +```sh +cd examples/looking-glass-client +CGO_ENABLED=0 go build -o looking-glass-client . +``` + +The module builds against the service API of the checkout it sits in +(`replace go.datum.net/galactic => ../..`). To pin a released API instead, +drop the `replace` and require the release's `go.datum.net/galactic` version. + +## Use + +Every subcommand takes `--addr` (the gateway, `host:port`), `--cell`, and the +service's `--cert`, `--key` and `--ca` PEM files. `--cert ""` presents no +client certificate. + +| Subcommand | Does | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `capabilities` | Calls `GetCapabilities` and prints the response, or the refusal, as JSON. | +| `run` | Runs one diagnostic and prints its record (below). `--events` also prints every event, payloads included, to stderr. | +| `load` | Runs a workload spec over one connection; prints one record per attempt, then one summary per group. SIGINT ends it and keeps results. | +| `invoke` | Calls any unary method with an empty request and prints its status, to show a method is closed to the service. | + +```sh +looking-glass-client run --addr gateway:9346 --cell dfw --cert tls.crt --key tls.key --ca ca.crt \ + -project 7f1c0e2a -type Ping -target 2c0f:40::2 # the containerlab lab's probe destination +``` + +`run` takes the request's fields (`-project`, `-request-id`, `-type`, +`-target`, `-family`, `-nodes`, `-require`, `-api-major`, `-expires-in`, +`-request-cell`), a transport `-deadline`, and consumer behaviour: +`-read-delay` before each read, `-stop-reading-after N` events, and +`-cancel-after` a duration or `-cancel-after-events N`. `-metadata key=value` +adds request metadata. A `-type` of `enum:N` sends the raw query type N. + +gRPC retries are off: the service must never replay an attempt. + +## Records + +A record holds what the consumer observed, never a diagnostic payload: + +- `status`: the gRPC code, and from `google.rpc.ErrorInfo` the reason and + metadata (`scope`, `execution_started`), and from `RetryInfo` the advised + delay in milliseconds (`-1` when absent). +- `stream`: the stream checked against the contract's grammar and coverage + rules. `delivered` is true only when a `complete` arrived, every rule held + and the stream ended with gRPC OK; `violations` lists every broken rule; + `pending` lists selected routers with no delivered outcome, whose result is + unknown. Each outcome keeps its node, result, failure reason and scope. +- `timing`: milliseconds from the call's start to the first event, the + snapshot, each outcome, the `complete`, a client cancellation, and the end. +- `bytes` and `maxEventBytes`: the serialized size of the stream and its + largest event. + +## Workload spec + +`load --spec FILE` (`-` reads stdin) takes JSON: + +```json +{ + "groups": [ + { + "name": "burst", + "count": 12, + "concurrency": 12, + "interval": "0s", + "startAfter": "0s", + "projects": ["p1", "p2", "p3"], + "targets": ["2c0f:40::2"], + "options": { "query": { "type": "Ping" }, "nodes": ["router-a"], "expiresIn": "30s" } + } + ] +} +``` + +Groups start together. A group runs `count` attempts, at most `concurrency` +at once, starting them `interval` apart after `startAfter`; attempts take +`projects` and `targets` in turn. `options` takes the `run` fields by their +JSON names (`query`, `nodes`, `expiresIn`, `deadline`, `requiredFeatures`, +`readDelay`, `stopReadingAfter`, `cancelAfterEvents`, `cancelAfter`, +`metadata`). Each group's summary counts attempts by status and terminal +reason and gives latency percentiles to the first outcome and the `complete`. diff --git a/examples/looking-glass-client/go.mod b/examples/looking-glass-client/go.mod new file mode 100644 index 000000000..f53598643 --- /dev/null +++ b/examples/looking-glass-client/go.mod @@ -0,0 +1,22 @@ +module github.com/datum-cloud/galactic/examples/looking-glass-client + +go 1.26.4 + +// The client is built against the service API in this checkout. It imports +// only go.datum.net/galactic/api/fabric/service/v1 and api/fabric/v1: its +// module path is outside go.datum.net/galactic, so the compiler refuses any +// import of the core's internal packages. +replace go.datum.net/galactic => ../.. + +require ( + go.datum.net/galactic v0.0.0-00010101000000-000000000000 + google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b + google.golang.org/grpc v1.84.0 + google.golang.org/protobuf v1.36.12 +) + +require ( + golang.org/x/net v0.61.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect +) diff --git a/examples/looking-glass-client/go.sum b/examples/looking-glass-client/go.sum new file mode 100644 index 000000000..1628b348e --- /dev/null +++ b/examples/looking-glass-client/go.sum @@ -0,0 +1,18 @@ +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= +golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b h1:6WNWHZk91dp4gZ25642OALkuG1UVNZbwENJD19DnB8k= +google.golang.org/genproto/googleapis/rpc v0.0.0-20261019180522-ce908d29299b/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/examples/looking-glass-client/internal/conn/conn.go b/examples/looking-glass-client/internal/conn/conn.go new file mode 100644 index 000000000..fad835dc4 --- /dev/null +++ b/examples/looking-glass-client/internal/conn/conn.go @@ -0,0 +1,102 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package conn dials a cell's fabric-api gateway as a Looking Glass service: +// mutual TLS with the service's certificate, accepting only a server that +// proves the cell's gateway identity. +package conn + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// GatewayURI is the URI SAN a cell's gateway certificate carries. +func GatewayURI(cell string) string { + return "spiffe://fabric-api.datumapis.com/cell/" + cell + "/gateway" +} + +// Options name the gateway and the service's credentials, as PEM files. +type Options struct { + Addr string + Cell string + CertFile string + KeyFile string + CAFile string +} + +// TLSConfig loads the credentials and returns a client config that checks +// the server's chain against the CA bundle and requires its only URI SAN to +// be the cell's gateway. An empty CertFile presents no client certificate. +func TLSConfig(o Options) (*tls.Config, error) { + var certs []tls.Certificate + if o.CertFile != "" { + cert, err := tls.LoadX509KeyPair(o.CertFile, o.KeyFile) + if err != nil { + return nil, fmt.Errorf("load service certificate: %w", err) + } + certs = append(certs, cert) + } + pem, err := os.ReadFile(o.CAFile) + if err != nil { + return nil, fmt.Errorf("read CA bundle: %w", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(pem) { + return nil, errors.New("CA bundle holds no certificate") + } + want := GatewayURI(o.Cell) + return &tls.Config{ + MinVersion: tls.VersionTLS13, + Certificates: certs, + // The gateway's certificate names it by URI SAN only, so the + // standard verifier's DNS check cannot apply; the chain and the + // URI are checked below instead. + InsecureSkipVerify: true, //nolint:gosec // verified in VerifyPeerCertificate + VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error { + if len(raw) == 0 { + return errors.New("gateway presented no certificate") + } + certs := make([]*x509.Certificate, len(raw)) + for i, der := range raw { + if certs[i], err = x509.ParseCertificate(der); err != nil { + return err + } + } + inter := x509.NewCertPool() + for _, c := range certs[1:] { + inter.AddCert(c) + } + if _, err := certs[0].Verify(x509.VerifyOptions{ + Roots: roots, Intermediates: inter, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }); err != nil { + return err + } + if uris := certs[0].URIs; len(uris) != 1 || uris[0].String() != want { + return fmt.Errorf("gateway identity %v is not %s", uris, want) + } + return nil + }, + }, nil +} + +// Dial opens a connection to the gateway. gRPC retries are off: the +// service must never replay an attempt. +func Dial(o Options, extra ...grpc.DialOption) (*grpc.ClientConn, error) { + cfg, err := TLSConfig(o) + if err != nil { + return nil, err + } + opts := append([]grpc.DialOption{ + grpc.WithTransportCredentials(credentials.NewTLS(cfg)), + grpc.WithDisableRetry(), + }, extra...) + return grpc.NewClient(o.Addr, opts...) +} diff --git a/examples/looking-glass-client/internal/conn/conn_test.go b/examples/looking-glass-client/internal/conn/conn_test.go new file mode 100644 index 000000000..fba36cdfc --- /dev/null +++ b/examples/looking-glass-client/internal/conn/conn_test.go @@ -0,0 +1,129 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package conn + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + + servicev1 "go.datum.net/galactic/api/fabric/service/v1" +) + +type pki struct { + key *ecdsa.PrivateKey + cert *x509.Certificate +} + +func newCA(t *testing.T) pki { + t.Helper() + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test CA"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + cert, _ := x509.ParseCertificate(der) + return pki{key, cert} +} + +// issue returns a leaf for uri with usage, signed by ca. +func (ca pki) issue(t *testing.T, uri string, usage x509.ExtKeyUsage) tls.Certificate { + t.Helper() + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + u, _ := url.Parse(uri) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), URIs: []*url.URL{u}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{usage}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.cert, &key.PublicKey, ca.key) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + +func writePEM(t *testing.T, path, kind string, der []byte) { + t.Helper() + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: kind, Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } +} + +// serve runs a TLS gRPC server presenting server and returns its address. +func serve(t *testing.T, server tls.Certificate) string { + t.Helper() + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{server}, ClientAuth: tls.RequireAnyClientCert, + }))) + servicev1.RegisterLookingGlassServiceServer(srv, servicev1.UnimplementedLookingGlassServiceServer{}) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return lis.Addr().String() +} + +func TestGatewayIdentity(t *testing.T) { + ca, other := newCA(t), newCA(t) + dir := t.TempDir() + client := ca.issue(t, "spiffe://fabric-api.datumapis.com/cell/c1/service/looking-glass", x509.ExtKeyUsageClientAuth) + key, _ := x509.MarshalECPrivateKey(client.PrivateKey.(*ecdsa.PrivateKey)) + writePEM(t, filepath.Join(dir, "tls.crt"), "CERTIFICATE", client.Certificate[0]) + writePEM(t, filepath.Join(dir, "tls.key"), "EC PRIVATE KEY", key) + writePEM(t, filepath.Join(dir, "ca.crt"), "CERTIFICATE", ca.cert.Raw) + + for _, tc := range []struct { + name string + server tls.Certificate + ok bool + }{ + {"cell gateway", ca.issue(t, GatewayURI("c1"), x509.ExtKeyUsageServerAuth), true}, + {"other cell", ca.issue(t, GatewayURI("c2"), x509.ExtKeyUsageServerAuth), false}, + { + "node identity", + ca.issue(t, "spiffe://fabric-api.datumapis.com/cell/c1/node/ns/pod", x509.ExtKeyUsageServerAuth), false, + }, + {"client-only usage", ca.issue(t, GatewayURI("c1"), x509.ExtKeyUsageClientAuth), false}, + {"untrusted CA", other.issue(t, GatewayURI("c1"), x509.ExtKeyUsageServerAuth), false}, + } { + t.Run(tc.name, func(t *testing.T) { + cc, err := Dial(Options{Addr: serve(t, tc.server), Cell: "c1", CertFile: filepath.Join(dir, "tls.crt"), + KeyFile: filepath.Join(dir, "tls.key"), CAFile: filepath.Join(dir, "ca.crt")}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = cc.Close() }() + _, err = servicev1.NewLookingGlassServiceClient(cc).GetCapabilities(t.Context(), &servicev1.GetCapabilitiesRequest{}) + // A handshake that succeeded reaches the unimplemented server. + if got := status.Code(err) == codes.Unimplemented; got != tc.ok { + t.Errorf("reached the server = %t, want %t (%v)", got, tc.ok, err) + } + }) + } +} diff --git a/examples/looking-glass-client/internal/load/load.go b/examples/looking-glass-client/internal/load/load.go new file mode 100644 index 000000000..6260d1d2e --- /dev/null +++ b/examples/looking-glass-client/internal/load/load.go @@ -0,0 +1,190 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package load runs a workload of concurrent attempts over one connection +// and summarizes what each group of them observed. +package load + +import ( + "context" + "fmt" + "slices" + "sort" + "sync" + "time" + + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/run" + "google.golang.org/grpc" +) + +// Group is a set of attempts sharing one shape. +type Group struct { + Name string `json:"name"` + // Count attempts run, at most Concurrency at once. + Count int `json:"count"` + Concurrency int `json:"concurrency"` + // StartAfter delays the group's first attempt from the workload's + // start; Interval spaces its attempts' starts. + StartAfter run.Duration `json:"startAfter,omitempty"` + Interval run.Duration `json:"interval,omitempty"` + // Projects, if set, attributes attempts to them in turn; otherwise + // every attempt uses Options.Project. + Projects []string `json:"projects,omitempty"` + // Targets, if set, gives attempts their query target in turn. + Targets []string `json:"targets,omitempty"` + Options run.Options `json:"options"` +} + +// Spec is a workload against one cell. +type Spec struct { + Cell string `json:"cell"` + Groups []Group `json:"groups"` +} + +// Line is one attempt's record, with its group and its start offset from +// the workload's start. +type Line struct { + Group string `json:"group"` + StartMS float64 `json:"startMs"` + run.Record +} + +// Run runs spec on cc, handing each record to emit as it finishes. emit is +// called from one goroutine at a time. +func Run(ctx context.Context, cc grpc.ClientConnInterface, spec Spec, emit func(Line)) error { + for _, g := range spec.Groups { + if g.Count < 1 || g.Concurrency < 1 { + return fmt.Errorf("group %q: count and concurrency must be positive", g.Name) + } + if _, err := g.Options.Query.Proto(); err != nil { + return fmt.Errorf("group %q: %w", g.Name, err) + } + } + var mu sync.Mutex + var wg sync.WaitGroup + t0 := time.Now() + for _, g := range spec.Groups { + sem := make(chan struct{}, g.Concurrency) + for i := range g.Count { + o := g.Options + o.Cell, o.RequestID = spec.Cell, "" + o.Nodes = slices.Clone(o.Nodes) + if len(g.Projects) > 0 { + o.Project = g.Projects[i%len(g.Projects)] + } + if len(g.Targets) > 0 { + o.Query.Target = g.Targets[i%len(g.Targets)] + } + at := t0.Add(time.Duration(g.StartAfter) + time.Duration(i)*time.Duration(g.Interval)) + wg.Go(func() { + if !sleepUntil(ctx, at) { + return + } + select { + case sem <- struct{}{}: + case <-ctx.Done(): + return + } + defer func() { <-sem }() + start := time.Since(t0) + rec, err := run.Do(ctx, cc, o, nil) + if err != nil { + return + } + mu.Lock() + defer mu.Unlock() + emit(Line{Group: g.Name, StartMS: float64(start.Microseconds()) / 1000, Record: rec}) + }) + } + } + wg.Wait() + return ctx.Err() +} + +func sleepUntil(ctx context.Context, at time.Time) bool { + t := time.NewTimer(time.Until(at)) + defer t.Stop() + select { + case <-t.C: + return true + case <-ctx.Done(): + return false + } +} + +// Stats summarize one group's attempts. +type Stats struct { + Group string `json:"group"` + Attempts int `json:"attempts"` + // Statuses counts attempts by "/", and Completes by + // terminal reason. + Statuses map[string]int `json:"statuses"` + Completes map[string]int `json:"completes"` + Delivered int `json:"delivered"` + Violations int `json:"violations"` + // CompleteMS and FirstOutcomeMS are latency percentiles over the + // attempts that reached each point, measured from each attempt's start. + CompleteMS Percentiles `json:"completeMs"` + FirstOutcomeMS Percentiles `json:"firstOutcomeMs"` +} + +// Percentiles of a sample, in its unit; N is its size. +type Percentiles struct { + N int `json:"n"` + P50 float64 `json:"p50"` + P95 float64 `json:"p95"` + Max float64 `json:"max"` +} + +// Of returns the percentiles of v (nearest rank). +func Of(v []float64) Percentiles { + if len(v) == 0 { + return Percentiles{} + } + s := slices.Clone(v) + sort.Float64s(s) + rank := func(p float64) float64 { + i := int(p*float64(len(s))+0.999999) - 1 + return s[max(0, min(i, len(s)-1))] + } + return Percentiles{N: len(s), P50: rank(0.5), P95: rank(0.95), Max: s[len(s)-1]} +} + +// Summarize groups lines by group, in first-seen order. +func Summarize(lines []Line) []Stats { + var order []string + by := map[string]*Stats{} + complete := map[string][]float64{} + first := map[string][]float64{} + for _, l := range lines { + s, ok := by[l.Group] + if !ok { + s = &Stats{Group: l.Group, Statuses: map[string]int{}, Completes: map[string]int{}} + by[l.Group] = s + order = append(order, l.Group) + } + s.Attempts++ + s.Statuses[l.Status.Code+"/"+l.Status.Reason]++ + if l.Stream.Reason != "" { + s.Completes[l.Stream.Reason]++ + } + if l.Stream.Delivered { + s.Delivered++ + } + s.Violations += len(l.Stream.Violations) + if l.Timing.CompleteMS >= 0 { + complete[l.Group] = append(complete[l.Group], l.Timing.CompleteMS) + } + if len(l.Timing.OutcomeMS) > 0 { + first[l.Group] = append(first[l.Group], l.Timing.OutcomeMS[0]) + } + } + out := make([]Stats, 0, len(order)) + for _, g := range order { + s := by[g] + s.CompleteMS, s.FirstOutcomeMS = Of(complete[g]), Of(first[g]) + out = append(out, *s) + } + return out +} diff --git a/examples/looking-glass-client/internal/load/load_test.go b/examples/looking-glass-client/internal/load/load_test.go new file mode 100644 index 000000000..3ff87a128 --- /dev/null +++ b/examples/looking-glass-client/internal/load/load_test.go @@ -0,0 +1,47 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package load + +import ( + "testing" + + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/rpcerr" + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/run" + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/stream" +) + +func TestPercentiles(t *testing.T) { + v := make([]float64, 0, 100) + for i := 100; i >= 1; i-- { + v = append(v, float64(i)) + } + if p := Of(v); p != (Percentiles{N: 100, P50: 50, P95: 95, Max: 100}) { + t.Errorf("percentiles = %+v", p) + } + if p := Of(nil); p != (Percentiles{}) { + t.Errorf("empty sample = %+v", p) + } +} + +func TestSummarize(t *testing.T) { + line := func(group, code, reason, complete string, ms float64) Line { + r := run.Record{Status: rpcerr.Error{Code: code, Reason: reason}, Timing: run.Timing{CompleteMS: -1}} + if complete != "" { + r.Stream = stream.Result{Delivered: true, Reason: complete} + r.Timing.CompleteMS, r.Timing.OutcomeMS = ms, []float64{ms / 2} + } + return Line{Group: group, Record: r} + } + got := Summarize([]Line{ + line("a", "OK", "", "Succeeded", 10), + line("b", "ResourceExhausted", "Throttled", "", 0), + line("a", "OK", "", "PartialResults", 30), + }) + if len(got) != 2 || got[0].Group != "a" || got[0].Attempts != 2 || got[0].Delivered != 2 || + got[0].CompleteMS.Max != 30 || got[0].FirstOutcomeMS.P50 != 5 || + got[1].Statuses["ResourceExhausted/Throttled"] != 1 || got[1].CompleteMS.N != 0 { + t.Errorf("summary = %+v", got) + } +} diff --git a/examples/looking-glass-client/internal/rpcerr/rpcerr.go b/examples/looking-glass-client/internal/rpcerr/rpcerr.go new file mode 100644 index 000000000..c8285d64a --- /dev/null +++ b/examples/looking-glass-client/internal/rpcerr/rpcerr.go @@ -0,0 +1,71 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package rpcerr decodes the typed errors of the Looking Glass service: a +// gRPC status whose google.rpc.ErrorInfo (domain fabric-api.datumapis.com) +// names the reason, with google.rpc.RetryInfo on a refusal that advises a +// delay. +package rpcerr + +import ( + "context" + "errors" + + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Domain is the ErrorInfo domain of every fabric-api error. +const Domain = "fabric-api.datumapis.com" + +// Error is a decoded RPC outcome. The zero value with Code "OK" is success. +type Error struct { + Code string `json:"code"` + // Reason is the ErrorInfo reason; empty when the status carried none, + // as for a transport failure. + Reason string `json:"reason,omitempty"` + // Domain is set when it is not Domain, which marks an error from + // something other than the service. + Domain string `json:"domain,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + // RetryDelayMS is RetryInfo's delay in milliseconds, or -1 when the + // status carried no RetryInfo. + RetryDelayMS int64 `json:"retryDelayMs"` + // MessageBytes is the length of the status message, which the contract + // caps at 256 bytes; the message itself is not recorded. + MessageBytes int `json:"messageBytes,omitempty"` +} + +// Decode returns err's status, ErrorInfo and RetryInfo. A nil err is OK. +func Decode(err error) Error { + out := Error{Code: codes.OK.String(), RetryDelayMS: -1} + if err == nil { + return out + } + st, ok := status.FromError(err) + if !ok { + switch { + case errors.Is(err, context.Canceled): + st = status.New(codes.Canceled, err.Error()) + case errors.Is(err, context.DeadlineExceeded): + st = status.New(codes.DeadlineExceeded, err.Error()) + default: + st = status.New(codes.Unknown, err.Error()) + } + } + out.Code, out.MessageBytes = st.Code().String(), len(st.Message()) + for _, d := range st.Details() { + switch d := d.(type) { + case *errdetails.ErrorInfo: + out.Reason, out.Metadata = d.GetReason(), d.GetMetadata() + if d.GetDomain() != Domain { + out.Domain = d.GetDomain() + } + case *errdetails.RetryInfo: + out.RetryDelayMS = d.GetRetryDelay().AsDuration().Milliseconds() + } + } + return out +} diff --git a/examples/looking-glass-client/internal/run/run.go b/examples/looking-glass-client/internal/run/run.go new file mode 100644 index 000000000..c3eefb69d --- /dev/null +++ b/examples/looking-glass-client/internal/run/run.go @@ -0,0 +1,301 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package run runs one RunDiagnostic attempt and records what a consumer +// observed: the status, the stream's grammar and coverage, and when each +// event arrived. A record never holds diagnostic payloads. +package run + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/rpcerr" + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/stream" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + servicev1 "go.datum.net/galactic/api/fabric/service/v1" + fabricv1 "go.datum.net/galactic/api/fabric/v1" +) + +// Query is a diagnostic in the contract's terms. +type Query struct { + // Type is RouteLookup, BGPSummary, Ping or Traceroute; any other name + // is sent as the enum value it names, for negative tests. + Type string `json:"type"` + Target string `json:"target,omitempty"` + // Family is IPv4, IPv6 or empty to let the target imply it. + Family string `json:"family,omitempty"` +} + +var queryTypes = map[string]fabricv1.QueryType{ + "RouteLookup": fabricv1.QueryType_QUERY_TYPE_ROUTE_LOOKUP, + "BGPSummary": fabricv1.QueryType_QUERY_TYPE_BGP_SUMMARY, + "Ping": fabricv1.QueryType_QUERY_TYPE_PING, + "Traceroute": fabricv1.QueryType_QUERY_TYPE_TRACEROUTE, + "ASPath": fabricv1.QueryType_QUERY_TYPE_AS_PATH, + "Community": fabricv1.QueryType_QUERY_TYPE_COMMUNITY, + "LargeCommunity": fabricv1.QueryType_QUERY_TYPE_LARGE_COMMUNITY, +} + +var families = map[string]fabricv1.AddressFamily{ + "": fabricv1.AddressFamily_ADDRESS_FAMILY_UNSPECIFIED, + "IPv4": fabricv1.AddressFamily_ADDRESS_FAMILY_IPV4, + "IPv6": fabricv1.AddressFamily_ADDRESS_FAMILY_IPV6, +} + +// Proto returns q as the published Query message. A Type of "enum:N" +// sends the raw enum value N. +func (q Query) Proto() (*fabricv1.Query, error) { + t, ok := queryTypes[q.Type] + if raw, isRaw := strings.CutPrefix(q.Type, "enum:"); isRaw { + n, err := strconv.ParseInt(raw, 10, 32) + if err != nil { + return nil, fmt.Errorf("query type %q: %w", q.Type, err) + } + t, ok = fabricv1.QueryType(n), true + } + if !ok { + return nil, fmt.Errorf("unknown query type %q", q.Type) + } + f, ok := families[q.Family] + if !ok { + return nil, fmt.Errorf("unknown address family %q", q.Family) + } + return &fabricv1.Query{Type: t, Target: q.Target, AddressFamily: f}, nil +} + +// Options describe one attempt and how the consumer behaves. +type Options struct { + Cell string `json:"cell"` + Project string `json:"project"` + RequestID string `json:"requestId,omitempty"` + Query Query `json:"query"` + Nodes []string `json:"nodes,omitempty"` + // APIMajor defaults to 1. + APIMajor uint32 `json:"apiMajor,omitempty"` + // ExpiresIn sets expires_at from now; it defaults to 30 s. + ExpiresIn Duration `json:"expiresIn,omitempty"` + // Deadline is the transport deadline; it defaults to ExpiresIn. + Deadline Duration `json:"deadline,omitempty"` + RequiredFeatures []string `json:"requiredFeatures,omitempty"` + Budgets *fabricv1.Budgets `json:"-"` + // ReadDelay sleeps before each read after the first event, as a slow + // consumer would. + ReadDelay Duration `json:"readDelay,omitempty"` + // StopReadingAfter stops reading after that many events and holds the + // stream open, unread, until the deadline: a stuck consumer. + StopReadingAfter int `json:"stopReadingAfter,omitempty"` + // CancelAfterEvents cancels the call once that many events arrived; + // CancelAfter cancels it that long after the call started. + CancelAfterEvents int `json:"cancelAfterEvents,omitempty"` + CancelAfter Duration `json:"cancelAfter,omitempty"` + // Metadata is sent as gRPC request metadata, e.g. to try forging the + // gateway's internal forwarding headers. + Metadata map[string]string `json:"metadata,omitempty"` +} + +// Duration is a time.Duration that reads and writes as a Go duration +// string. +type Duration time.Duration + +// MarshalText implements encoding.TextMarshaler. +func (d Duration) MarshalText() ([]byte, error) { return []byte(time.Duration(d).String()), nil } + +// UnmarshalText implements encoding.TextUnmarshaler. +func (d *Duration) UnmarshalText(b []byte) error { + v, err := time.ParseDuration(string(b)) + *d = Duration(v) + return err +} + +// Timing holds offsets in milliseconds from the call's start; -1 means the +// moment never came. +type Timing struct { + FirstEventMS float64 `json:"firstEventMs"` + SnapshotMS float64 `json:"snapshotMs"` + CompleteMS float64 `json:"completeMs"` + EndMS float64 `json:"endMs"` + // CancelMS is when the client cancelled, and CancelToEndMS how long the + // stream took to end after it. + CancelMS float64 `json:"cancelMs"` + CancelToEndMS float64 `json:"cancelToEndMs"` + // OutcomeMS holds each delivered outcome's arrival, in stream order. + OutcomeMS []float64 `json:"outcomeMs,omitempty"` +} + +// Record is what the consumer observed of one attempt. +type Record struct { + RequestID string `json:"requestId"` + Project string `json:"project"` + Query Query `json:"query"` + Nodes []string `json:"nodes,omitempty"` + StartedAt time.Time `json:"startedAt"` + Status rpcerr.Error `json:"status"` + Stream stream.Result `json:"stream"` + Events int `json:"events"` + // Bytes is the serialized size of every event received, and + // MaxEventBytes that of the largest. + Bytes int `json:"bytes"` + MaxEventBytes int `json:"maxEventBytes"` + Timing Timing `json:"timing"` +} + +// NewRequestID returns a random lowercase UUID. +func NewRequestID() string { + var b [16]byte + _, _ = rand.Read(b[:]) + b[6] = b[6]&0x0f | 0x40 + b[8] = b[8]&0x3f | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} + +func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } + +// Request builds the request o describes, expiring ExpiresIn after now. +func Request(o *Options, now time.Time) (*servicev1.RunDiagnosticRequest, error) { + if o.RequestID == "" { + o.RequestID = NewRequestID() + } + if o.ExpiresIn == 0 { + o.ExpiresIn = Duration(30 * time.Second) + } + if o.APIMajor == 0 { + o.APIMajor = 1 + } + q, err := o.Query.Proto() + if err != nil { + return nil, err + } + return &servicev1.RunDiagnosticRequest{ + ApiMajor: o.APIMajor, Cell: o.Cell, ProjectId: o.Project, RequestId: o.RequestID, + ExpiresAt: timestamppb.New(now.Add(time.Duration(o.ExpiresIn))), + Query: q, Nodes: o.Nodes, Budgets: o.Budgets, RequiredFeatures: o.RequiredFeatures, + }, nil +} + +// Do runs one attempt on cc. onEvent, if set, sees each event as it +// arrives; it is the only way to reach payloads. +func Do( + ctx context.Context, cc grpc.ClientConnInterface, o Options, onEvent func(*servicev1.DiagnosticEvent), +) (Record, error) { + start := time.Now() + req, err := Request(&o, start) + if err != nil { + return Record{}, err + } + rec := Record{ + RequestID: o.RequestID, Project: o.Project, Query: o.Query, Nodes: o.Nodes, StartedAt: start.UTC(), + Timing: Timing{FirstEventMS: -1, SnapshotMS: -1, CompleteMS: -1, CancelMS: -1, CancelToEndMS: -1}, + } + deadline := time.Duration(o.Deadline) + if deadline == 0 { + deadline = time.Duration(o.ExpiresIn) + } + ctx, cancel := context.WithTimeout(ctx, deadline) + defer cancel() + if len(o.Metadata) > 0 { + ctx = metadata.AppendToOutgoingContext(ctx, mdPairs(o.Metadata)...) + } + var cancelledAt time.Time + doCancel := func() { + if cancelledAt.IsZero() { + cancelledAt = time.Now() + cancel() + } + } + if o.CancelAfter > 0 { + t := time.AfterFunc(time.Duration(o.CancelAfter), doCancel) + defer t.Stop() + } + + check := stream.New(o.RequestID) + st, err := servicev1.NewLookingGlassServiceClient(cc).RunDiagnostic(ctx, req) + for err == nil { + if o.StopReadingAfter > 0 && rec.Events >= o.StopReadingAfter { + <-ctx.Done() + err = ctx.Err() + break + } + if o.ReadDelay > 0 && rec.Events > 0 { + time.Sleep(time.Duration(o.ReadDelay)) + } + var ev *servicev1.DiagnosticEvent + if ev, err = st.Recv(); err != nil { + break + } + at := ms(time.Since(start)) + rec.Events++ + size := proto.Size(ev) + rec.Bytes += size + rec.MaxEventBytes = max(rec.MaxEventBytes, size) + if rec.Timing.FirstEventMS < 0 { + rec.Timing.FirstEventMS = at + } + switch ev.GetEvent().(type) { + case *servicev1.DiagnosticEvent_Snapshot: + rec.Timing.SnapshotMS = at + case *servicev1.DiagnosticEvent_Outcome: + rec.Timing.OutcomeMS = append(rec.Timing.OutcomeMS, at) + case *servicev1.DiagnosticEvent_Complete: + rec.Timing.CompleteMS = at + } + check.Observe(ev) + if onEvent != nil { + onEvent(ev) + } + if o.CancelAfterEvents > 0 && rec.Events == o.CancelAfterEvents { + doCancel() + } + } + end := time.Now() + rec.Timing.EndMS = ms(end.Sub(start)) + if !cancelledAt.IsZero() { + rec.Timing.CancelMS = ms(cancelledAt.Sub(start)) + rec.Timing.CancelToEndMS = ms(end.Sub(cancelledAt)) + } + ok := errors.Is(err, io.EOF) + if ok { + err = nil + } + rec.Status = rpcerr.Decode(err) + rec.Stream = check.Finish(ok) + return rec, nil +} + +func mdPairs(md map[string]string) []string { + kv := make([]string, 0, 2*len(md)) + for k, v := range md { + kv = append(kv, k, v) + } + return kv +} + +// Summary is one line describing r for a person. +func Summary(r Record) string { + var b strings.Builder + fmt.Fprintf(&b, "%s %s", r.RequestID, r.Status.Code) + if r.Status.Reason != "" { + fmt.Fprintf(&b, " %s", r.Status.Reason) + } + if r.Stream.Complete != nil { + c := r.Stream.Complete + fmt.Fprintf(&b, " %s expected=%d successful=%d failed=%d omitted=%d", + r.Stream.Reason, c.Expected, c.Successful, c.Failed, c.Omitted) + } + fmt.Fprintf(&b, " delivered=%t", r.Stream.Delivered) + if n := len(r.Stream.Violations); n > 0 { + fmt.Fprintf(&b, " violations=%d", n) + } + return b.String() +} diff --git a/examples/looking-glass-client/internal/run/run_test.go b/examples/looking-glass-client/internal/run/run_test.go new file mode 100644 index 000000000..b2edcb329 --- /dev/null +++ b/examples/looking-glass-client/internal/run/run_test.go @@ -0,0 +1,183 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package run + +import ( + "context" + "net" + "testing" + "time" + + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/stream" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + + servicev1 "go.datum.net/galactic/api/fabric/service/v1" + fabricv1 "go.datum.net/galactic/api/fabric/v1" +) + +// eventStream is the server side of a RunDiagnostic stream. +type eventStream = grpc.ServerStreamingServer[servicev1.DiagnosticEvent] + +// fake is a LookingGlassService that runs serve for every RunDiagnostic. +type fake struct { + servicev1.UnimplementedLookingGlassServiceServer + serve func(*servicev1.RunDiagnosticRequest, eventStream) error +} + +func (f *fake) RunDiagnostic(req *servicev1.RunDiagnosticRequest, s eventStream) error { + return f.serve(req, s) +} + +func dial(t *testing.T, f *fake) *grpc.ClientConn { + t.Helper() + lis := bufconn.Listen(1 << 20) + srv := grpc.NewServer() + servicev1.RegisterLookingGlassServiceServer(srv, f) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + cc, err := grpc.NewClient("passthrough:///bufnet", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cc.Close() }) + return cc +} + +func snapshot(id string) *servicev1.DiagnosticEvent { + return &servicev1.DiagnosticEvent{RequestId: id, Sequence: 1, Event: &servicev1.DiagnosticEvent_Snapshot{ + Snapshot: &servicev1.Snapshot{SnapshotId: "s", Routers: []*servicev1.SnapshotRouter{ + {Node: "a", PodUid: "pa", Selected: true}, {Node: "b", PodUid: "pb", Selected: true}, + }}, + }} +} + +func outcome(id string, seq uint64, node string) *servicev1.DiagnosticEvent { + return &servicev1.DiagnosticEvent{RequestId: id, Sequence: seq, Event: &servicev1.DiagnosticEvent_Outcome{ + Outcome: &servicev1.RouterOutcome{Node: node, PodUid: "p" + node, CompletedAt: timestamppb.Now(), + Result: &servicev1.RouterOutcome_Observation{Observation: &fabricv1.Observation{ + Node: node, SampleTime: timestamppb.Now(), + }}}, + }} +} + +func opts() Options { + return Options{ + Cell: "c", Project: "p", Query: Query{Type: "BGPSummary", Family: "IPv6"}, ExpiresIn: Duration(5 * time.Second), + } +} + +func TestDelivered(t *testing.T) { + cc := dial(t, &fake{serve: func(req *servicev1.RunDiagnosticRequest, s eventStream) error { + if req.GetApiMajor() != 1 || req.GetQuery().GetType() != fabricv1.QueryType_QUERY_TYPE_BGP_SUMMARY || + req.GetQuery().GetAddressFamily() != fabricv1.AddressFamily_ADDRESS_FAMILY_IPV6 || req.GetProjectId() != "p" { + return status.Error(codes.InvalidArgument, "unexpected request") + } + id := req.GetRequestId() + for _, ev := range []*servicev1.DiagnosticEvent{snapshot(id), outcome(id, 2, "b"), outcome(id, 3, "a"), + {RequestId: id, Sequence: 4, Event: &servicev1.DiagnosticEvent_Complete{Complete: &servicev1.Complete{ + SnapshotId: "s", FinishedAt: timestamppb.Now(), Expected: 2, Successful: 2, Reason: stream.Succeeded, + }}}} { + if err := s.Send(ev); err != nil { + return err + } + } + return nil + }}) + rec, err := Do(t.Context(), cc, opts(), nil) + if err != nil { + t.Fatal(err) + } + if rec.Status.Code != "OK" || !rec.Stream.Delivered || rec.Events != 4 || len(rec.Timing.OutcomeMS) != 2 || + rec.Timing.SnapshotMS < 0 || rec.Timing.CompleteMS < rec.Timing.SnapshotMS || rec.Timing.CancelMS != -1 { + t.Errorf("record = %+v", rec) + } +} + +func TestTypedRefusal(t *testing.T) { + cc := dial(t, &fake{serve: func(*servicev1.RunDiagnosticRequest, eventStream) error { + st, _ := status.New(codes.ResourceExhausted, "throttled").WithDetails( + &errdetails.ErrorInfo{Reason: "Throttled", Domain: "fabric-api.datumapis.com", + Metadata: map[string]string{"scope": "project", "execution_started": "false"}}, + &errdetails.RetryInfo{RetryDelay: durationpb.New(1500 * time.Millisecond)}) + return st.Err() + }}) + rec, err := Do(t.Context(), cc, opts(), nil) + if err != nil { + t.Fatal(err) + } + s := rec.Status + if s.Code != "ResourceExhausted" || s.Reason != "Throttled" || s.Domain != "" || s.Metadata["scope"] != "project" || + s.RetryDelayMS != 1500 || rec.Stream.Delivered || rec.Events != 0 { + t.Errorf("record = %+v", rec) + } +} + +// block sends the snapshot and one outcome, then holds the stream until +// the client goes away. +func block(req *servicev1.RunDiagnosticRequest, s eventStream) error { + id := req.GetRequestId() + if err := s.Send(snapshot(id)); err != nil { + return err + } + if err := s.Send(outcome(id, 2, "a")); err != nil { + return err + } + <-s.Context().Done() + return s.Context().Err() +} + +func TestCancelAfterEvents(t *testing.T) { + cc := dial(t, &fake{serve: block}) + o := opts() + o.CancelAfterEvents = 2 + rec, err := Do(t.Context(), cc, o, nil) + if err != nil { + t.Fatal(err) + } + if rec.Status.Code != "Canceled" || rec.Stream.Delivered || rec.Timing.CancelMS < 0 || + rec.Timing.CancelToEndMS < 0 || rec.Timing.CancelToEndMS > 1000 || len(rec.Stream.Pending) != 1 { + t.Errorf("record = %+v", rec) + } +} + +func TestStuckConsumerHoldsToDeadline(t *testing.T) { + cc := dial(t, &fake{serve: block}) + o := opts() + o.StopReadingAfter, o.Deadline = 1, Duration(300*time.Millisecond) + rec, err := Do(t.Context(), cc, o, nil) + if err != nil { + t.Fatal(err) + } + if rec.Status.Code != "DeadlineExceeded" || rec.Events != 1 || rec.Timing.EndMS < 300 { + t.Errorf("record = %+v", rec) + } +} + +func TestRequestIDIsLowercaseUUID(t *testing.T) { + id := NewRequestID() + if len(id) != 36 || id[14] != '4' { + t.Errorf("request ID %q", id) + } + for _, r := range id { + if r >= 'A' && r <= 'Z' { + t.Errorf("request ID %q is not lowercase", id) + } + } +} + +func TestUnknownQueryType(t *testing.T) { + if _, err := (Query{Type: "Whois"}).Proto(); err == nil { + t.Error("an unknown query type was accepted") + } +} diff --git a/examples/looking-glass-client/internal/stream/check.go b/examples/looking-glass-client/internal/stream/check.go new file mode 100644 index 000000000..973b63406 --- /dev/null +++ b/examples/looking-glass-client/internal/stream/check.go @@ -0,0 +1,297 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package stream checks a RunDiagnostic stream against the contract's +// grammar and coverage rules (docs/designs/looking-glass-service.md, "Stream +// grammar and coverage") as a consumer sees them: from the events alone. +package stream + +import ( + "fmt" + + servicev1 "go.datum.net/galactic/api/fabric/service/v1" +) + +// The terminal reasons the contract defines. Stop reasons take precedence +// over the reason the counts imply. +const ( + Succeeded = "Succeeded" + PartialResults = "PartialResults" + ExecutionFailed = "ExecutionFailed" + NoNodesAvailable = "NoNodesAvailable" + DeadlineExceeded = "DeadlineExceeded" + Cancelled = "Cancelled" + OwnershipLost = "OwnershipLost" + OutputLimitExceeded = "OutputLimitExceeded" + + NotRequested = "NotRequested" +) + +var stopReasons = map[string]bool{ + DeadlineExceeded: true, Cancelled: true, OwnershipLost: true, OutputLimitExceeded: true, +} + +// Outcome is one router's outcome as delivered, without its payload. +type Outcome struct { + Node string `json:"node"` + // Result is "observation" or "failure". + Result string `json:"result"` + // Reason is the failure reason; empty for an observation. + Reason string `json:"reason,omitempty"` + Scope string `json:"scope,omitempty"` + Truncated bool `json:"truncated,omitempty"` + // Sequence is the event's sequence number, which is its completion + // handoff order. + Sequence uint64 `json:"sequence"` +} + +// Counts are a stream's coverage. +type Counts struct { + Expected uint32 `json:"expected"` + Successful uint32 `json:"successful"` + Failed uint32 `json:"failed"` + Omitted uint32 `json:"omitted"` + Truncated uint32 `json:"truncatedRouters"` +} + +// Result is what a consumer can claim about a stream. +type Result struct { + // Delivered is true only when a complete arrived, every rule held and + // the stream ended with OK: the server's coverage is then known. + Delivered bool `json:"delivered"` + // Violations lists every broken rule, in the order seen. + Violations []string `json:"violations,omitempty"` + // Snapshot holds the snapshot's counts, Selected the routers it + // selected, and OmittedReasons its omitted routers by reason. + SnapshotID string `json:"snapshotId,omitempty"` + Snapshot *Counts `json:"snapshot,omitempty"` + Selected []string `json:"selected,omitempty"` + OmittedReasons map[string]string `json:"omittedReasons,omitempty"` + Unscheduled uint32 `json:"unscheduledPodCount,omitempty"` + Outcomes []Outcome `json:"outcomes,omitempty"` + // Pending lists the selected routers with no delivered outcome: their + // result is unknown to the consumer. + Pending []string `json:"pending,omitempty"` + // Complete is the server's terminal coverage and Reason its reason; both + // are empty when no complete arrived. + Complete *Counts `json:"complete,omitempty"` + Reason string `json:"reason,omitempty"` +} + +// Checker accumulates one stream's events. +type Checker struct { + requestID string + next uint64 + snapshot *servicev1.Snapshot + routers map[string]*servicev1.SnapshotRouter + seen map[string]bool + complete *servicev1.Complete + unknown bool + res Result +} + +// New returns a Checker for the stream of request requestID. +func New(requestID string) *Checker { + return &Checker{requestID: requestID, next: 1, seen: map[string]bool{}} +} + +func (c *Checker) violate(format string, args ...any) { + c.res.Violations = append(c.res.Violations, fmt.Sprintf(format, args...)) +} + +// Observe checks one event in arrival order. +func (c *Checker) Observe(ev *servicev1.DiagnosticEvent) { + if ev.GetRequestId() != c.requestID { + c.violate("event %d: request_id %q, want %q", ev.GetSequence(), ev.GetRequestId(), c.requestID) + } + if ev.GetSequence() != c.next { + c.violate("event sequence %d, want %d", ev.GetSequence(), c.next) + } + c.next = ev.GetSequence() + 1 + if c.complete != nil { + c.violate("event %d after complete", ev.GetSequence()) + } + switch e := ev.GetEvent().(type) { + case *servicev1.DiagnosticEvent_Snapshot: + c.observeSnapshot(ev.GetSequence(), e.Snapshot) + case *servicev1.DiagnosticEvent_Outcome: + c.observeOutcome(ev.GetSequence(), e.Outcome) + case *servicev1.DiagnosticEvent_Complete: + c.observeComplete(ev.GetSequence(), e.Complete) + default: + // An unknown variant prevents claiming complete coverage. + c.unknown = true + c.violate("event %d: unknown event variant", ev.GetSequence()) + } +} + +func (c *Checker) observeSnapshot(seq uint64, s *servicev1.Snapshot) { + if c.snapshot != nil { + c.violate("event %d: second snapshot", seq) + return + } + if seq != 1 { + c.violate("event %d: snapshot is not the first event", seq) + } + c.snapshot = s + c.routers = map[string]*servicev1.SnapshotRouter{} + counts := &Counts{} + c.res.OmittedReasons = map[string]string{} + for _, r := range s.GetRouters() { + if _, dup := c.routers[r.GetNode()]; dup { + c.violate("snapshot lists router %s twice", r.GetNode()) + continue + } + c.routers[r.GetNode()] = r + counts.Expected++ + switch { + case r.GetSelected() && r.GetPodUid() == "": + c.violate("snapshot: selected router %s has no pod UID", r.GetNode()) + case r.GetSelected() && r.GetOmittedReason() != "": + c.violate("snapshot: selected router %s has omitted reason %s", r.GetNode(), r.GetOmittedReason()) + case !r.GetSelected() && r.GetOmittedReason() == "": + c.violate("snapshot: omitted router %s has no reason", r.GetNode()) + } + if r.GetSelected() { + c.res.Selected = append(c.res.Selected, r.GetNode()) + } else { + counts.Omitted++ + c.res.OmittedReasons[r.GetNode()] = r.GetOmittedReason() + } + } + c.res.SnapshotID = s.GetSnapshotId() + c.res.Snapshot = counts + c.res.Unscheduled = s.GetUnscheduledPodCount() +} + +func (c *Checker) observeOutcome(seq uint64, o *servicev1.RouterOutcome) { + if c.snapshot == nil { + c.violate("event %d: outcome before snapshot", seq) + return + } + r, ok := c.routers[o.GetNode()] + switch { + case !ok: + c.violate("event %d: outcome for router %s outside the snapshot", seq, o.GetNode()) + return + case !r.GetSelected(): + c.violate("event %d: outcome for omitted router %s", seq, o.GetNode()) + return + case c.seen[o.GetNode()]: + c.violate("event %d: second outcome for router %s", seq, o.GetNode()) + return + case o.GetPodUid() != r.GetPodUid(): + c.violate("event %d: router %s outcome pod %s, snapshot froze %s", seq, o.GetNode(), o.GetPodUid(), r.GetPodUid()) + } + if !o.GetCompletedAt().IsValid() { + c.violate("event %d: router %s outcome has no completion time", seq, o.GetNode()) + } + c.seen[o.GetNode()] = true + out := Outcome{Node: o.GetNode(), Sequence: seq} + switch { + case o.GetObservation() != nil: + obs := o.GetObservation() + out.Result, out.Truncated = "observation", obs.GetTruncated() + if obs.GetNode() != "" && obs.GetNode() != o.GetNode() { + c.violate("event %d: router %s carries router %s's observation", seq, o.GetNode(), obs.GetNode()) + } + if !obs.GetSampleTime().IsValid() { + c.violate("event %d: router %s observation has no sample time", seq, o.GetNode()) + } + case o.GetFailure() != nil: + f := o.GetFailure() + out.Result, out.Reason, out.Scope = "failure", f.GetReason(), f.GetScope() + if f.GetReason() == "" { + c.violate("event %d: router %s failure has no reason", seq, o.GetNode()) + } + if len(f.GetMessage()) > 256 { + c.violate("event %d: router %s failure message is %d bytes", seq, o.GetNode(), len(f.GetMessage())) + } + default: + c.violate("event %d: router %s outcome has neither observation nor failure", seq, o.GetNode()) + out.Result = "unknown" + } + c.res.Outcomes = append(c.res.Outcomes, out) +} + +func (c *Checker) observeComplete(seq uint64, k *servicev1.Complete) { + if c.snapshot == nil { + c.violate("event %d: complete before snapshot", seq) + } + c.complete = k + c.res.Reason = k.GetReason() + c.res.Complete = &Counts{ + Expected: k.GetExpected(), Successful: k.GetSuccessful(), Failed: k.GetFailed(), + Omitted: k.GetOmitted(), Truncated: k.GetTruncatedRouters(), + } + if c.snapshot == nil { + return + } + if k.GetSnapshotId() != c.snapshot.GetSnapshotId() { + c.violate("complete snapshot %q, stream snapshot %q", k.GetSnapshotId(), c.snapshot.GetSnapshotId()) + } + var got Counts + got.Expected, got.Omitted = c.res.Snapshot.Expected, c.res.Snapshot.Omitted + for _, o := range c.res.Outcomes { + switch o.Result { + case "observation": + got.Successful++ + if o.Truncated { + got.Truncated++ + } + default: + got.Failed++ + } + } + if got != *c.res.Complete { + c.violate("complete counts %+v, delivered events imply %+v", *c.res.Complete, got) + } + if k.GetExpected() != k.GetSuccessful()+k.GetFailed()+k.GetOmitted() { + c.violate("complete: expected %d != successful %d + failed %d + omitted %d", + k.GetExpected(), k.GetSuccessful(), k.GetFailed(), k.GetOmitted()) + } + if !k.GetFinishedAt().IsValid() { + c.violate("complete has no finish time") + } + if want := c.impliedReason(got); !stopReasons[k.GetReason()] && k.GetReason() != want { + c.violate("complete reason %s, counts imply %s", k.GetReason(), want) + } +} + +// impliedReason is the terminal reason the counts imply when no stop +// reason applies. +func (c *Checker) impliedReason(n Counts) string { + selected := n.Successful + n.Failed + partial := n.Failed > 0 || n.Truncated > 0 + for _, reason := range c.res.OmittedReasons { + partial = partial || reason != NotRequested + } + switch { + case selected == 0: + return NoNodesAvailable + case n.Successful == 0: + return ExecutionFailed + case partial: + return PartialResults + } + return Succeeded +} + +// Finish closes the stream with ok, whether the RPC ended with gRPC OK, and +// returns the result. +func (c *Checker) Finish(ok bool) Result { + for _, n := range c.res.Selected { + if !c.seen[n] { + c.res.Pending = append(c.res.Pending, n) + } + } + if c.complete != nil && len(c.res.Pending) > 0 { + c.violate("complete arrived with %d selected routers missing an outcome", len(c.res.Pending)) + } + if c.complete != nil && !ok { + c.violate("stream ended with an error after complete") + } + c.res.Delivered = ok && c.complete != nil && !c.unknown && len(c.res.Violations) == 0 + return c.res +} diff --git a/examples/looking-glass-client/internal/stream/check_test.go b/examples/looking-glass-client/internal/stream/check_test.go new file mode 100644 index 000000000..c1a953dca --- /dev/null +++ b/examples/looking-glass-client/internal/stream/check_test.go @@ -0,0 +1,177 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package stream + +import ( + "bufio" + "bytes" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + + servicev1 "go.datum.net/galactic/api/fabric/service/v1" +) + +const fixtureID = "b74e19ee-926f-43dc-b03d-c29550a827ab" + +// fixture returns the contract's published example stream: two selected +// routers (one observation, one NodeUnreachable) and one omitted for +// NoCertificatePod, ending PartialResults. +func fixture(t *testing.T) []*servicev1.DiagnosticEvent { + t.Helper() + path := filepath.Join("..", "..", "..", "..", "api", "fabric", "service", "v1", "testdata", "stream.jsonl") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var evs []*servicev1.DiagnosticEvent + sc := bufio.NewScanner(bytes.NewReader(data)) + for sc.Scan() { + ev := &servicev1.DiagnosticEvent{} + if err := protojson.Unmarshal(sc.Bytes(), ev); err != nil { + t.Fatal(err) + } + evs = append(evs, ev) + } + return evs +} + +func check(evs []*servicev1.DiagnosticEvent, ok bool) Result { + c := New(fixtureID) + for _, ev := range evs { + c.Observe(ev) + } + return c.Finish(ok) +} + +func TestFixtureDelivered(t *testing.T) { + r := check(fixture(t), true) + if !r.Delivered || len(r.Violations) > 0 { + t.Fatalf("fixture not delivered: %v", r.Violations) + } + if r.Reason != PartialResults || len(r.Outcomes) != 2 || len(r.Pending) != 0 || + r.OmittedReasons["router-c"] != "NoCertificatePod" || r.Complete.Expected != 3 { + t.Errorf("result = %+v", r) + } + if r.Outcomes[1].Result != "failure" || r.Outcomes[1].Reason != "NodeUnreachable" { + t.Errorf("second outcome = %+v", r.Outcomes[1]) + } +} + +func TestViolations(t *testing.T) { + cases := []struct { + name string + mutate func([]*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent + ok bool + want string + }{ + {"missing complete", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { return e[:3] }, true, ""}, + { + "error after complete", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { return e }, + false, "ended with an error", + }, + {"sequence gap", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[2].Sequence = 4 + return e + }, true, "sequence 4, want 3"}, + {"foreign request", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[1].RequestId = "other" + return e + }, true, `request_id "other"`}, + {"outcome first", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[0], e[1] = e[1], e[0] + e[0].Sequence, e[1].Sequence = 1, 2 + return e + }, true, "outcome before snapshot"}, + {"duplicate outcome", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[2].GetOutcome().Node, e[2].GetOutcome().PodUid = "router-a", "pod-a" + return e + }, true, "second outcome for router router-a"}, + {"omitted router outcome", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[2].GetOutcome().Node = "router-c" + return e + }, true, "outcome for omitted router router-c"}, + {"pod swapped", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[1].GetOutcome().PodUid = "pod-new" + return e + }, true, "snapshot froze pod-a"}, + {"counts disagree", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[3].GetComplete().Successful = 2 + e[3].GetComplete().Failed = 0 + return e + }, true, "delivered events imply"}, + {"wrong reason", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[3].GetComplete().Reason = Succeeded + return e + }, true, "counts imply PartialResults"}, + {"other snapshot", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[3].GetComplete().SnapshotId = "snap-2" + return e + }, true, `complete snapshot "snap-2"`}, + {"unknown variant", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + e[2].Event = nil + return e + }, true, "unknown event variant"}, + {"event after complete", func(e []*servicev1.DiagnosticEvent) []*servicev1.DiagnosticEvent { + extra := proto.Clone(e[2]).(*servicev1.DiagnosticEvent) + extra.Sequence = 5 + return append(e, extra) + }, true, "after complete"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := check(tc.mutate(fixture(t)), tc.ok) + if r.Delivered { + t.Fatalf("delivered despite %s", tc.name) + } + if tc.want != "" && !slices.ContainsFunc(r.Violations, func(v string) bool { return strings.Contains(v, tc.want) }) { + t.Errorf("violations %q do not mention %q", r.Violations, tc.want) + } + }) + } +} + +func TestMissingCompleteLeavesPending(t *testing.T) { + r := check(fixture(t)[:2], true) + if r.Delivered || r.Complete != nil || !slices.Equal(r.Pending, []string{"router-b"}) { + t.Errorf("result = %+v", r) + } +} + +func TestStopReasonOverridesCounts(t *testing.T) { + evs := fixture(t) + evs[3].GetComplete().Reason = Cancelled + if r := check(evs, true); !r.Delivered { + t.Errorf("a stop reason with consistent counts was refused: %v", r.Violations) + } +} + +func TestImpliedReasons(t *testing.T) { + c := New("x") + c.res.OmittedReasons = map[string]string{"r": NotRequested} + for _, tc := range []struct { + n Counts + want string + }{ + {Counts{Expected: 1, Omitted: 1}, NoNodesAvailable}, + {Counts{Expected: 2, Failed: 1, Omitted: 1}, ExecutionFailed}, + {Counts{Expected: 2, Successful: 1, Omitted: 1}, Succeeded}, + {Counts{Expected: 3, Successful: 1, Failed: 1, Omitted: 1}, PartialResults}, + {Counts{Expected: 2, Successful: 1, Truncated: 1, Omitted: 1}, PartialResults}, + } { + if got := c.impliedReason(tc.n); got != tc.want { + t.Errorf("%+v: got %s, want %s", tc.n, got, tc.want) + } + } + c.res.OmittedReasons["r"] = "NoSidecar" + if got := c.impliedReason(Counts{Expected: 2, Successful: 1, Omitted: 1}); got != PartialResults { + t.Errorf("an unavailable router's omission gave %s, want PartialResults", got) + } +} diff --git a/examples/looking-glass-client/main.go b/examples/looking-glass-client/main.go new file mode 100644 index 000000000..ed63633d8 --- /dev/null +++ b/examples/looking-glass-client/main.go @@ -0,0 +1,275 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Command looking-glass-client is an independent Looking Glass service +// client. It is built from the published service API alone +// (go.datum.net/galactic/api/fabric/service/v1 and api/fabric/v1), calls a +// cell's fabric-api gateway as an authorized service, and checks every +// stream against the contract's grammar and coverage rules. +// +// Subcommands: +// +// capabilities print GetCapabilities as JSON +// run run one diagnostic; print its record as JSON +// load run a workload from a JSON spec; print one record per +// attempt as JSON lines, then one summary line per group +// invoke call any unary method with an empty request and print its +// status, to show a method is closed to the service +// +// Records hold statuses, coverage and timings, never diagnostic payloads; +// run --events prints the events themselves to stderr. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/conn" + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/load" + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/rpcerr" + "github.com/datum-cloud/galactic/examples/looking-glass-client/internal/run" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/emptypb" + + servicev1 "go.datum.net/galactic/api/fabric/service/v1" +) + +const usage = `usage: looking-glass-client [flags] + +Every subcommand takes --addr, --cell, --cert, --key and --ca. Run +"looking-glass-client -h" for its flags.` + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, usage) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + var err error + switch os.Args[1] { + case "capabilities": + err = capabilities(ctx, os.Args[2:]) + case "run": + err = runOne(ctx, os.Args[2:]) + case "load": + err = runLoad(ctx, os.Args[2:]) + case "invoke": + err = invoke(ctx, os.Args[2:]) + default: + fmt.Fprintln(os.Stderr, usage) + os.Exit(2) + } + if err != nil { + fmt.Fprintln(os.Stderr, "looking-glass-client:", err) + os.Exit(1) + } +} + +func connFlags(fs *flag.FlagSet) *conn.Options { + o := &conn.Options{} + fs.StringVar(&o.Addr, "addr", "", "gateway address, host:port") + fs.StringVar(&o.Cell, "cell", "", "the cell of the gateway and of the service identity") + fs.StringVar(&o.CertFile, "cert", "tls.crt", "the service's certificate (PEM); empty presents none") + fs.StringVar(&o.KeyFile, "key", "tls.key", "the service's private key (PEM)") + fs.StringVar(&o.CAFile, "ca", "ca.crt", "the cell's trust bundle (PEM)") + return o +} + +func check(o *conn.Options) error { + if o.Addr == "" || o.Cell == "" { + return errors.New("--addr and --cell are required") + } + return nil +} + +// result is what capabilities and invoke print. +type result struct { + Method string `json:"method,omitempty"` + Status rpcerr.Error `json:"status"` + Capabilities json.RawMessage `json:"capabilities,omitempty"` +} + +func printJSON(v any) error { + enc := json.NewEncoder(os.Stdout) + return enc.Encode(v) +} + +func capabilities(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("capabilities", flag.ExitOnError) + co := connFlags(fs) + major := fs.Uint("api-major", 1, "api_major to request") + _ = fs.Parse(args) + if err := check(co); err != nil { + return err + } + cc, err := conn.Dial(*co) + if err != nil { + return err + } + defer func() { _ = cc.Close() }() + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + resp, err := servicev1.NewLookingGlassServiceClient(cc).GetCapabilities(ctx, + &servicev1.GetCapabilitiesRequest{Cell: co.Cell, ApiMajor: uint32(*major)}) + if err != nil { + // A refusal is a result, not a client failure. + return printJSON(result{Status: rpcerr.Decode(err)}) + } + b, err := protojson.Marshal(resp) + if err != nil { + return err + } + return printJSON(result{Status: rpcerr.Decode(nil), Capabilities: b}) +} + +type listFlag []string + +func (l *listFlag) String() string { return strings.Join(*l, ",") } + +func (l *listFlag) Set(v string) error { + *l = append(*l, strings.Split(v, ",")...) + return nil +} + +func runOne(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("run", flag.ExitOnError) + co := connFlags(fs) + var o run.Options + var nodes, features, md listFlag + var expires, deadline, readDelay, cancelAfter time.Duration + var major uint + fs.StringVar(&o.Project, "project", "", "project UID the request is attributed to") + fs.StringVar(&o.RequestID, "request-id", "", "request UUID; random when empty") + fs.StringVar(&o.Query.Type, "type", "BGPSummary", "RouteLookup, BGPSummary, Ping or Traceroute") + fs.StringVar(&o.Query.Target, "target", "", "the query's target address or prefix") + fs.StringVar(&o.Query.Family, "family", "", "IPv4, IPv6, or empty to infer it from the target") + fs.Var(&nodes, "nodes", "routers to select, by node name (repeatable or comma-separated)") + fs.Var(&features, "require", "features the request requires (repeatable or comma-separated)") + fs.Var(&md, "metadata", "request metadata as key=value (repeatable or comma-separated)") + fs.UintVar(&major, "api-major", 1, "api_major to send") + reqCell := fs.String("request-cell", "", "cell to name in the request; defaults to --cell") + fs.DurationVar(&expires, "expires-in", 30*time.Second, "expires_at, from now") + fs.DurationVar(&deadline, "deadline", 0, "transport deadline; defaults to --expires-in") + fs.DurationVar(&readDelay, "read-delay", 0, "delay before each read after the first, as a slow consumer") + fs.IntVar(&o.StopReadingAfter, "stop-reading-after", 0, "stop reading after N events and hold the stream") + fs.IntVar(&o.CancelAfterEvents, "cancel-after-events", 0, "cancel after N events") + fs.DurationVar(&cancelAfter, "cancel-after", 0, "cancel this long after the call starts") + events := fs.Bool("events", false, "print every event, payloads included, to stderr as JSON lines") + _ = fs.Parse(args) + if err := check(co); err != nil { + return err + } + o.Cell, o.Nodes, o.RequiredFeatures, o.APIMajor = co.Cell, nodes, features, uint32(major) + if *reqCell != "" { + o.Cell = *reqCell + } + for _, kv := range md { + k, v, ok := strings.Cut(kv, "=") + if !ok { + return fmt.Errorf("--metadata %q is not key=value", kv) + } + if o.Metadata == nil { + o.Metadata = map[string]string{} + } + o.Metadata[k] = v + } + o.ExpiresIn, o.Deadline, o.ReadDelay, o.CancelAfter = + run.Duration(expires), run.Duration(deadline), run.Duration(readDelay), run.Duration(cancelAfter) + cc, err := conn.Dial(*co) + if err != nil { + return err + } + defer func() { _ = cc.Close() }() + var onEvent func(*servicev1.DiagnosticEvent) + if *events { + onEvent = func(ev *servicev1.DiagnosticEvent) { + if b, err := protojson.Marshal(ev); err == nil { + fmt.Fprintln(os.Stderr, string(b)) + } + } + } + rec, err := run.Do(ctx, cc, o, onEvent) + if err != nil { + return err + } + return printJSON(rec) +} + +func runLoad(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("load", flag.ExitOnError) + co := connFlags(fs) + specFile := fs.String("spec", "", "workload spec (JSON); - reads stdin") + _ = fs.Parse(args) + if err := check(co); err != nil { + return err + } + var raw []byte + var err error + if *specFile == "-" { + raw, err = io.ReadAll(os.Stdin) + } else { + raw, err = os.ReadFile(*specFile) + } + if err != nil { + return fmt.Errorf("read spec: %w", err) + } + var spec load.Spec + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&spec); err != nil { + return fmt.Errorf("parse spec: %w", err) + } + spec.Cell = co.Cell + cc, err := conn.Dial(*co) + if err != nil { + return err + } + defer func() { _ = cc.Close() }() + var lines []load.Line + err = load.Run(ctx, cc, spec, func(l load.Line) { + lines = append(lines, l) + _ = printJSON(l) + }) + for _, s := range load.Summarize(lines) { + _ = printJSON(map[string]any{"summary": s}) + } + // An interrupt ends the workload early; what finished is the result. + if errors.Is(err, context.Canceled) { + return nil + } + return err +} + +func invoke(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("invoke", flag.ExitOnError) + co := connFlags(fs) + method := fs.String("method", "", "full method name, /package.Service/Method") + _ = fs.Parse(args) + if err := check(co); err != nil { + return err + } + if *method == "" { + return errors.New("--method is required") + } + cc, err := conn.Dial(*co) + if err != nil { + return err + } + defer func() { _ = cc.Close() }() + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + err = cc.Invoke(ctx, *method, &emptypb.Empty{}, &emptypb.Empty{}) + return printJSON(result{Method: *method, Status: rpcerr.Decode(err)}) +} diff --git a/tests/lgqualify/counter_test.go b/tests/lgqualify/counter_test.go new file mode 100644 index 000000000..661538dce --- /dev/null +++ b/tests/lgqualify/counter_test.go @@ -0,0 +1,247 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lgqualify_test + +import ( + "fmt" + "net/netip" + "strconv" + "strings" + "sync" + "time" +) + +// chain is the iptables chain, jumped to first from OUTPUT in every dfw +// node's network namespace, whose rules count ICMP echo requests (ping and +// traceroute probes alike) sent toward each watched destination. +const chain = "LG-QUALIFY" + +// sampleEvery is the counter's sampling period: the precision of every +// "packets stopped by" measurement. +const sampleEvery = 200 * time.Millisecond + +// sample is the cumulative packet count per destination, summed over the +// dfw nodes and per node, at one moment. +type sample struct { + At time.Time `json:"at"` + Count map[string]int64 `json:"count"` + PerNode map[string]map[string]int64 `json:"perNode"` +} + +// counter samples the chain's counters on every router. +type counter struct { + mu sync.Mutex + samples []sample + stop chan struct{} + done chan struct{} +} + +func iptablesFor(dst string) (string, string) { + if netip.MustParseAddr(dst).Is4() { + return iptables, "-p icmp --icmp-type echo-request" + } + return ip6tables, "-p ipv6-icmp --icmpv6-type echo-request" +} + +// installChain creates the chain and its OUTPUT jump on every router, +// replacing any left by an earlier run. +func installChain() error { + for _, node := range routers { + for _, ipt := range []string{iptables, ip6tables} { + script := fmt.Sprintf(`%[1]s -w -D OUTPUT -j %[2]s 2>/dev/null +%[1]s -w -F %[2]s 2>/dev/null +%[1]s -w -X %[2]s 2>/dev/null +%[1]s -w -N %[2]s && %[1]s -w -I OUTPUT 1 -j %[2]s`, ipt, chain) + if _, err := nodeExec(node, "sh", "-c", script); err != nil { + return err + } + } + } + return nil +} + +// removeChain removes the chain and its jump from every router. +func removeChain() error { + var errs []string + for _, node := range routers { + for _, ipt := range []string{iptables, ip6tables} { + script := fmt.Sprintf(`%[1]s -w -D OUTPUT -j %[2]s; %[1]s -w -F %[2]s; %[1]s -w -X %[2]s`, ipt, chain) + if _, err := nodeExec(node, "sh", "-c", script); err != nil { + errs = append(errs, err.Error()) + } + } + } + if len(errs) > 0 { + return fmt.Errorf("remove %s: %s", chain, strings.Join(errs, "; ")) + } + return nil +} + +// watch adds counting rules for dsts on every router. +func watch(dsts ...string) error { + for _, node := range routers { + for _, dst := range dsts { + ipt, match := iptablesFor(dst) + script := fmt.Sprintf("%[1]s -w -C %[2]s -d %[3]s %[4]s 2>/dev/null || %[1]s -w -A %[2]s -d %[3]s %[4]s", + ipt, chain, dst, match) + if _, err := nodeExec(node, "sh", "-c", script); err != nil { + return err + } + } + } + return nil +} + +// readCounts returns the cumulative echo requests per destination, summed +// over the routers and per router. +func readCounts() (map[string]int64, map[string]map[string]int64, error) { + type res struct { + out string + err error + } + results := make([]res, len(routers)) + var wg sync.WaitGroup + for i, node := range routers { + wg.Go(func() { + out, err := nodeExec(node, "sh", "-c", fmt.Sprintf("iptables -w -nvx -L %[1]s; ip6tables -w -nvx -L %[1]s", chain)) + results[i] = res{out, err} + }) + } + wg.Wait() + total := map[string]int64{} + perNode := map[string]map[string]int64{} + for i, r := range results { + if r.err != nil { + return nil, nil, r.err + } + perNode[routers[i]] = map[string]int64{} + for line := range strings.SplitSeq(r.out, "\n") { + f := strings.Fields(line) + // pkts bytes target prot opt in out source destination ...; the + // IPv6 listing omits opt, so take the field that parses as an + // address after the source. + if len(f) < 8 { + continue + } + pkts, err := strconv.ParseInt(f[0], 10, 64) + if err != nil { + continue + } + var addrs []string + for _, x := range f[2:] { + if a, err := netip.ParsePrefix(x); err == nil { + addrs = append(addrs, a.Addr().String()) + } else if a, err := netip.ParseAddr(x); err == nil { + addrs = append(addrs, a.String()) + } + } + if len(addrs) < 2 { + continue + } + total[addrs[1]] += pkts + perNode[routers[i]][addrs[1]] += pkts + } + } + return total, perNode, nil +} + +// startCounter samples every sampleEvery until stopped. +func startCounter() *counter { + c := &counter{stop: make(chan struct{}), done: make(chan struct{})} + go func() { + defer close(c.done) + t := time.NewTicker(sampleEvery) + defer t.Stop() + for { + if counts, perNode, err := readCounts(); err == nil { + c.mu.Lock() + c.samples = append(c.samples, sample{At: time.Now(), Count: counts, PerNode: perNode}) + c.mu.Unlock() + } + select { + case <-c.stop: + return + case <-t.C: + } + } + }() + return c +} + +// close stops sampling and returns every sample. +func (c *counter) close() []sample { + close(c.stop) + <-c.done + c.mu.Lock() + defer c.mu.Unlock() + return c.samples +} + +// lastIncrease returns the time of the first sample at which dst's count +// reached its final value, i.e. when its packets stopped, and that value. +// The true stop lies within sampleEvery before it. +func lastIncrease(samples []sample, dst string) (time.Time, int64) { + if len(samples) == 0 { + return time.Time{}, 0 + } + final := samples[len(samples)-1].Count[dst] + for _, s := range samples { + if s.Count[dst] == final { + return s.At, final + } + } + return time.Time{}, final +} + +// between returns dst's packets counted between from and to. +func between(samples []sample, dst string, from, to time.Time) int64 { + var first, last int64 + for _, s := range samples { + if s.At.After(to) { + break + } + if s.At.Before(from) { + first = s.Count[dst] + } + last = s.Count[dst] + } + return max(0, last-first) +} + +// nodeTotal returns node's final count toward dst. +func nodeTotal(samples []sample, node, dst string) int64 { + if len(samples) == 0 { + return 0 + } + return samples[len(samples)-1].PerNode[node][dst] +} + +// worstWindow checks every window of the series for dsts' summed counts +// against B + r(T + 32) and returns the window with the largest count +// minus its bound (positive is a violation). +func worstWindow( + samples []sample, dsts []string, burst, rate float64, +) (excess float64, from, to time.Time, count int64) { + excess = -1e18 + sum := func(s sample) int64 { + var n int64 + for _, d := range dsts { + n += s.Count[d] + } + return n + } + for i := range samples { + for j := i + 1; j < len(samples); j++ { + // Packets counted in (samples[i].At, samples[j].At], an interval + // no longer than the samples' spacing plus one sampling period. + n := sum(samples[j]) - sum(samples[i]) + T := samples[j].At.Sub(samples[i].At).Seconds() + sampleEvery.Seconds() + if e := float64(n) - (burst + rate*(T+32)); e > excess { + excess, from, to, count = e, samples[i].At, samples[j].At, n + } + } + } + return excess, from, to, count +} diff --git a/tests/lgqualify/fault_test.go b/tests/lgqualify/fault_test.go new file mode 100644 index 000000000..73d5ba83c --- /dev/null +++ b/tests/lgqualify/fault_test.go @@ -0,0 +1,759 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lgqualify_test + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// twoRouters selects two routers, so one project's traceroute holds both +// of its node slots and none of its routers queues. +var twoRouters = routers[0] + "," + routers[1] + +// runAsync runs one diagnostic in the background. +func (l *lab) runAsync(ctx context.Context, args ...string) <-chan Record { + ch := make(chan Record, 1) + go func() { + r, err := l.run(ctx, idService, args...) + if err != nil { + r.Status.Code, r.Status.Reason = "ClientError", err.Error() + } + ch <- r + }() + return ch +} + +// leaderPod returns the gateway pod holding the lease. +func (l *lab) leaderPod(ctx context.Context) (corev1.Pod, string, error) { + name, holder, err := l.leader(ctx) + if err != nil { + return corev1.Pod{}, "", err + } + p, err := l.cs.CoreV1().Pods(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return corev1.Pod{}, "", err + } + return *p, holder, nil +} + +// gatewayPID returns the host PID of pod's gateway process, through the +// node's CRI. +// sandboxPID returns the PID of pod's sandbox, whose network namespace is +// the pod's for its whole life. +func sandboxPID(pod corev1.Pod) (int, error) { + sandbox, err := nodeExec(pod.Spec.NodeName, "crictl", "pods", + "--name", pod.Name, "--namespace", ns, "-q", "--state", "ready") + if err != nil { + return 0, err + } + sandbox = strings.TrimSpace(strings.Split(strings.TrimSpace(sandbox), "\n")[0]) + out, err := nodeExec(pod.Spec.NodeName, "crictl", "inspectp", + "-o", "go-template", "--template", "{{.info.pid}}", sandbox) + if err != nil { + return 0, err + } + return strconv.Atoi(strings.TrimSpace(out)) +} + +func gatewayPID(pod corev1.Pod) (int, error) { + sandbox, err := nodeExec(pod.Spec.NodeName, "crictl", "pods", + "--name", pod.Name, "--namespace", ns, "-q", "--state", "ready") + if err != nil { + return 0, err + } + sandbox = strings.TrimSpace(strings.Split(strings.TrimSpace(sandbox), "\n")[0]) + cid, err := nodeExec(pod.Spec.NodeName, "crictl", "ps", + "--pod", sandbox, "--name", "gateway", "-q", "--state", "running") + if err != nil { + return 0, err + } + cid = strings.TrimSpace(strings.Split(strings.TrimSpace(cid), "\n")[0]) + out, err := nodeExec(pod.Spec.NodeName, "crictl", "inspect", "-o", "go-template", "--template", "{{.info.pid}}", cid) + if err != nil { + return 0, err + } + return strconv.Atoi(strings.TrimSpace(out)) +} + +// gauge polls a gateway pod's metric every sampleEvery until cond holds +// or timeout, returning when it held. +func (l *lab) gauge( + ctx context.Context, pod corev1.Pod, timeout time.Duration, cond func(text string) bool, +) (time.Time, bool) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if text, err := l.gatewayMetrics(ctx, pod); err == nil && cond(text) { + return time.Now(), true + } + time.Sleep(sampleEvery) + } + return time.Time{}, false +} + +// peakDuring returns the largest value of a gateway pod's gauge while fn +// runs. +func (l *lab) peakDuring(ctx context.Context, pod corev1.Pod, name string, fn func()) float64 { + stop := make(chan struct{}) + peak := make(chan float64, 1) + go func() { + best := 0.0 + for { + if text, err := l.gatewayMetrics(ctx, pod); err == nil { + if v, ok := metricValue(text, name); ok { + best = max(best, v) + } + } + select { + case <-stop: + peak <- best + return + case <-time.After(sampleEvery): + } + } + }() + fn() + close(stop) + return <-peak +} + +func gaugeAtMost(name string, limit float64) func(string) bool { + return func(text string) bool { + v, ok := metricValue(text, name) + return ok && v <= limit + } +} + +// at is r's offset ms as a wall time. +func at(r Record, ms float64) time.Time { + return r.StartedAt.Add(time.Duration(ms * float64(time.Millisecond))) +} + +// judgeStop checks packets toward dst stopped by bound and records it. +func judgeStop( + t *testing.T, ev *evidence, name string, samples []sample, dst string, origin time.Time, bound time.Duration, +) { + t.Helper() + stop, total := lastIncrease(samples, dst) + after := stop.Sub(origin) + // A negative stop is the last probe before the trigger: probes toward + // the sink go about 1 s apart, under every bound judged here, so a job + // still running would have sent another inside the bound. + ev.Observed[name] = map[string]any{"packets": total, "stoppedAfter": after.String(), "bound": bound.String()} + switch { + case total == 0: + t.Errorf("%s: no probe packet was sent toward %s, so nothing was stopped", name, dst) + case after > bound+sampleEvery: + t.Errorf("%s: packets stopped %s after the trigger, bound %s", name, after, bound) + } +} + +// cancellation: a cancelled request's running and queued router jobs stop +// within 2 s of the cancellation (A12). +func cancellation(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + var r Record + leader, _, err := l.leaderPod(ctx) + if err != nil { + t.Fatal(err) + } + running, queued := l.sinkHost(6), l.sinkHost(4) + if err := watch(running, queued); err != nil { + t.Fatal(err) + } + cnt := startCounter() + // No gauge counts service router jobs in their slots; the leader's live + // streams, which hold a request's jobs and buffers, stand in for them. + peak := l.peakDuring(ctx, leader, "fabric_api_gateway_service_streams", func() { + r = mustRun(ctx, t, l, idService, fProject, l.project(), fType, traceroute, fTarget, running, + fNodes, twoRouters, fExpires, "60s", "-cancel-after", "4s") + }) + ev.Observed["running: live streams before cancellation"] = peak + if peak < 1 { + t.Errorf("running: %v live streams before the cancellation, want 1", peak) + } + cancelAt := at(r, r.Timing.CancelMS) + idle, ok := l.gauge(ctx, leader, 5*time.Second, gaugeAtMost("fabric_api_gateway_service_streams", 0)) + ev.Observed["running: stream"] = brief(r) + ev.Observed["running: stream released after"] = idle.Sub(cancelAt).String() + if !ok || idle.Sub(cancelAt) > stopBound+sampleEvery { + t.Errorf("running: leader's live stream not released within %s of the cancellation", stopBound) + } + + // Three routers from one project: two run, one queues for a slot. + peak = l.peakDuring(ctx, leader, "fabric_api_gateway_queue_depth", func() { + r = mustRun(ctx, t, l, idService, fProject, l.project(), fType, traceroute, fTarget, queued, + fExpires, "60s", "-cancel-after", "3s") + }) + ev.Observed["queued: queue depth before cancellation"] = peak + if peak < 1 { + t.Errorf("queued: queue depth %v before the cancellation, want a queued router", peak) + } + queuedCancel := at(r, r.Timing.CancelMS) + empty, ok := l.gauge(ctx, leader, 5*time.Second, gaugeAtMost("fabric_api_gateway_queue_depth", 0)) + ev.Observed["queued: queue emptied after"] = empty.Sub(queuedCancel).String() + if !ok || empty.Sub(queuedCancel) > stopBound+sampleEvery { + t.Errorf("queued: leader's queue not emptied within %s of the cancellation", stopBound) + } + time.Sleep(5 * time.Second) + samples := cnt.close() + judgeStop(t, ev, "running: probes", samples, running, cancelAt, stopBound) + judgeStop(t, ev, "queued: probes", samples, queued, queuedCancel, stopBound) + sent := 0 + for _, n := range routers { + if nodeTotal(samples, n, queued) > 0 { + sent++ + } + } + ev.Observed["queued: routers that probed"] = sent + if sent != 2 { + t.Errorf("queued: %d routers probed, want the 2 holding slots", sent) + } + ev.Notes = append(ev.Notes, "Cancellation during discovery is not timed: discovery takes milliseconds, below "+ + "what an external client can aim a cancellation at.") +} + +// deadline: a request whose expiry passes mid-execution ends with +// DeadlineExceeded, and its probes stop within 2 s of expiry (A12). +func deadline(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + target := l.sinkHost(4) + if err := watch(target); err != nil { + t.Fatal(err) + } + cnt := startCounter() + const expiresIn = 6 * time.Second + r := mustRun(ctx, t, l, idService, fProject, l.project(), fType, traceroute, fTarget, target, + fNodes, twoRouters, fExpires, expiresIn.String(), "-deadline", "12s") + time.Sleep(4 * time.Second) + samples := cnt.close() + ev.Observed["stream"] = brief(r) + ev.Observed["stream ended after"] = time.Duration(r.Timing.EndMS * float64(time.Millisecond)).String() + // At expires_at the deadline no longer allows delivery, so the stream + // may end with complete DeadlineExceeded or, as the contract allows, + // with the transport's DEADLINE_EXCEEDED and the routers unknown. + switch { + case r.Stream.Reason == "DeadlineExceeded" && r.Stream.Delivered: + case r.Status.Code == "DeadlineExceeded" && r.Timing.SnapshotMS >= 0 && r.Stream.Complete == nil: + default: + t.Errorf("expired request: %s complete %q, want DeadlineExceeded", r.status(), r.Stream.Reason) + } + end := time.Duration(r.Timing.EndMS * float64(time.Millisecond)) + if end < expiresIn-200*time.Millisecond || end > expiresIn+time.Second { + t.Errorf("expired request ended %s after it started, want about %s", end, expiresIn) + } + judgeStop(t, ev, "probes", samples, target, r.StartedAt.Add(expiresIn), stopBound) +} + +// serviceSwitch: turning the switch off cancels running requests within +// 2 s of the read, refuses new ones, leaves operator diagnostics working, +// and turning it on admits again at once (A12, A15). +func serviceSwitch(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + t.Cleanup(func() { _ = l.setSwitch(context.Background(), "enabled") }) + target := l.sinkHost(6) + if err := watch(target); err != nil { + t.Fatal(err) + } + cnt := startCounter() + running := l.runAsync(ctx, fProject, l.project(), fType, traceroute, fTarget, target, + fNodes, twoRouters, fExpires, "60s") + time.Sleep(3 * time.Second) + offAt := time.Now() + if err := l.setSwitch(ctx, "disabled"); err != nil { + t.Fatal(err) + } + r := <-running + ev.Observed["running"] = brief(r) + expectDelivered(t, "running request", r, "Cancelled") + if end := r.endAt().Sub(offAt); end > switchRead+stopBound+500*time.Millisecond { + t.Errorf("running request ended %s after the switch, bound %s", end, switchRead+stopBound) + } + time.Sleep(2 * time.Second) + samples := cnt.close() + judgeStop(t, ev, "probes", samples, target, offAt, switchRead+stopBound) + + r = mustRun(ctx, t, l, idService, fProject, l.project(), fType, bgpSummary, fFamily, ipv4) + expectRefused(t, "new request while off", r, codeFailedPrecondition, "ServiceDisabled") + ev.Observed["new request while off"] = brief(r) + lines, err := l.lgc(ctx, idService, "capabilities") + if err != nil { + t.Fatal(err) + } + var caps struct { + Capabilities struct { + Enabled bool `json:"enabled"` + } `json:"capabilities"` + } + _ = json.Unmarshal(lines[0], &caps) + ev.Observed["capabilities enabled while off"] = caps.Capabilities.Enabled + if caps.Capabilities.Enabled { + t.Error("capabilities report enabled while the switch is off") + } + out, err := l.operator(ctx, l.viaGateway(bgpSummary, "--family", ipv4)...) + obs, codes, _ := operatorResults(out) + ev.Observed["operator while off"] = answers{Observations: obs, Errors: codes} + if err != nil || obs != len(routers) { + t.Errorf("operator BGPSummary while the service is off: %d observations, %v: %s", obs, err, out) + } + + onAt := time.Now() + if err := l.setSwitch(ctx, "enabled"); err != nil { + t.Fatal(err) + } + var first Record + for range 10 { + first = mustRun(ctx, t, l, idService, fProject, l.project(), fType, bgpSummary, fFamily, ipv4) + if first.ok() { + break + } + } + ev.Observed["admitting again after"] = first.StartedAt.Sub(onAt).String() + if !first.ok() || first.StartedAt.Sub(onAt) > switchRead+3*pace { + t.Errorf("after switching on: %s at %s, want admission within %s", + first.status(), first.StartedAt.Sub(onAt), switchRead+3*pace) + } +} + +// blackhole: a client whose connection silently stops passing packets has +// its request's probes stopped within transport liveness detection plus +// cleanup (A12). +func blackhole(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + target := l.sinkHost(4) + if err := watch(target); err != nil { + t.Fatal(err) + } + gws, err := l.gatewayPods(ctx) + if err != nil { + t.Fatal(err) + } + cnt := startCounter() + reqID := "00000000-0000-4000-8000-" + fmt.Sprintf("%012x", time.Now().UnixNano()&0xffffffffffff) + done := l.runAsync(ctx, fProject, l.project(), "-request-id", reqID, fType, traceroute, fTarget, target, + fNodes, twoRouters, fExpires, "60s") + time.Sleep(3 * time.Second) + port, err := clientPort() + if err != nil { + t.Fatal(err) + } + rules := []string{ + fmt.Sprintf("INPUT -p tcp --dport %d -j DROP", port), + fmt.Sprintf("OUTPUT -p tcp --sport %d -j DROP", port), + } + blackAt := time.Now() + for _, rule := range rules { + if _, err := nodeExec(clientNode, "sh", "-c", "ip6tables -w -I "+rule); err != nil { + t.Fatal(err) + } + } + unblock := func() { + for _, rule := range rules { + _, _ = nodeExec(clientNode, "sh", "-c", "ip6tables -w -D "+rule) + } + } + t.Cleanup(unblock) + // The client is unaware; streams on either replica drain once the + // gateway detects the dead peer. + var released time.Time + for time.Since(blackAt) < 2*blackholeBound { + total := 0.0 + for _, p := range gws { + if text, err := l.gatewayMetrics(ctx, p); err == nil { + v, _ := metricValue(text, "fabric_api_gateway_service_streams") + total += v + } + } + if total == 0 { + released = time.Now() + break + } + time.Sleep(sampleEvery) + } + time.Sleep(2 * time.Second) + samples := cnt.close() + unblock() + _, _ = nodeExec(clientNode, "pkill", "-f", reqID) + r := <-done + ev.Observed["client"] = brief(r) + ev.Observed["client port"] = port + judgeStop(t, ev, "probes", samples, target, blackAt, blackholeBound) + if released.IsZero() || released.Sub(blackAt) > blackholeBound+sampleEvery { + t.Errorf("service streams not released within %s of the blackhole", blackholeBound) + } else { + ev.Observed["streams released after"] = released.Sub(blackAt).String() + } +} + +// clientPort returns the local port of the client's connection to the +// gateway on clientNode. +func clientPort() (int, error) { + out, err := nodeExec(clientNode, "ss", "-tnpH", "state", "established", fmt.Sprintf("( dport = :%d )", gatewayPort)) + if err != nil { + return 0, err + } + for line := range strings.SplitSeq(out, "\n") { + if !strings.Contains(line, `"lgc"`) { + continue + } + f := strings.Fields(line) + if len(f) < 3 { + continue + } + local := f[2] + p, err := strconv.Atoi(local[strings.LastIndex(local, ":")+1:]) + if err == nil { + return p, nil + } + } + return 0, fmt.Errorf("no client connection in %q", out) +} + +// failoverWatch measures one leadership change: the new leader's +// acquisition, its recovery hold (no admission, no probe packet from an +// operator or the service), and when admission resumes. +func failoverWatch(ctx context.Context, t *testing.T, l *lab, ev *evidence, oldHolder string) { + newHolder, acquired, err := l.waitNewLeader(ctx, oldHolder, 90*time.Second) + if err != nil { + t.Fatal(err) + } + ev.Observed["new leader"] = newHolder + ev.Observed["acquired"] = acquired + probe := l.sinkHost(6) + if err := watch(probe); err != nil { + t.Fatal(err) + } + cnt := startCounter() + var firstOK time.Time + refusals := map[string]int{} + lastOperator := time.Time{} + for time.Since(acquired) < recoveryHold+resumeSlack+10*time.Second { + r, err := l.run(ctx, idService, fProject, l.project(), fType, ping, fTarget, probe) + if err == nil && r.ok() { + firstOK = r.StartedAt + break + } + refusals[r.status()]++ + if time.Since(lastOperator) > 15*time.Second { + out, _ := l.operator(ctx, l.viaGateway(ping, probe)...) + _, codes, _ := operatorResults(out) + for _, c := range codes { + refusals["operator/"+c]++ + } + lastOperator = time.Now() + } + time.Sleep(3 * time.Second) + } + samples := cnt.close() + held := between(samples, probe, acquired, acquired.Add(recoveryHold)) + ev.Observed["hold refusals"] = refusals + ev.Observed["hold probe packets"] = held + ev.Observed["first admission after acquire"] = firstOK.Sub(acquired).String() + switch { + case firstOK.IsZero(): + t.Errorf("no admission within %s of the new leader's acquisition", recoveryHold+resumeSlack+10*time.Second) + case firstOK.Before(acquired.Add(recoveryHold - 500*time.Millisecond)): + t.Errorf("admitted %s after acquisition, inside the %s recovery hold", firstOK.Sub(acquired), recoveryHold) + case firstOK.After(acquired.Add(recoveryHold + resumeSlack)): + t.Errorf("first admission %s after acquisition, later than %s", firstOK.Sub(acquired), recoveryHold+resumeSlack) + } + if held > 0 { + t.Errorf("%d probe packets sent during the recovery hold", held) + } +} + +// leaderCrash: the leader process is killed while its request runs. The +// running probes end by their node deadline, the stream is not replayed, +// and the replacement serves its hold (A11, A14). +func leaderCrash(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + leader, holder, err := l.leaderPod(ctx) + if err != nil { + t.Fatal(err) + } + // The isolated gateway exits when it loses the Lease and its container + // restarts in the same pod, so the rules go in the pod sandbox's + // network namespace, which outlives every container. + pid, err := sandboxPID(leader) + if err != nil { + t.Fatal(err) + } + target := l.sinkHost(4) + if err := watch(target); err != nil { + t.Fatal(err) + } + cnt := startCounter() + running := l.runAsync(ctx, fProject, l.project(), fType, traceroute, fTarget, target, + fNodes, twoRouters, fExpires, "60s") + time.Sleep(3 * time.Second) + crashAt := time.Now() + if _, err := nodeExec(leader.Spec.NodeName, "kill", "-9", strconv.Itoa(pid)); err != nil { + t.Fatal(err) + } + ev.Observed["crashed"] = fault{Pod: leader.Name, At: crashAt} + r := <-running + ev.Observed["running stream"] = brief(r) + if r.Stream.Delivered && r.Stream.Reason == "Succeeded" { + t.Error("the crashed leader's request claims full delivery") + } + failoverWatch(ctx, t, l, ev, holder) + samples := cnt.close() + dispatch := at(r, r.Timing.SnapshotMS) + if r.Timing.SnapshotMS < 0 { + dispatch = crashAt.Add(-3 * time.Second) + } + judgeStop(t, ev, "running probes", samples, target, dispatch, maxEmission) +} + +// leaderPause: the leader is stopped, replaced, then resumed. Nothing it +// had queued or dispatched runs after it resumes (A14). +func leaderPause(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + leader, holder, err := l.leaderPod(ctx) + if err != nil { + t.Fatal(err) + } + pid, err := gatewayPID(leader) + if err != nil { + t.Fatal(err) + } + target := l.sinkHost(6) + if err := watch(target); err != nil { + t.Fatal(err) + } + cnt := startCounter() + // Three routers from one project: two dispatched, one queued when the + // leader stops. + running := l.runAsync(ctx, fProject, l.project(), fType, traceroute, fTarget, target, fExpires, "60s") + time.Sleep(3 * time.Second) + pausedAt := time.Now() + if _, err := nodeExec(leader.Spec.NodeName, "kill", "-STOP", strconv.Itoa(pid)); err != nil { + t.Fatal(err) + } + resume := func() { _, _ = nodeExec(leader.Spec.NodeName, "kill", "-CONT", strconv.Itoa(pid)) } + t.Cleanup(resume) + newHolder, acquired, err := l.waitNewLeader(ctx, holder, 90*time.Second) + if err != nil { + resume() + t.Fatal(err) + } + time.Sleep(5 * time.Second) + resumedAt := time.Now() + resume() + ev.Observed["paused"] = fault{Pod: leader.Name, At: pausedAt} + ev.Observed["new leader"] = newHolder + ev.Observed["acquired after pause"] = acquired.Sub(pausedAt).String() + ev.Observed["resumed after pause"] = resumedAt.Sub(pausedAt).String() + time.Sleep(maxEmission + 3*time.Second) + samples := cnt.close() + r := <-running + ev.Observed["running stream"] = brief(r) + late := between(samples, target, resumedAt, time.Now()) + probed := 0 + for _, n := range routers { + if nodeTotal(samples, n, target) > 0 { + probed++ + } + } + ev.Observed["packets after resume"] = late + ev.Observed["routers that probed"] = probed + if late > 0 { + t.Errorf("%d probe packets sent after the replaced leader resumed", late) + } + if probed > 2 { + t.Errorf("%d routers probed: the paused leader's queued router was dispatched", probed) + } + if nm, err := l.nodeMetrics(ctx); err == nil { + permits := map[string]string{} + for node, text := range nm { + for line := range strings.SplitSeq(text, "\n") { + if strings.HasPrefix(line, "fabric_api_node_permits_total{") { + permits[node] += line + "\n" + } + } + } + ev.Observed["node permit outcomes"] = permits + } + // The new leader's hold is measured from its acquisition, which is + // already past; wait it out so the next scenario starts admitting. + if _, err := l.waitReady(ctx, recoveryHold+resumeSlack); err != nil { + t.Fatal(err) + } +} + +// apiIsolation: the leader loses the API server while its request runs. +// It stops admitting and dispatching on detection, its running probes stop +// within 2 s of detection, its queued router never runs, and the new +// leader serves its hold (A14). +func apiIsolation(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + leader, holder, err := l.leaderPod(ctx) + if err != nil { + t.Fatal(err) + } + // The isolated gateway exits when it loses the Lease and its container + // restarts in the same pod, so the rules go in the pod sandbox's + // network namespace, which outlives every container. + pid, err := sandboxPID(leader) + if err != nil { + t.Fatal(err) + } + target := l.sinkHost(4) + if err := watch(target); err != nil { + t.Fatal(err) + } + cnt := startCounter() + running := l.runAsync(ctx, fProject, l.project(), fType, traceroute, fTarget, target, fExpires, "60s") + time.Sleep(3 * time.Second) + rules := make([]string, 0, 4) + for _, ipt := range []string{iptables, ip6tables} { + for _, port := range []string{"443", "6443"} { + rules = append(rules, fmt.Sprintf("%s -w -%%s OUTPUT -p tcp --dport %s -j DROP", ipt, port)) + } + } + nsenter := func(action string) error { + for _, rule := range rules { + if _, err := nodeExec(leader.Spec.NodeName, "nsenter", "-t", strconv.Itoa(pid), "-n", "sh", "-c", + fmt.Sprintf(rule, action)); err != nil { + return err + } + } + return nil + } + isolatedAt := time.Now() + if err := nsenter("I"); err != nil { + t.Fatal(err) + } + var restoreOnce sync.Once + restore := func() { + restoreOnce.Do(func() { + if err := nsenter("D"); err != nil { + t.Errorf("remove the isolation rules from %s: %v", leader.Name, err) + } + }) + } + t.Cleanup(restore) + detected, ok := l.gauge(ctx, leader, 60*time.Second, func(text string) bool { + v, found := metricValue(text, "fabric_api_gateway_limiter_state", `state="ready"`) + return found && v == 0 + }) + ev.Observed["isolated"] = fault{Pod: leader.Name, At: isolatedAt} + if ok { + ev.Observed["detected after"] = detected.Sub(isolatedAt).String() + } else { + t.Error("the isolated leader's limiter never left ready") + } + // New work is refused from detection on; a Lease read already under + // way at detection ends within its bound. + time.Sleep(leaseReadBound) + r, err := l.run(ctx, idService, fProject, l.project(), fType, bgpSummary, fFamily, ipv4) + if err == nil { + ev.Observed["request after detection"] = brief(r) + if r.ok() { + t.Error("a request was admitted after the leader lost the API") + } + } + newHolder, acquired, err := l.waitNewLeader(ctx, holder, 90*time.Second) + if err != nil { + t.Fatal(err) + } + ev.Observed["new leader"] = map[string]any{"holder": newHolder, "acquiredAfter": acquired.Sub(isolatedAt).String()} + time.Sleep(10 * time.Second) + restore() + ev.Observed["restored after"] = time.Since(isolatedAt).String() + run := <-running + ev.Observed["running stream"] = brief(run) + time.Sleep(maxEmission) + samples := cnt.close() + dispatch := at(run, run.Timing.SnapshotMS) + if run.Timing.SnapshotMS < 0 { + dispatch = isolatedAt.Add(-3 * time.Second) + } + if ok { + judgeStop(t, ev, "running probes after detection", samples, target, detected, stopBound) + } + judgeStop(t, ev, "running probes after dispatch", samples, target, dispatch, maxEmission) + probed := 0 + for _, n := range routers { + if nodeTotal(samples, n, target) > 0 { + probed++ + } + } + ev.Observed["routers that probed"] = probed + if probed > 2 { + t.Errorf("%d routers probed: the isolated leader's queued router was dispatched", probed) + } + if _, err := l.waitReady(ctx, recoveryHold+resumeSlack); err != nil { + t.Fatal(err) + } +} + +// cleanup: no fault the suite injected outlives it. No dfw node, nor any +// gateway pod, keeps a DROP rule in its INPUT or OUTPUT chain, and every +// gateway and fabric-router pod is Ready again within cleanupReadyBound +// (a container that crashed while isolated may wait out its restart +// back-off). +func cleanup(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + const list = "for t in iptables ip6tables; do $t -w -S INPUT; $t -w -S OUTPUT; done" + residue := map[string][]string{} + scan := func(where, node string, args ...string) { + out, err := nodeExec(node, append(args, "sh", "-c", list)...) + if err != nil { + t.Errorf("list %s's rules: %v", where, err) + return + } + for line := range strings.SplitSeq(out, "\n") { + if strings.Contains(line, "-j DROP") { + residue[where] = append(residue[where], strings.TrimSpace(line)) + } + } + } + for _, n := range routers { + scan(n, n) + } + gws, err := l.gatewayPods(ctx) + if err != nil { + t.Fatal(err) + } + for _, p := range gws { + pid, err := sandboxPID(p) + if err != nil { + t.Errorf("sandbox of %s: %v", p.Name, err) + continue + } + scan(p.Name, p.Spec.NodeName, "nsenter", "-t", strconv.Itoa(pid), "-n") + } + ev.Observed["rules left"] = residue + for where, rules := range residue { + t.Errorf("%s keeps %d DROP rules, e.g. %s", where, len(rules), rules[0]) + } + + start := time.Now() + var notReady []string + for { + notReady = notReady[:0] + for _, sel := range []string{"app.kubernetes.io/name=fabric-api-gateway", fabricRouterSelector} { + pods, err := l.cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: sel}) + if err != nil { + t.Fatal(err) + } + for _, p := range pods.Items { + if !podReady(p) { + notReady = append(notReady, p.Name) + } + } + } + if len(notReady) == 0 || time.Since(start) > cleanupReadyBound { + break + } + time.Sleep(5 * time.Second) + } + ev.Observed["pods ready after"] = time.Since(start).Round(time.Second).String() + if len(notReady) > 0 { + t.Errorf("pods not Ready %s after the run: %v", cleanupReadyBound, notReady) + } +} diff --git a/tests/lgqualify/functional_test.go b/tests/lgqualify/functional_test.go new file mode 100644 index 000000000..36d03512f --- /dev/null +++ b/tests/lgqualify/functional_test.go @@ -0,0 +1,932 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lgqualify_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os/exec" + "slices" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// authorization: only an allowed same-cell service identity reaches the +// service, only its two methods, and forwarding metadata it sends is +// ignored (A1, A2). +func authorization(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + q := []string{fType, bgpSummary, fFamily, ipv4} + r := mustRun(ctx, t, l, idService, append([]string{fProject, l.project()}, q...)...) + expectDelivered(t, "allowed service", r, "Succeeded") + ev.Observed["allowed service"] = brief(r) + + for _, c := range []struct { + name string + id identity + code string + }{ + {"no client certificate", idNone, ""}, + {"expired certificate", idExpired, ""}, + {"service of another cell", idOtherCell, ""}, + {"service not allowlisted", idUnlisted, ""}, + {"operator identity", idOperator, "PermissionDenied"}, + } { + r := mustRun(ctx, t, l, c.id, append([]string{fProject, l.project()}, q...)...) + expectRefused(t, c.name, r, c.code, "") + ev.Observed[c.name] = brief(r) + lines, err := l.lgc(ctx, c.id, "capabilities") + if err != nil { + t.Fatal(err) + } + var caps struct { + Status struct{ Code string } `json:"status"` + Capabilities json.RawMessage `json:"capabilities"` + } + _ = json.Unmarshal(lines[0], &caps) + if caps.Status.Code == "OK" || caps.Capabilities != nil { + t.Errorf("%s: capabilities answered: %s", c.name, lines[0]) + } + ev.Observed[c.name+" capabilities"] = lines[0] + } + + r = mustRun(ctx, t, l, idService, append([]string{fProject, l.project(), "-request-cell", "iad"}, q...)...) + expectRefused(t, "request for another cell", r, "PermissionDenied", "Unauthorized") + ev.Observed["request for another cell"] = brief(r) + + // Every other method on the gateway's listener is closed to the + // service, the leader-only forwarding method included. + closed := map[string]any{} + for _, m := range []string{ + "/fabric.v1.FabricGatewayService/Query", "/fabric.v1.FabricGatewayService/ListNodes", + "/fabric.v1.FabricService/Execute", "/fabric.v1.FabricService/Info", + "/fabric.forward.v1.ForwardService/ForwardDiagnostic", + "/fabric.permit.v1.PermitService/RequestPermit", "/fabric.permit.v1.PermitService/ReleasePermit", + } { + lines, err := l.lgc(ctx, idService, "invoke", "--method", m) + if err != nil { + t.Fatal(err) + } + var res struct { + Status struct{ Code, Reason string } `json:"status"` + } + _ = json.Unmarshal(lines[0], &res) + closed[m] = res.Status + if res.Status.Code != "PermissionDenied" && res.Status.Code != "Unimplemented" { + t.Errorf("%s: %s/%s, want PermissionDenied", m, res.Status.Code, res.Status.Reason) + } + } + ev.Observed["other methods"] = closed + + // Forged forwarding metadata: the request runs under its own + // attribution, which the leader's log line records. + project, since := l.project(), time.Now() + r = mustRun(ctx, t, l, idService, append([]string{fProject, project, "-metadata", + "x-fabric-api-project=lgq-forged,x-fabric-api-service=spiffe://fabric-api.datumapis.com/cell/dfw/service/forged," + + "x-fabric-api-request-id=00000000-0000-4000-8000-000000000000,x-fabric-api-deadline=2099-01-01T00:00:00Z"}, q...)...) + expectDelivered(t, "forged forwarding metadata", r, "Succeeded") + ev.Observed["forged forwarding metadata"] = brief(r) + line, err := waitFinished(ctx, l, r.RequestID, since) + if err != nil { + t.Error(err) + } else { + ev.Observed["forged forwarding metadata log"] = line + text, _ := json.Marshal(line) + if !bytes.Contains(text, []byte(project)) || bytes.Contains(text, []byte("forged")) { + t.Errorf("forged metadata changed attribution: %s", text) + } + } + + // One connection carries several streams in turn. + recs, _, err := l.load(ctx, spec{Groups: []group{{ + Name: "reused-connection", Count: 3, Concurrency: 1, Interval: "600ms", + Projects: []string{l.project(), l.project(), l.project()}, + Options: options{Query: query{Type: bgpSummary, Family: ipv6}}, + }}}) + if err != nil { + t.Fatal(err) + } + for _, r := range recs { + expectDelivered(t, "reused connection", r, "Succeeded") + } + ev.Observed["reused connection"] = len(recs) + + rotation(ctx, t, l, ev, q) + ev.Notes = append(ev.Notes, "CA rotation (overlapping bundle) is not exercised on the lab: rotating the "+ + "lab CA re-keys every node and gateway identity in the cell; the reload/overlap mechanism is covered "+ + "by internal/fabric/identity tests.") +} + +// rotation reissues the service certificate and checks the new one and +// the still-valid old one are both accepted. +func rotation(ctx context.Context, t *testing.T, l *lab, ev *evidence, q []string) { + const secret = "lg-qualify-looking-glass" + old, err := l.cs.CoreV1().Secrets(ns).Get(ctx, secret, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if _, err := l.kubectl("-n", ns, "delete", "secret", secret); err != nil { + t.Fatal(err) + } + var renewed *corev1.Secret + for range 60 { + time.Sleep(time.Second) + s, err := l.cs.CoreV1().Secrets(ns).Get(ctx, secret, metav1.GetOptions{}) + if err == nil && len(s.Data[certFile]) > 0 && !bytes.Equal(s.Data[certFile], old.Data[certFile]) { + renewed = s + break + } + } + if renewed == nil { + t.Fatal("cert-manager did not reissue the service certificate within 60 s") + } + for _, f := range []string{certFile, keyFile, caFile} { + if err := clientWrite(fmt.Sprintf("%s/rotated/%s", clientDir, f), renewed.Data[f], 0o600); err != nil { + t.Fatal(err) + } + } + for _, id := range []identity{"rotated", idService} { + r := mustRun(ctx, t, l, id, append([]string{fProject, l.project()}, q...)...) + expectDelivered(t, "certificate "+string(id), r, "Succeeded") + ev.Observed["rotation: "+string(id)] = brief(r) + } +} + +// waitFinished polls for the leader's log line of requestID. +func waitFinished(ctx context.Context, l *lab, requestID string, since time.Time) (map[string]any, error) { + var err error + for range 10 { + var line map[string]any + if line, err = l.finishedLine(ctx, requestID, since); err == nil { + return line, nil + } + time.Sleep(time.Second) + } + return nil, err +} + +// compatibility: an incompatible major, unknown feature or unsupported +// operation is refused before any work, never downgraded (A7). +func compatibility(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + lines, err := l.lgc(ctx, idService, "capabilities", "--api-major", "2") + if err != nil { + t.Fatal(err) + } + var caps struct { + Status struct{ Code, Reason string } `json:"status"` + } + _ = json.Unmarshal(lines[0], &caps) + if caps.Status.Code != codeFailedPrecondition || caps.Status.Reason != "VersionUnsupported" { + t.Errorf("capabilities api_major 2: %s", lines[0]) + } + ev.Observed["capabilities api_major 2"] = caps.Status + + for _, c := range []struct { + name string + args []string + code, reason string + }{ + { + "api_major 2", []string{"-api-major", "2", fType, bgpSummary, fFamily, ipv4}, + codeFailedPrecondition, "VersionUnsupported", + }, + { + "unknown required feature", []string{"-require", "teleport", fType, bgpSummary, fFamily, ipv4}, + codeFailedPrecondition, "", + }, + {"ASPath", []string{fType, "ASPath", fTarget, "_65000_"}, "", ""}, + {"Community", []string{fType, "Community", fTarget, "65000:1"}, "", ""}, + {"unknown operation enum", []string{fType, "enum:99"}, "", ""}, + } { + r := mustRun(ctx, t, l, idService, append([]string{fProject, l.project()}, c.args...)...) + expectRefused(t, c.name, r, c.code, c.reason) + ev.Observed[c.name] = brief(r) + } + r := mustRun(ctx, t, l, idService, fProject, l.project(), fType, bgpSummary, fFamily, ipv4, + "-require", "router-outcomes,terminal-coverage,aggregate-probe-budgets") + expectDelivered(t, "every served feature required", r, "Succeeded") + ev.Observed["every served feature required"] = brief(r) + ev.Notes = append(ev.Notes, "The client decodes the contract's published fixtures "+ + "(api/fabric/service/v1/testdata) in its own module's tests: examples/looking-glass-client/internal/stream.") +} + +// event is the part of a delivered event the payload checks read. +type event struct { + Outcome *struct { + Node string `json:"node"` + Observation *struct { + Truncated bool `json:"truncated"` + Ping *struct { + Sent int `json:"sent"` + Received int `json:"received"` + } `json:"ping"` + Traceroute *struct { + Reached bool `json:"reached"` + Hops []json.RawMessage `json:"hops"` + } `json:"traceroute"` + Summary *struct { + Peers []json.RawMessage `json:"peers"` + } `json:"summary"` + Routes json.RawMessage `json:"routes"` + } `json:"observation"` + } `json:"outcome"` +} + +func observations(events []json.RawMessage) []event { + var out []event + for _, raw := range events { + var e event + if json.Unmarshal(raw, &e) == nil && e.Outcome != nil && e.Outcome.Observation != nil { + out = append(out, e) + } + } + return out +} + +// diagnostics: all four operations on both families return typed +// observations from every router; loss, silent hops and an empty lookup +// are observations, not failures (A3, A6). +func diagnostics(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + type check func(e event) string + pingReplies := func(want int) check { + return func(e event) string { + if p := e.Outcome.Observation.Ping; p == nil || p.Sent != pingPackets || p.Received != want { + return fmt.Sprintf("ping %+v, want %d of %d replies", p, want, pingPackets) + } + return "" + } + } + reached := func(want bool) check { + return func(e event) string { + if tr := e.Outcome.Observation.Traceroute; tr == nil || tr.Reached != want || len(tr.Hops) == 0 { + return fmt.Sprintf("traceroute %+v, want reached=%t", tr, want) + } + return "" + } + } + summary := func(e event) string { + if s := e.Outcome.Observation.Summary; s == nil || len(s.Peers) == 0 { + return "summary has no peers" + } + return "" + } + routes := func(e event) string { + if len(e.Outcome.Observation.Routes) == 0 { + return "lookup has no route result" + } + return "" + } + // queued marks a three-router traceroute that outlasts the queue wait: + // a project holds two node slots, so its third router waits and fails + // QueueTimeout at 10 s, the default behaviour the contract requires + // qualification to report. + cases := []struct { + name string + args []string + check check + queued bool + }{ + {"BGPSummary IPv4", []string{fType, bgpSummary, fFamily, ipv4}, summary, false}, + {"BGPSummary IPv6", []string{fType, bgpSummary, fFamily, ipv6}, summary, false}, + {"RouteLookup IPv4 longest match", []string{fType, routeLookup, fTarget, answer4}, routes, false}, + {"RouteLookup IPv6 longest match", []string{fType, routeLookup, fTarget, answer6}, routes, false}, + {"RouteLookup IPv4 exact", []string{fType, routeLookup, fTarget, "11.1.40.0/24"}, routes, false}, + {"RouteLookup IPv6 exact", []string{fType, routeLookup, fTarget, "2c0f:40::/64"}, routes, false}, + {"RouteLookup with no route", []string{fType, routeLookup, fTarget, "198.51.100.7"}, routes, false}, + {"Ping IPv4", []string{fType, ping, fTarget, answer4}, pingReplies(pingPackets), false}, + {"Ping IPv6", []string{fType, ping, fTarget, answer6}, pingReplies(pingPackets), false}, + {"Traceroute IPv4", []string{fType, traceroute, fTarget, answer4}, reached(true), false}, + {"Traceroute IPv6", []string{fType, traceroute, fTarget, answer6}, reached(true), false}, + {"Ping IPv4 with loss", []string{fType, ping, fTarget, l.sinkHost(4)}, pingReplies(0), false}, + {"Ping IPv6 with loss", []string{fType, ping, fTarget, l.sinkHost(6)}, pingReplies(0), false}, + { + "Traceroute IPv4 with silent hops", []string{fType, traceroute, fTarget, l.sinkHost(4), fExpires, "60s"}, + reached(false), true, + }, + { + "Traceroute IPv6 with silent hops", []string{fType, traceroute, fTarget, l.sinkHost(6), fExpires, "60s"}, + reached(false), true, + }, + } + sizes := map[string]int{} + for _, c := range cases { + time.Sleep(pace) + r, events, err := l.runEvents(ctx, idService, append([]string{fProject, l.project()}, c.args...)...) + if err != nil { + t.Fatal(err) + } + ev.Observed[c.name] = brief(r) + sizes[c.name] = r.MaxEventBytes + obs := observations(events) + want := len(routers) + if c.queued { + want-- + expectDelivered(t, c.name, r, "PartialResults") + checkQueueTimeout(t, c.name, r, ev) + } else { + expectDelivered(t, c.name, r, "Succeeded") + } + if len(obs) != want { + t.Errorf("%s: %d observations, want %d", c.name, len(obs), want) + } + for _, e := range obs { + if msg := c.check(e); msg != "" { + t.Errorf("%s on %s: %s", c.name, e.Outcome.Node, msg) + } + } + } + ev.Observed["largest event bytes"] = sizes +} + +// checkQueueTimeout checks r holds exactly one QueueTimeout router outcome, +// delivered within the queue wait (measured from the snapshot) plus 1.5 s. +func checkQueueTimeout(t *testing.T, name string, r Record, ev *evidence) { + t.Helper() + n := 0 + for i, o := range r.Stream.Outcomes { + if o.Result != failure { + continue + } + n++ + after := time.Duration((r.Timing.OutcomeMS[i] - r.Timing.SnapshotMS) * float64(time.Millisecond)) + ev.Observed[name+" queue timeout after snapshot"] = after.String() + if o.Reason != "QueueTimeout" || after < queueWait-500*time.Millisecond || after > queueWait+1500*time.Millisecond { + t.Errorf("%s: %s failed %s %s after the snapshot, want QueueTimeout at %s", name, o.Node, o.Reason, after, queueWait) + } + } + if n != 1 { + t.Errorf("%s: %d failed routers, want 1", name, n) + } +} + +// coverage: fixed membership and exact counts for an explicit subset, an +// unreachable router and an omitted one; a fast router's outcome is not +// held behind a slow one (A4, A5). +func coverage(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + r := mustRun(ctx, t, l, idService, fProject, l.project(), fType, bgpSummary, fFamily, ipv4, fNodes, routers[0]) + expectDelivered(t, "explicit subset", r, "Succeeded") + if c := r.Stream.Complete; c == nil || c.Expected != 3 || c.Successful != 1 || c.Omitted != 2 || + r.Stream.OmittedReasons[routers[1]] != reasonNotRequested || + r.Stream.OmittedReasons[routers[2]] != reasonNotRequested { + t.Errorf("explicit subset: %+v %v", r.Stream.Complete, r.Stream.OmittedReasons) + } + ev.Observed["explicit subset"] = brief(r) + + many := make([]string, 33) + for i := range many { + many[i] = fmt.Sprintf("router-%d", i) + } + for _, c := range []struct { + name string + nodes string + }{ + {"unknown router", "no-such-router"}, + {"duplicate router", routers[0] + "," + routers[0]}, + {"33 routers", strings.Join(many, ",")}, + } { + r := mustRun(ctx, t, l, idService, fProject, l.project(), fType, bgpSummary, fFamily, ipv4, fNodes, c.nodes) + expectRefused(t, c.name, r, "InvalidArgument", reasonInvalidQuery) + ev.Observed[c.name] = brief(r) + } + + unreachableRouter(ctx, t, l, ev) + omittedRouter(ctx, t, l, ev) +} + +// unreachableRouter drops the gateway's connections to dfw-worker3's +// sidecar and checks the other routers' outcomes arrive first, with the +// unreachable one a failure, not an omission. +func unreachableRouter(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + slow := routers[2] + rule := "-p tcp --dport 9344 -j DROP" + for _, ipt := range []string{iptables, ip6tables} { + if _, err := nodeExec(slow, "sh", "-c", fmt.Sprintf("%s -w -I INPUT %s", ipt, rule)); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { + for _, ipt := range []string{iptables, ip6tables} { + _, _ = nodeExec(slow, "sh", "-c", fmt.Sprintf("%s -w -D INPUT %s", ipt, rule)) + } + }) + r := mustRun(ctx, t, l, idService, fProject, l.project(), fType, bgpSummary, fFamily, ipv4, fExpires, "60s") + for _, ipt := range []string{iptables, ip6tables} { + _, _ = nodeExec(slow, "sh", "-c", fmt.Sprintf("%s -w -D INPUT %s", ipt, rule)) + } + expectDelivered(t, "unreachable router", r, "PartialResults") + ev.Observed["unreachable router"] = brief(r) + var fastLast, slowAt float64 = -1, -1 + for i, o := range r.Stream.Outcomes { + at := r.Timing.OutcomeMS[i] + switch { + case o.Node == slow && o.Result == failure: + slowAt = at + case o.Node != slow && o.Result == observation: + fastLast = max(fastLast, at) + default: + t.Errorf("unreachable router: unexpected outcome %s:%s:%s", o.Node, o.Result, o.Reason) + } + } + ev.Observed["unreachable router timing"] = map[string]float64{ + "lastReachableOutcomeMs": fastLast, "unreachableOutcomeMs": slowAt, + } + if fastLast < 0 || slowAt < 0 || slowAt-fastLast < 1000 { + t.Errorf("reachable outcomes at %.0f ms were not delivered ahead of the unreachable router's at %.0f ms", + fastLast, slowAt) + } +} + +// omittedRouter keeps the fabric-api-certs pod off dfw-worker3 and checks +// that router is omitted, not failed, and the result partial. +func omittedRouter(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + target := routers[2] + dsc := l.cs.AppsV1().DaemonSets(ns) + ds, err := dsc.Get(ctx, "fabric-api-certs", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + original := ds.Spec.Template.Spec.Affinity.DeepCopy() + restore := func() { + cur, err := dsc.Get(context.Background(), "fabric-api-certs", metav1.GetOptions{}) + if err != nil { + t.Errorf("restore fabric-api-certs: %v", err) + return + } + cur.Spec.Template.Spec.Affinity = original + if _, err := dsc.Update(context.Background(), cur, metav1.UpdateOptions{}); err != nil { + t.Errorf("restore fabric-api-certs: %v", err) + } + } + t.Cleanup(restore) + excludeNode(ds, target) + if _, err := dsc.Update(ctx, ds, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := l.certsRolledOut(); err != nil { + t.Fatal(err) + } + others := func(nodes []string) bool { return !slices.Contains(nodes, target) && len(nodes) == len(routers)-1 } + if err := waitCertsPods(ctx, l, others); err != nil { + t.Fatal(err) + } + if err := l.waitRouters(ctx, routers[:2]); err != nil { + t.Fatal(err) + } + r := mustRun(ctx, t, l, idService, fProject, l.project(), fType, bgpSummary, fFamily, ipv4) + expectDelivered(t, "omitted router", r, "PartialResults") + if reason := r.Stream.OmittedReasons[target]; reason == "" || reason == reasonNotRequested { + t.Errorf("omitted router: %s omitted for %q", target, reason) + } + if c := r.Stream.Complete; c == nil || c.Expected != 3 || c.Successful != 2 || c.Omitted != 1 { + t.Errorf("omitted router: counts %+v", r.Stream.Complete) + } + ev.Observed["omitted router"] = brief(r) + ev.Observed["omitted reason"] = r.Stream.OmittedReasons[target] + restore() + if err := l.certsRolledOut(); err != nil { + t.Fatal(err) + } + if err := l.waitRouters(ctx, routers); err != nil { + t.Fatal(err) + } +} + +// certsRolledOut waits for the fabric-api-certs DaemonSet's rollout. +func (l *lab) certsRolledOut() error { + _, err := l.kubectl("-n", ns, "rollout", "status", "daemonset/fabric-api-certs", "--timeout=180s") + return err +} + +// waitRouters waits until a summary answers from exactly the routers in +// want, which happens once each sidecar has loaded its certs pod's new +// certificate. It runs no probe, so it charges no packet budget. +func (l *lab) waitRouters(ctx context.Context, want []string) error { + var last Record + for range 60 { + time.Sleep(3 * time.Second) + r, err := l.run(ctx, idService, fProject, l.project(), fType, bgpSummary, fFamily, ipv4) + if err != nil { + return err + } + last = r + var got []string + for _, o := range r.Stream.Outcomes { + if o.Result == observation { + got = append(got, o.Node) + } + } + slices.Sort(got) + if slices.Equal(got, want) { + return nil + } + } + return fmt.Errorf("routers did not converge on %v: last %v", want, outcomeSummary(last)) +} + +// excludeNode adds a required node affinity term keeping ds off node. +func excludeNode(ds *appsv1.DaemonSet, node string) { + spec := &ds.Spec.Template.Spec + if spec.Affinity == nil { + spec.Affinity = &corev1.Affinity{} + } + if spec.Affinity.NodeAffinity == nil { + spec.Affinity.NodeAffinity = &corev1.NodeAffinity{} + } + na := spec.Affinity.NodeAffinity + if na.RequiredDuringSchedulingIgnoredDuringExecution == nil { + na.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{}}, + } + } + not := corev1.NodeSelectorRequirement{ + Key: "kubernetes.io/hostname", Operator: corev1.NodeSelectorOpNotIn, Values: []string{node}, + } + terms := na.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + for i := range terms { + terms[i].MatchExpressions = append(terms[i].MatchExpressions, not) + } +} + +// waitCertsPods waits until the running, ready fabric-api-certs pods' +// nodes satisfy ok. +func waitCertsPods(ctx context.Context, l *lab, ok func([]string) bool) error { + for range 120 { + pods, err := l.cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: certsSelector}) + if err == nil { + var nodes []string + for _, p := range pods.Items { + if p.DeletionTimestamp == nil && p.Status.Phase == corev1.PodRunning && podReady(p) { + nodes = append(nodes, p.Spec.NodeName) + } + } + if ok(nodes) { + return nil + } + } + time.Sleep(time.Second) + } + return errors.New("fabric-api-certs pods did not converge") +} + +func podReady(p corev1.Pod) bool { + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +// destinations: the public probe policy and target syntax apply to every +// service probe, with no operator exception (A3). +func destinations(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + for _, c := range []struct { + name, typ, target, family, reason string + }{ + {"private IPv4", ping, "10.0.0.1", "", reasonDestinationNotAllowed}, + {"documentation IPv6", ping, "2001:db8:1:40::2", "", reasonDestinationNotAllowed}, + {"operator-allowed fabric loopback", ping, "fc00:0:2::1", "", reasonDestinationNotAllowed}, + {"operator-allowed IPv4 loopback", traceroute, "10.255.255.2", "", reasonDestinationNotAllowed}, + {"multicast", traceroute, "224.0.0.1", "", reasonDestinationNotAllowed}, + {"IPv4-mapped IPv6", ping, "::ffff:" + answer4, "", ""}, + {"hostname", ping, "remote.example", "", reasonInvalidQuery}, + {"family mismatch", ping, answer4, ipv6, reasonInvalidQuery}, + {"prefix as probe target", ping, "11.1.40.0/24", "", reasonInvalidQuery}, + {"lookup prefix with host bits", routeLookup, "11.1.40.1/24", "", reasonInvalidQuery}, + {"summary with target", bgpSummary, answer4, ipv4, reasonInvalidQuery}, + } { + args := []string{fProject, l.project(), fType, c.typ, fTarget, c.target} + if c.family != "" { + args = append(args, fFamily, c.family) + } + r := mustRun(ctx, t, l, idService, args...) + expectRefused(t, c.name, r, "InvalidArgument", c.reason) + ev.Observed[c.name] = brief(r) + } +} + +// fabricQuery creates a FabricQuery for this cell and returns its status +// once Complete. +func (l *lab) fabricQuery(ctx context.Context, name, query string) (map[string]any, error) { + manifest := fmt.Sprintf(`apiVersion: galactic.datumapis.com/v1alpha1 +kind: FabricQuery +metadata: + name: %[1]s + namespace: fabric-api-verify +spec: + requestID: %[1]s + source: {uid: lg-qualify, cluster: lab, namespace: default, name: %[1]s} + query: %[2]s + site: %[3]s + clusterName: %[3]s + expiresAt: %[4]q +`, name, query, cell, time.Now().UTC().Add(90*time.Second).Format(time.RFC3339)) + _, err := l.kubectl("create", "namespace", "fabric-api-verify") + if err != nil && !strings.Contains(err.Error(), "AlreadyExists") { + return nil, err + } + cmd := exec.CommandContext(ctx, "kubectl", "--kubeconfig", l.kubeconfig, "apply", "-f", "-") + cmd.Stdin = strings.NewReader(manifest) + if out, err := cmd.CombinedOutput(); err != nil { + return nil, fmt.Errorf("apply FabricQuery: %w: %s", err, out) + } + defer func() { _, _ = l.kubectl("-n", "fabric-api-verify", "delete", "fabricquery", name, "--wait=false") }() + _, err = l.kubectl("-n", "fabric-api-verify", "wait", "--for=condition=Complete", "--timeout=90s", "fabricquery/"+name) + if err != nil { + return nil, err + } + out, err := l.kubectl("-n", "fabric-api-verify", "get", "fabricquery", name, "-o", "jsonpath={.status}") + if err != nil { + return nil, err + } + var st map[string]any + return st, json.Unmarshal([]byte(out), &st) +} + +// fqSummary is a FabricQuery status's Complete reason and its observation +// and error counts. +func fqSummary(st map[string]any) (string, int, []string) { + reason := "" + if conds, ok := st["conditions"].([]any); ok { + for _, c := range conds { + if m, ok := c.(map[string]any); ok && m["type"] == "Complete" { + reason, _ = m["reason"].(string) + } + } + } + obs := 0 + var errs []string + if list, ok := st["observations"].([]any); ok { + for _, o := range list { + m, _ := o.(map[string]any) + if code, ok := m["errorCode"].(string); ok && code != "" { + errs = append(errs, code) + } else { + obs++ + } + } + } + return reason, obs, errs +} + +// nodeResult is one node's answer in the query CLI's JSON. +type nodeResult struct { + Observation json.RawMessage `json:"observation"` + ErrorCode string `json:"errorCode"` +} + +// operatorResults parses the query CLI's JSON: per node, an observation or +// an error code. A gateway query prints {"results": [...]}, a direct node +// query that node's single result. +func operatorResults(out string) (int, []string, error) { + i := strings.Index(out, "{") + if i < 0 { + return 0, nil, fmt.Errorf("no JSON in %q", out) + } + var res struct { + Results []nodeResult `json:"results"` + nodeResult + } + if err := json.Unmarshal([]byte(out[i:]), &res); err != nil { + return 0, nil, err + } + if res.Results == nil { + res.Results = []nodeResult{res.nodeResult} + } + obs := 0 + var codes []string + for _, r := range res.Results { + if r.ErrorCode != "" { + codes = append(codes, r.ErrorCode) + } else if r.Observation != nil { + obs++ + } + } + return obs, codes, nil +} + +// legacy: the operator debug service, direct operator node probes and +// FabricQuery keep working while the service is on; their probes are now +// charged and permitted (A16). +func legacy(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + for _, c := range []struct { + name string + args []string + wantObs int + wantCode string + }{ + {"operator gateway BGPSummary", l.viaGateway(bgpSummary, "--family", ipv6), 3, ""}, + {"operator gateway Ping", l.viaGateway(ping, answer4), 3, ""}, + {"operator gateway Ping of a fabric loopback", l.viaGateway(ping, "fc00:0:a::1"), 0, reasonDestinationNotAllowed}, + } { + time.Sleep(pace) + out, err := l.operator(ctx, c.args...) + obs, codes, perr := operatorResults(out) + ev.Observed[c.name] = answers{Observations: obs, Errors: codes} + if err != nil || perr != nil { + t.Errorf("%s: %v %v: %s", c.name, err, perr, out) + continue + } + if obs != c.wantObs || (c.wantCode != "" && (len(codes) != len(routers) || codes[0] != c.wantCode)) { + t.Errorf("%s: %d observations, errors %v", c.name, obs, codes) + } + } + + // A direct operator probe of a fabric loopback, on each node: the + // sidecar must obtain a permit from the gateway first. + certs, err := l.cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: certsSelector}) + if err != nil { + t.Fatal(err) + } + frs, err := l.cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: fabricRouterSelector}) + if err != nil { + t.Fatal(err) + } + direct := map[string]any{} + for _, fr := range frs.Items { + certPod := certsPodOn(certs.Items, fr.Spec.NodeName) + out, obs, codes, err := l.directProbe(ctx, fr, certPod) + res := map[string]any{"answers": answers{Observations: obs, Errors: codes, Failed: err != nil}} + if err != nil { + res["output"] = strings.TrimSpace(out) + } + direct[fr.Spec.NodeName] = res + // Only clientNode's host network reaches the cluster's DNS and the + // gateway Service in the lab, so only its sidecar can ask for a + // permit; the edge nodes' results are recorded, not judged. + if fr.Spec.NodeName == clientNode && (err != nil || obs != 1) { + t.Errorf("direct operator probe on %s: %v: %s", fr.Spec.NodeName, err, strings.TrimSpace(out)) + } + if fr.Spec.NodeName == clientNode { + direct[fr.Spec.NodeName+" after certs pod replacement"] = l.certsReplacement(ctx, t, fr, certPod) + } + } + ev.Observed["direct operator probe"] = direct + ev.Notes = append(ev.Notes, "Direct operator probes are judged on "+clientNode+" only: the lab's edge nodes' host "+ + "network reaches neither the cluster DNS server nor the gateway Service, so their sidecars cannot ask for a permit.") + + for _, c := range []struct{ name, query string }{ + {"FabricQuery BGPSummary", "{type: BGPSummary, addressFamily: IPv6}"}, + {"FabricQuery Ping", fmt.Sprintf("{type: Ping, target: %q, addressFamily: IPv4}", answer4)}, + } { + name := fmt.Sprintf("%s%s-%d", projectPrefix, strings.ToLower(strings.Fields(c.name)[1]), time.Now().Unix()) + st, err := l.fabricQuery(ctx, name, c.query) + if err != nil { + t.Errorf("%s: %v", c.name, err) + continue + } + reason, obs, errs := fqSummary(st) + ev.Observed[c.name] = answers{Reason: reason, Observations: obs, Errors: errs} + if reason != "Succeeded" || obs != len(routers) { + t.Errorf("%s: %s with %d observations, errors %v", c.name, reason, obs, errs) + } + } +} + +func certsPodOn(pods []corev1.Pod, node string) string { + for _, c := range pods { + if c.Spec.NodeName == node && c.DeletionTimestamp == nil { + return c.Name + } + } + return "" +} + +// directProbe pings a fabric loopback directly on fr's node, as an +// operator; the sidecar must obtain a permit from the gateway first. +func (l *lab) directProbe(ctx context.Context, fr corev1.Pod, certPod string) (string, int, []string, error) { + time.Sleep(pace) + out, err := l.operator(ctx, "--node", fmt.Sprintf("[%s]:9344", fr.Status.HostIP), "--node-name", fr.Spec.NodeName, + "--node-pod", certPod, "--timeout", "10s", ping, "fc00:0:7::1") + obs, codes, perr := operatorResults(out) + return out, obs, codes, errors.Join(err, perr) +} + +// certsReplacement replaces the fabric-api-certs pod on fr's node, which +// changes the node's identity without restarting its sidecar, and requires +// a direct operator probe there to succeed within certsReplacementBound of +// the new pod turning Ready: the sidecar's permit requests must present +// the new certificate. +func (l *lab) certsReplacement(ctx context.Context, t *testing.T, fr corev1.Pod, old string) map[string]any { + res := map[string]any{"bound": certsReplacementBound.String()} + if err := l.cs.CoreV1().Pods(ns).Delete(ctx, old, metav1.DeleteOptions{}); err != nil { + t.Errorf("delete certs pod %s: %v", old, err) + return res + } + var replacement string + var ready time.Time + waitCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + for replacement == "" { + select { + case <-waitCtx.Done(): + t.Errorf("certs pod on %s not replaced within 2m", fr.Spec.NodeName) + return res + case <-time.After(time.Second): + } + pods, err := l.cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: certsSelector, + FieldSelector: "spec.nodeName=" + fr.Spec.NodeName}) + if err != nil { + continue + } + for _, p := range pods.Items { + if p.Name != old && p.DeletionTimestamp == nil && podReady(p) { + replacement, ready = p.Name, time.Now() + } + } + } + var out string + var obs int + var err error + for { + out, obs, _, err = l.directProbe(ctx, fr, replacement) + if err == nil && obs == 1 { + res["succeededAfter"] = time.Since(ready).Round(time.Millisecond).String() + return res + } + if time.Since(ready) > certsReplacementBound { + break + } + } + res["output"] = strings.TrimSpace(out) + t.Errorf("direct operator probe on %s still failing %s after its certs pod was replaced: %v: %s", + fr.Spec.NodeName, certsReplacementBound, err, strings.TrimSpace(out)) + return res +} + +// targetPattern matches the probe destinations the suite uses. +var targetPatterns = []string{answer4, answer6, "11.1.42.", "2c0f:42:"} + +// confidentiality: no service target or result reaches a routine log line +// or a metric label, and no metric label carries a project (A15). +func confidentiality(ctx context.Context, t *testing.T, l *lab, ev *evidence, since time.Time) { + gws, err := l.gatewayPods(ctx) + if err != nil { + t.Fatal(err) + } + found := map[string][]string{} + scan := func(where, text string, patterns []string, levelFilter bool) { + for line := range strings.SplitSeq(text, "\n") { + if levelFilter && (strings.Contains(line, `"level":"DEBUG"`) || strings.Contains(line, "level=DEBUG")) { + continue + } + // Operator and FabricQuery traffic may name its own targets; + // only service request lines are bound by the contract. + if levelFilter && !strings.Contains(line, "Looking Glass") && !strings.Contains(line, "/service/") { + continue + } + for _, p := range patterns { + if strings.Contains(line, p) { + found[where] = append(found[where], line) + break + } + } + } + } + for _, p := range gws { + logs, err := l.podLogs(ctx, p.Name, "gateway", since) + if err != nil { + t.Errorf("logs of %s: %v", p.Name, err) + continue + } + scan(p.Name+" log", logs, targetPatterns, true) + text, err := l.gatewayMetrics(ctx, p) + if err != nil { + t.Errorf("metrics of %s: %v", p.Name, err) + continue + } + scan(p.Name+" metrics", text, append([]string{projectPrefix}, targetPatterns...), false) + } + nm, err := l.nodeMetrics(ctx) + if err != nil { + t.Error(err) + } + for node, text := range nm { + scan(node+" sidecar metrics", text, append([]string{projectPrefix}, targetPatterns...), false) + } + frs, err := l.cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: fabricRouterSelector}) + if err != nil { + t.Fatal(err) + } + for _, fr := range frs.Items { + logs, err := l.podLogs(ctx, fr.Name, "fabric-api", since) + if err != nil { + t.Errorf("logs of %s: %v", fr.Name, err) + continue + } + scan(fr.Spec.NodeName+" sidecar log", logs, append([]string{projectPrefix}, targetPatterns...), true) + } + ev.Observed["leaks"] = found + for where, lines := range found { + t.Errorf("%s: %d lines name a service target or project, e.g. %s", where, len(lines), lines[0]) + } +} diff --git a/tests/lgqualify/harness_test.go b/tests/lgqualify/harness_test.go new file mode 100644 index 000000000..dc12248d9 --- /dev/null +++ b/tests/lgqualify/harness_test.go @@ -0,0 +1,757 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package lgqualify_test qualifies the Looking Glass service backend +// (docs/designs/looking-glass-service.md) on real fabric routers: the +// containerlab lab's dfw cell, with the service enabled by +// deploy:looking-glass. Every request goes through the independent client +// in examples/looking-glass-client, built from the published service API +// alone; this package only drives the lab around it (certificates, faults, +// packet counters, metrics) and judges the client's records against +// thresholds fixed in thresholds.go before any run. +// +// It skips unless LG_QUALIFY_KUBECONFIG names dfw's kubeconfig. The client +// runs inside the dfw-worker node container, the node both gateway +// replicas run on, and reaches them through the gateway Service. Probe +// packets are counted by iptables rules in every dfw node's network +// namespace, where the fabric-api sidecars send them. Run it with +// task verify:looking-glass, which writes evidence to bin/lg-qualify/. +package lgqualify_test + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + ns = "galactic-system" + cell = "dfw" + gatewayPort = 9346 + // clientNode runs the client: both gateway replicas run there, and it + // is the dfw node that reaches the gateway Service. + clientNode = "dfw-worker" + clientDir = "/opt/lg-qualify" + leaseName = "fabric-api-gateway" + switchName = "fabric-api-service" + // The lab's probe destinations (deploy/containerlab/node_files/tr4): + // remote-host answers on the first two; the sink prefixes are dropped + // silently, so probes toward them are lost and traceroutes run long. + answer4 = "11.1.40.2" + answer6 = "2c0f:40::2" +) + +// The client's query types and outcome results. +const ( + bgpSummary = "BGPSummary" + routeLookup = "RouteLookup" + ping = "Ping" + traceroute = "Traceroute" + observation = "observation" + failure = "failure" +) + +// The client's run flags the suite sets. +const ( + fProject = "-project" + fType = "-type" + fTarget = "-target" + fFamily = "-family" + fNodes = "-nodes" + fExpires = "-expires-in" +) + +const ( + ipv4 = "IPv4" + ipv6 = "IPv6" +) + +// Status codes and reasons the suite expects by name. +const ( + codeFailedPrecondition = "FailedPrecondition" + reasonInvalidQuery = "InvalidQuery" + reasonDestinationNotAllowed = "DestinationNotAllowed" + reasonNotRequested = "NotRequested" +) + +// The contract's acceptance rows (looking-glass-service.md, "Acceptance matrix +// and evidence") each scenario evidences. +const ( + rowA1 = "A1" + rowA2 = "A2" + rowA3 = "A3" + rowA4 = "A4" + rowA5 = "A5" + rowA6 = "A6" + rowA7 = "A7" + rowA8 = "A8" + rowA9 = "A9" + rowA10 = "A10" + rowA11 = "A11" + rowA12 = "A12" + rowA13 = "A13" + rowA14 = "A14" + rowA15 = "A15" + rowA16 = "A16" + rowA17 = "A17" +) + +const ( + fabricRouterSelector = "app.kubernetes.io/name=fabric-router" + certsSelector = "app.kubernetes.io/name=fabric-api-certs" + iptables = "iptables" + ip6tables = "ip6tables" + certFile = "tls.crt" + keyFile = "tls.key" + caFile = "ca.crt" + dockerExec = "exec" + // projectPrefix starts every project UID the suite sends. + projectPrefix = "lgq-" + // commandTimeout bounds every command the suite runs on a node. + commandTimeout = 2 * time.Minute +) + +// spec is the client's workload spec (examples/looking-glass-client, +// internal/load). +type spec struct { + Groups []group `json:"groups"` +} + +type group struct { + Name string `json:"name"` + Count int `json:"count"` + Concurrency int `json:"concurrency"` + Interval string `json:"interval,omitempty"` + StartAfter string `json:"startAfter,omitempty"` + Projects []string `json:"projects,omitempty"` + Options options `json:"options"` +} + +type options struct { + Query query `json:"query"` + Nodes []string `json:"nodes,omitempty"` +} + +type query struct { + Type string `json:"type"` + Target string `json:"target,omitempty"` + Family string `json:"family,omitempty"` +} + +// routers are dfw's fabric routers. +var routers = []string{"dfw-worker", "dfw-worker2", "dfw-worker3"} + +// sink returns a distinct address in the lab's probe sink for family 4 or +// 6, so tests do not share a destination host bucket. +func sink(family, n int) string { + if family == 4 { + return fmt.Sprintf("11.1.42.%d", n) + } + return fmt.Sprintf("2c0f:42::%x", n) +} + +// lab is the cell under test. +type lab struct { + kubeconfig string + cs kubernetes.Interface + root string + out string + runID string + clusterIP string + + mu sync.Mutex + seq int + sink int +} + +var ( + theLab *lab + labOnce sync.Once + errLabInit error + skipReason string +) + +func getLab(t *testing.T) *lab { + t.Helper() + labOnce.Do(func() { + kc := os.Getenv("LG_QUALIFY_KUBECONFIG") + if kc == "" { + skipReason = "LG_QUALIFY_KUBECONFIG is not set; run task verify:looking-glass" + return + } + theLab, errLabInit = newLab(kc) + }) + if skipReason != "" { + t.Skip(skipReason) + } + if errLabInit != nil { + t.Fatalf("lab: %v", errLabInit) + } + return theLab +} + +func newLab(kubeconfig string) (*lab, error) { + cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, err + } + cs, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, err + } + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + return nil, err + } + runID := time.Now().UTC().Format("20060102t150405") + out := os.Getenv("LG_QUALIFY_ARTIFACTS") + if out == "" { + out = filepath.Join(root, "bin", "lg-qualify", runID) + } + if err := os.MkdirAll(out, 0o755); err != nil { + return nil, err + } + svc, err := cs.CoreV1().Services(ns).Get(context.Background(), "fabric-api-gateway", metav1.GetOptions{}) + if err != nil { + return nil, err + } + return &lab{ + kubeconfig: kubeconfig, cs: cs, root: root, out: out, runID: runID, clusterIP: svc.Spec.ClusterIP, sink: 10, + }, nil +} + +// project returns a project UID no earlier request of the run used, so a +// test starts with a full project bucket. +func (l *lab) project() string { + l.mu.Lock() + defer l.mu.Unlock() + l.seq++ + return fmt.Sprintf("%s%s-%d", projectPrefix, l.runID, l.seq) +} + +// sinkHost returns a probe sink address no earlier test of the run used. +func (l *lab) sinkHost(family int) string { + l.mu.Lock() + defer l.mu.Unlock() + l.sink++ + return sink(family, l.sink) +} + +// nodeExec runs a command in a Kind node container. +func nodeExec(node string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "docker", append([]string{dockerExec, node}, args...)...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + if err != nil { + return stdout.String(), fmt.Errorf("%s %v: %w: %s", node, args, err, strings.TrimSpace(stderr.String())) + } + return stdout.String(), nil +} + +// clientWrite writes data to path on clientNode. docker cp cannot write +// below a tmpfs mount, which Kind nodes have, so it streams through a shell. +func clientWrite(path string, data []byte, mode os.FileMode) error { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "docker", dockerExec, "-i", clientNode, "sh", "-c", + fmt.Sprintf("mkdir -p %q && cat > %q && chmod %o %q", filepath.Dir(path), path, mode, path)) + cmd.Stdin = bytes.NewReader(data) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("write %s:%s: %w: %s", clientNode, path, err, out) + } + return nil +} + +// kubectl runs kubectl against the cell. +func (l *lab) kubectl(args ...string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "kubectl", append([]string{"--kubeconfig", l.kubeconfig}, args...)...) + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("kubectl %v: %w: %s", args, err, out) + } + return string(out), nil +} + +// identity names a set of credentials installed under clientDir on +// clientNode. +type identity string + +const ( + idService identity = "service" + idUnlisted identity = "unlisted" + idOtherCell identity = "other-cell" + idOperator identity = "operator" + idExpired identity = "expired" + idNone identity = "none" +) + +// installClient copies the client binary, the fabric-api CLI and every +// identity's credentials onto clientNode. +func (l *lab) installClient(clientBin, fabricAPIBin string, expired [2][]byte) error { + for path, src := range map[string]string{"lgc": clientBin, "fabric-api": fabricAPIBin} { + data, err := os.ReadFile(src) + if err != nil { + return err + } + if err := clientWrite(clientDir+"/"+path, data, 0o755); err != nil { + return err + } + } + secrets := map[identity]string{ + idService: "lg-qualify-looking-glass", idUnlisted: "lg-qualify-unlisted", + idOtherCell: "lg-qualify-other-cell", idOperator: "fabric-api-operator", + } + for id, name := range secrets { + s, err := l.cs.CoreV1().Secrets(ns).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + return err + } + for _, f := range []string{certFile, keyFile, caFile} { + if err := clientWrite(fmt.Sprintf("%s/%s/%s", clientDir, id, f), s.Data[f], 0o600); err != nil { + return err + } + } + } + ca, err := l.cs.CoreV1().Secrets(ns).Get(context.Background(), "fabric-api-ca", metav1.GetOptions{}) + if err != nil { + return err + } + for f, data := range map[string][]byte{certFile: expired[0], keyFile: expired[1], caFile: ca.Data[certFile]} { + if err := clientWrite(fmt.Sprintf("%s/%s/%s", clientDir, idExpired, f), data, 0o600); err != nil { + return err + } + } + return nil +} + +// lgc runs the client on clientNode as id and returns its JSON lines. +func (l *lab) lgc(ctx context.Context, id identity, sub string, args ...string) ([]json.RawMessage, error) { + lines, _, err := l.lgcEvents(ctx, id, sub, args...) + return lines, err +} + +// lgcEvents is lgc that also returns what the client printed to stderr: +// with run --events, every event as a JSON line. +func (l *lab) lgcEvents( + ctx context.Context, id identity, sub string, args ...string, +) ([]json.RawMessage, []json.RawMessage, error) { + cred := id + if id == idNone { + cred = idService + } + full := []string{dockerExec, clientNode, clientDir + "/lgc", sub, + "--addr", fmt.Sprintf("[%s]:%d", l.clusterIP, gatewayPort), "--cell", cell, + "--ca", fmt.Sprintf("%s/%s/ca.crt", clientDir, cred)} + if id == idNone { + full = append(full, "--cert", "") + } else { + full = append(full, + "--cert", fmt.Sprintf("%s/%s/%s", clientDir, id, certFile), + "--key", fmt.Sprintf("%s/%s/%s", clientDir, id, keyFile)) + } + cmd := exec.CommandContext(ctx, "docker", append(full, args...)...) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + return nil, nil, fmt.Errorf("client %s %v: %w: %s", sub, args, err, strings.TrimSpace(stderr.String())) + } + lines, err := jsonLines(&stdout) + if err != nil { + return nil, nil, err + } + events, _ := jsonLines(&stderr) + return lines, events, nil +} + +func jsonLines(b *bytes.Buffer) ([]json.RawMessage, error) { + var lines []json.RawMessage + sc := bufio.NewScanner(b) + sc.Buffer(make([]byte, 1<<20), 16<<20) + for sc.Scan() { + if len(sc.Bytes()) > 0 && sc.Bytes()[0] == '{' { + lines = append(lines, json.RawMessage(slices.Clone(sc.Bytes()))) + } + } + return lines, sc.Err() +} + +// Record mirrors the client's record (examples/looking-glass-client, +// internal/run.Record) as far as the suite judges it. +type Record struct { + Group string `json:"group,omitempty"` + StartMS float64 `json:"startMs,omitempty"` + RequestID string `json:"requestId"` + Project string `json:"project"` + StartedAt time.Time `json:"startedAt"` + Status struct { + Code string `json:"code"` + Reason string `json:"reason"` + Domain string `json:"domain"` + Metadata map[string]string `json:"metadata"` + RetryDelayMS int64 `json:"retryDelayMs"` + MessageBytes int `json:"messageBytes"` + } `json:"status"` + Stream struct { + Delivered bool `json:"delivered"` + Violations []string `json:"violations"` + Selected []string `json:"selected"` + OmittedReasons map[string]string `json:"omittedReasons"` + Outcomes []struct { + Node string `json:"node"` + Result string `json:"result"` + Reason string `json:"reason"` + Scope string `json:"scope"` + Truncated bool `json:"truncated"` + } `json:"outcomes"` + Pending []string `json:"pending"` + Complete *struct { + Expected int `json:"expected"` + Successful int `json:"successful"` + Failed int `json:"failed"` + Omitted int `json:"omitted"` + } `json:"complete"` + Reason string `json:"reason"` + } `json:"stream"` + Events int `json:"events"` + Bytes int `json:"bytes"` + MaxEventBytes int `json:"maxEventBytes"` + Timing struct { + FirstEventMS float64 `json:"firstEventMs"` + SnapshotMS float64 `json:"snapshotMs"` + CompleteMS float64 `json:"completeMs"` + EndMS float64 `json:"endMs"` + CancelMS float64 `json:"cancelMs"` + CancelToEndMS float64 `json:"cancelToEndMs"` + OutcomeMS []float64 `json:"outcomeMs"` + } `json:"timing"` +} + +// ok reports whether r ended with gRPC OK. +func (r Record) ok() bool { return r.Status.Code == "OK" } + +// status is "/". +func (r Record) status() string { return r.Status.Code + "/" + r.Status.Reason } + +// endAt is when r's stream ended, on the wall clock. +func (r Record) endAt() time.Time { + return r.StartedAt.Add(time.Duration(r.Timing.EndMS * float64(time.Millisecond))) +} + +// run runs one diagnostic as id with the client's run flags. +func (l *lab) run(ctx context.Context, id identity, args ...string) (Record, error) { + lines, err := l.lgc(ctx, id, "run", args...) + if err != nil { + return Record{}, err + } + if len(lines) != 1 { + return Record{}, fmt.Errorf("client printed %d lines", len(lines)) + } + var r Record + return r, json.Unmarshal(lines[0], &r) +} + +// runEvents runs one diagnostic as id and also returns its events, +// payloads included. +func (l *lab) runEvents(ctx context.Context, id identity, args ...string) (Record, []json.RawMessage, error) { + lines, events, err := l.lgcEvents(ctx, id, "run", append([]string{"-events"}, args...)...) + if err != nil { + return Record{}, nil, err + } + if len(lines) != 1 { + return Record{}, nil, fmt.Errorf("client printed %d lines", len(lines)) + } + var r Record + return r, events, json.Unmarshal(lines[0], &r) +} + +// load runs a workload and returns its records and summaries. +func (l *lab) load(ctx context.Context, s spec) ([]Record, []json.RawMessage, error) { + data, err := json.Marshal(s) + if err != nil { + return nil, nil, err + } + path := fmt.Sprintf("%s/spec-%d.json", clientDir, time.Now().UnixNano()) + if err := clientWrite(path, data, 0o600); err != nil { + return nil, nil, err + } + lines, err := l.lgc(ctx, idService, "load", "--spec", path) + if err != nil { + return nil, nil, err + } + var recs []Record + var sums []json.RawMessage + for _, line := range lines { + var probe struct { + Summary json.RawMessage `json:"summary"` + } + if json.Unmarshal(line, &probe) == nil && probe.Summary != nil { + sums = append(sums, probe.Summary) + continue + } + var r Record + if err := json.Unmarshal(line, &r); err != nil { + return nil, nil, err + } + recs = append(recs, r) + } + return recs, sums, nil +} + +// operator runs the fabric-api CLI's query subcommand on clientNode with +// the lab's operator identity, as the legacy operator interface. +func (l *lab) operator(ctx context.Context, args ...string) (string, error) { + full := append([]string{dockerExec, clientNode, clientDir + "/fabric-api", + "--log-format", "text", "--log-level", "warn", + "query", "--cell", cell, "--operator", "lab-verify", + "--tls-cert", clientDir + "/operator/tls.crt", "--tls-key", clientDir + "/operator/tls.key", + "--tls-ca", clientDir + "/operator/ca.crt"}, args...) + out, err := exec.CommandContext(ctx, "docker", full...).CombinedOutput() + return string(out), err +} + +// viaGateway prefixes the query CLI's arguments with the gateway's +// address, for an operator call through the gateway's debug service. +func (l *lab) viaGateway(args ...string) []string { + return append([]string{"--gateway", fmt.Sprintf("[%s]:%d", l.clusterIP, gatewayPort)}, args...) +} + +// answers is a legacy query's result in the evidence. +type answers struct { + Reason string `json:"reason,omitempty"` + Observations int `json:"observations"` + Errors []string `json:"errors,omitempty"` + Failed bool `json:"failed,omitempty"` +} + +// fault records which gateway pod a fault hit, and when. +type fault struct { + Pod string `json:"pod"` + At time.Time `json:"at"` +} + +// waitReady waits until a request is admitted: after a gateway start the +// leader serves its 125 s recovery hold first. +func (l *lab) waitReady(ctx context.Context, timeout time.Duration) (time.Time, error) { + deadline := time.Now().Add(timeout) + for { + r, err := l.run(ctx, idService, fProject, l.project(), fType, bgpSummary, fFamily, "IPv4", fNodes, routers[0]) + if err == nil && r.ok() { + return r.StartedAt, nil + } + if time.Now().After(deadline) { + return time.Time{}, fmt.Errorf("not admitting after %s: %+v: %w", timeout, r.Status, err) + } + time.Sleep(2 * time.Second) + } +} + +// settle waits for the cell's request bucket to refill: 8 tokens at 2/s. +func settle() { time.Sleep(4500 * time.Millisecond) } + +// leader returns the gateway pod holding the lease and its holder identity. +func (l *lab) leader(ctx context.Context) (string, string, error) { + lease, err := l.cs.CoordinationV1().Leases(ns).Get(ctx, leaseName, metav1.GetOptions{}) + if err != nil { + return "", "", err + } + return leaseHolder(lease) +} + +func leaseHolder(lease *coordinationv1.Lease) (string, string, error) { + if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "" { + return "", "", errors.New("lease has no holder") + } + h := *lease.Spec.HolderIdentity + pod, _, _ := strings.Cut(h, "_") + return pod, h, nil +} + +// waitNewLeader waits until the lease names a holder other than old and +// returns it with the moment it was observed. +func (l *lab) waitNewLeader(ctx context.Context, old string, timeout time.Duration) (string, time.Time, error) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if _, holder, err := l.leader(ctx); err == nil && holder != old { + return holder, time.Now(), nil + } + time.Sleep(200 * time.Millisecond) + } + return "", time.Time{}, fmt.Errorf("no new leader within %s", timeout) +} + +// gatewayPods returns the gateway pods. +func (l *lab) gatewayPods(ctx context.Context) ([]corev1.Pod, error) { + pods, err := l.cs.CoreV1().Pods(ns).List(ctx, + metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=fabric-api-gateway"}) + if err != nil { + return nil, err + } + return pods.Items, nil +} + +// setSwitch sets the service switch. +func (l *lab) setSwitch(ctx context.Context, value string) error { + patch := fmt.Sprintf(`{"data":{"service":%q}}`, value) + _, err := l.cs.CoreV1().ConfigMaps(ns).Patch(ctx, switchName, types.MergePatchType, []byte(patch), + metav1.PatchOptions{}) + return err +} + +// metrics scrapes a /metrics endpoint from clientNode, which reaches both +// the gateway pods and every node's sidecar. +func metrics(ctx context.Context, url string) (string, error) { + out, err := exec.CommandContext(ctx, "docker", dockerExec, clientNode, "curl", "-sf", "--max-time", "3", url).Output() + return string(out), err +} + +// metricValue sums the samples of name whose labels include every label in +// match (`key="value"` strings). +func metricValue(text, name string, match ...string) (float64, bool) { + var sum float64 + found := false + for line := range strings.SplitSeq(text, "\n") { + if !strings.HasPrefix(line, name) || strings.HasPrefix(line, "#") { + continue + } + rest := line[len(name):] + if rest == "" || (rest[0] != '{' && rest[0] != ' ') { + continue + } + if !allIn(rest, match) { + continue + } + f := strings.Fields(rest[strings.LastIndex(rest, "}")+1:]) + if len(f) == 0 { + continue + } + v, err := strconv.ParseFloat(f[0], 64) + if err == nil { + sum += v + found = true + } + } + return sum, found +} + +func allIn(s string, subs []string) bool { + for _, m := range subs { + if !strings.Contains(s, m) { + return false + } + } + return true +} + +// gatewayMetrics scrapes pod's gateway metrics. +func (l *lab) gatewayMetrics(ctx context.Context, pod corev1.Pod) (string, error) { + return metrics(ctx, fmt.Sprintf("http://[%s]:9347/metrics", pod.Status.PodIP)) +} + +// nodeMetrics scrapes the fabric-api sidecar's metrics on each router. +func (l *lab) nodeMetrics(ctx context.Context) (map[string]string, error) { + pods, err := l.cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: fabricRouterSelector}) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, p := range pods.Items { + ip, err := netip.ParseAddr(p.Status.HostIP) + if err != nil { + return nil, err + } + text, err := metrics(ctx, "http://"+netip.AddrPortFrom(ip, 9345).String()+"/metrics") + if err != nil { + return nil, fmt.Errorf("%s metrics: %w", p.Spec.NodeName, err) + } + out[p.Spec.NodeName] = text + } + return out, nil +} + +// evidence is one scenario's record in the run's artifacts. +type evidence struct { + Scenario string `json:"scenario"` + Rows []string `json:"rows"` + Thresholds map[string]any `json:"thresholds,omitempty"` + Observed map[string]any `json:"observed"` + Outcome string `json:"outcome"` + Notes []string `json:"notes,omitempty"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished"` +} + +// scenarioFunc is one scenario of the suite. +type scenarioFunc func(ctx context.Context, t *testing.T, l *lab, ev *evidence) + +// scenario runs fn as a subtest and writes its evidence whatever happens, +// naming the acceptance rows it evidences. A scenario fails through +// t.Errorf/Fatalf; its outcome is "passed" only when the subtest did not +// fail or skip. +func (l *lab) scenario(ctx context.Context, t *testing.T, name string, fn scenarioFunc, rows ...string) { + t.Run(name, func(t *testing.T) { + ev := &evidence{Scenario: name, Rows: rows, Observed: map[string]any{}, Started: time.Now().UTC()} + defer func() { + ev.Finished = time.Now().UTC() + switch { + case t.Skipped(): + ev.Outcome = "not run" + case t.Failed(): + ev.Outcome = "failed" + default: + ev.Outcome = "passed" + } + data, _ := json.MarshalIndent(ev, "", " ") + _ = os.WriteFile(filepath.Join(l.out, name+".json"), data, 0o644) + }() + fn(ctx, t, l, ev) + }) +} + +// saveRecords writes records to the run's artifacts as JSON lines. +func (l *lab) saveRecords(name string, recs []Record) { + var b bytes.Buffer + enc := json.NewEncoder(&b) + for _, r := range recs { + _ = enc.Encode(r) + } + _ = os.WriteFile(filepath.Join(l.out, name+".records.jsonl"), b.Bytes(), 0o644) +} + +var ansi = regexp.MustCompile(`\x1b\[[0-9;]*m`) + +// podLogs returns a pod container's log since t, its previous instance's +// first when it restarted. +func (l *lab) podLogs(ctx context.Context, pod, container string, since time.Time) (string, error) { + st := metav1.NewTime(since) + pods := l.cs.CoreV1().Pods(ns) + prev, _ := pods.GetLogs(pod, &corev1.PodLogOptions{Container: container, SinceTime: &st, Previous: true}).DoRaw(ctx) + raw, err := pods.GetLogs(pod, &corev1.PodLogOptions{Container: container, SinceTime: &st}).DoRaw(ctx) + return ansi.ReplaceAllString(string(prev)+string(raw), ""), err +} diff --git a/tests/lgqualify/load_test.go b/tests/lgqualify/load_test.go new file mode 100644 index 000000000..021c345ea --- /dev/null +++ b/tests/lgqualify/load_test.go @@ -0,0 +1,342 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lgqualify_test + +import ( + "context" + "fmt" + "math" + "testing" + "time" +) + +// spread is the time between the first and last start in recs, in +// seconds, plus allowance for the hop from the client to the leader. +func spread(recs []Record) float64 { + if len(recs) == 0 { + return 0 + } + lo, hi := recs[0].StartedAt, recs[0].StartedAt + for _, r := range recs { + if r.StartedAt.Before(lo) { + lo = r.StartedAt + } + if r.StartedAt.After(hi) { + hi = r.StartedAt + } + } + return hi.Sub(lo).Seconds() + 0.5 +} + +// countStatuses counts recs by "/". +func countStatuses(recs []Record) map[string]int { + out := map[string]int{} + for _, r := range recs { + out[r.status()]++ + } + return out +} + +// checkThrottled checks a refused record carries the throttling contract: +// scope, execution_started=false and a positive retry delay of at most +// maxDelay. +func checkThrottled(t *testing.T, name string, r Record, scope string, maxDelay time.Duration) { + t.Helper() + md := r.Status.Metadata + d := time.Duration(r.Status.RetryDelayMS) * time.Millisecond + if r.Status.Code != "ResourceExhausted" || r.Status.Reason != "Throttled" || md["scope"] != scope || + md["execution_started"] != "false" || d <= 0 || d > maxDelay || r.Events != 0 { + t.Errorf("%s: refusal %s scope=%q execution_started=%q retry=%s events=%d, want Throttled scope=%s retry in (0, %s]", + name, r.status(), md["scope"], md["execution_started"], d, r.Events, scope, maxDelay) + } +} + +// bursts: the cell and project request buckets admit at most their burst +// plus refill, and refuse the rest with retry guidance (A8). +func bursts(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + q := query{Type: bgpSummary, Family: ipv4} + projects := make([]string, 12) + for i := range projects { + projects[i] = l.project() + } + settle() + recs, _, err := l.load(ctx, spec{Groups: []group{{ + Name: "cell-burst", Count: 12, Concurrency: 12, Projects: projects, + Options: options{Query: q, Nodes: []string{routers[0]}}, + }}}) + if err != nil { + t.Fatal(err) + } + l.saveRecords("bursts-cell", recs) + admitted := 0 + // A token shortage is advised as the deficit over the rate, at most one + // token's worth: 500 ms at 2/s, rounded up to the millisecond. + maxDelay := time.Duration(math.Ceil(1000/requestCellRate)+1) * time.Millisecond + for _, r := range recs { + if r.ok() { + admitted++ + expectDelivered(t, "cell burst", r, "Succeeded") + continue + } + checkThrottled(t, "cell burst", r, "cell", maxDelay) + } + limit := requestCellBurst + int(math.Ceil(requestCellRate*spread(recs))) + ev.Thresholds = map[string]any{"cellBurstAdmittedMax": limit, "cellRetryMaxMs": maxDelay.Milliseconds()} + ev.Observed["cell burst"] = countStatuses(recs) + ev.Observed["cell burst admitted"] = admitted + ev.Observed["cell burst spread s"] = spread(recs) + if admitted > limit || admitted == len(recs) { + t.Errorf("cell burst: %d of %d admitted, want at most %d and at least one refusal", admitted, len(recs), limit) + } + + settle() + project := l.project() + recs, _, err = l.load(ctx, spec{Groups: []group{{ + Name: "project-burst", Count: 5, Concurrency: 5, Projects: []string{project}, + Options: options{Query: q, Nodes: []string{routers[0]}}, + }}}) + if err != nil { + t.Fatal(err) + } + l.saveRecords("bursts-project", recs) + admitted = 0 + for _, r := range recs { + if r.ok() { + admitted++ + continue + } + // One project token is 5 s at 0.2/s. + checkThrottled(t, "project burst", r, "project", time.Duration(1000/requestProjectRate+1)*time.Millisecond) + } + limit = requestProjectBurst + int(math.Floor(requestProjectRate*spread(recs))) + ev.Thresholds["projectBurstAdmittedMax"] = limit + ev.Observed["project burst"] = countStatuses(recs) + ev.Observed["project burst admitted"] = admitted + if admitted > limit || admitted == 0 { + t.Errorf("project burst: %d of %d admitted, want 1 to %d", admitted, len(recs), limit) + } +} + +// fairness: a project sending faster than its rate is limited to its own +// bucket and never delays or refuses a project within its rate; a project +// holding its two node slots does not delay another project's routers +// (A8). +func fairness(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + const duration = 60 * time.Second + heavy, light := l.project(), l.project() + opts := options{Query: query{Type: bgpSummary, Family: ipv4}, Nodes: []string{routers[0]}} + settle() + recs, sums, err := l.load(ctx, spec{Groups: []group{ + { + Name: "over-rate", Count: int(duration / time.Second), Concurrency: 4, Interval: "1s", + Projects: []string{heavy}, Options: opts, + }, + { + Name: "within-rate", Count: int(duration / (5 * time.Second)), Concurrency: 1, Interval: "5s", + StartAfter: "500ms", Projects: []string{light}, Options: opts, + }, + }}) + if err != nil { + t.Fatal(err) + } + l.saveRecords("fairness-rate", recs) + var over, within []Record + for _, r := range recs { + if r.Group == "over-rate" { + over = append(over, r) + } else { + within = append(within, r) + } + } + overAdmitted := 0 + for _, r := range over { + if r.ok() { + overAdmitted++ + } else if r.Status.Metadata["scope"] != "project" { + t.Errorf("over-rate project refused by scope %q (%s), want project", r.Status.Metadata["scope"], r.status()) + } + } + maxOver := requestProjectBurst + int(requestProjectRate*spread(over)) + 1 + minOver := int(requestProjectRate*spread(over)) - 1 + ev.Thresholds = map[string]any{"overRateAdmittedMax": maxOver, "overRateAdmittedMin": minOver, "withinRateRefusals": 0} + ev.Observed["rate"] = map[string]any{ + "overRate": countStatuses(over), "withinRate": countStatuses(within), "summaries": sums, + } + if overAdmitted > maxOver || overAdmitted < minOver { + t.Errorf("over-rate project: %d admitted, want %d to %d", overAdmitted, minOver, maxOver) + } + for _, r := range within { + if !r.ok() || !r.Stream.Delivered { + t.Errorf("within-rate project refused or undelivered while another project exceeded its rate: %s", r.status()) + } + } + + // Slots: one project's three-router traceroute holds its two slots for + // about 20 s; other projects' probes meanwhile run at once. + settle() + holder := l.project() + target := l.sinkHost(4) + holdCh := make(chan Record, 1) + go func() { + r, _ := l.run(ctx, idService, fProject, holder, fType, traceroute, fTarget, target, fExpires, "60s") + holdCh <- r + }() + time.Sleep(2 * time.Second) + others := make([]Record, 0, 3) + for range 3 { + others = append(others, mustRun(ctx, t, l, idService, fProject, l.project(), fType, ping, fTarget, answer4)) + } + held := <-holdCh + l.saveRecords("fairness-slots", append([]Record{held}, others...)) + ev.Observed["slot holder"] = brief(held) + for i, r := range others { + ev.Observed[fmt.Sprintf("probe beside the slot holder %d", i)] = brief(r) + } + checkQueueTimeout(t, "slot holder", held, ev) + for _, r := range others { + expectDelivered(t, "probe beside a slot holder", r, "Succeeded") + if r.Timing.CompleteMS > 5000 { + t.Errorf("probe beside a slot holder took %.0f ms", r.Timing.CompleteMS) + } + } +} + +// destinationBudget: traceroutes from many projects to one host drain its +// destination bucket; refusals name the destination scope, reservations +// stay within B + rT and emissions within B + r(T + 32); operator and +// FabricQuery probes to the same host are charged to the same bucket and +// refused with it (A9, A10). +func destinationBudget(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + gws, err := l.gatewayPods(ctx) + if err != nil { + t.Fatal(err) + } + refusals := func() map[string]float64 { + out := map[string]float64{} + for _, p := range gws { + text, err := l.gatewayMetrics(ctx, p) + if err != nil { + continue + } + for _, class := range []string{"service", "operator", "fabricquery"} { + v, _ := metricValue(text, "fabric_api_gateway_admission_refusals_total", + `class="`+class+`"`, `scope="destination"`) + out[class] += v + c, _ := metricValue(text, "fabric_api_gateway_probe_packets_charged_total", `class="`+class+`"`) + out["charged/"+class] += c + } + } + return out + } + if err := watch(answer4); err != nil { + t.Fatal(err) + } + cnt := startCounter() + before := refusals() + settle() + + // 60 two-router traceroutes at the cell's request rate reserve 80 + // packets/s against the host's 20/s and burst 640 for 30 s. + projects := make([]string, 60) + for i := range projects { + projects[i] = l.project() + } + loadDone := make(chan []Record, 1) + loadStart := time.Now() + go func() { + recs, _, err := l.load(ctx, spec{Groups: []group{{ + Name: "drain", Count: len(projects), Concurrency: 8, Interval: "500ms", Projects: projects, + Options: options{Query: query{Type: traceroute, Target: answer4}, Nodes: []string{routers[0], routers[1]}}, + }}}) + if err != nil { + t.Error(err) + } + loadDone <- recs + }() + + // Once the drain has emptied the host bucket, other callers probe the + // same host in turn while it continues. + time.Sleep(10 * time.Second) + cross := map[string]any{} + for i := range 3 { + out, err := l.operator(ctx, l.viaGateway(traceroute, answer4)...) + obs, codes, _ := operatorResults(out) + cross[fmt.Sprintf("operator gateway traceroute %d", i)] = answers{ + Observations: obs, Errors: codes, Failed: err != nil, + } + st, err := l.fabricQuery(ctx, fmt.Sprintf("lgq-budget-%d-%d", time.Now().Unix(), i), + fmt.Sprintf("{type: Traceroute, target: %q, addressFamily: IPv4}", answer4)) + if err == nil { + reason, obs, errs := fqSummary(st) + cross[fmt.Sprintf("FabricQuery traceroute %d", i)] = answers{Reason: reason, Observations: obs, Errors: errs} + } else { + cross[fmt.Sprintf("FabricQuery traceroute %d", i)] = err.Error() + } + } + recs := <-loadDone + loadEnd := time.Now() + // Emissions continue up to 32 s after the last dispatch. + time.Sleep(5 * time.Second) + samples := cnt.close() + after := refusals() + l.saveRecords("destination-budget", recs) + + destRefused, dispatched := 0, 0 + for _, r := range recs { + for _, o := range r.Stream.Outcomes { + switch { + case o.Result == failure && o.Reason == "Throttled": + if o.Scope != "destination" { + t.Errorf("drain: router throttled with scope %q, want destination", o.Scope) + } + destRefused++ + case o.Result == observation || (o.Result == failure && o.Reason != "QueueTimeout"): + dispatched++ + } + } + if r.Status.Code == "ResourceExhausted" && r.Status.Metadata["scope"] == "destination" { + destRefused++ + } + } + T := loadEnd.Sub(loadStart).Seconds() + reservedMax := packetHostBurst + packetHostRate*T + excess, wFrom, wTo, wCount := worstWindow(samples, []string{answer4}, packetHostBurst, packetHostRate) + delta := map[string]float64{} + for k, v := range after { + delta[k] = v - before[k] + } + ev.Thresholds = map[string]any{ + "reservedPacketsMax": reservedMax, "emissionBound": "640 + 20(T+32) per window", + "crossCallerRefusals": "operator and fabricquery destination refusals > 0", + } + ev.Observed["drain"] = map[string]any{ + "statuses": countStatuses(recs), "routersDispatched": dispatched, "routersThrottledDestination": destRefused, + "reservedPackets": dispatched * traceroutePackets, "durationS": T, + } + ev.Observed["cross caller"] = cross + ev.Observed["metric deltas"] = delta + ev.Observed["emission"] = map[string]any{ + "totalPackets": between(samples, answer4, loadStart.Add(-time.Second), time.Now()), + "worstWindowExcess": excess, "worstWindowCount": wCount, "worstWindowS": wTo.Sub(wFrom).Seconds(), + } + if destRefused == 0 { + t.Error("drain: no router was throttled by the destination budget") + } + if float64(dispatched*traceroutePackets) > reservedMax { + t.Errorf("drain: %d packets reserved in %.1f s, over the host bucket's %.0f", + dispatched*traceroutePackets, T, reservedMax) + } + if excess > 0 { + t.Errorf("emission: %d packets in %s, over B + r(T + 32) by %.0f", wCount, wTo.Sub(wFrom), excess) + } + for _, class := range []string{"operator", "fabricquery"} { + if delta[class] <= 0 { + t.Errorf("no %s probe was refused by the destination budget the service drained", class) + } + } + ev.Notes = append(ev.Notes, fmt.Sprintf("Emissions are bounded far below the bucket by node slots (8 per cell, 2 per "+ + "project) and probe duration in a 3-router cell; the emission check holds with margin %.0f packets, "+ + "it is not driven to its limit.", -excess)) +} diff --git a/tests/lgqualify/qualify_test.go b/tests/lgqualify/qualify_test.go new file mode 100644 index 000000000..0da5373df --- /dev/null +++ b/tests/lgqualify/qualify_test.go @@ -0,0 +1,296 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lgqualify_test + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net/url" + "os" + "os/exec" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// TestQualify runs every scenario in order against one cell. The order +// matters: the scenarios that restart or isolate the gateway leader come +// last, since each costs a 125 s recovery hold, and the routing-impact +// trials run after the functional ones so a broken service fails fast. +func TestQualify(t *testing.T) { + l := getLab(t) + ctx := t.Context() + + clientBin := os.Getenv("LG_QUALIFY_CLIENT") + fabricAPIBin := os.Getenv("LG_QUALIFY_FABRIC_API") + if clientBin == "" || fabricAPIBin == "" { + t.Fatal("LG_QUALIFY_CLIENT and LG_QUALIFY_FABRIC_API must name the client and fabric-api binaries") + } + expired, err := expiredCert(ctx, l) + if err != nil { + t.Fatal(err) + } + if err := l.installClient(clientBin, fabricAPIBin, expired); err != nil { + t.Fatal(err) + } + if err := installChain(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := removeChain(); err != nil { + t.Error(err) + } + _, _ = nodeExec(clientNode, "rm", "-rf", clientDir) + }) + logsFrom := time.Now() + use := l.startSampler(ctx) + t.Logf("evidence: %s", l.out) + + l.scenario(ctx, t, "preflight", preflight, rowA17) + l.scenario(ctx, t, "authorization", authorization, rowA1, rowA2) + l.scenario(ctx, t, "compatibility", compatibility, rowA7) + l.scenario(ctx, t, "diagnostics", diagnostics, rowA3, rowA6) + l.scenario(ctx, t, "coverage", coverage, rowA4, rowA5) + l.scenario(ctx, t, "destinations", destinations, rowA3) + l.scenario(ctx, t, "legacy", legacy, rowA16) + l.scenario(ctx, t, "bursts", bursts, rowA8) + l.scenario(ctx, t, "fairness", fairness, rowA8) + l.scenario(ctx, t, "destination-budget", destinationBudget, rowA9, rowA10) + l.scenario(ctx, t, "cancellation", cancellation, rowA12) + l.scenario(ctx, t, "deadline", deadline, rowA12) + l.scenario(ctx, t, "service-switch", serviceSwitch, rowA12, rowA15) + l.scenario(ctx, t, "blackhole", blackhole, rowA12) + l.scenario(ctx, t, "routing-impact", routingImpact, rowA17) + l.scenario(ctx, t, "leader-crash", leaderCrash, rowA11, rowA14) + l.scenario(ctx, t, "leader-pause", leaderPause, rowA14) + l.scenario(ctx, t, "api-isolation", apiIsolation, rowA11, rowA14) + l.scenario(ctx, t, "resources", func(_ context.Context, t *testing.T, _ *lab, ev *evidence) { + resources(t, ev, use.close()) + }, rowA13, rowA17) + l.scenario(ctx, t, "confidentiality", func(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + confidentiality(ctx, t, l, ev, logsFrom) + }, rowA15, rowA17) + l.scenario(ctx, t, "cleanup", cleanup, rowA17) +} + +// expiredCert issues, from the cell's lab CA, a looking-glass service +// certificate that expired an hour ago: cert-manager will not issue one. +func expiredCert(ctx context.Context, l *lab) ([2][]byte, error) { + ca, err := l.cs.CoreV1().Secrets(ns).Get(ctx, "fabric-api-ca", metav1.GetOptions{}) + if err != nil { + return [2][]byte{}, err + } + pair, err := tls.X509KeyPair(ca.Data[certFile], ca.Data[keyFile]) + if err != nil { + return [2][]byte{}, err + } + caCert, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + return [2][]byte{}, err + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return [2][]byte{}, err + } + u, _ := url.Parse("spiffe://fabric-api.datumapis.com/cell/" + cell + "/service/looking-glass") + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), URIs: []*url.URL{u}, + NotBefore: time.Now().Add(-2 * time.Hour), NotAfter: time.Now().Add(-time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &key.PublicKey, pair.PrivateKey) + if err != nil { + return [2][]byte{}, err + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return [2][]byte{}, err + } + return [2][]byte{ + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), + }, nil +} + +// preflight records what is under test and waits for admission. +func preflight(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + lines, err := l.lgc(ctx, idService, "capabilities") + if err != nil { + t.Fatal(err) + } + var caps struct { + Status struct{ Code string } `json:"status"` + Capabilities struct { + Enabled bool `json:"enabled"` + ContractRevision string `json:"contractRevision"` + PolicyRevision string `json:"policyRevision"` + Features []string `json:"features"` + Operations []json.RawMessage `json:"operations"` + Limits json.RawMessage `json:"limits"` + } `json:"capabilities"` + } + if err := json.Unmarshal(lines[0], &caps); err != nil { + t.Fatal(err) + } + c := caps.Capabilities + ev.Observed["capabilities"] = lines[0] + if caps.Status.Code != "OK" || !c.Enabled || len(c.Features) != 3 || len(c.Operations) != 4 { + t.Fatalf("capabilities: %s", lines[0]) + } + + images := map[string]string{} + pods, err := l.cs.CoreV1().Pods(ns).List(ctx, + metav1.ListOptions{LabelSelector: "app.kubernetes.io/name in (fabric-api-gateway,fabric-router)"}) + if err != nil { + t.Fatal(err) + } + for _, p := range pods.Items { + for _, cs := range p.Status.ContainerStatuses { + if cs.Name == "gateway" || cs.Name == "fabric-api" { + images[p.Name+"/"+cs.Name] = cs.ImageID + } + } + } + ev.Observed["images"] = images + head, _ := exec.CommandContext(ctx, "git", "-C", l.root, "rev-parse", "HEAD").Output() + dirty, _ := exec.CommandContext(ctx, "git", "-C", l.root, "status", "--porcelain").Output() + ev.Observed["checkout"] = map[string]any{"commit": strings.TrimSpace(string(head)), "dirty": len(dirty) > 0} + if nm, err := l.nodeMetrics(ctx); err == nil { + frr := map[string]string{} + for node, text := range nm { + for line := range strings.SplitSeq(text, "\n") { + if strings.HasPrefix(line, "fabric_api_node_frr_info{") { + frr[node] = line + } + } + } + ev.Observed["frr"] = frr + } + ready, err := l.waitReady(ctx, recoveryHold+resumeSlack) + if err != nil { + t.Fatal(err) + } + ev.Observed["admittingAt"] = ready + settle() +} + +// expectRefused checks r was refused before its snapshot with code and, +// when reason is not empty, that reason. +func expectRefused(t *testing.T, name string, r Record, code, reason string) { + t.Helper() + switch { + case r.Events != 0: + t.Errorf("%s: %d events delivered to a refused request", name, r.Events) + case code != "" && r.Status.Code != code: + t.Errorf("%s: status %s, want %s", name, r.status(), code) + case reason != "" && r.Status.Reason != reason: + t.Errorf("%s: status %s, want reason %s", name, r.status(), reason) + case r.ok(): + t.Errorf("%s: admitted", name) + } +} + +// expectDelivered checks r delivered a valid stream ending reason. +func expectDelivered(t *testing.T, name string, r Record, reason string) { + t.Helper() + if !r.ok() || !r.Stream.Delivered || len(r.Stream.Violations) > 0 { + t.Errorf("%s: status %s, delivered %t, violations %v", name, r.status(), r.Stream.Delivered, r.Stream.Violations) + return + } + if reason != "" && r.Stream.Reason != reason { + t.Errorf("%s: complete %s, want %s", name, r.Stream.Reason, reason) + } +} + +// outcomeSummary is "node:result[:reason]" per delivered outcome. +func outcomeSummary(r Record) []string { + out := make([]string, 0, len(r.Stream.Outcomes)) + for _, o := range r.Stream.Outcomes { + s := o.Node + ":" + o.Result + if o.Reason != "" { + s += ":" + o.Reason + } + out = append(out, s) + } + return out +} + +// brief is what the evidence keeps of a record. +func brief(r Record) map[string]any { + m := map[string]any{"status": r.status(), "events": r.Events, "bytes": r.Bytes} + if r.Stream.Reason != "" { + m["complete"] = r.Stream.Reason + m["counts"] = r.Stream.Complete + } + if len(r.Stream.Outcomes) > 0 { + m["outcomes"] = outcomeSummary(r) + } + if len(r.Status.Metadata) > 0 { + m["metadata"] = r.Status.Metadata + } + if r.Status.RetryDelayMS >= 0 { + m["retryDelayMs"] = r.Status.RetryDelayMS + } + if r.Timing.CompleteMS >= 0 { + m["completeMs"] = r.Timing.CompleteMS + } + if len(r.Stream.Violations) > 0 { + m["violations"] = r.Stream.Violations + } + return m +} + +// pace spaces requests so a scenario never draws down the cell's request +// bucket (2 tokens/s): its refusals are then its own. +const pace = 550 * time.Millisecond + +// mustRun runs one diagnostic, paced, or fails the test on a client error. +func mustRun(ctx context.Context, t *testing.T, l *lab, id identity, args ...string) Record { + t.Helper() + time.Sleep(pace) + r, err := l.run(ctx, id, args...) + if err != nil { + t.Fatal(err) + } + return r +} + +var errNoLine = errors.New("no matching log line") + +// finishedLine returns the gateway's "Looking Glass request finished" log +// line for requestID from any replica. +func (l *lab) finishedLine(ctx context.Context, requestID string, since time.Time) (map[string]any, error) { + pods, err := l.gatewayPods(ctx) + if err != nil { + return nil, err + } + for _, p := range pods { + logs, err := l.podLogs(ctx, p.Name, "gateway", since) + if err != nil { + continue + } + for line := range strings.SplitSeq(logs, "\n") { + if !strings.Contains(line, requestID) || !strings.Contains(line, "Looking Glass request finished") { + continue + } + var m map[string]any + if json.Unmarshal([]byte(line), &m) == nil { + return m, nil + } + } + } + return nil, fmt.Errorf("%w for %s", errNoLine, requestID) +} diff --git a/tests/lgqualify/resources_test.go b/tests/lgqualify/resources_test.go new file mode 100644 index 000000000..35c30e574 --- /dev/null +++ b/tests/lgqualify/resources_test.go @@ -0,0 +1,126 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lgqualify_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" +) + +// maxJobGoroutines is the contract's cap on service job goroutines. +const maxJobGoroutines = 512 + +// usage is one process's peak resident memory and goroutines over the +// run, with its first goroutine count as a baseline. +type usage struct { + PeakRSS float64 `json:"peakRssBytes"` + PeakGoroutines float64 `json:"peakGoroutines"` + BaseGoroutines float64 `json:"baseGoroutines"` + LimitBytes int64 `json:"memoryLimitBytes,omitempty"` + Samples int `json:"samples"` +} + +// sampler records every gateway's and sidecar's usage every 5 s. +type sampler struct { + mu sync.Mutex + use map[string]*usage + stop chan struct{} + done chan struct{} +} + +func (s *sampler) observe(key, text string, limit int64) { + rss, ok1 := metricValue(text, "process_resident_memory_bytes") + gr, ok2 := metricValue(text, "go_goroutines") + if !ok1 || !ok2 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + u, ok := s.use[key] + if !ok { + u = &usage{BaseGoroutines: gr, LimitBytes: limit} + s.use[key] = u + } + u.PeakRSS, u.PeakGoroutines = max(u.PeakRSS, rss), max(u.PeakGoroutines, gr) + u.Samples++ +} + +func memoryLimit(p corev1.Pod, container string) int64 { + for _, c := range p.Spec.Containers { + if c.Name == container { + return c.Resources.Limits.Memory().Value() + } + } + return 0 +} + +// startSampler samples until the returned sampler is closed. +func (l *lab) startSampler(ctx context.Context) *sampler { + s := &sampler{use: map[string]*usage{}, stop: make(chan struct{}), done: make(chan struct{})} + go func() { + defer close(s.done) + t := time.NewTicker(5 * time.Second) + defer t.Stop() + for { + if gws, err := l.gatewayPods(ctx); err == nil { + for _, p := range gws { + if text, err := l.gatewayMetrics(ctx, p); err == nil { + s.observe("gateway/"+p.Name, text, memoryLimit(p, "gateway")) + } + } + } + if frs, err := l.frrPods(ctx); err == nil { + if nm, err := l.nodeMetrics(ctx); err == nil { + for node, text := range nm { + s.observe("sidecar/"+node, text, memoryLimit(frs[node], "fabric-api")) + } + } + } + select { + case <-s.stop: + return + case <-t.C: + } + } + }() + return s +} + +func (s *sampler) close() map[string]usage { + close(s.stop) + <-s.done + s.mu.Lock() + defer s.mu.Unlock() + out := map[string]usage{} + for k, v := range s.use { + out[k] = *v + } + return out +} + +// resources judges the run's peak usage: under each container's memory +// limit, and goroutines within the service's job cap of their baseline. +func resources(t *testing.T, ev *evidence, use map[string]usage) { + ev.Thresholds = map[string]any{ + "rss": "below the container memory limit", "goroutines": fmt.Sprintf("baseline + %d", maxJobGoroutines), + } + ev.Observed["usage"] = use + if len(use) == 0 { + t.Fatal("no usage sample was taken") + } + for k, u := range use { + if u.LimitBytes > 0 && u.PeakRSS >= float64(u.LimitBytes) { + t.Errorf("%s: peak RSS %.0f reached its limit %d", k, u.PeakRSS, u.LimitBytes) + } + if u.PeakGoroutines > u.BaseGoroutines+maxJobGoroutines { + t.Errorf("%s: %0.f goroutines, over baseline %.0f + %d", k, u.PeakGoroutines, u.BaseGoroutines, maxJobGoroutines) + } + } +} diff --git a/tests/lgqualify/routing_test.go b/tests/lgqualify/routing_test.go new file mode 100644 index 000000000..dbadd97b6 --- /dev/null +++ b/tests/lgqualify/routing_test.go @@ -0,0 +1,310 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lgqualify_test + +import ( + "context" + "encoding/json" + "fmt" + "math" + "math/rand/v2" + "os/exec" + "slices" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// The convergence trial: dfw-worker's IPv6 iBGP session to its edge node +// dfw-worker2, hard reset, timed until it is Established with its accepted +// prefixes back. Each router sees it as its own session to the other's +// link address. +const ( + trialRouter = "dfw-worker" + trialPeer = "2001:db8:1:10::1" + trialPeerRouter = "dfw-worker2" + trialRouterAddr = "2001:db8:1:10::2" + trialFamily = "ipv6Unicast" + trialBlock = 10 + trialsEach = 40 +) + +// trialScript runs in the router's frr container. It prints the +// convergence time in milliseconds, measured on that container's clock. +const trialScript = ` +import json, subprocess, sys, time +peer, family = sys.argv[1], sys.argv[2] +def state(): + out = subprocess.run(["vtysh", "-c", "show bgp neighbors %s json" % peer], capture_output=True, text=True).stdout + try: + n = json.loads(out)[peer] + except Exception: + return None, None, None + return (n.get("bgpState"), n.get("addressFamilyInfo", {}).get(family, {}).get("acceptedPrefixCounter"), + n.get("bgpTimerUpMsec")) +st, base, _ = state() +if st != "Established" or not base: + print(json.dumps({"error": "not established before the reset", "state": st})) + sys.exit(0) +t0 = time.monotonic() +subprocess.run(["vtysh", "-c", "clear bgp %s" % peer], capture_output=True) +# The session counts as reset once seen down, or Established for less time +# than has passed since the reset. +down = False +while time.monotonic() - t0 < 60: + st, n, up = state() + elapsed = (time.monotonic() - t0) * 1000 + if st != "Established" or (up is not None and up < elapsed): + down = True + if down and st == "Established" and n is not None and n >= base: + print(json.dumps({"ms": (time.monotonic() - t0) * 1000, "prefixes": base})) + sys.exit(0) + time.sleep(0.02) +print(json.dumps({"error": "did not converge in 60 s", "down": down})) +` + +// sessions is a router's BGP peers' uptime and accepted prefixes. +const sessionsScript = ` +import json, subprocess +out = subprocess.run(["vtysh", "-c", "show bgp summary json"], capture_output=True, text=True).stdout +dec, i, res = json.JSONDecoder(), 0, {} +while i < len(out): + while i < len(out) and out[i].isspace(): + i += 1 + if i >= len(out): + break + obj, i = dec.raw_decode(out, i) + for af, v in obj.items(): + for peer, p in (v.get("peers") or {}).items(): + res[af + "/" + peer] = { + "state": p.get("state"), "uptimeMs": p.get("peerUptimeMsec"), "pfx": p.get("pfxRcd"), + } +print(json.dumps(res)) +` + +type session struct { + State string `json:"state"` + UptimeMS int64 `json:"uptimeMs"` + Pfx int64 `json:"pfx"` +} + +// frrPods returns each dfw router's fabric-router pod. +func (l *lab) frrPods(ctx context.Context) (map[string]corev1.Pod, error) { + pods, err := l.cs.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: fabricRouterSelector}) + if err != nil { + return nil, err + } + out := map[string]corev1.Pod{} + for _, p := range pods.Items { + out[p.Spec.NodeName] = p + } + return out, nil +} + +func (l *lab) frrPython(ctx context.Context, pod string, script string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "kubectl", append([]string{"--kubeconfig", l.kubeconfig, "-n", ns, "exec", pod, + "-c", "frr", "--", "python3", "-c", script}, args...)...) + return cmd.Output() +} + +// sessions returns every dfw router's sessions. +func (l *lab) sessions(ctx context.Context, pods map[string]corev1.Pod) (map[string]session, error) { + out := map[string]session{} + for node, p := range pods { + raw, err := l.frrPython(ctx, p.Name, sessionsScript) + if err != nil { + return nil, fmt.Errorf("%s sessions: %w", node, err) + } + var m map[string]session + if err := json.Unmarshal(raw, &m); err != nil { + return nil, err + } + for k, v := range m { + out[node+"/"+k] = v + } + } + return out, nil +} + +// trial resets the session once and returns its convergence time. +func (l *lab) trial(ctx context.Context, pod string) (float64, error) { + raw, err := l.frrPython(ctx, pod, trialScript, trialPeer, trialFamily) + if err != nil { + return 0, err + } + var res struct { + MS float64 `json:"ms"` + Error string `json:"error"` + } + if err := json.Unmarshal(raw, &res); err != nil { + return 0, err + } + if res.Error != "" { + return 0, fmt.Errorf("trial: %s", res.Error) + } + return res.MS, nil +} + +// backgroundLoad starts a service workload on clientNode at the cell's +// admissible request rate and returns a function that stops it. +func (l *lab) backgroundLoad(ctx context.Context, tag string) (func() []Record, error) { + // Each group's 20 projects stay within their own 0.2/s; together the + // groups ask 2 requests/s of the cell. + mix := func(name, start string, q query) group { + projects := make([]string, 20) + for i := range projects { + projects[i] = l.project() + } + return group{ + Name: name, Count: 1000, Concurrency: 8, Interval: "2s", StartAfter: start, + Projects: projects, Options: options{Query: q}, + } + } + workload := spec{Groups: []group{ + mix("lookup", "0s", query{Type: routeLookup, Target: answer6}), + mix("summary", "500ms", query{Type: bgpSummary, Family: ipv6}), + mix("ping", "1s", query{Type: ping, Target: answer4}), + mix("traceroute", "1500ms", query{Type: traceroute, Target: answer6}), + }} + done := make(chan []Record, 1) + path := fmt.Sprintf("%s/spec-%s.json", clientDir, tag) + data, _ := json.Marshal(workload) + if err := clientWrite(path, data, 0o600); err != nil { + return nil, err + } + go func() { + lines, _ := l.lgc(context.WithoutCancel(ctx), idService, "load", "--spec", path) + var recs []Record + for _, line := range lines { + var r Record + if json.Unmarshal(line, &r) == nil && r.RequestID != "" { + recs = append(recs, r) + } + } + done <- recs + }() + return func() []Record { + // SIGINT ends the client's workload and prints what finished. + _, _ = nodeExec(clientNode, "pkill", "-INT", "-f", path) + select { + case recs := <-done: + return recs + case <-time.After(90 * time.Second): + return nil + } + }, nil +} + +// p95 is the nearest-rank 95th percentile. +func p95(v []float64) float64 { + s := slices.Clone(v) + slices.Sort(s) + i := int(math.Ceil(0.95*float64(len(s)))) - 1 + return s[max(0, i)] +} + +// bootstrapHalfWidth is half the width of a bootstrap 95 % interval of +// v's p95, from a fixed seed. +func bootstrapHalfWidth(v []float64) float64 { + rng := rand.New(rand.NewPCG(1048, 1048)) + est := make([]float64, 2000) + sample := make([]float64, len(v)) + for i := range est { + for j := range sample { + sample[j] = v[rng.IntN(len(v))] + } + est[i] = p95(sample) + } + slices.Sort(est) + return (est[int(0.975*float64(len(est)))-1] - est[int(0.025*float64(len(est)))]) / 2 +} + +// routingImpact: diagnostics at the cell's admissible rate neither reset a +// session nor slow the control plane's convergence measurably (contract +// "Acceptance matrix and evidence", routing-impact gate). +func routingImpact(ctx context.Context, t *testing.T, l *lab, ev *evidence) { + pods, err := l.frrPods(ctx) + if err != nil { + t.Fatal(err) + } + pod := pods[trialRouter].Name + var control, loaded []float64 + var loadRecs []Record + resets := map[string]any{} + for block := 0; len(control) < trialsEach || len(loaded) < trialsEach; block++ { + isLoaded := block%2 == 1 + var stop func() []Record + var before map[string]session + blockStart := time.Now() + if isLoaded { + if stop, err = l.backgroundLoad(ctx, fmt.Sprintf("routing-%d", block)); err != nil { + t.Fatal(err) + } + time.Sleep(5 * time.Second) + if before, err = l.sessions(ctx, pods); err != nil { + t.Fatal(err) + } + blockStart = time.Now() + } + for range trialBlock { + ms, err := l.trial(ctx, pod) + if err != nil { + t.Fatalf("block %d: %v", block, err) + } + if isLoaded { + loaded = append(loaded, ms) + } else { + control = append(control, ms) + } + time.Sleep(3 * time.Second) + } + if isLoaded { + after, err := l.sessions(ctx, pods) + if err != nil { + t.Fatal(err) + } + elapsed := time.Since(blockStart).Milliseconds() + for k, b := range before { + if k == trialRouter+"/"+trialFamily+"/"+trialPeer || k == trialPeerRouter+"/"+trialFamily+"/"+trialRouterAddr { + continue + } + a, ok := after[k] + if !ok || a.State != "Established" || a.UptimeMS < b.UptimeMS+elapsed-5000 || a.Pfx < b.Pfx { + resets[fmt.Sprintf("block %d %s", block, k)] = map[string]any{"before": b, "after": a} + } + } + loadRecs = append(loadRecs, stop()...) + } + } + l.saveRecords("routing-load", loadRecs) + cp, lp, hw := p95(control), p95(loaded), bootstrapHalfWidth(control) + ev.Thresholds = map[string]any{"loadedP95MaxRatio": convergenceRatio, "trialsEach": trialsEach, + "inconclusiveIfControlP95HalfWidthOver": 0.10} + ev.Observed["controlMs"] = control + ev.Observed["loadedMs"] = loaded + ev.Observed["controlP95Ms"] = cp + ev.Observed["loadedP95Ms"] = lp + ev.Observed["ratio"] = lp / cp + ev.Observed["controlP95HalfWidthMs"] = hw + ev.Observed["unexpected session changes"] = resets + ev.Observed["load"] = countStatuses(loadRecs) + if len(resets) > 0 { + t.Errorf("%d sessions reset or lost prefixes under diagnostic load", len(resets)) + } + if len(loadRecs) == 0 { + t.Error("the background load recorded no request") + } + if hw > 0.10*cp { + ev.Notes = append(ev.Notes, fmt.Sprintf("inconclusive: the control p95's 95 %% bootstrap half-width is %.0f ms, "+ + "%.0f %% of its p95 %.0f ms, so a 10 %% regression cannot be distinguished", hw, 100*hw/cp, cp)) + t.Errorf("routing impact inconclusive: control variability (±%.0f ms on a %.0f ms p95) exceeds 10 %%", hw, cp) + return + } + if lp > convergenceRatio*cp { + t.Errorf("loaded convergence p95 %.0f ms exceeds %.2f × control p95 %.0f ms", lp, convergenceRatio, cp) + } +} diff --git a/tests/lgqualify/thresholds_test.go b/tests/lgqualify/thresholds_test.go new file mode 100644 index 000000000..9d30f8eea --- /dev/null +++ b/tests/lgqualify/thresholds_test.go @@ -0,0 +1,61 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lgqualify_test + +import "time" + +// The pass thresholds, fixed from docs/designs/looking-glass-service.md +// ("Initial defaults and hard bounds" and "Cancellation, delivery, and +// failure assumptions") before any qualification run. A failure against +// one is a qualification failure, never a reason to change it here. +const ( + // stopBound is how soon reachable node probes stop after an explicit + // cancellation, disconnect, ownership loss or service disable is + // received; switchRead is how often each replica reads the switch. + stopBound = 2 * time.Second + switchRead = time.Second + // blackholeBound is transport liveness detection plus local cleanup. + blackholeBound = 10*time.Second + 2*time.Second + // leaseReadBound bounds the leader's uncached Lease read. + leaseReadBound = time.Second + // recoveryHold is the hold after every leadership acquisition. Buckets + // then start empty and a new project key refills from the hold's end, + // so a new project's first admission comes 1/0.2 s = 5 s after it, plus + // the cell's 0.5 s; resumeSlack adds the suite's 3 s polling period. + recoveryHold = 125 * time.Second + resumeSlack = 5*time.Second + time.Second + 3*time.Second + // maxEmission is the latest a node job may emit after dispatch: its 30 s + // deadline plus the 2 s allowance. + maxEmission = 32 * time.Second + // queueWait bounds a router job's wait for a slot. + queueWait = 10 * time.Second + // certsReplacementBound is how soon after a replaced fabric-api-certs + // pod turns Ready a direct operator probe on its node must succeed: + // certsync copies the new certificate within 10 s, the sidecar reloads + // it within another 10 s, plus 10 s for the probe and its permit. + certsReplacementBound = 30 * time.Second + // cleanupReadyBound is how long after the last scenario every gateway + // and fabric-router pod must be Ready: kubelet's 5 minute ceiling on a + // crashed container's restart back-off, plus a minute. + cleanupReadyBound = 6 * time.Minute + + // The contract's default limits, which the lab runs. + requestCellRate = 2.0 + requestCellBurst = 8 + requestProjectRate = 0.2 + requestProjectBurst = 2 + packetHostRate = 20.0 + packetHostBurst = 640.0 + packetPrefixRate = 40.0 + packetPrefixBurst = 640.0 + pingPackets = 3 + traceroutePackets = 20 + + // convergenceTrials is the minimum number of controlled convergence + // trials in each of the control and loaded series; convergenceRatio is + // how much slower the loaded p95 may be. + convergenceTrials = 20 + convergenceRatio = 1.10 +)