diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c0d044..c5c5cc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ env: GOLANGCI_VERSION: "v2.12.2" GOVULNCHECK_VERSION: "v1.6.0" GORELEASER_VERSION: "v2.17.0" + HELM_VERSION: "v4.2.2" + HELM_LINUX_AMD64_SHA256: "9adafecab4d406853bba163a70e9f104f47dbbf65ce24b7653bae7e36150bcb6" SYFT_VERSION: "v1.46.0" jobs: @@ -25,7 +27,7 @@ jobs: name: build · vet · gofmt · lint · test · e2e runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -82,6 +84,22 @@ jobs: - name: Binary integration smoke test run: go test -race -count=1 -tags=e2e ./tests/e2e + - name: Immutable OCI image contract + run: make e2e-oci + + - name: Install pinned Helm + run: | + set -euo pipefail + archive="$RUNNER_TEMP/helm.tar.gz" + curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 --retry 3 --retry-all-errors \ + --output "$archive" "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" + echo "${HELM_LINUX_AMD64_SHA256} ${archive}" | sha256sum --check --status + tar -xzf "$archive" -C "$RUNNER_TEMP" + install -m 0755 "$RUNNER_TEMP/linux-amd64/helm" "$RUNNER_TEMP/helm" + + - name: Fail-closed Helm hub chart contract + run: make e2e-helm HELM="$RUNNER_TEMP/helm" + - name: Install pinned kind run: go install sigs.k8s.io/kind@v0.32.0 @@ -92,7 +110,7 @@ jobs: name: reproducible archives · SPDX SBOM · Homebrew formula runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a213b1..9cd7b2e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: id-token: write attestations: write steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..55c6476 --- /dev/null +++ b/Containerfile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 + +# The manifest-list digest is intentionally pinned so every supported Linux architecture resolves +# to the reviewed distroless static runtime, never a floating tag. +FROM gcr.io/distroless/static-debian12@sha256:b7bb25d9f7c31d2bdd1982feb4dafcaf137703c7075dbe2febb41c24212b946f + +ARG TARGETARCH + +# The build context contains only a static Linux binary assembled by the test or release tooling. +COPY --chown=65532:65532 --chmod=0555 bin/linux/${TARGETARCH}/sith /usr/local/bin/sith + +USER 65532:65532 +ENTRYPOINT ["/usr/local/bin/sith"] diff --git a/Makefile b/Makefile index 40b62f7..a190737 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,12 @@ BIN_DIR := bin GOLANGCI ?= golangci-lint GOVULNCHECK ?= govulncheck KIND ?= kind +HELM ?= helm GORELEASER ?= goreleaser DOCKER ?= docker +KUBECTL ?= kubectl +OCM_SCRATCH_ROOT ?= /Volumes/EXTENDED/tmp/sith-m0 +OCM_PREFIX ?= sith-m0 KIND_NODE_IMAGE ?= kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5 POSTGRES_IMAGE ?= postgres:18.4-alpine3.23@sha256:996d0920e4ff9df1fc19dacb904492f3c1ec0ec1cc338f0ad7123be7731c5f5e @@ -25,7 +29,7 @@ LDFLAGS := -s -w \ -X $(PKG)/internal/buildinfo.Commit=$(COMMIT) \ -X $(PKG)/internal/buildinfo.Date=$(DATE) -.PHONY: all build test test-scripts perf e2e e2e-kind e2e-postgres e2e-isolation lint vuln fmt fmt-check vet tidy clean run ci release-check help +.PHONY: all build test test-scripts perf e2e e2e-helm e2e-oci e2e-kind e2e-ocm e2e-postgres e2e-isolation lint vuln fmt fmt-check vet tidy clean run ci release-check help all: build @@ -45,9 +49,37 @@ perf: ## Enforce the warm-cache TUI p95 latency budget without race overhead e2e: ## Build and exercise the real binary as a subprocess go test -race -count=1 -tags=e2e ./tests/e2e +e2e-helm: ## Validate the fail-closed Helm hub chart with the pinned Helm CLI + HELM_BIN="$(HELM)" go test -race -count=1 -timeout=5m -tags='e2e helm' -run '^TestHelmHubChartContract$$' ./tests/e2e + +e2e-oci: ## Build and inspect the local immutable OCI image contract for linux/amd64 and linux/arm64 + go test -race -count=1 -timeout=10m -tags='e2e oci' -run '^Test(OCIImageCrossPlatformContract|ContainerfileInstructionGuard)$$' ./tests/e2e + e2e-kind: ## Exercise adapter and binary against two real kind clusters KIND_BIN="$(KIND)" KIND_NODE_IMAGE="$(KIND_NODE_IMAGE)" \ - go test -race -count=1 -timeout=15m -tags='e2e kind' -run '^TestKindFleetFanout$$' ./tests/e2e + go test -race -count=1 -timeout=15m -tags='e2e kind' -run '^Test(KindFleetFanout|KindOCIImageContract)$$' ./tests/e2e + +e2e-ocm: ## Prove direct ClusterProxy transport in the pinned two-spoke M0 lab + @set -euo pipefail; \ + run_required_e2e_test() { \ + local test_name="$$1"; shift; \ + local output; \ + if ! output="$$("$$@" 2>&1)"; then \ + printf '%s\n' "$$output" >&2; return 1; \ + fi; \ + printf '%s\n' "$$output"; \ + grep -Fq -- "--- PASS: $${test_name}" <<<"$$output" || { \ + echo "required M0 test $${test_name} did not run" >&2; return 1; \ + }; \ + }; \ + trap 'KIND_BIN="$(KIND)" SITH_M0_SCRATCH_ROOT="$(OCM_SCRATCH_ROOT)" SITH_M0_PREFIX="$(OCM_PREFIX)" hack/experiments/m0-ocm-falsification.sh cleanup' EXIT; \ + KIND_BIN="$(KIND)" SITH_M0_SCRATCH_ROOT="$(OCM_SCRATCH_ROOT)" SITH_M0_PREFIX="$(OCM_PREFIX)" SITH_M0_KEEP_CLUSTERS=1 \ + hack/experiments/m0-ocm-falsification.sh run; \ + export KUBECTL_BIN="$(KUBECTL)" SITH_OCM_HUB_KUBECONFIG="$(OCM_SCRATCH_ROOT)/kubeconfig" SITH_OCM_HUB_CONTEXT="kind-$(OCM_PREFIX)-hub"; \ + run_required_e2e_test TestDirectClusterProxyM0 \ + go test -v -race -count=1 -timeout=8m -tags='e2e ocm' -run '^TestDirectClusterProxyM0$$' ./internal/hubocm; \ + run_required_e2e_test TestHubRuntimeDirectClusterProxyM0 \ + go test -v -race -count=1 -timeout=8m -tags='e2e ocm' -run '^TestHubRuntimeDirectClusterProxyM0$$' ./internal/hubruntime e2e-postgres: ## Prove forced RLS against a temporary digest-pinned PostgreSQL container DOCKER_BIN="$(DOCKER)" POSTGRES_IMAGE="$(POSTGRES_IMAGE)" \ @@ -92,7 +124,7 @@ ci: fmt-check vet lint vuln test test-scripts perf e2e build ## Run the full CI release-check: ## Build and verify the reproducible multi-platform release snapshot twice @command -v "$(GORELEASER)" >/dev/null || { echo "goreleaser is required" >&2; exit 1; } @command -v syft >/dev/null || { echo "syft is required" >&2; exit 1; } - @tmp="$$(mktemp -d)"; trap 'rm -rf "$$tmp"' EXIT; \ + @set -e; tmp="$$(mktemp -d)"; trap 'rm -rf "$$tmp"' EXIT; \ go mod download; \ go mod verify; \ "$(GORELEASER)" check .goreleaser.yaml; \ diff --git a/README.md b/README.md index c8b289b..47b631a 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,10 @@ revoked immediately. Exchange responses, including generic failures, are non-cac handler includes a bounded per-process attempt limiter; a replicated hub must additionally enforce a shared limit at its ingress or gateway. Deployments must provide the HMAC pepper and Ed25519 private key through a secret manager, keep both out of logs and configuration repositories, and -rotate them under an explicit operational procedure. These are E1 library and HTTP boundaries; -the `sith hub` runtime remains staged behind later hub epics rather than exposing an incomplete -service. +rotate them under an explicit operational procedure. These are E1 library and HTTP boundaries. +The P1 `sith hub` runtime now mounts only the session-authenticated fleet read/refresh surface +below; API-key, OIDC, and cloud-proof exchange handlers remain intentionally unmounted until their +ingress and operator lifecycle are composed. Pinned OIDC federation uses the same exchange model. Each endpoint is fixed to one requested workspace, and each provider configuration allowlists an exact HTTPS issuer, audience, token type, @@ -159,13 +160,100 @@ only the workspace boundary and registered managed-cluster reference; it never r kubeconfig, endpoint, or token through the Sith collector contract. Only normalized `inventory` and `health` facts are accepted, source-stamped, freshness-bounded, and stored behind forced RLS. Failed refreshes retain the last snapshot as explicitly stale evidence and record only a closed -failure category. The concrete OCM ClusterGateway transport is deliberately not exposed by the -`sith hub` stub until its projected-token lifecycle is wired and exercised as a product adapter. -The same model now answers a read-only, exact cross-cluster correlation such as “every deployment +failure category. The pinned direct OCM ClusterProxy adapter reads the exact rotating +`sith-reader` managed-serviceaccount Secret for a registered spoke, opens a short-lived +Konnectivity tunnel only to that spoke, and verifies both proxy mTLS and the spoke Kubernetes +certificate; it never forwards a caller `Authorization` header, stores a credential, disables +TLS verification, lists or watches Secrets, or carries raw Kubernetes objects across the +collector seam. Its executable two-spoke M0 gate is `make e2e-ocm`, which now also drives a +signed-session request through the TLS hub runtime across both spokes. The same model now answers +a read-only, exact cross-cluster correlation such as “every deployment named `payments` that is not Healthy” within one workspace. Matching is by exact kind/name/namespace -rather than a prefix, and every answer retains full stale/unreachable coverage rather than claiming +rather than a prefix. Every returned cluster and matching fact retains its source identity and +observation time, and every answer retains full stale/unreachable coverage rather than claiming that a partial fleet is complete. +### Governed hub runtime (P1) + +`sith hub` is an in-cluster, TLS-only process. It has no listener default and exits non-zero before +opening a listener, database pool, or Kubernetes client unless all of these deployment inputs are +present and valid: + +- `SITH_HUB_LISTEN_ADDR` and `SITH_HUB_DATABASE_URL` (the database must use the existing + non-owner, forced-RLS application role and TLS); +- `SITH_HUB_SESSION_ISSUER`, `SITH_HUB_SESSION_AUDIENCE`, `SITH_HUB_SESSION_KEY_ID`, and + `SITH_HUB_SESSION_PUBLIC_KEY_FILE` (a static Ed25519 PKIX public key; no remote discovery); +- `SITH_HUB_SERVER_TLS_CERT_FILE` and `SITH_HUB_SERVER_TLS_KEY_FILE` for the hub HTTPS listener; +- `SITH_HUB_PROXY_ADDRESS`, `SITH_HUB_PROXY_SERVER_NAME`, `SITH_HUB_PROXY_CA_FILE`, + `SITH_HUB_PROXY_CERT_FILE`, `SITH_HUB_PROXY_KEY_FILE`, and `SITH_HUB_KUBE_API_SERVER_NAME` for + the direct ClusterProxy mTLS path. + +Every referenced key, certificate, or CA file must be a read-only regular file from a deployment +mount. The runtime obtains its Kubernetes identity only with in-cluster configuration; it has no +kubeconfig fallback and uses that identity through the fixed `sith-reader` Secret reader. It serves +only `POST /v1/workspaces/{workspace}/fleet:refresh`, +`GET /v1/workspaces/{workspace}/fleet`, and +`GET /v1/workspaces/{workspace}/fleet/images/{sha256:<64-lowercase-hex>}`. Every route requires an +exact signed Sith session, derives the workspace scope from its signed memberships, carries that +scope through the PEP and RLS seams, accepts no query parameters, and returns only normalized +coverage/fleet data under `Cache-Control: no-store`. + +After deriving the signed scope, the hub mints one opaque local trace ID for the governed request. +It strips common caller-supplied trace and correlation carriers, never echoes or forwards them, +and carries the local ID through the PEP audit record and each snapshot transport attempt. The hub +logs only local trace ID, fixed stage, fixed outcome, and bounded duration; it records no workspace, +actor, spoke, endpoint, resource, selector, argument digest, credential, raw error, or returned +data in trace events. This is not a telemetry exporter: it adds no OpenTelemetry SDK, listener, +network egress, queue, trace store, persistence, or action-intent protocol. + +Before a signed scope exists, the authentication gate emits one local WARN record for every +refusal with only the fixed `hub-auth` surface and `refused` outcome. It deliberately does not +distinguish credential failure modes or carry a trace/correlation ID, token, header, path, client +address, workspace, principal, or verifier error. The record is a passive alerting signal, not an +audit record, rate limiter, telemetry export, or additional authentication decision. + +The image route answers one exact, immutable runtime digest question across registered spokes. The +direct reader accepts only canonical digests normalized from ordinary +`Pod.Status.ContainerStatuses[].ImageID`; PodSpec image strings, init and ephemeral container +statuses, mutable tags, malformed values, and ambiguous runtime IDs abstain. Sith makes no registry +request, image pull, SBOM retrieval, vulnerability-feed lookup, or credential use for this read. +The result remains coverage-honest: matching Pod inventory facts retain source and freshness, and +unreachable or stale spokes are reported rather than assumed clean. + +### Hub schema migration + +Run `sith hub migrate` as a short-lived deployment Job before starting `sith hub`. It accepts only +`SITH_HUB_MIGRATION_OWNER_DATABASE_URL` and `SITH_HUB_APPLICATION_DATABASE_ROLE`; mount the owner +database URL from the deployment secret provider and set the application role explicitly. The +command requires TLS for any non-local database target, applies the checksum-locked serializable +migration ledger, audits forced RLS, attempts to close its one owner connection, and exits. It never opens the +hub listener, creates a Kubernetes client, or starts collection. + +The normal hub process continues to use only `SITH_HUB_DATABASE_URL` for the non-owner application +role. Do not reuse the migration-owner credential in the hub Deployment or place either database +URL, certificates, tokens, or private keys in chart values or logs. + +### OCI image deployment contract + +The hub OCI recipe uses the digest-pinned distroless static Debian 12 runtime and contains only a +static Linux Sith binary running as UID/GID `65532`. It has no shell, package manager, default +configuration, Kubernetes credential, certificate, database URL, or secret. The source test builds +and inspects both `linux/amd64` and `linux/arm64` variants without publishing; the native image must +also run with a read-only filesystem, no network, no Linux capabilities, and no privilege +escalation, then complete the same contract as a hardened Job on each of two Kind clusters. +The no-network setting applies only to those isolated image checks. A deployed hub needs narrowly +allowlisted egress to its configured runtime dependencies, including its database and, when +enabled, the pinned OIDC discovery and JWKS endpoints. + +This is not a published image reference. The fail-closed [`charts/sith-hub`](charts/sith-hub) +chart requires an explicit immutable `repository@sha256:...` image reference and refuses tags, +especially `latest`; it invokes `sith hub migrate` in a separate short-lived Job before the +non-owner hub Deployment starts. Its defaults intentionally cannot install until a release-bound +hub image and operator-provided Secret references exist; it never renders secret material. The +chart permits only fixed `light` and `heavy` resource profiles, which retain identical security, +credential, and RBAC controls. This first F9.3a profile slice does not claim a public image, +in-chart database, or HA; those parent-F9.3 topology and custody capabilities need later evidence. + `sith serve --mcp` exposes `fleet.inventory`, `fleet.health`, `fleet.correlate`, and `fleet.cve-search` over MCP Streamable HTTP. All four tools are cache-only and carry `readOnlyHint:true`; they use the exact workspace-required query path used by the CLI, TUI, and web @@ -260,7 +348,8 @@ make release-check The gate also compiles the binary under a functional HTTP/HTTPS egress sentinel and exercises local commands, deterministic investigation, plus the running web UI and MCP server with an official SDK client. A source boundary exact-allowlists production network, filesystem-write, and subprocess imports, confines -client-go transport to the kubeconfig adapter, and rejects known telemetry SDKs and low-level +local-mode client-go transport to the kubeconfig adapter, permits only the separately reviewed +tenant-scoped direct OCM adapter in governed mode, and rejects known telemetry SDKs and low-level network bypasses. Together these checks prove the reviewed paths; they are regression controls rather than an operating-system network sandbox. diff --git a/charts/sith-hub/Chart.yaml b/charts/sith-hub/Chart.yaml new file mode 100644 index 0000000..253d79e --- /dev/null +++ b/charts/sith-hub/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: sith-hub +description: Fail-closed deployment contract for the Sith governed fleet hub +type: application +version: 0.2.0 +appVersion: "unpublished" diff --git a/charts/sith-hub/README.md b/charts/sith-hub/README.md new file mode 100644 index 0000000..6489e37 --- /dev/null +++ b/charts/sith-hub/README.md @@ -0,0 +1,32 @@ +# Sith hub Helm chart + +This chart is a fail-closed deployment contract for a released Sith hub image. The repository does not currently publish that image, so the chart has deliberately invalid default values and cannot be installed until an operator supplies an immutable `repository@sha256:<64 lowercase hex>` reference. Tags, including `latest`, are rejected by both the value schema and template logic. + +This `F9.3a` slice provides only fixed resource envelopes. It does not claim the parent F9.3 end state of a minimal in-chart Postgres for light or an HA hub with external Postgres/cloud KMS for heavy: those need separate E3 custody and topology evidence before they can be rendered safely. + +The chart creates no `Secret`, `data`, or `stringData` block. An E3-approved KMS/ExternalSecret materializer must create these existing Secret objects before an installation: + +| Value | Required Secret keys | Consumer | +| --- | --- | --- | +| `runtime.existingSecret` | `database-url`, `session-public.pem`, `server-tls.crt`, `server-tls.key`, `proxy-ca.crt`, `proxy-tls.crt`, `proxy-tls.key` | long-running `sith hub` Deployment | +| `migration.existingSecret` | `owner-database-url` | short-lived `sith hub migrate` hook Job | + +`migration.applicationRole` is a non-secret PostgreSQL role name. The migration hook runs before install and upgrade, blocks the release if it fails, and receives no Kubernetes service-account token or runtime TLS material. The Deployment receives an in-cluster token only to read the fixed `sith-reader` managed-serviceaccount Secret; its ClusterRole permits exactly `get` on that one resource name and no list/watch or write verbs. + +The chart permits exactly two fixed profiles for both the hub and its migration hook: + +| Profile | Requests | Limits | Intended envelope | +| --- | --- | --- | --- | +| `light` | 100m CPU, 128Mi memory | 500m CPU, 512Mi memory | development and lab scheduling envelope | +| `heavy` | 500m CPU, 512Mi memory | 2 CPU, 2Gi memory | larger production-like scheduling envelope | + +The heavy profile reserves five times the requested CPU and four times the requested memory, so it carries a correspondingly higher node-pool cost. These are fixed scheduling bounds, not measured capacity claims; no arbitrary resource override or third profile is accepted. Both profiles use the same immutable image requirement, existing Secret references, migration isolation, RBAC, probes, and pod/container hardening. They do not change replica count, database custody, or KMS materialization, so `heavy` does not claim unproven high availability. + +The chart pins workload hardening (UID/GID 65532, read-only root filesystem, RuntimeDefault seccomp, no privilege escalation, and all Linux capabilities dropped). It deliberately does not create a broad egress NetworkPolicy: the database and pinned OCM endpoints are deployment-specific, so operators must place the release in a namespace with an appropriate least-privilege egress policy. KMS provider resources, release-bound image publication, real install/upgrade proof, air-gap bundles, and addon packaging are later E9/E3 slices. + +Validate supplied values before applying anything: + +```bash +helm lint charts/sith-hub -f operator-values.yaml +helm template sith-hub charts/sith-hub --namespace sith-system -f operator-values.yaml +``` diff --git a/charts/sith-hub/templates/NOTES.txt b/charts/sith-hub/templates/NOTES.txt new file mode 100644 index 0000000..391924b --- /dev/null +++ b/charts/sith-hub/templates/NOTES.txt @@ -0,0 +1,5 @@ +The Sith hub chart intentionally has no runnable default values. + +Set image.reference to a released immutable repository@sha256 reference, and provide the existing +runtime and migration Secrets described in charts/sith-hub/README.md. This chart never creates +Secrets or materializes KMS values; an approved external secret materializer owns that boundary. diff --git a/charts/sith-hub/templates/_helpers.tpl b/charts/sith-hub/templates/_helpers.tpl new file mode 100644 index 0000000..e59b55b --- /dev/null +++ b/charts/sith-hub/templates/_helpers.tpl @@ -0,0 +1,67 @@ +{{- define "sith-hub.name" -}} +{{- .Chart.Name -}} +{{- end -}} + +{{- define "sith-hub.fullname" -}} +{{- printf "%s-%s" .Release.Name (include "sith-hub.name" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "sith-hub.labels" -}} +app.kubernetes.io/name: {{ include "sith-hub.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +{{- end -}} + +{{- define "sith-hub.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sith-hub.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{- define "sith-hub.imageReference" -}} +{{- $reference := required "image.reference must be an immutable repository@sha256 reference" .Values.image.reference -}} +{{- if not (regexMatch "^[^@[:space:]]+@sha256:[a-f0-9]{64}$" $reference) -}} +{{- fail "image.reference must be an immutable repository@sha256:<64 lowercase hex> reference; image tags are forbidden" -}} +{{- end -}} +{{- $reference -}} +{{- end -}} + +{{- define "sith-hub.runtimeSecretName" -}} +{{- required "runtime.existingSecret must name an operator-provided Secret" .Values.runtime.existingSecret -}} +{{- end -}} + +{{- define "sith-hub.migrationSecretName" -}} +{{- required "migration.existingSecret must name an operator-provided Secret" .Values.migration.existingSecret -}} +{{- end -}} + +{{- define "sith-hub.profile" -}} +{{- if hasKey .Values "resources" -}} +{{- fail "resources is not configurable; select the fixed light or heavy profile" -}} +{{- end -}} +{{- $profile := required "profile must be light or heavy" .Values.profile -}} +{{- if not (has $profile (list "light" "heavy")) -}} +{{- fail "profile must be light or heavy; arbitrary resource profiles are forbidden" -}} +{{- end -}} +{{- $profile -}} +{{- end -}} + +{{- define "sith-hub.resources" -}} +{{- $profile := include "sith-hub.profile" . -}} +{{- if eq $profile "light" }} +requests: + cpu: "100m" + memory: "128Mi" +limits: + cpu: "500m" + memory: "512Mi" +{{- else if eq $profile "heavy" }} +requests: + cpu: "500m" + memory: "512Mi" +limits: + cpu: "2" + memory: "2Gi" +{{- else -}} +{{- fail "profile must be light or heavy" -}} +{{- end -}} +{{- end -}} diff --git a/charts/sith-hub/templates/clusterrole.yaml b/charts/sith-hub/templates/clusterrole.yaml new file mode 100644 index 0000000..fdaa8fd --- /dev/null +++ b/charts/sith-hub/templates/clusterrole.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "sith-hub.fullname" . }} + labels: + {{- include "sith-hub.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["sith-reader"] + verbs: ["get"] diff --git a/charts/sith-hub/templates/clusterrolebinding.yaml b/charts/sith-hub/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..fba4a36 --- /dev/null +++ b/charts/sith-hub/templates/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "sith-hub.fullname" . }} + labels: + {{- include "sith-hub.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "sith-hub.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "sith-hub.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/sith-hub/templates/deployment.yaml b/charts/sith-hub/templates/deployment.yaml new file mode 100644 index 0000000..924f64d --- /dev/null +++ b/charts/sith-hub/templates/deployment.yaml @@ -0,0 +1,108 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "sith-hub.fullname" . }} + labels: + {{- include "sith-hub.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "sith-hub.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "sith-hub.selectorLabels" . | nindent 8 }} + spec: + automountServiceAccountToken: true + serviceAccountName: {{ include "sith-hub.fullname" . }} + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: hub + image: {{ include "sith-hub.imageReference" . | quote }} + imagePullPolicy: IfNotPresent + args: ["hub"] + ports: + - name: https + containerPort: 8443 + protocol: TCP + env: + - name: SITH_HUB_LISTEN_ADDR + value: "0.0.0.0:8443" + - name: SITH_HUB_DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "sith-hub.runtimeSecretName" . }} + key: database-url + - name: SITH_HUB_SESSION_ISSUER + value: {{ required "runtime.sessionIssuer is required" .Values.runtime.sessionIssuer | quote }} + - name: SITH_HUB_SESSION_AUDIENCE + value: {{ required "runtime.sessionAudience is required" .Values.runtime.sessionAudience | quote }} + - name: SITH_HUB_SESSION_KEY_ID + value: {{ required "runtime.sessionKeyID is required" .Values.runtime.sessionKeyID | quote }} + - name: SITH_HUB_SESSION_PUBLIC_KEY_FILE + value: /var/run/sith/runtime/session-public.pem + - name: SITH_HUB_SERVER_TLS_CERT_FILE + value: /var/run/sith/runtime/server-tls.crt + - name: SITH_HUB_SERVER_TLS_KEY_FILE + value: /var/run/sith/runtime/server-tls.key + - name: SITH_HUB_PROXY_ADDRESS + value: {{ required "runtime.proxyAddress is required" .Values.runtime.proxyAddress | quote }} + - name: SITH_HUB_PROXY_SERVER_NAME + value: {{ required "runtime.proxyServerName is required" .Values.runtime.proxyServerName | quote }} + - name: SITH_HUB_PROXY_CA_FILE + value: /var/run/sith/runtime/proxy-ca.crt + - name: SITH_HUB_PROXY_CERT_FILE + value: /var/run/sith/runtime/proxy-tls.crt + - name: SITH_HUB_PROXY_KEY_FILE + value: /var/run/sith/runtime/proxy-tls.key + - name: SITH_HUB_KUBE_API_SERVER_NAME + value: {{ required "runtime.kubeAPIServerName is required" .Values.runtime.kubeAPIServerName | quote }} + livenessProbe: + tcpSocket: + port: https + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + tcpSocket: + port: https + initialDelaySeconds: 2 + periodSeconds: 5 + resources: + {{- include "sith-hub.resources" . | nindent 12 }} + securityContext: + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: runtime + mountPath: /var/run/sith/runtime + readOnly: true + volumes: + - name: runtime + secret: + secretName: {{ include "sith-hub.runtimeSecretName" . }} + defaultMode: 288 + items: + - key: session-public.pem + path: session-public.pem + - key: server-tls.crt + path: server-tls.crt + - key: server-tls.key + path: server-tls.key + - key: proxy-ca.crt + path: proxy-ca.crt + - key: proxy-tls.crt + path: proxy-tls.crt + - key: proxy-tls.key + path: proxy-tls.key diff --git a/charts/sith-hub/templates/migration-job.yaml b/charts/sith-hub/templates/migration-job.yaml new file mode 100644 index 0000000..681c35d --- /dev/null +++ b/charts/sith-hub/templates/migration-job.yaml @@ -0,0 +1,48 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "sith-hub.fullname" . }}-migrate + labels: + {{- include "sith-hub.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 0 + activeDeadlineSeconds: 300 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + {{- include "sith-hub.selectorLabels" . | nindent 8 }} + spec: + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: migrate + image: {{ include "sith-hub.imageReference" . | quote }} + imagePullPolicy: IfNotPresent + args: ["hub", "migrate"] + env: + - name: SITH_HUB_MIGRATION_OWNER_DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "sith-hub.migrationSecretName" . }} + key: owner-database-url + - name: SITH_HUB_APPLICATION_DATABASE_ROLE + value: {{ required "migration.applicationRole is required" .Values.migration.applicationRole | quote }} + resources: + {{- include "sith-hub.resources" . | nindent 12 }} + securityContext: + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] diff --git a/charts/sith-hub/templates/service.yaml b/charts/sith-hub/templates/service.yaml new file mode 100644 index 0000000..6bed062 --- /dev/null +++ b/charts/sith-hub/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "sith-hub.fullname" . }} + labels: + {{- include "sith-hub.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + {{- include "sith-hub.selectorLabels" . | nindent 4 }} diff --git a/charts/sith-hub/templates/serviceaccount.yaml b/charts/sith-hub/templates/serviceaccount.yaml new file mode 100644 index 0000000..7565464 --- /dev/null +++ b/charts/sith-hub/templates/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "sith-hub.fullname" . }} + labels: + {{- include "sith-hub.labels" . | nindent 4 }} +automountServiceAccountToken: true diff --git a/charts/sith-hub/values.schema.json b/charts/sith-hub/values.schema.json new file mode 100644 index 0000000..23111ef --- /dev/null +++ b/charts/sith-hub/values.schema.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Sith hub chart values", + "type": "object", + "additionalProperties": false, + "required": ["profile", "image", "runtime", "migration"], + "properties": { + "profile": { + "type": "string", + "enum": ["light", "heavy"] + }, + "image": { + "type": "object", + "additionalProperties": false, + "required": ["reference"], + "properties": { + "reference": { + "type": "string", + "pattern": "^[^@[:space:]]+@sha256:[a-f0-9]{64}$" + } + } + }, + "runtime": { + "type": "object", + "additionalProperties": false, + "required": ["existingSecret", "sessionIssuer", "sessionAudience", "sessionKeyID", "proxyAddress", "proxyServerName", "kubeAPIServerName"], + "properties": { + "existingSecret": {"type": "string", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"}, + "sessionIssuer": {"type": "string", "minLength": 1}, + "sessionAudience": {"type": "string", "minLength": 1}, + "sessionKeyID": {"type": "string", "minLength": 1}, + "proxyAddress": {"type": "string", "minLength": 1}, + "proxyServerName": {"type": "string", "minLength": 1}, + "kubeAPIServerName": {"type": "string", "minLength": 1} + } + }, + "migration": { + "type": "object", + "additionalProperties": false, + "required": ["existingSecret", "applicationRole"], + "properties": { + "existingSecret": {"type": "string", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"}, + "applicationRole": {"type": "string", "pattern": "^[a-z_][a-z0-9_]{0,62}$"} + } + } + } +} diff --git a/charts/sith-hub/values.yaml b/charts/sith-hub/values.yaml new file mode 100644 index 0000000..beedab8 --- /dev/null +++ b/charts/sith-hub/values.yaml @@ -0,0 +1,28 @@ +# image.reference is an explicit immutable repository@sha256 reference for a released Sith hub image. +# It is intentionally empty: this repository does not publish a hub image yet. +profile: light + +image: + reference: "" + +# runtime.existingSecret is an operator-provided Secret materialized outside this chart. It contains +# database-url, session-public.pem, server-tls.crt, server-tls.key, proxy-ca.crt, proxy-tls.crt, +# and proxy-tls.key. The chart never creates or renders secret data. +runtime: + existingSecret: "" + sessionIssuer: "" + sessionAudience: "" + sessionKeyID: "" + proxyAddress: "" + proxyServerName: "" + kubeAPIServerName: "" + +# migration.existingSecret is an operator-provided Secret containing owner-database-url only. +# migration.applicationRole is the non-owner Postgres role enforced by the migration ledger. +migration: + existingSecret: "" + applicationRole: "" + +# profile selects a fixed, reviewed resource envelope for both the hub and its migration hook. +# light requests 100m CPU / 128Mi memory and caps at 500m / 512Mi; heavy requests 500m / 512Mi +# and caps at 2 CPU / 2Gi. Arbitrary resource overrides are intentionally rejected. diff --git a/docs/CONVENTIONS.md b/docs/CONVENTIONS.md index 2d4b306..684c4bf 100644 --- a/docs/CONVENTIONS.md +++ b/docs/CONVENTIONS.md @@ -18,8 +18,8 @@ Related: [`ARCHITECTURE.md`](ARCHITECTURE.md), [`adr/0002-stack-and-language.md` | Branch | Role | Rules | |---|---|---| -| `main` | **Release.** Seed + tagged releases only. | Protected. No direct pushes. Only receives merges from `dev` at release time. Tags (`vX.Y.Z`) are cut here. **Do not touch `main` during Phase L.** | -| `dev` | **Integration.** The default PR target and the trunk all feature work merges into. | Protected. All feature PRs target `dev`. Must stay green. | +| `main` | **Release.** Seed + tagged releases only. | Protected from deletion and force-push. No direct pushes by process. Only receives merges from `dev` at release time. Tags (`vX.Y.Z`) are cut here. **Do not touch `main` during Phase L.** | +| `dev` | **Integration.** The default PR target and the trunk all feature work merges into. | Protected from deletion and force-push. All feature PRs target `dev`. Must stay green. | | `feat/*`, `fix/*`, `docs/*`, `chore/*`, `refactor/*`, `test/*`, `ci/*`, `build/*` | **Feature branches.** One slice or one coherent change each. | Branched **off `dev`**. PR **into `dev`**. Deleted after merge. | **Naming.** `feat/-` — e.g. `feat/slice-0-foundation`, @@ -38,6 +38,8 @@ identify the slice from `BUILD-SEQUENCE.md`. **Release flow (not Phase L, documented for completeness).** PR `dev → main`, then tag `main` with `vX.Y.Z`. Release artifacts (cosign signature, SLSA provenance, SBOM) attach to the tag per E9/#27. +`dev` is the durable integration source: never use `--delete-branch` when merging this release PR. +Only a merged feature branch is eligible for automatic deletion. --- @@ -278,13 +280,15 @@ workflow): 4. **build** — `go build ./...` succeeds; `cmd/sith` produces a runnable binary. 5. **test** — `go test -race -count=1 ./...` passes. -Additional merge requirements (branch protection): +Additional merge requirements (repository discipline): - Every commit is **DCO signed-off** (a DCO check verifies each commit has a matching `Signed-off-by`). - Every commit is **SSH-signed** and verifies. - At least one approving review (owner review counts). - Branch is up to date with `dev` (rebased) before merge. - No `--no-verify`, no `--no-gpg-sign`, no squash-merge (§1). +- Branch protection rejects deletion and force-pushes for both `dev` and `main`; it does not replace + these merge requirements. Toolchain is **pinned**: the Go version in CI matches `go.mod`'s `go` directive; the golangci-lint version is pinned in the workflow. Bumps to either are their own `ci:`/`build:` commit, reviewed diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 6967288..af3260c 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -84,10 +84,34 @@ These checks establish producer identity, artifact integrity, build provenance, binding. They do not prove that every dependency is vulnerability-free; consumers must still evaluate the attached SBOM against their own policy and current advisory data. +## OCI image contract + +Sith's deployment image recipe is validated locally before any registry publication. It assembles +the existing static Linux binary into a digest-pinned distroless runtime, uses non-root UID/GID +`65532`, and exposes only the `sith` entrypoint. The test builds and inspects both `linux/amd64` +and `linux/arm64` variants without pushing, then runs the native image with a read-only filesystem, +no network, no capabilities, and `no-new-privileges`; the same image must complete a hardened Job +on two real Kind clusters. + +No network is an isolated image-check constraint, not the operational hub policy. A deployment +must allow only narrowly scoped egress to configured runtime dependencies, including the database +and, where enabled, the pinned OIDC discovery and JWKS endpoints. + +No OCI image is published by this repository yet. Consumers must not infer a mutable image tag +from a release archive. The [`charts/sith-hub`](../charts/sith-hub) chart accepts only an explicit +immutable `repository@sha256:...` reference, and its defaults intentionally fail until an operator +provides that reference and existing Secret names. Image publishing/signing/attestation will be +added as a separate release-boundary change before any public deployment guidance. Its fixed +`light` and `heavy` profiles alter only the reviewed resource envelope; both preserve the same +digest, Secret-reference, migration, RBAC, and workload-hardening contract. This first F9.3a +slice is not a claim of the parent feature's future in-chart database, HA, or cloud-KMS topology. + ## Maintainer release procedure 1. Merge the feature PR into `dev`, ensure the full CI and release-snapshot jobs are green, then - merge a reviewed `dev` to `main` release PR. + merge a reviewed `dev` to `main` release PR. `dev` is the durable integration source: never use + `--delete-branch` for this release PR. Automatic branch deletion is reserved for merged feature + branches. 2. From an up-to-date `main`, run `make ci` and `make release-check`. The latter compares archive SHA-256 digests across two complete builds; SBOM creation timestamps and Sigstore signatures are intentionally not expected to be byte-for-byte reproducible. @@ -97,6 +121,8 @@ evaluate the attached SBOM against their own policy and current advisory data. 5. Verify one archive with the commands above, dispatch the `ArdurAI/homebrew-tap` sync workflow, and prove a clean `brew install sith && sith version` before announcing the release. 6. Check Dependabot, code-scanning, and secret-scanning alerts after publication. +7. Confirm `dev` still exists at the intended integration tip before starting the next feature + branch. Published versions are immutable. A bad public release is corrected with a new patch version; do not replace its tag or silently rewrite assets. The release job uses only short-lived GitHub OIDC diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index fbb7ffa..b84a9b0 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -27,6 +27,28 @@ executable evidence and dependency caveats are in [`experiments/M0-ocm-falsification.md`](experiments/M0-ocm-falsification.md). The bespoke transport/agent scope is deleted; the hub track proceeds to Phase 1. +> **Phase-1 ClusterGateway authorization gate (2026-07-13).** M0 proves reverse-tunnel +> connectivity and scoped-token RBAC; it does **not** authorize a Sith transport to use a +> ClusterGateway proxy that forwards a hub caller's inbound `Authorization` header. The tracked +> upstream remediation ([oam-dev/cluster-gateway#171](https://github.com/oam-dev/cluster-gateway/pull/171)) +> removes that header before client-go applies the selected managed-service-account credential and +> adds a header-precedence regression. It is verified green but remains open, and no official +> ClusterGateway release contains it. Keep [#103](https://github.com/ArdurAI/sith/issues/103) +> blocked by [#104](https://github.com/ArdurAI/sith/issues/104) until an official upstream release +> includes the fix; then rerun the Sith two-spoke negative route and require `403` for the scoped +> Secrets denial without logging credentials or response bodies. + +> **Phase-1 direct ClusterProxy alternative (2026-07-13).** [#123](https://github.com/ArdurAI/sith/issues/123) +> delivers the same bounded read contract without consuming ClusterGateway: it uses the released +> ClusterProxy Konnectivity client directly, the exact rotating `sith-reader` MSA projection, and +> a fixed registered managed-cluster target. The adapter does not forward caller authorization, +> disables neither proxy nor Kubernetes TLS verification, and returns only normalized +> Pods/Deployments/Rollouts inventory plus health. `make e2e-ocm` proves the direct route across +> both M0 spokes, its `403` Secrets negative control, an MSA projection replacement, and the +> authenticated TLS runtime refresh/read composition. This does not unblock [#103](https://github.com/ArdurAI/sith/issues/103); that ClusterGateway-specific +> transport remains blocked by [#104](https://github.com/ArdurAI/sith/issues/104) pending an +> official upstream release. + **Assumption under test:** OCM `cluster-proxy` + `managed-serviceaccount` really do deliver outbound-only, cross-network, reach-cluster-local-services connectivity — so we do **not** need to build a bespoke tunnel/agent. diff --git a/docs/experiments/M0-ocm-falsification.md b/docs/experiments/M0-ocm-falsification.md index 829a6a5..0998aaf 100644 --- a/docs/experiments/M0-ocm-falsification.md +++ b/docs/experiments/M0-ocm-falsification.md @@ -172,6 +172,28 @@ Replay the committed terminal capture locally: asciinema play docs/experiments/M0-ocm-falsification.cast ``` +### Direct ClusterProxy adapter gate + +The Phase-1 direct adapter is tested by the following target. It creates the same pinned M0 lab, +retains it only long enough for the direct Go test, and always invokes the owned-scratch cleanup +path on exit: + +```bash +KIND=/Volumes/EXTENDED/MacData/tools/bin/kind \ + make e2e-ocm +``` + +The test uses a loopback-only temporary port-forward to the hub's `proxy-entrypoint` only because +it runs from the developer host. It loads the hub proxy CA/client certificate fixture only to model +the read-only deployment mount; its actual per-spoke path reads the exact `sith-reader` projection +through the narrow `get` reader. It requires direct TLS-verified snapshots from both spokes, an +MSA-token `Forbidden` for a cluster-wide Secrets list, and a replacement projection with a changed +token before a subsequent snapshot. No token, CA, response body, or port-forward output is printed. + +To support that product read boundary, each M0 `sith-reader` gets only cluster-wide `list` on +Pods, Deployments, and Rollouts. The existing namespaced service-proxy Role is separate; there is +still no grant for Secrets, Nodes, writes, watches, or hub API access. + ## What the runner proves ### Registration and addon health diff --git a/go.mod b/go.mod index c1e3eb8..0b98af2 100644 --- a/go.mod +++ b/go.mod @@ -2,24 +2,31 @@ module github.com/ArdurAI/sith go 1.26.0 +toolchain go1.26.5 + require ( charm.land/bubbletea/v2 v2.0.8 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/jackc/pgx/v5 v5.10.0 github.com/modelcontextprotocol/go-sdk v1.6.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 + github.com/prometheus/client_golang v1.23.2 github.com/spf13/cobra v1.10.2 github.com/zalando/go-keyring v0.2.8 go.yaml.in/yaml/v3 v3.0.4 golang.org/x/term v0.45.0 + google.golang.org/grpc v1.79.3 k8s.io/api v0.36.2 k8s.io/apimachinery v0.36.2 k8s.io/client-go v0.36.2 k8s.io/streaming v0.36.2 + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 sigs.k8s.io/yaml v1.6.0 ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/ultraviolet v0.0.0-20260703014108-f5a850f9c2b7 // indirect github.com/charmbracelet/x/ansi v0.11.7 // indirect @@ -55,6 +62,9 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect @@ -69,6 +79,7 @@ require ( golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 4b3023e..d80cef2 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/ultraviolet v0.0.0-20260703014108-f5a850f9c2b7 h1:3FmWoGNWK4STvqg0O0Aeav2T7rodWJAPeF0QpH+8gFw= @@ -36,6 +40,8 @@ github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sa github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= @@ -48,6 +54,8 @@ github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -73,6 +81,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -80,6 +90,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -103,6 +115,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -136,6 +156,20 @@ github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zI github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -158,6 +192,12 @@ golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -184,6 +224,8 @@ k8s.io/streaming v0.36.2 h1:NSKthPPg9UFSKsRauVJUVGH2Dvn8fhKmY4qrMkw/p98= k8s.io/streaming v0.36.2/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= diff --git a/hack/experiments/m0-ocm-falsification.sh b/hack/experiments/m0-ocm-falsification.sh index 9fb019e..97f20cd 100755 --- a/hack/experiments/m0-ocm-falsification.sh +++ b/hack/experiments/m0-ocm-falsification.sh @@ -456,6 +456,22 @@ verify_chart_digest() { die "chart digest mismatch for $(basename "${archive}")" } +wait_for_addon_creation() { + local cluster=$1 + local addon=$2 + local deadline=$((SECONDS + 300)) + + while ! "${KUBECTL_BIN}" --context "${HUB_CONTEXT}" -n "${cluster}" get \ + "managedclusteraddon/${addon}" >/dev/null 2>&1; do + if (( SECONDS >= deadline )); then + die "timed out waiting for ${cluster} managedclusteraddon/${addon} creation" + fi + sleep 1 + done + "${KUBECTL_BIN}" --context "${HUB_CONTEXT}" -n "${cluster}" wait \ + "managedclusteraddon/${addon}" --for=condition=Available --timeout=300s +} + install_addons() { local cluster_proxy_chart="${SCRATCH_NAME}/charts/cluster-proxy-${CLUSTER_PROXY_VERSION}.tgz" local msa_chart="${SCRATCH_NAME}/charts/managed-serviceaccount-${MANAGED_SERVICEACCOUNT_VERSION}.tgz" @@ -490,10 +506,8 @@ install_addons() { --kube-context "${HUB_CONTEXT}" --wait --timeout 5m for cluster in spoke-a spoke-b; do - "${KUBECTL_BIN}" --context "${HUB_CONTEXT}" -n "${cluster}" wait \ - managedclusteraddon/cluster-proxy --for=condition=Available --timeout=300s - "${KUBECTL_BIN}" --context "${HUB_CONTEXT}" -n "${cluster}" wait \ - managedclusteraddon/managed-serviceaccount --for=condition=Available --timeout=300s + wait_for_addon_creation "${cluster}" cluster-proxy + wait_for_addon_creation "${cluster}" managed-serviceaccount done } @@ -613,6 +627,39 @@ roleRef: name: sith-reader-svcproxy EOF + # Cross-namespace inventory is intentionally limited to the three resource kinds + # normalized by Sith. This does not grant Secrets, Nodes, writes, list/watch of the + # hub API, or an arbitrary Kubernetes API surface. + "${KUBECTL_BIN}" --context "${context}" apply -f - <<'EOF' +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sith-reader-inventory +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["list"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["list"] +- apiGroups: ["argoproj.io"] + resources: ["rollouts"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sith-reader-inventory +subjects: +- kind: ServiceAccount + name: sith-reader + namespace: open-cluster-management-agent-addon +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sith-reader-inventory +EOF + # Projected secrets may contain only the token and CA, never a kubeconfig. "${KUBECTL_BIN}" --context "${HUB_CONTEXT}" -n "${cluster}" get secret sith-reader -o json | "${JQ_BIN}" -e '(.data | keys | sort) == ["ca.crt", "token"]' >/dev/null @@ -633,6 +680,18 @@ verify_cluster_registration() { done } +proxy_health_port_available() { + "${PYTHON_BIN}" - <<'PY' +import socket + +try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 8090)) +except OSError: + raise SystemExit(1) +PY +} + verify_scoped_proxy() { local cluster local response @@ -781,12 +840,19 @@ verify_outbound_only() { } verify_lab() { + local transport_mode="clusteradm-scoped" + verify_cluster_registration - "${CLUSTERADM_BIN}" proxy health --context "${HUB_CONTEXT}" - verify_scoped_proxy + if proxy_health_port_available; then + "${CLUSTERADM_BIN}" proxy health --context "${HUB_CONTEXT}" + verify_scoped_proxy + else + transport_mode="direct-e2e-required" + log "local port 8090 is occupied; deferring clusteradm proxy checks to the mandatory direct e2e gate" + fi verify_spoke_ingress_boundary verify_outbound_only - log "M0_RESULT=PASS topology=hub+2-spokes identity=scoped-msa transport=outbound-only boundary=active-deny" + log "M0_RESULT=PASS topology=hub+2-spokes identity=scoped-msa transport=${transport_mode} boundary=active-deny" } run_lab() { diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 1ab40ed..ab4e354 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -186,10 +186,20 @@ func TestUIRequiresLocalBackend(t *testing.T) { } } -func TestHubStub(t *testing.T) { - stdout, _, exitCode := runCLI(t, []string{"hub"}, fleet.StubSource{}) - if exitCode != 0 || stdout != "sith hub: not yet implemented — hub mode is phase-1+ (E1–E10).\n" { - t.Fatalf("exit/stdout = %d/%q", exitCode, stdout) +func TestHubRequiresCompleteSecureConfiguration(t *testing.T) { + stdout, stderr, exitCode := runCLI(t, []string{"hub"}, fleet.StubSource{}) + if exitCode == 0 || stdout != "" || !strings.Contains(stderr, "SITH_HUB_LISTEN_ADDR is required") { + t.Fatalf("exit/stdout/stderr = %d/%q/%q", exitCode, stdout, stderr) + } +} + +func TestHubMigrateRequiresOwnerConfigurationWithoutStartingHub(t *testing.T) { + stdout, stderr, exitCode := runCLI(t, []string{"hub", "migrate"}, fleet.StubSource{}) + if exitCode == 0 || stdout != "" || !strings.Contains(stderr, "SITH_HUB_MIGRATION_OWNER_DATABASE_URL is required") { + t.Fatalf("exit/stdout/stderr = %d/%q/%q", exitCode, stdout, stderr) + } + if strings.Contains(stderr, "SITH_HUB_LISTEN_ADDR") { + t.Fatalf("migration stderr = %q, want no hub-server configuration", stderr) } } @@ -240,6 +250,15 @@ func runCLI(t *testing.T, args []string, source fleet.Source) (stdout, stderr st t.Setenv("SITH_LOG_LEVEL", "") t.Setenv("SITH_LOG_FORMAT", "") t.Setenv("SITH_KUBECONFIG", "") + for _, name := range []string{ + "SITH_HUB_LISTEN_ADDR", "SITH_HUB_DATABASE_URL", "SITH_HUB_SESSION_ISSUER", "SITH_HUB_SESSION_AUDIENCE", + "SITH_HUB_SESSION_KEY_ID", "SITH_HUB_SESSION_PUBLIC_KEY_FILE", "SITH_HUB_SERVER_TLS_CERT_FILE", "SITH_HUB_SERVER_TLS_KEY_FILE", + "SITH_HUB_PROXY_ADDRESS", "SITH_HUB_PROXY_SERVER_NAME", "SITH_HUB_PROXY_CA_FILE", "SITH_HUB_PROXY_CERT_FILE", + "SITH_HUB_PROXY_KEY_FILE", "SITH_HUB_KUBE_API_SERVER_NAME", "SITH_HUB_MIGRATION_OWNER_DATABASE_URL", + "SITH_HUB_APPLICATION_DATABASE_ROLE", + } { + t.Setenv(name, "") + } var stdoutBuffer bytes.Buffer var stderrBuffer bytes.Buffer diff --git a/internal/cli/hub.go b/internal/cli/hub.go index eb0c2fa..5555ca4 100644 --- a/internal/cli/hub.go +++ b/internal/cli/hub.go @@ -6,18 +6,42 @@ import ( "fmt" "github.com/spf13/cobra" + + "github.com/ArdurAI/sith/internal/hubruntime" ) func newHubCommand() *cobra.Command { - return &cobra.Command{ + command := &cobra.Command{ Use: "hub", Short: "Start the governed fleet hub", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { - if _, err := fmt.Fprintln(command.OutOrStdout(), "sith hub: not yet implemented — hub mode is phase-1+ (E1–E10)."); err != nil { - return fmt.Errorf("write hub status: %w", err) + state, ok := command.Context().Value(runtimeKey{}).(runtimeState) + if !ok || state.logger == nil { + return fmt.Errorf("start hub: runtime logging is unavailable") + } + runtime, err := hubruntime.NewFromEnvironment(command.Context(), state.logger) + if err != nil { + return fmt.Errorf("start hub: %w", err) + } + return runtime.Run(command.Context()) + }, + } + command.AddCommand(newHubMigrateCommand()) + return command +} + +func newHubMigrateCommand() *cobra.Command { + return &cobra.Command{ + Use: "migrate", + Short: "Apply hub schema migrations with the owner credential", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if err := hubruntime.MigrateFromEnvironment(command.Context()); err != nil { + return fmt.Errorf("migrate hub schema: %w", err) } - return nil + _, err := fmt.Fprintln(command.OutOrStdout(), "Hub schema migrations completed.") + return err }, } } diff --git a/internal/fleet/graph.go b/internal/fleet/graph.go index 7e2ef3f..aa7a381 100644 --- a/internal/fleet/graph.go +++ b/internal/fleet/graph.go @@ -135,6 +135,29 @@ func ImageDigestFromRepoDigest(repoDigest string) (string, error) { return digest, nil } +// ImageDigestFromRuntimeImageID extracts one immutable digest from a Kubernetes runtime-resolved +// ContainerStatus.ImageID. It accepts an exact digest, an optional runtime scheme, or a +// repository@digest form, and deliberately rejects mutable image references. +func ImageDigestFromRuntimeImageID(imageID string) (string, error) { + if imageID == "" || strings.TrimSpace(imageID) != imageID { + return "", fmt.Errorf("runtime image ID is required") + } + value := imageID + if runtimeName, remainder, found := strings.Cut(value, "://"); found { + if !validRuntimeScheme(runtimeName) || remainder == "" { + return "", fmt.Errorf("runtime image ID has an invalid runtime scheme") + } + value = remainder + } + if strings.Contains(value, "@") { + return ImageDigestFromRepoDigest(value) + } + if err := ValidateImageDigest(value); err != nil { + return "", fmt.Errorf("runtime image ID: %w", err) + } + return value, nil +} + func validRepository(repo string) bool { if repo == "" || strings.TrimSpace(repo) != repo || len(repo) > 255 || strings.ContainsAny(repo, "\x00\r\n@") { return false @@ -143,7 +166,8 @@ func validRepository(repo string) bool { return !strings.Contains(repo[lastPathSeparator+1:], ":") } -func validateImageDigest(digest string) error { +// ValidateImageDigest rejects every image reference other than one lowercase immutable sha256 digest. +func ValidateImageDigest(digest string) error { if len(digest) != len("sha256:")+64 || !strings.HasPrefix(digest, "sha256:") { return fmt.Errorf("image digest must be one immutable sha256 digest") } @@ -155,6 +179,17 @@ func validateImageDigest(digest string) error { return nil } +func validateImageDigest(digest string) error { return ValidateImageDigest(digest) } + +func validRuntimeScheme(value string) bool { + switch value { + case "containerd", "docker-pullable", "cri-o", "docker": + return true + default: + return false + } +} + func validateEntityText(label, value string) error { if trimmed := strings.TrimSpace(value); trimmed == "" || trimmed != value || len(value) > 253 || strings.ContainsAny(value, "/\x00\r\n") { return fmt.Errorf("entity %s is invalid", label) diff --git a/internal/fleet/graph_test.go b/internal/fleet/graph_test.go index f1e68cd..0652b0c 100644 --- a/internal/fleet/graph_test.go +++ b/internal/fleet/graph_test.go @@ -130,6 +130,37 @@ func TestImageDigestFromRepoDigest(t *testing.T) { } } +func TestImageDigestFromRuntimeImageID(t *testing.T) { + t.Parallel() + + digest := testGraphDigest + for _, imageID := range []string{ + digest, + "containerd://" + digest, + "cri-o://" + digest, + "docker-pullable://registry.example/payments@" + digest, + } { + got, err := ImageDigestFromRuntimeImageID(imageID) + if err != nil || got != digest { + t.Fatalf("ImageDigestFromRuntimeImageID(%q) = %q, %v", imageID, got, err) + } + } + for _, imageID := range []string{ + "registry.example/payments:latest", + "containerd://registry.example/payments:latest", + "containerd://" + "sha256:ABC", + "Containerd://" + digest, + "http://" + digest, + "custom-runtime://" + digest, + "docker-pullable://registry.example/payments:latest@" + digest, + "containerd://" + digest + "@" + digest, + } { + if _, err := ImageDigestFromRuntimeImageID(imageID); err == nil { + t.Fatalf("ImageDigestFromRuntimeImageID(%q) unexpectedly succeeded", imageID) + } + } +} + func TestEntityRefKeyIncludesEveryLocalDimension(t *testing.T) { t.Parallel() diff --git a/internal/hubdb/fleet.go b/internal/hubdb/fleet.go index 064ec29..4d84032 100644 --- a/internal/hubdb/fleet.go +++ b/internal/hubdb/fleet.go @@ -332,6 +332,9 @@ func queryFacts(ctx context.Context, tx pgx.Tx, workspaceID tenancy.WorkspaceID, conditions = append(conditions, "fact.payload ? 'status'") conditions = append(conditions, "fact.payload->>'status' <> "+placeholder(query.Selector.HealthNot)) } + if query.Selector.Image != "" { + conditions = append(conditions, "fact.payload->'image_digests' ? "+placeholder(query.Selector.Image)) + } limit := query.Limit if limit == 0 { limit = defaultFleetFactLimit @@ -426,13 +429,22 @@ func normalizeFleetQuery(query fleet.Query) (fleet.Query, []string, error) { return fleet.Query{}, nil, fmt.Errorf("fact kind %q is not available from persisted spoke snapshots", kind) } } - if (query.Selector.Health != "" || query.Selector.HealthNot != "") && + if len(query.Selector.Labels) != 0 || query.Selector.CVE != "" { + return fleet.Query{}, nil, fmt.Errorf("requested selector is not available from persisted spoke snapshots") + } + if query.Selector.Image != "" { + if err := fleet.ValidateImageDigest(query.Selector.Image); err != nil { + return fleet.Query{}, nil, fmt.Errorf("image selector: %w", err) + } + if len(query.Kinds) != 1 || query.Kinds[0] != fleet.FactInventory || query.Selector.ResourceKind != "Pod" || + query.Selector.Namespace != "" || query.Selector.Name != "" || query.Selector.NamePrefix != "" || + query.Selector.Health != "" || query.Selector.HealthNot != "" { + return fleet.Query{}, nil, fmt.Errorf("image selector requires exactly the Pod inventory fact kind") + } + } else if (query.Selector.Health != "" || query.Selector.HealthNot != "") && (len(query.Kinds) != 1 || query.Kinds[0] != fleet.FactHealth) { return fleet.Query{}, nil, fmt.Errorf("health selectors require exactly the health fact kind") } - if len(query.Selector.Labels) != 0 || query.Selector.Image != "" || query.Selector.CVE != "" { - return fleet.Query{}, nil, fmt.Errorf("requested selector is not available from persisted spoke snapshots") - } if query.Limit > maxFleetFactLimit { return fleet.Query{}, nil, fmt.Errorf("fact limit exceeds %d", maxFleetFactLimit) } diff --git a/internal/hubdb/fleet_query_test.go b/internal/hubdb/fleet_query_test.go new file mode 100644 index 0000000..e740b99 --- /dev/null +++ b/internal/hubdb/fleet_query_test.go @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubdb + +import ( + "strings" + "testing" + + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestNormalizeFleetQueryAllowsOnlyExactPodImageInventory(t *testing.T) { + t.Parallel() + + digest := "sha256:" + strings.Repeat("a", 64) + query, scopes, err := normalizeFleetQuery(fleet.Query{ + Kinds: []fleet.FactKind{fleet.FactInventory}, + Scopes: []string{"spoke-b", "spoke-a", "spoke-a"}, + Selector: fleet.Selector{ + ResourceKind: "Pod", + Image: digest, + }, + }) + if err != nil { + t.Fatalf("normalize exact image query: %v", err) + } + if query.Selector.Image != digest || len(scopes) != 2 || scopes[0] != "spoke-a" || scopes[1] != "spoke-b" { + t.Fatalf("normalized image query = %#v, scopes = %#v", query, scopes) + } +} + +func TestNormalizeFleetQueryRejectsBroadOrUnsafeImageSelectors(t *testing.T) { + t.Parallel() + + digest := "sha256:" + strings.Repeat("a", 64) + for _, query := range []fleet.Query{ + {Kinds: []fleet.FactKind{fleet.FactInventory}, Selector: fleet.Selector{ResourceKind: "Pod", Image: "registry.example/payments:latest"}}, + {Kinds: []fleet.FactKind{fleet.FactHealth}, Selector: fleet.Selector{ResourceKind: "Pod", Image: digest}}, + {Kinds: []fleet.FactKind{fleet.FactInventory}, Selector: fleet.Selector{ResourceKind: "Deployment", Image: digest}}, + {Kinds: []fleet.FactKind{fleet.FactInventory}, Selector: fleet.Selector{ResourceKind: "Pod", Image: digest, NamePrefix: "payments"}}, + {Kinds: []fleet.FactKind{fleet.FactInventory}, Selector: fleet.Selector{ResourceKind: "Pod", Image: digest, Labels: map[string]string{"app": "payments"}}}, + {Kinds: []fleet.FactKind{fleet.FactInventory}, Selector: fleet.Selector{ResourceKind: "Pod", Image: digest, CVE: "CVE-2026-0001"}}, + } { + if _, _, err := normalizeFleetQuery(query); err == nil { + t.Fatalf("normalizeFleetQuery(%#v) unexpectedly succeeded", query) + } + } +} diff --git a/internal/hubdb/migrate.go b/internal/hubdb/migrate.go index 616b85c..1df88ef 100644 --- a/internal/hubdb/migrate.go +++ b/internal/hubdb/migrate.go @@ -10,16 +10,72 @@ import ( "fmt" "io/fs" "strings" + "time" "unicode" "github.com/jackc/pgx/v5" ) -const migrationLockID int64 = 0x53495448524c53 +const ( + migrationLockID int64 = 0x53495448524c53 + migrationCloseWindow = 5 * time.Second +) //go:embed migrations/*.sql var migrationFiles embed.FS +// MigrationConfig defines the short-lived owner-credential boundary used only to evolve the hub +// schema. The application database role is intentionally distinct and is never used here as the +// migration owner. +type MigrationConfig struct { + OwnerURL string + ApplicationRole string + AllowInsecureLocal bool +} + +// Migrate connects once with the deployment-provided schema-owner URL, applies the embedded +// migrations, and attempts to close that connection. Production callers leave +// AllowInsecureLocal false; the exception exists solely for hermetic local PostgreSQL integration +// tests. +func Migrate(ctx context.Context, config MigrationConfig) error { + if ctx == nil { + return fmt.Errorf("migrate hub database: context is required") + } + if config.OwnerURL == "" || strings.TrimSpace(config.OwnerURL) != config.OwnerURL { + return fmt.Errorf("migrate hub database: owner database URL is required") + } + if err := validateRoleName(config.ApplicationRole); err != nil { + return fmt.Errorf("migrate hub database: %w", err) + } + + ownerConfig, err := pgx.ParseConfig(config.OwnerURL) + if err != nil || ownerConfig.User == "" || strings.TrimSpace(ownerConfig.User) != ownerConfig.User { + return fmt.Errorf("migrate hub database: owner database URL is invalid") + } + if ownerConfig.User == config.ApplicationRole { + return fmt.Errorf("migrate hub database: owner and application roles must differ") + } + if !secureTransport(ownerConfig) && (!config.AllowInsecureLocal || !localTransport(ownerConfig)) { + return fmt.Errorf("migrate hub database: TLS without plaintext fallback is required for non-local connections") + } + + owner, err := pgx.ConnectConfig(ctx, ownerConfig) + if err != nil { + return fmt.Errorf("migrate hub database: owner connection is unavailable") + } + defer func() { + // Commit is the migration success boundary. Closing this short-lived connection is best-effort + // because a transport error after commit cannot be retried or rolled back. + closeCtx, cancel := context.WithTimeout(context.Background(), migrationCloseWindow) + defer cancel() + _ = owner.Close(closeCtx) + }() + if err := ApplyMigrations(ctx, owner, config.ApplicationRole); err != nil { + return fmt.Errorf("migrate hub database: %w", err) + } + return nil +} + // ApplyMigrations applies versioned schema changes as a role distinct from the application role. func ApplyMigrations(ctx context.Context, owner *pgx.Conn, appRole string) error { if ctx == nil || owner == nil { diff --git a/internal/hubdb/migrate_test.go b/internal/hubdb/migrate_test.go new file mode 100644 index 0000000..7af1ccb --- /dev/null +++ b/internal/hubdb/migrate_test.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubdb + +import ( + "context" + "strings" + "testing" +) + +func TestMigrateRejectsUnsafeConfigurationBeforeConnecting(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + config MigrationConfig + want string + }{ + {name: "nil context", config: MigrationConfig{}, want: "context is required"}, + {name: "missing owner URL", config: MigrationConfig{ApplicationRole: "sith_app"}, want: "owner database URL is required"}, + {name: "invalid application role", config: MigrationConfig{OwnerURL: "postgres://sith_owner:password@db.example/sith?sslmode=require"}, want: "application role"}, + {name: "malformed owner URL", config: MigrationConfig{OwnerURL: "postgres://%", ApplicationRole: "sith_app"}, want: "owner database URL is invalid"}, + {name: "application role as owner", config: MigrationConfig{OwnerURL: "postgres://sith_app:password@db.example/sith?sslmode=require", ApplicationRole: "sith_app"}, want: "roles must differ"}, + {name: "remote plaintext", config: MigrationConfig{OwnerURL: "postgres://sith_owner:password@192.0.2.1/sith?sslmode=disable", ApplicationRole: "sith_app", AllowInsecureLocal: true}, want: "TLS"}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + if test.name == "nil context" { + ctx = nil + } + err := Migrate(ctx, test.config) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Migrate() error = %v, want %q", err, test.want) + } + }) + } +} diff --git a/internal/hubdb/migrations/0006_fleet_image_digests.sql b/internal/hubdb/migrations/0006_fleet_image_digests.sql new file mode 100644 index 0000000..82d6ac2 --- /dev/null +++ b/internal/hubdb/migrations/0006_fleet_image_digests.sql @@ -0,0 +1,7 @@ +-- SPDX-License-Identifier: Apache-2.0 + +-- Exact immutable runtime image lookup is constrained to normalized inventory facts. The +-- expression index supports JSONB array-element membership without indexing raw Pod objects. +CREATE INDEX fleet_facts_inventory_image_digests_idx + ON sith.fleet_facts USING GIN ((payload -> 'image_digests')) + WHERE kind = 'inventory'; diff --git a/internal/hubdb/postgres_integration_test.go b/internal/hubdb/postgres_integration_test.go index 2e965f4..c822dee 100644 --- a/internal/hubdb/postgres_integration_test.go +++ b/internal/hubdb/postgres_integration_test.go @@ -58,11 +58,11 @@ func TestPostgresRLSBackstop(t *testing.T) { ownerURL := databaseURL(adminURL, ownerRole, ownerPassword) owner := connectPostgres(t, ctx, ownerURL) defer owner.Close(context.Background()) - if err := ApplyMigrations(ctx, owner, appRole); err != nil { - t.Fatalf("ApplyMigrations() error = %v", err) + if err := Migrate(ctx, MigrationConfig{OwnerURL: ownerURL, ApplicationRole: appRole, AllowInsecureLocal: true}); err != nil { + t.Fatalf("Migrate() error = %v", err) } - if err := ApplyMigrations(ctx, owner, appRole); err != nil { - t.Fatalf("idempotent ApplyMigrations() error = %v", err) + if err := Migrate(ctx, MigrationConfig{OwnerURL: ownerURL, ApplicationRole: appRole, AllowInsecureLocal: true}); err != nil { + t.Fatalf("idempotent Migrate() error = %v", err) } seedTenantRows(t, ctx, admin) @@ -514,6 +514,7 @@ func assertCloudIdentityStoreIntegration(t *testing.T, ctx context.Context, data func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *AppDB) { t.Helper() now := time.Date(2026, time.July, 12, 15, 0, 0, 0, time.UTC) + digest := "sha256:" + strings.Repeat("a", 64) principal, err := tenancy.NewPrincipal("user:alice", map[tenancy.WorkspaceID]tenancy.Role{"workspace-a": tenancy.RoleReader}) if err != nil { t.Fatal(err) @@ -551,6 +552,14 @@ func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *Ap Source: "cluster-a", Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, }, + { + Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a", Kind: "Pod", Namespace: "payments", Name: "payments-a"}, + Kind: fleet.FactInventory, + Observed: []byte(`{"resource":"Pod","ready":1,"generation":1,"image_digests":["` + digest + `"]}`), + ObservedAt: now, + Source: "cluster-a", + Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, + }, }} if err := database.ReplaceSnapshot(ctx, scope, spokes[0], snapshot, now); err != nil { t.Fatalf("replace workspace-a snapshot: %v", err) @@ -581,14 +590,24 @@ func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *Ap if err != nil || len(spokes) != 2 { t.Fatalf("two workspace-a registered spokes = %#v, error = %v", spokes, err) } - secondSnapshot := hubfleet.Snapshot{ObservedAt: now, Facts: []fleet.Evidence{{ - Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a2", Kind: "Deployment", Namespace: "payments", Name: "payments"}, - Kind: fleet.FactHealth, - Observed: []byte(`{"status":"Healthy"}`), - ObservedAt: now, - Source: "cluster-a2", - Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, - }}} + secondSnapshot := hubfleet.Snapshot{ObservedAt: now, Facts: []fleet.Evidence{ + { + Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a2", Kind: "Deployment", Namespace: "payments", Name: "payments"}, + Kind: fleet.FactHealth, + Observed: []byte(`{"status":"Healthy"}`), + ObservedAt: now, + Source: "cluster-a2", + Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, + }, + { + Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a2", Kind: "Pod", Namespace: "payments", Name: "payments-b"}, + Kind: fleet.FactInventory, + Observed: []byte(`{"resource":"Pod","ready":1,"generation":1,"image_digests":["` + digest + `"]}`), + ObservedAt: now, + Source: "cluster-a2", + Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, + }, + }} if err := database.ReplaceSnapshot(ctx, scope, spokes[1], secondSnapshot, now); err != nil { t.Fatalf("replace second workspace-a snapshot: %v", err) } @@ -598,6 +617,17 @@ func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *Ap if err != nil { t.Fatal(err) } + imageSearcher, err := hubfleet.NewImageSearcher(hubfleet.ImageSearcherConfig{ + Querier: database, PEP: postgresReadPEP(t), Freshness: time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + images, err := imageSearcher.Search(ctx, scope, hubfleet.ImageSearchRequest{Digest: digest}) + if err != nil || len(images.Facts) != 2 || images.Facts[0].Ref.Scope != "cluster-a" || images.Facts[1].Ref.Scope != "cluster-a2" || + images.Coverage.Requested != 2 || images.Coverage.Reachable != 2 || len(images.Coverage.Stale) != 0 { + t.Fatalf("two-spoke exact image search = %#v, error = %v", images, err) + } correlated, err := correlator.Correlate(ctx, scope, hubfleet.CorrelationRequest{ ResourceKind: "Deployment", Name: "payments", Namespace: "payments", HealthNot: "Healthy", }) @@ -619,6 +649,10 @@ func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *Ap if err != nil || len(foreignCorrelation.Facts) != 0 { t.Fatalf("cross-workspace correlation = %#v, error = %v", foreignCorrelation, err) } + foreignImages, err := imageSearcher.Search(ctx, foreignScope, hubfleet.ImageSearchRequest{Digest: digest}) + if err != nil || len(foreignImages.Facts) != 0 || foreignImages.Coverage.Requested != 1 || len(foreignImages.Coverage.Unreachable) != 1 { + t.Fatalf("cross-workspace image search = %#v, error = %v", foreignImages, err) + } foreignResult, err := database.QueryFleet(ctx, foreignScope, fleet.Query{Scopes: []string{"cluster-a"}}, time.Minute, now) if err != nil || len(foreignResult.Facts) != 0 || foreignResult.Coverage.Requested != 1 || len(foreignResult.Coverage.Unreachable) != 1 || foreignResult.Coverage.Unreachable[0] != "cluster-a" { @@ -651,7 +685,7 @@ func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *Ap t.Fatalf("stale two-spoke correlation = %#v, error = %v", staleCorrelation, err) } staleResult, err := database.QueryFleet(ctx, scope, fleet.Query{Scopes: []string{"cluster-a"}}, time.Minute, now.Add(time.Second)) - if err != nil || len(staleResult.Facts) != 3 || !staleResult.Facts[0].Stale || staleResult.Facts[0].StaleFor != "collection failed" || + if err != nil || len(staleResult.Facts) != 4 || !staleResult.Facts[0].Stale || staleResult.Facts[0].StaleFor != "collection failed" || staleResult.Coverage.Reachable != 0 || len(staleResult.Coverage.Unreachable) != 1 || len(staleResult.Coverage.Stale) != 1 { t.Fatalf("retained stale query = %#v, error = %v", staleResult, err) } diff --git a/internal/hubfleet/collector.go b/internal/hubfleet/collector.go index 22cc22d..c1983e2 100644 --- a/internal/hubfleet/collector.go +++ b/internal/hubfleet/collector.go @@ -18,6 +18,7 @@ import ( "github.com/ArdurAI/sith/internal/fleet" "github.com/ArdurAI/sith/internal/pep" "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" ) const ( @@ -42,6 +43,7 @@ var observedKeys = map[fleet.FactKind]map[string]struct{}{ "available_replicas": {}, "ready": {}, "generation": {}, + "image_digests": {}, }, fleet.FactHealth: {"status": {}}, } @@ -117,6 +119,8 @@ type CollectorConfig struct { Store Store Transport Transport PEP *pep.Enforcer + Observer SnapshotObserver + TraceObserver tracing.Observer SpokeTimeout time.Duration MaxSnapshotAge time.Duration Now func() time.Time @@ -127,6 +131,8 @@ type Collector struct { store Store transport Transport pep *pep.Enforcer + observer SnapshotObserver + tracer tracing.Observer spokeTimeout time.Duration maxSnapshotAge time.Duration now func() time.Time @@ -152,10 +158,18 @@ func NewCollector(config CollectorConfig) (*Collector, error) { if config.Now == nil { config.Now = time.Now } + if config.Observer == nil { + config.Observer = noopSnapshotObserver{} + } + if config.TraceObserver == nil { + config.TraceObserver = tracing.NoopObserver() + } return &Collector{ store: config.Store, transport: config.Transport, pep: config.PEP, + observer: config.Observer, + tracer: config.TraceObserver, spokeTimeout: config.SpokeTimeout, maxSnapshotAge: config.MaxSnapshotAge, now: config.Now, @@ -174,6 +188,11 @@ func (collector *Collector) Collect(ctx context.Context, scope tenancy.Scope) (f if tenancy.ValidateWorkspaceID(scope.WorkspaceID()) != nil { return fleet.Coverage{}, fmt.Errorf("collect spoke snapshots: validated workspace scope is required") } + traceContext, _, err := tracing.Ensure(ctx) + if err != nil { + return fleet.Coverage{}, fmt.Errorf("collect spoke snapshots: establish trace context: %w", err) + } + ctx = traceContext if err := collector.pep.AuthorizeRead(ctx, scope, pep.NewReadInput(pep.VerbSpokeSnapshotRefresh, nil)); err != nil { return fleet.Coverage{}, fmt.Errorf("collect spoke snapshots: %w", err) } @@ -192,11 +211,14 @@ func (collector *Collector) Collect(ctx context.Context, scope tenancy.Scope) (f return coverage, fmt.Errorf("collect spoke snapshots: %w", err) } attemptedAt := collector.now().UTC() + startedAt := time.Now() spokeContext, cancel := context.WithTimeout(ctx, collector.spokeTimeout) snapshot, collectionErr := collector.transport.Snapshot(spokeContext, scope.WorkspaceID(), cloneSpoke(spoke)) deadlineErr := spokeContext.Err() cancel() if err := ctx.Err(); err != nil { + collector.observeSnapshot(SnapshotOutcomeCanceled, time.Since(startedAt)) + collector.observeTrace(ctx, tracing.OutcomeCanceled, time.Since(startedAt)) return coverage, fmt.Errorf("collect spoke snapshots: %w", err) } if collectionErr == nil && deadlineErr != nil { @@ -204,20 +226,32 @@ func (collector *Collector) Collect(ctx context.Context, scope tenancy.Scope) (f } if collectionErr != nil { if err := collector.recordFailure(ctx, scope, spoke, failureFor(collectionErr), attemptedAt, &coverage); err != nil { + collector.observeSnapshot(SnapshotOutcomeStoreError, time.Since(startedAt)) + collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) return coverage, err } + collector.observeSnapshot(snapshotOutcomeForFailure(failureFor(collectionErr)), time.Since(startedAt)) + collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) continue } if err := validateSnapshot(spoke, snapshot, attemptedAt, collector.maxSnapshotAge); err != nil { if failureErr := collector.recordFailure(ctx, scope, spoke, FailureInvalidSnapshot, attemptedAt, &coverage); failureErr != nil { + collector.observeSnapshot(SnapshotOutcomeStoreError, time.Since(startedAt)) + collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) return coverage, failureErr } + collector.observeSnapshot(SnapshotOutcomeInvalidSnapshot, time.Since(startedAt)) + collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) continue } if err := collector.store.ReplaceSnapshot(ctx, scope, spoke, cloneSnapshot(snapshot), attemptedAt); err != nil { + collector.observeSnapshot(SnapshotOutcomeStoreError, time.Since(startedAt)) + collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) return coverage, fmt.Errorf("collect spoke snapshots: persist %q: %w", spoke.ID, err) } coverage.Reachable++ + collector.observeSnapshot(SnapshotOutcomeSuccess, time.Since(startedAt)) + collector.observeTrace(ctx, tracing.OutcomeSuccess, time.Since(startedAt)) } sort.Strings(coverage.Unreachable) sort.Strings(coverage.Stale) @@ -331,6 +365,11 @@ func validateEvidence(spoke Spoke, evidence fleet.Evidence, snapshotObservedAt, if err := validateObserved(evidence.Kind, evidence.Observed); err != nil { return err } + if evidence.Kind == fleet.FactInventory { + if err := validateInventoryImageDigests(evidence); err != nil { + return err + } + } return validateDisplay(evidence.Display) } @@ -365,6 +404,35 @@ func validateObserved(kind fleet.FactKind, observed json.RawMessage) error { return nil } +func validateInventoryImageDigests(evidence fleet.Evidence) error { + var observed map[string]json.RawMessage + if err := json.Unmarshal(evidence.Observed, &observed); err != nil { + return fmt.Errorf("decode normalized inventory: %w", err) + } + rawDigests, present := observed["image_digests"] + if !present { + return nil + } + if evidence.Ref.Kind != "Pod" { + return fmt.Errorf("image digests are allowed only on normalized Pod inventory") + } + var digests []string + if err := json.Unmarshal(rawDigests, &digests); err != nil || len(digests) == 0 || len(digests) > 64 { + return fmt.Errorf("image digests must be a non-empty bounded string array") + } + previous := "" + for _, digest := range digests { + if err := fleet.ValidateImageDigest(digest); err != nil { + return fmt.Errorf("normalized Pod image digest: %w", err) + } + if previous != "" && previous >= digest { + return fmt.Errorf("normalized Pod image digests must be unique and sorted") + } + previous = digest + } + return nil +} + func validateJSONObject(decoder *json.Decoder, allowed map[string]struct{}) error { seen := make(map[string]struct{}) for decoder.More() { diff --git a/internal/hubfleet/image_search.go b/internal/hubfleet/image_search.go new file mode 100644 index 0000000..70ead8a --- /dev/null +++ b/internal/hubfleet/image_search.go @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +// ImageSearchRequest names one immutable runtime image digest across every registered spoke. +type ImageSearchRequest struct { + Digest string `json:"digest"` + Limit int `json:"limit,omitempty"` +} + +// ImageSearcherConfig defines a read-only, tenant-scoped immutable image evidence service. +type ImageSearcherConfig struct { + Querier FleetQuerier + PEP *pep.Enforcer + Freshness time.Duration + Now func() time.Time +} + +// ImageSearcher resolves one exact immutable image digest across normalized Pod inventory. +type ImageSearcher struct { + querier FleetQuerier + pep *pep.Enforcer + freshness time.Duration + now func() time.Time +} + +// NewImageSearcher constructs a bounded read-only immutable image evidence service. +func NewImageSearcher(config ImageSearcherConfig) (*ImageSearcher, error) { + if config.Querier == nil || config.PEP == nil { + return nil, fmt.Errorf("new fleet image searcher: querier and policy enforcer are required") + } + if config.Freshness == 0 { + config.Freshness = defaultSnapshotAge + } + if config.Freshness < time.Second || config.Freshness > maxSnapshotAge { + return nil, fmt.Errorf("new fleet image searcher: freshness must be between 1s and %s", maxSnapshotAge) + } + if config.Now == nil { + config.Now = time.Now + } + return &ImageSearcher{querier: config.Querier, pep: config.PEP, freshness: config.Freshness, now: config.Now}, nil +} + +// Search returns coverage-honest Pod inventory containing the requested immutable digest. +func (searcher *ImageSearcher) Search( + ctx context.Context, + scope tenancy.Scope, + request ImageSearchRequest, +) (fleet.QueryResult, error) { + if searcher == nil || searcher.querier == nil || searcher.pep == nil || ctx == nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet image: searcher, policy enforcer, and context are required") + } + traceContext, _, err := tracing.Ensure(ctx) + if err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet image: establish trace context: %w", err) + } + ctx = traceContext + if err := scope.Authorize(tenancy.ActionRead); err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet image: %w", err) + } + if err := request.validate(); err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet image: %w", err) + } + canonicalArguments := request.Digest + "\x00" + strconv.Itoa(request.Limit) + if err := searcher.pep.AuthorizeRead(ctx, scope, pep.NewReadInput(pep.VerbFleetImageSearch, []byte(canonicalArguments))); err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet image: %w", err) + } + result, err := searcher.querier.QueryFleet(ctx, scope, fleet.Query{ + Kinds: []fleet.FactKind{fleet.FactInventory}, + Selector: fleet.Selector{ + ResourceKind: "Pod", + Image: request.Digest, + }, + Limit: request.Limit, + }, searcher.freshness, searcher.now().UTC()) + if err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet image: %w", err) + } + return result, nil +} + +func (request ImageSearchRequest) validate() error { + if err := fleet.ValidateImageDigest(request.Digest); err != nil { + return fmt.Errorf("image digest: %w", err) + } + if request.Limit < 0 || request.Limit > 1_000 { + return fmt.Errorf("limit must be between 0 and 1000") + } + return nil +} diff --git a/internal/hubfleet/image_search_test.go b/internal/hubfleet/image_search_test.go new file mode 100644 index 0000000..de6c488 --- /dev/null +++ b/internal/hubfleet/image_search_test.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestImageSearcherUsesExactTenantScopedPodInventoryQuery(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 14, 15, 0, 0, 0, time.UTC) + digest := "sha256:" + strings.Repeat("a", 64) + querier := &recordingFleetQuerier{result: fleet.QueryResult{Facts: []fleet.Fact{{ + Evidence: fleet.Evidence{Ref: fleet.ResourceRef{Scope: "spoke-b", Kind: "Pod", Name: "payments"}}, + Workspace: "workspace-a", + }}}} + searcher, err := NewImageSearcher(ImageSearcherConfig{ + Querier: querier, PEP: testReadPEP(t), Freshness: time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + result, err := searcher.Search(context.Background(), readerScope(t, "workspace-a"), ImageSearchRequest{Digest: digest, Limit: 12}) + if err != nil || len(result.Facts) != 1 { + t.Fatalf("Search() result = %#v, error = %v", result, err) + } + if querier.scope.WorkspaceID() != "workspace-a" || querier.query.Kinds[0] != fleet.FactInventory || + querier.query.Selector.ResourceKind != "Pod" || querier.query.Selector.Image != digest || + querier.query.Selector.Name != "" || querier.query.Selector.Namespace != "" || querier.query.Limit != 12 || + querier.freshness != time.Minute || !querier.now.Equal(now) { + t.Fatalf("querier call = %#v, freshness = %s, now = %s", querier.query, querier.freshness, querier.now) + } +} + +func TestImageSearcherRejectsUnsafeRequestsBeforeQuery(t *testing.T) { + t.Parallel() + + searcher, err := NewImageSearcher(ImageSearcherConfig{ + Querier: fleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + return fleet.QueryResult{}, errors.New("unexpected query") + }), + PEP: testReadPEP(t), + }) + if err != nil { + t.Fatal(err) + } + for _, request := range []ImageSearchRequest{ + {Digest: "registry.example/payments:latest"}, + {Digest: "sha256:" + strings.Repeat("A", 64)}, + {Digest: "sha256:" + strings.Repeat("a", 64), Limit: 1_001}, + } { + if _, err := searcher.Search(context.Background(), readerScope(t, "workspace-a"), request); err == nil { + t.Fatalf("Search(%#v) unexpectedly succeeded", request) + } + } +} + +func TestNewImageSearcherRejectsUnsafeConfiguration(t *testing.T) { + t.Parallel() + + if _, err := NewImageSearcher(ImageSearcherConfig{}); err == nil { + t.Fatal("NewImageSearcher accepted missing dependencies") + } +} diff --git a/internal/hubfleet/metrics.go b/internal/hubfleet/metrics.go new file mode 100644 index 0000000..421f520 --- /dev/null +++ b/internal/hubfleet/metrics.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "time" + + "github.com/ArdurAI/sith/internal/tracing" +) + +// SnapshotOutcome is the bounded self-observability result of one spoke snapshot attempt. It does +// not include spoke, workspace, endpoint, token, raw error, or snapshot data. +type SnapshotOutcome string + +// Closed snapshot-observability outcomes. +const ( + SnapshotOutcomeSuccess SnapshotOutcome = "success" + SnapshotOutcomeTransport SnapshotOutcome = "transport" + SnapshotOutcomeDeadline SnapshotOutcome = "deadline" + SnapshotOutcomeInvalidSnapshot SnapshotOutcome = "invalid-snapshot" + SnapshotOutcomeStoreError SnapshotOutcome = "store-error" + SnapshotOutcomeCanceled SnapshotOutcome = "canceled" +) + +// SnapshotObserver receives passive, bounded measurements for completed snapshot attempts. +// Implementations must not block or mutate collection behavior. The collector isolates observer +// panics defensively. +type SnapshotObserver interface { + ObserveSpokeSnapshot(outcome SnapshotOutcome, duration time.Duration) +} + +type noopSnapshotObserver struct{} + +func (noopSnapshotObserver) ObserveSpokeSnapshot(SnapshotOutcome, time.Duration) {} + +func (collector *Collector) observeSnapshot(outcome SnapshotOutcome, duration time.Duration) { + if collector == nil || collector.observer == nil { + return + } + defer func() { + _ = recover() + }() + collector.observer.ObserveSpokeSnapshot(outcome, duration) +} + +func (collector *Collector) observeTrace(ctx context.Context, outcome tracing.Outcome, duration time.Duration) { + if collector == nil || collector.tracer == nil { + return + } + traceID, ok := tracing.FromContext(ctx) + if !ok { + return + } + tracing.Observe(collector.tracer, tracing.Event{ + TraceID: traceID, Stage: tracing.StageSpokeSnapshot, Outcome: outcome, Duration: duration, + }) +} + +func snapshotOutcomeForFailure(failure FailureKind) SnapshotOutcome { + switch failure { + case FailureDeadline: + return SnapshotOutcomeDeadline + case FailureInvalidSnapshot: + return SnapshotOutcomeInvalidSnapshot + default: + return SnapshotOutcomeTransport + } +} diff --git a/internal/hubfleet/metrics_test.go b/internal/hubfleet/metrics_test.go new file mode 100644 index 0000000..8a6e5d2 --- /dev/null +++ b/internal/hubfleet/metrics_test.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestCollectorObservesClosedSnapshotOutcomes(t *testing.T) { + now := time.Date(2026, time.July, 12, 20, 0, 0, 0, time.UTC) + tests := []struct { + name string + transport transportFunc + wantOutcome SnapshotOutcome + }{ + { + name: "success", + transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { + return validSnapshot("spoke-a", now), nil + }), + wantOutcome: SnapshotOutcomeSuccess, + }, + { + name: "transport failure", + transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { + return Snapshot{}, errors.New("proxy unavailable") + }), + wantOutcome: SnapshotOutcomeTransport, + }, + { + name: "invalid snapshot", + transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { + return validSnapshot("spoke-b", now), nil + }), + wantOutcome: SnapshotOutcomeInvalidSnapshot, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + observer := &recordingSnapshotObserver{} + store := &memoryStore{ + spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind), + } + collector, err := NewCollector(CollectorConfig{ + Store: store, Transport: test.transport, PEP: testReadPEP(t), Observer: observer, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if _, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")); err != nil { + t.Fatal(err) + } + if len(observer.events) != 1 || observer.events[0].outcome != test.wantOutcome || observer.events[0].duration < 0 { + t.Fatalf("observations = %#v", observer.events) + } + }) + } +} + +func TestCollectorRecoversFromPanickingSnapshotObserver(t *testing.T) { + now := time.Date(2026, time.July, 12, 20, 0, 0, 0, time.UTC) + collector, err := NewCollector(CollectorConfig{ + Store: &memoryStore{ + spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind), + }, + Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { + return validSnapshot("spoke-a", now), nil + }), + PEP: testReadPEP(t), + Observer: snapshotObserverFunc(func(SnapshotOutcome, time.Duration) { panic("metrics fault") }), + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + coverage, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")) + if err != nil || coverage.Reachable != 1 { + t.Fatalf("Collect() coverage = %#v, error = %v", coverage, err) + } +} + +type snapshotObservation struct { + outcome SnapshotOutcome + duration time.Duration +} + +type recordingSnapshotObserver struct { + events []snapshotObservation +} + +func (observer *recordingSnapshotObserver) ObserveSpokeSnapshot(outcome SnapshotOutcome, duration time.Duration) { + observer.events = append(observer.events, snapshotObservation{outcome: outcome, duration: duration}) +} + +type snapshotObserverFunc func(SnapshotOutcome, time.Duration) + +func (function snapshotObserverFunc) ObserveSpokeSnapshot(outcome SnapshotOutcome, duration time.Duration) { + function(outcome, duration) +} diff --git a/internal/hubfleet/policy_test.go b/internal/hubfleet/policy_test.go index dcce249..1653581 100644 --- a/internal/hubfleet/policy_test.go +++ b/internal/hubfleet/policy_test.go @@ -5,6 +5,7 @@ package hubfleet import ( "context" "errors" + "strings" "testing" "time" @@ -59,7 +60,18 @@ func TestHubReadEntrypointsStopBeforeDependenciesWhenPolicyRefuses(t *testing.T) if querier.calls != 0 { t.Fatalf("correlator reached fleet query %d times after refusal", querier.calls) } - if got, want := refusal.verbs, []pep.Verb{pep.VerbSpokeSnapshotRefresh, pep.VerbFleetRead, pep.VerbFleetCorrelate}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] { + + imageSearcher, err := NewImageSearcher(ImageSearcherConfig{Querier: querier, PEP: refusal.enforcer(t)}) + if err != nil { + t.Fatal(err) + } + if _, err := imageSearcher.Search(context.Background(), scope, ImageSearchRequest{Digest: "sha256:" + strings.Repeat("a", 64)}); err == nil { + t.Fatal("Search() unexpectedly bypassed policy refusal") + } + if querier.calls != 0 { + t.Fatalf("image search reached fleet query %d times after refusal", querier.calls) + } + if got, want := refusal.verbs, []pep.Verb{pep.VerbSpokeSnapshotRefresh, pep.VerbFleetRead, pep.VerbFleetCorrelate, pep.VerbFleetImageSearch}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] || got[3] != want[3] { t.Fatalf("policy verbs = %q, want %q", got, want) } } diff --git a/internal/hubfleet/source.go b/internal/hubfleet/source.go index 8031ed1..2fff97c 100644 --- a/internal/hubfleet/source.go +++ b/internal/hubfleet/source.go @@ -10,6 +10,7 @@ import ( "github.com/ArdurAI/sith/internal/fleet" "github.com/ArdurAI/sith/internal/pep" "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" ) // FleetReader provides a tenant-scoped fleet snapshot from persisted spoke observations. @@ -65,6 +66,11 @@ func (source *Source) Fleet(ctx context.Context) (fleet.FleetResult, error) { if source == nil || source.reader == nil || source.pep == nil || ctx == nil { return fleet.FleetResult{}, fmt.Errorf("read OCM spoke fleet: source, policy enforcer, and context are required") } + traceContext, _, err := tracing.Ensure(ctx) + if err != nil { + return fleet.FleetResult{}, fmt.Errorf("read OCM spoke fleet: establish trace context: %w", err) + } + ctx = traceContext if err := source.pep.AuthorizeRead(ctx, source.scope, pep.NewReadInput(pep.VerbFleetRead, nil)); err != nil { return fleet.FleetResult{}, fmt.Errorf("read OCM spoke fleet: %w", err) } diff --git a/internal/hubfleet/tracing_test.go b/internal/hubfleet/tracing_test.go new file mode 100644 index 0000000..90dd5cd --- /dev/null +++ b/internal/hubfleet/tracing_test.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +func TestCollectorPropagatesOneTraceToPEPAndSpokeTransport(t *testing.T) { + now := time.Date(2026, time.July, 14, 13, 0, 0, 0, time.UTC) + var audits []pep.AuditEvent + var events []tracing.Event + tracer := tracing.ObserverFunc(func(event tracing.Event) { events = append(events, event) }) + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, + Auditor: pep.AuditFunc(func(_ context.Context, event pep.AuditEvent) error { + audits = append(audits, event) + return nil + }), + TraceObserver: tracer, + }) + if err != nil { + t.Fatal(err) + } + var transportTrace tracing.ID + store := &memoryStore{ + spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind), + } + collector, err := NewCollector(CollectorConfig{ + Store: store, + Transport: transportFunc(func(ctx context.Context, _ tenancy.WorkspaceID, spoke Spoke) (Snapshot, error) { + var ok bool + transportTrace, ok = tracing.FromContext(ctx) + if !ok { + t.Fatal("snapshot transport received no trace context") + } + return validSnapshot(spoke.ID, now), nil + }), + PEP: enforcer, TraceObserver: tracer, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if coverage, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")); err != nil || coverage.Reachable != 1 { + t.Fatalf("Collect() coverage = %#v, error = %v", coverage, err) + } + if !transportTrace.Valid() || len(audits) != 1 || audits[0].TraceID != transportTrace { + t.Fatalf("trace/audit propagation = transport %q audits %#v", transportTrace, audits) + } + if len(events) != 2 || events[0].Stage != tracing.StagePEPDecision || events[1].Stage != tracing.StageSpokeSnapshot || + events[0].TraceID != transportTrace || events[1].TraceID != transportTrace || events[1].Outcome != tracing.OutcomeSuccess { + t.Fatalf("trace events = %#v", events) + } +} + +func TestCollectorSurvivesPanickingTraceObserver(t *testing.T) { + now := time.Date(2026, time.July, 14, 13, 0, 0, 0, time.UTC) + tracer := tracing.ObserverFunc(func(tracing.Event) { panic("trace recorder fault") }) + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, Auditor: pep.AuditFunc(func(context.Context, pep.AuditEvent) error { return nil }), TraceObserver: tracer, + }) + if err != nil { + t.Fatal(err) + } + collector, err := NewCollector(CollectorConfig{ + Store: &memoryStore{spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind)}, + Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { + return validSnapshot("spoke-a", now), nil + }), + PEP: enforcer, TraceObserver: tracer, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + if coverage, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")); err != nil || coverage.Reachable != 1 { + t.Fatalf("Collect() changed because tracing panicked: coverage %#v, error %v", coverage, err) + } +} diff --git a/internal/hubocm/credentials.go b/internal/hubocm/credentials.go new file mode 100644 index 0000000..a3c53de --- /dev/null +++ b/internal/hubocm/credentials.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubocm + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + + "github.com/ArdurAI/sith/internal/tenancy" +) + +const ( + managedClusterRefPrefix = "ocm/" + managedServiceAccount = "sith-reader" + maxTokenBytes = 16 * 1024 + maxCABundleBytes = 256 * 1024 +) + +// CredentialReader returns the current scoped managed-serviceaccount material for one +// already registered managed cluster. The returned value is adapter-internal only. +type CredentialReader interface { + Read(context.Context, tenancy.WorkspaceID, string) (projectedCredential, error) +} + +// ManagedServiceAccountReader reads exactly the projected sith-reader Secret. It does +// not use list or watch operations, and it intentionally has no configurable Secret name. +type ManagedServiceAccountReader struct { + secrets corev1client.CoreV1Interface +} + +// NewManagedServiceAccountReader constructs the narrow Secret reader used by the direct +// transport. The caller's Kubernetes RBAC must grant get on resourceName sith-reader only +// in each managed-cluster namespace. +func NewManagedServiceAccountReader(secrets corev1client.CoreV1Interface) (*ManagedServiceAccountReader, error) { + if secrets == nil { + return nil, fmt.Errorf("new managed-serviceaccount reader: Kubernetes core client is required") + } + return &ManagedServiceAccountReader{secrets: secrets}, nil +} + +// Read obtains a fresh credential on every snapshot; it intentionally keeps no cache. +func (reader *ManagedServiceAccountReader) Read( + ctx context.Context, + workspaceID tenancy.WorkspaceID, + managedCluster string, +) (projectedCredential, error) { + if reader == nil || reader.secrets == nil || ctx == nil { + return projectedCredential{}, fmt.Errorf("read managed-serviceaccount credential: reader and context are required") + } + if err := tenancy.ValidateWorkspaceID(workspaceID); err != nil { + return projectedCredential{}, fmt.Errorf("read managed-serviceaccount credential: workspace is invalid") + } + if err := validateManagedClusterName(managedCluster); err != nil { + return projectedCredential{}, fmt.Errorf("read managed-serviceaccount credential: managed cluster is invalid") + } + secret, err := reader.secrets.Secrets(managedCluster).Get(ctx, managedServiceAccount, metav1.GetOptions{}) + if err != nil { + return projectedCredential{}, contextOrGeneric(ctx, "read projected managed-serviceaccount credential") + } + if secret == nil || secret.Namespace != managedCluster { + return projectedCredential{}, fmt.Errorf("read managed-serviceaccount credential: projected Secret is invalid") + } + return credentialFromSecret(secret) +} + +type projectedCredential struct { + token []byte + ca []byte +} + +func credentialFromSecret(secret *corev1.Secret) (projectedCredential, error) { + if secret == nil || secret.Name != managedServiceAccount || secret.Namespace == "" || + len(validation.IsDNS1123Label(secret.Namespace)) != 0 { + return projectedCredential{}, fmt.Errorf("read managed-serviceaccount credential: projected Secret is invalid") + } + if len(secret.Data) != 2 { + return projectedCredential{}, fmt.Errorf("read managed-serviceaccount credential: projected Secret keys are invalid") + } + token, hasToken := secret.Data["token"] + ca, hasCA := secret.Data["ca.crt"] + if !hasToken || !hasCA || len(token) == 0 || len(token) > maxTokenBytes || len(ca) == 0 || len(ca) > maxCABundleBytes { + return projectedCredential{}, fmt.Errorf("read managed-serviceaccount credential: projected Secret payload is invalid") + } + for key := range secret.Data { + if key != "token" && key != "ca.crt" { + return projectedCredential{}, fmt.Errorf("read managed-serviceaccount credential: projected Secret keys are invalid") + } + } + return projectedCredential{token: append([]byte(nil), token...), ca: append([]byte(nil), ca...)}, nil +} + +func parseManagedClusterRef(reference string) (string, error) { + if !strings.HasPrefix(reference, managedClusterRefPrefix) { + return "", fmt.Errorf("managed cluster reference must use the %q prefix", managedClusterRefPrefix) + } + name := strings.TrimPrefix(reference, managedClusterRefPrefix) + if err := validateManagedClusterName(name); err != nil { + return "", err + } + return name, nil +} + +func validateManagedClusterName(name string) error { + if len(validation.IsDNS1123Label(name)) != 0 { + return fmt.Errorf("managed cluster name is invalid") + } + return nil +} diff --git a/internal/hubocm/direct.go b/internal/hubocm/direct.go new file mode 100644 index 0000000..630197c --- /dev/null +++ b/internal/hubocm/direct.go @@ -0,0 +1,523 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubocm + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/http" + "sort" + "strconv" + "strings" + "sync" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "google.golang.org/grpc" + grpccredentials "google.golang.org/grpc/credentials" + konnectivity "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/tenancy" +) + +const ( + protocolVersion = "1.0.0" + maxResources = 500 + listPageSize = 100 +) + +var rolloutGVR = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "rollouts"} + +// Config configures the direct ClusterProxy transport. ProxyTLSConfig must be constructed +// from deployment-mounted proxy mTLS material; it is cloned and never persisted by Sith. +type Config struct { + CredentialReader CredentialReader + ProxyAddress string + ProxyTLSConfig *tls.Config + KubeAPIServerName string + Now func() time.Time +} + +// Adapter implements hubfleet.Transport through OCM ClusterProxy's released direct +// Konnectivity client. It never forwards a caller Authorization header. +type Adapter struct { + credentials CredentialReader + proxyAddress string + proxyTLS *tls.Config + kubeAPIServerName string + now func() time.Time + tunnels tunnelFactory + clients snapshotClientFactory +} + +var _ hubfleet.Transport = (*Adapter)(nil) + +// New constructs a fail-closed direct ClusterProxy adapter. +func New(config Config) (*Adapter, error) { + if config.CredentialReader == nil { + return nil, fmt.Errorf("new direct OCM transport: credential reader is required") + } + if err := validateProxyAddress(config.ProxyAddress); err != nil { + return nil, fmt.Errorf("new direct OCM transport: %w", err) + } + if err := validateProxyTLS(config.ProxyTLSConfig); err != nil { + return nil, fmt.Errorf("new direct OCM transport: %w", err) + } + if len(validation.IsDNS1123Subdomain(config.KubeAPIServerName)) != 0 { + return nil, fmt.Errorf("new direct OCM transport: Kubernetes TLS server name is invalid") + } + if config.Now == nil { + config.Now = time.Now + } + proxyTLS := config.ProxyTLSConfig.Clone() + return &Adapter{ + credentials: config.CredentialReader, + proxyAddress: config.ProxyAddress, + proxyTLS: proxyTLS, + kubeAPIServerName: config.KubeAPIServerName, + now: config.Now, + tunnels: grpcTunnelFactory{address: config.ProxyAddress, tls: proxyTLS}, + clients: defaultSnapshotClientFactory, + }, nil +} + +// Snapshot reads the bounded inventory and health projection for one registered spoke. +func (adapter *Adapter) Snapshot( + ctx context.Context, + workspaceID tenancy.WorkspaceID, + spoke hubfleet.Spoke, +) (hubfleet.Snapshot, error) { + if adapter == nil || adapter.credentials == nil || adapter.tunnels == nil || adapter.clients == nil || ctx == nil { + return hubfleet.Snapshot{}, fmt.Errorf("direct OCM snapshot: adapter and context are required") + } + if err := tenancy.ValidateWorkspaceID(workspaceID); err != nil { + return hubfleet.Snapshot{}, fmt.Errorf("direct OCM snapshot: workspace is invalid") + } + if err := spoke.Validate(); err != nil { + return hubfleet.Snapshot{}, fmt.Errorf("direct OCM snapshot: spoke is invalid") + } + managedCluster, err := parseManagedClusterRef(spoke.ManagedClusterRef) + if err != nil { + return hubfleet.Snapshot{}, fmt.Errorf("direct OCM snapshot: managed cluster reference is invalid") + } + credential, err := adapter.credentials.Read(ctx, workspaceID, managedCluster) + if err != nil { + return hubfleet.Snapshot{}, contextOrGeneric(ctx, "read direct OCM credential") + } + defer clearCredential(&credential) + + config := adapter.restConfig(ctx, managedCluster, credential) + defer func() { + clear(config.CAData) + config.BearerToken = "" + }() + client, err := adapter.clients(config) + if err != nil { + return hubfleet.Snapshot{}, contextOrGeneric(ctx, "construct direct OCM client") + } + defer client.Close() + + observedAt := adapter.now().UTC() + facts, err := collectFacts(ctx, client, spoke, observedAt) + if err != nil { + return hubfleet.Snapshot{}, contextOrGeneric(ctx, "collect direct OCM snapshot") + } + return hubfleet.Snapshot{ObservedAt: observedAt, Facts: facts}, nil +} + +func (adapter *Adapter) restConfig(ctx context.Context, managedCluster string, credential projectedCredential) *rest.Config { + target := net.JoinHostPort(managedCluster, "443") + return &rest.Config{ + Host: "https://" + managedCluster, + BearerToken: string(credential.token), + TLSClientConfig: rest.TLSClientConfig{ + CAData: append([]byte(nil), credential.ca...), + ServerName: adapter.kubeAPIServerName, + }, + Dial: adapter.dialContext(ctx, target), + } +} + +func (adapter *Adapter) dialContext(snapshotCtx context.Context, target string) func(context.Context, string, string) (net.Conn, error) { + return func(requestCtx context.Context, network, address string) (net.Conn, error) { + if network != "tcp" || address != target { + return nil, fmt.Errorf("direct OCM tunnel rejected an unpinned dial target") + } + if err := requestCtx.Err(); err != nil { + return nil, err + } + tunnelCtx, cancel := context.WithCancel(snapshotCtx) + tunnel, err := adapter.tunnels.Open(requestCtx, tunnelCtx) + if err != nil { + cancel() + return nil, contextOrGeneric(requestCtx, "open direct OCM tunnel") + } + connection, err := tunnel.DialContext(requestCtx, network, target) + if err != nil { + cancel() + return nil, contextOrGeneric(requestCtx, "dial direct OCM tunnel") + } + return &tunnelConnection{Conn: connection, cancel: cancel}, nil + } +} + +func validateProxyAddress(address string) error { + host, port, err := net.SplitHostPort(address) + if err != nil || host == "" || strings.ContainsAny(host, "/\\@") { + return fmt.Errorf("proxy address must be a host and port") + } + value, err := strconv.ParseUint(port, 10, 16) + if err != nil || value == 0 { + return fmt.Errorf("proxy address must use a valid port") + } + return nil +} + +func validateProxyTLS(config *tls.Config) error { + if config == nil || config.InsecureSkipVerify || config.MinVersion < tls.VersionTLS12 || config.ServerName == "" || + config.RootCAs == nil || len(config.Certificates) != 1 || len(config.Certificates[0].Certificate) == 0 || + config.Certificates[0].PrivateKey == nil || config.GetClientCertificate != nil { + return fmt.Errorf("proxy TLS configuration must pin CA, server name, TLS 1.2+, and one client certificate") + } + return nil +} + +type tunnelFactory interface { + Open(createCtx, tunnelCtx context.Context) (konnectivity.Tunnel, error) +} + +type grpcTunnelFactory struct { + address string + tls *tls.Config +} + +func (factory grpcTunnelFactory) Open(createCtx, tunnelCtx context.Context) (konnectivity.Tunnel, error) { + return konnectivity.CreateSingleUseGrpcTunnelWithContext( + createCtx, + tunnelCtx, + factory.address, + //nolint:staticcheck // Konnectivity has no NewClient-compatible constructor; blocking preserves the caller-bounded creation deadline. + grpc.WithBlock(), + grpc.WithTransportCredentials(grpccredentials.NewTLS(factory.tls.Clone())), + ) +} + +type tunnelConnection struct { + net.Conn + once sync.Once + cancel context.CancelFunc +} + +func (connection *tunnelConnection) Close() error { + connection.once.Do(connection.cancel) + return connection.Conn.Close() +} + +type snapshotClient interface { + ListDeployments(context.Context, metav1.ListOptions) (*appsv1.DeploymentList, error) + ListPods(context.Context, metav1.ListOptions) (*corev1.PodList, error) + ListRollouts(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error) + Close() +} + +type snapshotClientFactory func(*rest.Config) (snapshotClient, error) + +type kubeSnapshotClient struct { + kube kubernetes.Interface + dynamic dynamic.Interface + http *http.Client +} + +func defaultSnapshotClientFactory(config *rest.Config) (snapshotClient, error) { + transport, err := rest.TransportFor(config) + if err != nil { + return nil, err + } + httpClient := &http.Client{Transport: transport} + kubeClient, err := kubernetes.NewForConfigAndClient(config, httpClient) + if err != nil { + httpClient.CloseIdleConnections() + return nil, err + } + dynamicClient, err := dynamic.NewForConfigAndClient(config, httpClient) + if err != nil { + httpClient.CloseIdleConnections() + return nil, err + } + return &kubeSnapshotClient{kube: kubeClient, dynamic: dynamicClient, http: httpClient}, nil +} + +func (client *kubeSnapshotClient) ListDeployments(ctx context.Context, options metav1.ListOptions) (*appsv1.DeploymentList, error) { + return client.kube.AppsV1().Deployments("").List(ctx, options) +} + +func (client *kubeSnapshotClient) ListPods(ctx context.Context, options metav1.ListOptions) (*corev1.PodList, error) { + return client.kube.CoreV1().Pods("").List(ctx, options) +} + +func (client *kubeSnapshotClient) ListRollouts(ctx context.Context, options metav1.ListOptions) (*unstructured.UnstructuredList, error) { + return client.dynamic.Resource(rolloutGVR).Namespace("").List(ctx, options) +} + +func (client *kubeSnapshotClient) Close() { + if client != nil && client.http != nil { + client.http.CloseIdleConnections() + } +} + +func collectFacts(ctx context.Context, client snapshotClient, spoke hubfleet.Spoke, observedAt time.Time) ([]fleet.Evidence, error) { + if client == nil { + return nil, fmt.Errorf("snapshot client is required") + } + remaining := maxResources + facts := make([]fleet.Evidence, 0, maxResources*2) + deployments, err := listDeployments(ctx, client, &remaining) + if err != nil { + return nil, err + } + for index := range deployments { + facts = append(facts, deploymentFacts(spoke.ID, deployments[index], observedAt)...) + } + pods, err := listPods(ctx, client, &remaining) + if err != nil { + return nil, err + } + for index := range pods { + facts = append(facts, podFacts(spoke.ID, pods[index], observedAt)...) + } + rollouts, err := listRollouts(ctx, client, &remaining) + if err != nil { + return nil, err + } + for index := range rollouts { + facts = append(facts, rolloutFacts(spoke.ID, rollouts[index], observedAt)...) + } + return facts, nil +} + +func listDeployments(ctx context.Context, client snapshotClient, remaining *int) ([]appsv1.Deployment, error) { + items := make([]appsv1.Deployment, 0) + continueToken := "" + for { + page, err := client.ListDeployments(ctx, listOptions(continueToken, *remaining)) + if err != nil { + return nil, contextOrGeneric(ctx, "list deployments") + } + if err := appendPage(&items, page.Items, page.Continue, remaining); err != nil { + return nil, err + } + continueToken = page.Continue + if continueToken == "" { + return items, nil + } + } +} + +func listPods(ctx context.Context, client snapshotClient, remaining *int) ([]corev1.Pod, error) { + if *remaining <= 0 { + return nil, fmt.Errorf("direct OCM snapshot exceeds the bounded resource limit") + } + items := make([]corev1.Pod, 0) + continueToken := "" + for { + page, err := client.ListPods(ctx, listOptions(continueToken, *remaining)) + if err != nil { + return nil, contextOrGeneric(ctx, "list pods") + } + if err := appendPage(&items, page.Items, page.Continue, remaining); err != nil { + return nil, err + } + continueToken = page.Continue + if continueToken == "" { + return items, nil + } + } +} + +func listRollouts(ctx context.Context, client snapshotClient, remaining *int) ([]unstructured.Unstructured, error) { + if *remaining <= 0 { + return nil, fmt.Errorf("direct OCM snapshot exceeds the bounded resource limit") + } + items := make([]unstructured.Unstructured, 0) + continueToken := "" + for { + page, err := client.ListRollouts(ctx, listOptions(continueToken, *remaining)) + if apierrors.IsNotFound(err) && continueToken == "" { + return items, nil + } + if err != nil { + return nil, contextOrGeneric(ctx, "list rollouts") + } + if err := appendPage(&items, page.Items, page.GetContinue(), remaining); err != nil { + return nil, err + } + continueToken = page.GetContinue() + if continueToken == "" { + return items, nil + } + } +} + +func listOptions(continueToken string, remaining int) metav1.ListOptions { + limit := remaining + if limit > listPageSize { + limit = listPageSize + } + return metav1.ListOptions{Limit: int64(limit), Continue: continueToken} +} + +func appendPage[T any](items *[]T, page []T, continueToken string, remaining *int) error { + if len(page) > *remaining || (len(page) == *remaining && continueToken != "") { + return fmt.Errorf("direct OCM snapshot exceeds the bounded resource limit") + } + *items = append(*items, page...) + *remaining -= len(page) + return nil +} + +func deploymentFacts(spokeID string, deployment appsv1.Deployment, observedAt time.Time) []fleet.Evidence { + desired := int32(1) + if deployment.Spec.Replicas != nil { + desired = *deployment.Spec.Replicas + } + health := "Progressing" + if deployment.Status.AvailableReplicas >= desired && deployment.Status.ObservedGeneration >= deployment.Generation { + health = "Healthy" + } else if deployment.Status.UnavailableReplicas > 0 { + health = "Degraded" + } + return resourceFacts(spokeID, "Deployment", deployment.Namespace, deployment.Name, observedAt, + map[string]any{"resource": "Deployment", "replicas": desired, "available_replicas": deployment.Status.AvailableReplicas, "generation": deployment.Generation}, health) +} + +func podFacts(spokeID string, pod corev1.Pod, observedAt time.Time) []fleet.Evidence { + ready := int32(0) + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + ready = 1 + } + } + health := podHealth(pod) + inventory := map[string]any{"resource": "Pod", "ready": ready, "generation": pod.Generation} + if digests := podImageDigests(pod); len(digests) > 0 { + inventory["image_digests"] = digests + } + return resourceFacts(spokeID, "Pod", pod.Namespace, pod.Name, observedAt, + inventory, health) +} + +func podImageDigests(pod corev1.Pod) []string { + seen := make(map[string]struct{}, len(pod.Status.ContainerStatuses)) + for _, status := range pod.Status.ContainerStatuses { + digest, err := fleet.ImageDigestFromRuntimeImageID(status.ImageID) + if err != nil { + continue + } + seen[digest] = struct{}{} + } + digests := make([]string, 0, len(seen)) + for digest := range seen { + digests = append(digests, digest) + } + sort.Strings(digests) + return digests +} + +func rolloutFacts(spokeID string, rollout unstructured.Unstructured, observedAt time.Time) []fleet.Evidence { + replicas, _, _ := unstructured.NestedInt64(rollout.Object, "status", "replicas") + available, _, _ := unstructured.NestedInt64(rollout.Object, "status", "availableReplicas") + phase, _, _ := unstructured.NestedString(rollout.Object, "status", "phase") + health := rolloutHealth(phase, replicas, available) + return resourceFacts(spokeID, "Rollout", rollout.GetNamespace(), rollout.GetName(), observedAt, + map[string]any{"resource": "Rollout", "replicas": replicas, "available_replicas": available, "generation": rollout.GetGeneration()}, health) +} + +func resourceFacts( + spokeID, kind, namespace, name string, + observedAt time.Time, + inventory map[string]any, + health string, +) []fleet.Evidence { + ref := fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: spokeID, Kind: kind, Namespace: namespace, Name: name} + provenance := fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: protocolVersion} + return []fleet.Evidence{ + {Ref: ref, Kind: fleet.FactInventory, Observed: mustObserved(inventory), ObservedAt: observedAt, Source: spokeID, Provenance: provenance}, + {Ref: ref, Kind: fleet.FactHealth, Observed: mustObserved(map[string]any{"status": health}), ObservedAt: observedAt, Source: spokeID, Provenance: provenance}, + } +} + +func mustObserved(value map[string]any) json.RawMessage { + encoded, err := json.Marshal(value) + if err != nil { + panic("direct OCM observed projection is not serializable") + } + return encoded +} + +func podHealth(pod corev1.Pod) string { + if pod.Status.Phase == corev1.PodFailed { + return "Degraded" + } + for _, status := range append(append([]corev1.ContainerStatus(nil), pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) { + if status.State.Waiting != nil && (status.State.Waiting.Reason == "CrashLoopBackOff" || status.State.Waiting.Reason == "ImagePullBackOff" || status.State.Waiting.Reason == "ErrImagePull") { + return "Degraded" + } + } + if pod.Status.Phase == corev1.PodRunning { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return "Healthy" + } + } + return "Progressing" + } + if pod.Status.Phase == corev1.PodSucceeded { + return "Healthy" + } + return "Unknown" +} + +func rolloutHealth(phase string, replicas, available int64) string { + switch strings.ToLower(phase) { + case "healthy": + return "Healthy" + case "degraded", "error": + return "Degraded" + case "progressing", "paused": + return "Progressing" + } + if replicas > 0 && available >= replicas { + return "Healthy" + } + return "Unknown" +} + +func clearCredential(credential *projectedCredential) { + if credential == nil { + return + } + clear(credential.token) + clear(credential.ca) +} + +func contextOrGeneric(ctx context.Context, operation string) error { + if ctx != nil && ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("%s failed", operation) +} diff --git a/internal/hubocm/direct_test.go b/internal/hubocm/direct_test.go new file mode 100644 index 0000000..52b0830 --- /dev/null +++ b/internal/hubocm/direct_test.go @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubocm + +import ( + "context" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "errors" + "net" + "slices" + "strconv" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + ktesting "k8s.io/client-go/testing" + + konnectivity "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestManagedServiceAccountReaderGetsOnlyPinnedSecret(t *testing.T) { + t.Parallel() + + client := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: managedServiceAccount, Namespace: "spoke-a"}, + Data: map[string][]byte{"token": []byte("scoped-token"), "ca.crt": []byte("scoped-ca")}, + }) + var actions []ktesting.Action + client.PrependReactor("get", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + actions = append(actions, action) + return false, nil, nil + }) + reader, err := NewManagedServiceAccountReader(client.CoreV1()) + if err != nil { + t.Fatal(err) + } + credential, err := reader.Read(context.Background(), "workspace-a", "spoke-a") + if err != nil { + t.Fatal(err) + } + if string(credential.token) != "scoped-token" || string(credential.ca) != "scoped-ca" { + t.Fatal("projected credential did not contain the expected fixed material") + } + if len(actions) != 1 || actions[0].GetVerb() != "get" || actions[0].GetResource().Resource != "secrets" || + actions[0].GetNamespace() != "spoke-a" || actions[0].(ktesting.GetAction).GetName() != managedServiceAccount { + t.Fatalf("Secret actions = %#v, want one exact get", actions) + } +} + +func TestManagedServiceAccountReaderRejectsUnsafeProjection(t *testing.T) { + t.Parallel() + + for _, secret := range []*corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: managedServiceAccount, Namespace: "spoke-a"}, Data: map[string][]byte{"token": []byte("x")}}, + {ObjectMeta: metav1.ObjectMeta{Name: managedServiceAccount, Namespace: "spoke-a"}, Data: map[string][]byte{"token": []byte("x"), "ca.crt": []byte("ca"), "kubeconfig": []byte("forbidden")}}, + {ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: "spoke-a"}, Data: map[string][]byte{"token": []byte("x"), "ca.crt": []byte("ca")}}, + } { + if _, err := credentialFromSecret(secret); err == nil { + t.Fatal("credentialFromSecret unexpectedly accepted an unsafe Secret") + } + } +} + +func TestNewRejectsUnsafeProxyConfiguration(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + mutate func(*Config) + }{ + {name: "missing reader", mutate: func(config *Config) { config.CredentialReader = nil }}, + {name: "unqualified address", mutate: func(config *Config) { config.ProxyAddress = "proxy.example" }}, + {name: "insecure TLS", mutate: func(config *Config) { config.ProxyTLSConfig.InsecureSkipVerify = true }}, + {name: "missing CA pin", mutate: func(config *Config) { config.ProxyTLSConfig.RootCAs = nil }}, + {name: "weak TLS minimum", mutate: func(config *Config) { config.ProxyTLSConfig.MinVersion = tls.VersionTLS11 }}, + {name: "unconfigured Kubernetes name", mutate: func(config *Config) { config.KubeAPIServerName = "" }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config := testConfig(credentialReaderFunc(func(context.Context, tenancy.WorkspaceID, string) (projectedCredential, error) { + return projectedCredential{}, nil + })) + test.mutate(&config) + if _, err := New(config); err == nil { + t.Fatal("New() unexpectedly accepted unsafe configuration") + } + }) + } +} + +func TestSnapshotPinsMSACredentialTLSAndNormalizedFacts(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 13, 18, 0, 0, 0, time.UTC) + reader := &rotatingCredentialReader{} + deployment := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "payments", Namespace: "apps", Generation: 4}, + Spec: appsv1.DeploymentSpec{Replicas: pointer[int32](2)}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 4, AvailableReplicas: 2}, + } + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "apps", Generation: 5}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{{ImageID: "containerd://sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}, + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + } + client := &fakeSnapshotClient{ + deployments: []*appsv1.DeploymentList{{Items: []appsv1.Deployment{deployment}}}, + pods: []*corev1.PodList{{Items: []corev1.Pod{pod}}}, + rollouts: []*unstructured.UnstructuredList{{}}, + } + adapter := testAdapter(t, reader, client, now) + + snapshot, err := adapter.Snapshot(context.Background(), "workspace-a", hubfleet.Spoke{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}) + if err != nil { + t.Fatal(err) + } + if err := hubfleet.ValidateSnapshot(hubfleet.Spoke{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}, snapshot, now); err != nil { + t.Fatalf("ValidateSnapshot() error = %v", err) + } + if len(snapshot.Facts) != 4 || client.closed != 1 { + t.Fatalf("snapshot facts/close = %d/%d, want 4/1", len(snapshot.Facts), client.closed) + } + if len(client.configs) != 1 { + t.Fatalf("client configs = %d, want 1", len(client.configs)) + } + config := client.configs[0] + if config.Host != "https://spoke-a" || config.BearerToken != "rotated-token-1" || config.Insecure || + config.ServerName != "kubernetes" || string(config.CAData) != "rotated-ca-1" { + t.Fatal("rest config was not the pinned projected-credential path") + } + for _, fact := range snapshot.Facts { + if fact.Ref.SourceKind != hubfleet.SourceKind || fact.Ref.Scope != "spoke-a" || fact.Source != "spoke-a" || + fact.Provenance.NativeID != "" || strings.Contains(string(fact.Observed), "token") || strings.Contains(string(fact.Observed), "endpoint") { + t.Fatalf("unsafe normalized fact: %#v", fact) + } + } + for _, fact := range snapshot.Facts { + if fact.Kind == fleet.FactInventory && fact.Ref.Kind == "Pod" && !strings.Contains(string(fact.Observed), "\"image_digests\":[\"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]") { + t.Fatalf("pod inventory did not retain the canonical runtime digest: %s", fact.Observed) + } + } +} + +func TestPodImageDigestsAbstainsFromMutableOrNonWorkloadStatus(t *testing.T) { + t.Parallel() + + digest := "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + pod := corev1.Pod{Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{{ImageID: "docker-pullable://registry.example/api@" + digest}, {ImageID: "registry.example/api:latest"}, {ImageID: "docker-pullable://registry.example/api@" + digest}}, + InitContainerStatuses: []corev1.ContainerStatus{{ImageID: "containerd://sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}}, + EphemeralContainerStatuses: []corev1.ContainerStatus{{ + ImageID: "containerd://sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + }}, + }} + if got := podImageDigests(pod); !slices.Equal(got, []string{digest}) { + t.Fatalf("podImageDigests() = %#v, want one ordinary-container digest", got) + } +} + +func TestSnapshotReadsRotatedCredentialForEachCall(t *testing.T) { + t.Parallel() + + reader := &rotatingCredentialReader{} + client := &fakeSnapshotClient{deployments: []*appsv1.DeploymentList{{}, {}}, pods: []*corev1.PodList{{}, {}}, rollouts: []*unstructured.UnstructuredList{{}, {}}} + adapter := testAdapter(t, reader, client, time.Date(2026, time.July, 13, 18, 0, 0, 0, time.UTC)) + spoke := hubfleet.Spoke{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"} + for range 2 { + if _, err := adapter.Snapshot(context.Background(), "workspace-a", spoke); err != nil { + t.Fatal(err) + } + } + if reader.calls != 2 || len(client.configs) != 2 || client.configs[0].BearerToken != "rotated-token-1" || client.configs[1].BearerToken != "rotated-token-2" { + t.Fatalf("credential rotation calls/configs = %d/%#v", reader.calls, client.configs) + } +} + +func TestSnapshotClearsAdapterOwnedRestConfigCredentialBuffers(t *testing.T) { + t.Parallel() + + reader := credentialReaderFunc(func(context.Context, tenancy.WorkspaceID, string) (projectedCredential, error) { + return projectedCredential{token: []byte("test-token"), ca: []byte("test-ca")}, nil + }) + client := &fakeSnapshotClient{} + adapter := testAdapter(t, reader, client, time.Now().UTC()) + var constructed *rest.Config + adapter.clients = func(config *rest.Config) (snapshotClient, error) { + constructed = config + return client, nil + } + if _, err := adapter.Snapshot(context.Background(), "workspace-a", hubfleet.Spoke{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}); err != nil { + t.Fatal(err) + } + if constructed == nil || constructed.BearerToken != "" || string(constructed.CAData) != "\x00\x00\x00\x00\x00\x00\x00" { + t.Fatal("snapshot retained adapter-owned rest-config credential material") + } +} + +func TestSnapshotDoesNotExposeDependencyCredentialDetails(t *testing.T) { + t.Parallel() + + secret := "eyJnot-a-real-token" + adapter := testAdapter(t, credentialReaderFunc(func(context.Context, tenancy.WorkspaceID, string) (projectedCredential, error) { + return projectedCredential{}, errors.New(secret) + }), &fakeSnapshotClient{}, time.Now().UTC()) + _, err := adapter.Snapshot(context.Background(), "workspace-a", hubfleet.Spoke{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}) + if err == nil || strings.Contains(err.Error(), secret) { + t.Fatal("snapshot error leaked a dependency credential detail") + } +} + +func TestDialRejectsUnpinnedTargetsAndClosesTunnel(t *testing.T) { + t.Parallel() + + factory := &fakeTunnelFactory{} + adapter := testAdapter(t, credentialReaderFunc(func(context.Context, tenancy.WorkspaceID, string) (projectedCredential, error) { + return projectedCredential{}, nil + }), &fakeSnapshotClient{}, time.Now().UTC()) + adapter.tunnels = factory + dial := adapter.dialContext(context.Background(), "spoke-a:443") + if _, err := dial(context.Background(), "tcp", "other:443"); err == nil || factory.opens != 0 { + t.Fatal("unpinned direct dial was accepted") + } + connection, err := dial(context.Background(), "tcp", "spoke-a:443") + if err != nil { + t.Fatal(err) + } + if factory.opens != 1 || factory.tunnel.target != "spoke-a:443" { + t.Fatalf("tunnel open/dial = %d/%q", factory.opens, factory.tunnel.target) + } + if err := connection.Close(); err != nil { + t.Fatal(err) + } + select { + case <-factory.tunnel.canceled: + case <-time.After(time.Second): + t.Fatal("closing direct connection did not close its single-use tunnel") + } +} + +func TestGRPCTunnelFactoryRejectsCanceledCreationContext(t *testing.T) { + t.Parallel() + + proxyTLS := testConfig(nil).ProxyTLSConfig + creationContext, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := (grpcTunnelFactory{address: "127.0.0.1:1", tls: proxyTLS}).Open(creationContext, context.Background()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Open() error = %v, want context cancellation", err) + } +} + +func TestParseManagedClusterRefRejectsEndpointInjection(t *testing.T) { + t.Parallel() + + for _, reference := range []string{"", "spoke-a", "ocm/https://spoke-a", "ocm/spoke-a/other", "ocm/Spoke-A"} { + if _, err := parseManagedClusterRef(reference); err == nil { + t.Fatalf("parseManagedClusterRef(%q) unexpectedly succeeded", reference) + } + } +} + +func TestSnapshotFailsBeforeAnUnboundedFollowupList(t *testing.T) { + t.Parallel() + + deployments := make([]appsv1.Deployment, maxResources) + for index := range deployments { + deployments[index] = appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{ + Name: "deployment-" + strconv.Itoa(index), + Namespace: "apps", + }} + } + client := &fakeSnapshotClient{deployments: []*appsv1.DeploymentList{{Items: deployments}}} + _, err := collectFacts(context.Background(), client, hubfleet.Spoke{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}, time.Now().UTC()) + if err == nil || client.podCalls != 0 || client.rolloutCalls != 0 { + t.Fatal("snapshot did not fail before an unbounded follow-up resource list") + } +} + +type credentialReaderFunc func(context.Context, tenancy.WorkspaceID, string) (projectedCredential, error) + +func (function credentialReaderFunc) Read(ctx context.Context, workspaceID tenancy.WorkspaceID, cluster string) (projectedCredential, error) { + return function(ctx, workspaceID, cluster) +} + +type rotatingCredentialReader struct{ calls int } + +func (reader *rotatingCredentialReader) Read(_ context.Context, _ tenancy.WorkspaceID, _ string) (projectedCredential, error) { + reader.calls++ + return projectedCredential{token: []byte("rotated-token-" + strconv.Itoa(reader.calls)), ca: []byte("rotated-ca-" + strconv.Itoa(reader.calls))}, nil +} + +type fakeSnapshotClient struct { + deployments []*appsv1.DeploymentList + pods []*corev1.PodList + rollouts []*unstructured.UnstructuredList + configs []*rest.Config + closed int + podCalls int + rolloutCalls int +} + +func (client *fakeSnapshotClient) ListDeployments(_ context.Context, _ metav1.ListOptions) (*appsv1.DeploymentList, error) { + if len(client.deployments) == 0 { + return &appsv1.DeploymentList{}, nil + } + page := client.deployments[0] + client.deployments = client.deployments[1:] + return page, nil +} + +func (client *fakeSnapshotClient) ListPods(_ context.Context, _ metav1.ListOptions) (*corev1.PodList, error) { + client.podCalls++ + if len(client.pods) == 0 { + return &corev1.PodList{}, nil + } + page := client.pods[0] + client.pods = client.pods[1:] + return page, nil +} + +func (client *fakeSnapshotClient) ListRollouts(_ context.Context, _ metav1.ListOptions) (*unstructured.UnstructuredList, error) { + client.rolloutCalls++ + if len(client.rollouts) == 0 { + return &unstructured.UnstructuredList{}, nil + } + page := client.rollouts[0] + client.rollouts = client.rollouts[1:] + return page, nil +} + +func (client *fakeSnapshotClient) Close() { client.closed++ } + +type fakeTunnelFactory struct { + opens int + tunnel *fakeTunnel +} + +func (factory *fakeTunnelFactory) Open(_ context.Context, tunnelCtx context.Context) (konnectivity.Tunnel, error) { + factory.opens++ + factory.tunnel = &fakeTunnel{canceled: tunnelCtx.Done()} + return factory.tunnel, nil +} + +type fakeTunnel struct { + target string + canceled <-chan struct{} +} + +func (tunnel *fakeTunnel) DialContext(_ context.Context, _ string, address string) (net.Conn, error) { + tunnel.target = address + client, server := net.Pipe() + go server.Close() + return client, nil +} + +func (tunnel *fakeTunnel) Done() <-chan struct{} { return tunnel.canceled } + +func testConfig(reader CredentialReader) Config { + return Config{ + CredentialReader: reader, + ProxyAddress: "proxy.example:8090", + ProxyTLSConfig: &tls.Config{ + RootCAs: x509.NewCertPool(), MinVersion: tls.VersionTLS12, ServerName: "proxy.example", + Certificates: []tls.Certificate{{Certificate: [][]byte{{1}}, PrivateKey: &rsa.PrivateKey{}}}, + }, + KubeAPIServerName: "kubernetes", + } +} + +func testAdapter(t *testing.T, reader CredentialReader, client *fakeSnapshotClient, now time.Time) *Adapter { + t.Helper() + config := testConfig(reader) + config.Now = func() time.Time { return now } + adapter, err := New(config) + if err != nil { + t.Fatal(err) + } + adapter.clients = func(config *rest.Config) (snapshotClient, error) { + captured := rest.CopyConfig(config) + captured.CAData = append([]byte(nil), config.CAData...) + client.configs = append(client.configs, captured) + return client, nil + } + return adapter +} + +func pointer[T any](value T) *T { return &value } diff --git a/internal/hubocm/doc.go b/internal/hubocm/doc.go new file mode 100644 index 0000000..971dfd4 --- /dev/null +++ b/internal/hubocm/doc.go @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package hubocm implements the pinned, direct OCM ClusterProxy read adapter. +// +// It deliberately keeps the OCM reverse tunnel and managed-serviceaccount identity +// substrate outside Sith's persistence model. The adapter reads one fixed projected +// credential per registered managed cluster, opens short-lived Konnectivity tunnels +// to that exact cluster, and returns only normalized inventory and health facts. +// +// Proxy client mTLS material belongs in a read-only deployment mount. It is supplied +// as a TLS configuration at construction time and is never read from, or persisted to, +// Sith-managed storage. +package hubocm diff --git a/internal/hubocm/ocm_integration_test.go b/internal/hubocm/ocm_integration_test.go new file mode 100644 index 0000000..72f2afd --- /dev/null +++ b/internal/hubocm/ocm_integration_test.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 +//go:build e2e && ocm + +package hubocm + +import ( + "context" + "crypto/subtle" + "net/http" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/tests/testutil/ocmlab" +) + +const ( + m0WorkspaceID = "workspace-m0" +) + +// TestDirectClusterProxyM0 proves the direct Konnectivity path against the retained +// M0 lab. The test deliberately never reads an admin kubeconfig for either spoke. +func TestDirectClusterProxyM0(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) + defer cancel() + + hubConfig := ocmlab.HubConfig(t) + hubClient, err := kubernetes.NewForConfig(hubConfig) + if err != nil { + t.Fatal("construct M0 hub client failed") + } + proxyAddress := ocmlab.StartProxyPortForward(ctx, t) + proxyTLS := ocmlab.ProxyTLS(ctx, t, hubClient) + reader, err := NewManagedServiceAccountReader(hubClient.CoreV1()) + if err != nil { + t.Fatal("construct scoped MSA reader failed") + } + adapter, err := New(Config{ + CredentialReader: reader, + ProxyAddress: proxyAddress, + ProxyTLSConfig: proxyTLS, + KubeAPIServerName: "kubernetes", + }) + if err != nil { + t.Fatal("construct direct OCM transport failed") + } + + for _, spoke := range []hubfleet.Spoke{ + {ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}, + {ID: "spoke-b", ManagedClusterRef: "ocm/spoke-b"}, + } { + snapshot, err := adapter.Snapshot(ctx, m0WorkspaceID, spoke) + if err != nil { + t.Fatal("direct OCM snapshot failed") + } + if err := hubfleet.ValidateSnapshot(spoke, snapshot, time.Now().UTC()); err != nil { + t.Fatal("direct OCM snapshot did not meet the fleet contract") + } + if !hasInventoryFor(snapshot, "Deployment") || !hasInventoryFor(snapshot, "Pod") { + t.Fatal("direct OCM snapshot did not contain the scoped deployment and pod inventory") + } + } + + assertDirectSecretsForbidden(ctx, t, adapter, reader) + assertMSARotation(ctx, t, hubClient, reader, adapter) +} + +func assertDirectSecretsForbidden( + ctx context.Context, + t *testing.T, + adapter *Adapter, + reader CredentialReader, +) { + t.Helper() + credential, err := reader.Read(ctx, m0WorkspaceID, "spoke-a") + if err != nil { + t.Fatal("read scoped MSA credential for negative control failed") + } + defer clearCredential(&credential) + config := adapter.restConfig(ctx, "spoke-a", credential) + transport, err := rest.TransportFor(config) + if err != nil { + t.Fatal("construct direct negative-control transport failed") + } + httpClient := &http.Client{Transport: transport} + defer httpClient.CloseIdleConnections() + directClient, err := kubernetes.NewForConfigAndClient(config, httpClient) + if err != nil { + t.Fatal("construct direct negative-control client failed") + } + if _, err := directClient.CoreV1().Secrets("").List(ctx, metav1.ListOptions{Limit: 1}); !apierrors.IsForbidden(err) { + t.Fatal("direct MSA path did not fail closed for Secrets") + } +} + +func assertMSARotation( + ctx context.Context, + t *testing.T, + hubClient kubernetes.Interface, + reader CredentialReader, + adapter *Adapter, +) { + t.Helper() + before, err := reader.Read(ctx, m0WorkspaceID, "spoke-a") + if err != nil { + t.Fatal("read MSA credential before rotation failed") + } + previousToken := append([]byte(nil), before.token...) + clearCredential(&before) + defer clear(previousToken) + if err := hubClient.CoreV1().Secrets("spoke-a").Delete(ctx, managedServiceAccount, metav1.DeleteOptions{}); err != nil { + t.Fatal("request MSA projection rotation failed") + } + + deadline := time.NewTimer(90 * time.Second) + defer deadline.Stop() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + t.Fatal("MSA projection rotation exceeded the direct-test deadline") + case <-deadline.C: + t.Fatal("MSA projection rotation did not produce a new token") + case <-ticker.C: + next, err := reader.Read(ctx, m0WorkspaceID, "spoke-a") + if err != nil { + continue + } + rotated := subtle.ConstantTimeCompare(previousToken, next.token) != 1 + clearCredential(&next) + if !rotated { + continue + } + snapshot, err := adapter.Snapshot(ctx, m0WorkspaceID, hubfleet.Spoke{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}) + if err != nil || len(snapshot.Facts) == 0 { + t.Fatal("direct transport did not use the rotated MSA credential") + } + return + } + } +} + +func hasInventoryFor(snapshot hubfleet.Snapshot, kind string) bool { + for _, fact := range snapshot.Facts { + if fact.Kind == "inventory" && fact.Ref.Kind == kind { + return true + } + } + return false +} diff --git a/internal/hubruntime/config.go b/internal/hubruntime/config.go new file mode 100644 index 0000000..22c64ce --- /dev/null +++ b/internal/hubruntime/config.go @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubruntime + +import ( + "context" + "crypto/ed25519" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "log/slog" + "net" + "os" + "strconv" + "strings" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/ArdurAI/sith/internal/hubauth" + "github.com/ArdurAI/sith/internal/hubdb" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/hubocm" + "github.com/ArdurAI/sith/internal/hubserver" + "github.com/ArdurAI/sith/internal/observability" + "github.com/ArdurAI/sith/internal/pep" +) + +const ( + maxMountedCertificateBytes = 256 * 1024 + maxMountedKeyBytes = 64 * 1024 + maxMountedCABundleBytes = 256 * 1024 + maxMountedPublicKeyBytes = 16 * 1024 +) + +// Runtime owns the configured application database while the hub server runs. +type Runtime struct { + server *Server + close func() +} + +// NewFromEnvironment constructs the production hub only from complete, deployment-mounted +// configuration and in-cluster Kubernetes identity. It never loads a kubeconfig or persists a +// token, certificate, or database credential. +func NewFromEnvironment(ctx context.Context, logger *slog.Logger) (*Runtime, error) { + if ctx == nil || logger == nil { + return nil, fmt.Errorf("construct hub runtime: context and logger are required") + } + config, err := loadDeploymentConfig(os.LookupEnv) + if err != nil { + return nil, err + } + serverTLS, err := loadServerTLS(config) + if err != nil { + return nil, err + } + proxyTLS, err := loadProxyTLS(config) + if err != nil { + return nil, err + } + publicKey, err := loadSessionPublicKey(config.sessionPublicKeyFile) + if err != nil { + return nil, err + } + verifier, err := hubauth.NewJWTVerifier(hubauth.JWTConfig{ + Issuer: config.sessionIssuer, Audience: config.sessionAudience, Keys: map[string]ed25519.PublicKey{config.sessionKeyID: publicKey}, + }) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: session verification configuration is invalid") + } + auditor, err := pep.NewSlogAuditor(logger) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: policy audit configuration is invalid") + } + tracer, err := observability.NewSlogTraceObserver(logger) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: trace configuration is invalid") + } + authObserver, err := observability.NewSlogAuthObserver(logger) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: authentication observability configuration is invalid") + } + enforcer, err := pep.NewEnforcer(pep.Config{Hook: pep.AllowReadHook{}, Auditor: auditor, TraceObserver: tracer}) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: policy configuration is invalid") + } + + inClusterConfig, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("construct hub runtime: in-cluster Kubernetes identity is required") + } + kubeClient, err := kubernetes.NewForConfig(inClusterConfig) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: Kubernetes client is unavailable") + } + credentialReader, err := hubocm.NewManagedServiceAccountReader(kubeClient.CoreV1()) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: scoped credential reader is unavailable") + } + transport, err := hubocm.New(hubocm.Config{ + CredentialReader: credentialReader, + ProxyAddress: config.proxyAddress, + ProxyTLSConfig: proxyTLS, + KubeAPIServerName: config.kubeAPIServerName, + }) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: direct OCM transport configuration is invalid") + } + database, err := hubdb.OpenAppDB(ctx, hubdb.AppConfig{URL: config.databaseURL}) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: database is unavailable") + } + cleanup := database.Close + collector, err := hubfleet.NewCollector(hubfleet.CollectorConfig{Store: database, Transport: transport, PEP: enforcer, TraceObserver: tracer}) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: collector configuration is invalid") + } + imageSearcher, err := hubfleet.NewImageSearcher(hubfleet.ImageSearcherConfig{Querier: database, PEP: enforcer}) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: image search configuration is invalid") + } + handler, err := hubserver.NewFleetHandler(hubserver.FleetHandlerConfig{ + Verifier: verifier, AuthObserver: authObserver, Collector: collector, Reader: database, ImageSearcher: imageSearcher, PEP: enforcer, + }) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: HTTP handler configuration is invalid") + } + listener, err := net.Listen("tcp", config.listenAddress) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: listener is unavailable") + } + server, err := NewServer(ServerConfig{Listener: listener, Handler: handler, TLSConfig: serverTLS}) + if err != nil { + _ = listener.Close() + cleanup() + return nil, err + } + return &Runtime{server: server, close: cleanup}, nil +} + +// Run serves the configured hub and releases its application database pool on exit. +func (runtime *Runtime) Run(ctx context.Context) error { + if runtime == nil || runtime.server == nil || runtime.close == nil { + return fmt.Errorf("run hub runtime: runtime is required") + } + defer runtime.close() + return runtime.server.Run(ctx) +} + +type deploymentConfig struct { + listenAddress string + databaseURL string + sessionIssuer string + sessionAudience string + sessionKeyID string + sessionPublicKeyFile string + serverCertFile string + serverKeyFile string + proxyAddress string + proxyServerName string + proxyCAFile string + proxyCertFile string + proxyKeyFile string + kubeAPIServerName string +} + +func loadDeploymentConfig(lookup func(string) (string, bool)) (deploymentConfig, error) { + if lookup == nil { + return deploymentConfig{}, fmt.Errorf("load hub configuration: environment lookup is required") + } + config := deploymentConfig{} + var err error + for _, value := range []struct { + name string + target *string + }{ + {"SITH_HUB_LISTEN_ADDR", &config.listenAddress}, + {"SITH_HUB_DATABASE_URL", &config.databaseURL}, + {"SITH_HUB_SESSION_ISSUER", &config.sessionIssuer}, + {"SITH_HUB_SESSION_AUDIENCE", &config.sessionAudience}, + {"SITH_HUB_SESSION_KEY_ID", &config.sessionKeyID}, + {"SITH_HUB_SESSION_PUBLIC_KEY_FILE", &config.sessionPublicKeyFile}, + {"SITH_HUB_SERVER_TLS_CERT_FILE", &config.serverCertFile}, + {"SITH_HUB_SERVER_TLS_KEY_FILE", &config.serverKeyFile}, + {"SITH_HUB_PROXY_ADDRESS", &config.proxyAddress}, + {"SITH_HUB_PROXY_SERVER_NAME", &config.proxyServerName}, + {"SITH_HUB_PROXY_CA_FILE", &config.proxyCAFile}, + {"SITH_HUB_PROXY_CERT_FILE", &config.proxyCertFile}, + {"SITH_HUB_PROXY_KEY_FILE", &config.proxyKeyFile}, + {"SITH_HUB_KUBE_API_SERVER_NAME", &config.kubeAPIServerName}, + } { + *value.target, err = requiredEnvironment(lookup, value.name) + if err != nil { + return deploymentConfig{}, err + } + } + if err := validateListenAddress(config.listenAddress); err != nil { + return deploymentConfig{}, fmt.Errorf("load hub configuration: listen address is invalid") + } + return config, nil +} + +func requiredEnvironment(lookup func(string) (string, bool), name string) (string, error) { + value, present := lookup(name) + if !present || value == "" || strings.TrimSpace(value) != value || len(value) > 4096 { + return "", fmt.Errorf("load hub configuration: %s is required", name) + } + return value, nil +} + +func validateListenAddress(address string) error { + host, port, err := net.SplitHostPort(address) + if err != nil || host == "" { + return fmt.Errorf("listen address must include host and port") + } + value, err := strconv.ParseUint(port, 10, 16) + if err != nil || value == 0 { + return fmt.Errorf("listen address must use a non-zero port") + } + return nil +} + +func loadServerTLS(config deploymentConfig) (*tls.Config, error) { + certificate, err := loadMountedCertificate("hub server certificate", config.serverCertFile, config.serverKeyFile) + if err != nil { + return nil, err + } + return &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{certificate}}, nil +} + +func loadProxyTLS(config deploymentConfig) (*tls.Config, error) { + caPEM, err := readMountedFile("proxy CA bundle", config.proxyCAFile, maxMountedCABundleBytes) + if err != nil { + return nil, err + } + defer clear(caPEM) + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("load hub configuration: proxy CA bundle is invalid") + } + certificate, err := loadMountedCertificate("proxy client certificate", config.proxyCertFile, config.proxyKeyFile) + if err != nil { + return nil, err + } + return &tls.Config{ + RootCAs: pool, MinVersion: tls.VersionTLS12, ServerName: config.proxyServerName, Certificates: []tls.Certificate{certificate}, + }, nil +} + +func loadMountedCertificate(label, certificateFile, keyFile string) (tls.Certificate, error) { + certificatePEM, err := readMountedFile(label, certificateFile, maxMountedCertificateBytes) + if err != nil { + return tls.Certificate{}, err + } + defer clear(certificatePEM) + keyPEM, err := readMountedFile(label+" key", keyFile, maxMountedKeyBytes) + if err != nil { + return tls.Certificate{}, err + } + defer clear(keyPEM) + certificate, err := tls.X509KeyPair(certificatePEM, keyPEM) + if err != nil || len(certificate.Certificate) == 0 || certificate.PrivateKey == nil { + return tls.Certificate{}, fmt.Errorf("load hub configuration: %s is invalid", label) + } + return certificate, nil +} + +func loadSessionPublicKey(path string) (ed25519.PublicKey, error) { + encoded, err := readMountedFile("session public key", path, maxMountedPublicKeyBytes) + if err != nil { + return nil, err + } + defer clear(encoded) + block, rest := pem.Decode(encoded) + if block == nil || block.Type != "PUBLIC KEY" || len(strings.TrimSpace(string(rest))) != 0 { + return nil, fmt.Errorf("load hub configuration: session public key is invalid") + } + parsed, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("load hub configuration: session public key is invalid") + } + key, ok := parsed.(ed25519.PublicKey) + if !ok || len(key) != ed25519.PublicKeySize { + return nil, fmt.Errorf("load hub configuration: session public key is not Ed25519") + } + return append(ed25519.PublicKey(nil), key...), nil +} + +func readMountedFile(label, path string, maxBytes int) ([]byte, error) { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o222 != 0 { + return nil, fmt.Errorf("load hub configuration: %s must be a read-only regular file", label) + } + // #nosec G304 -- path is a required deployment input, validated as a bounded read-only regular file immediately above. + contents, err := os.ReadFile(path) + if err != nil || len(contents) == 0 || len(contents) > maxBytes { + clear(contents) + return nil, fmt.Errorf("load hub configuration: %s is unavailable", label) + } + return contents, nil +} diff --git a/internal/hubruntime/doc.go b/internal/hubruntime/doc.go new file mode 100644 index 0000000..9779b52 --- /dev/null +++ b/internal/hubruntime/doc.go @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package hubruntime composes the in-cluster, governed Sith hub process. +package hubruntime diff --git a/internal/hubruntime/migrate.go b/internal/hubruntime/migrate.go new file mode 100644 index 0000000..57d12c7 --- /dev/null +++ b/internal/hubruntime/migrate.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubruntime + +import ( + "context" + "fmt" + "os" + + "github.com/ArdurAI/sith/internal/hubdb" +) + +type migrationConfig struct { + ownerDatabaseURL string + applicationRole string +} + +// MigrateFromEnvironment applies the hub schema from a short-lived owner-credential process. It +// deliberately does not construct the TLS hub server, a Kubernetes client, or any read transport. +func MigrateFromEnvironment(ctx context.Context) error { + config, err := loadMigrationConfig(os.LookupEnv) + if err != nil { + return err + } + return hubdb.Migrate(ctx, hubdb.MigrationConfig{ + OwnerURL: config.ownerDatabaseURL, + ApplicationRole: config.applicationRole, + }) +} + +func loadMigrationConfig(lookup func(string) (string, bool)) (migrationConfig, error) { + if lookup == nil { + return migrationConfig{}, fmt.Errorf("load hub migration configuration: environment lookup is required") + } + ownerDatabaseURL, err := requiredEnvironment(lookup, "SITH_HUB_MIGRATION_OWNER_DATABASE_URL") + if err != nil { + return migrationConfig{}, err + } + applicationRole, err := requiredEnvironment(lookup, "SITH_HUB_APPLICATION_DATABASE_ROLE") + if err != nil { + return migrationConfig{}, err + } + return migrationConfig{ownerDatabaseURL: ownerDatabaseURL, applicationRole: applicationRole}, nil +} diff --git a/internal/hubruntime/migrate_test.go b/internal/hubruntime/migrate_test.go new file mode 100644 index 0000000..13dd0de --- /dev/null +++ b/internal/hubruntime/migrate_test.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubruntime + +import "testing" + +func TestLoadMigrationConfigRequiresDistinctDeploymentInputs(t *testing.T) { + t.Parallel() + + if _, err := loadMigrationConfig(func(string) (string, bool) { return "", false }); err == nil { + t.Fatal("loadMigrationConfig accepted missing inputs") + } + values := map[string]string{ + "SITH_HUB_MIGRATION_OWNER_DATABASE_URL": "postgres://sith_owner@db.sith.svc/sith?sslmode=require", + "SITH_HUB_APPLICATION_DATABASE_ROLE": "sith_app", + } + config, err := loadMigrationConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }) + if err != nil || config.ownerDatabaseURL != values["SITH_HUB_MIGRATION_OWNER_DATABASE_URL"] || config.applicationRole != "sith_app" { + t.Fatalf("config/error = %#v/%v", config, err) + } + values["SITH_HUB_APPLICATION_DATABASE_ROLE"] = " sith_app" + if _, err := loadMigrationConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }); err == nil { + t.Fatal("loadMigrationConfig accepted whitespace-padded application role") + } +} diff --git a/internal/hubruntime/ocm_integration_test.go b/internal/hubruntime/ocm_integration_test.go new file mode 100644 index 0000000..71b893a --- /dev/null +++ b/internal/hubruntime/ocm_integration_test.go @@ -0,0 +1,360 @@ +// SPDX-License-Identifier: Apache-2.0 +//go:build e2e && ocm + +package hubruntime + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "fmt" + "net" + "net/http" + "slices" + "sync" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + "k8s.io/client-go/kubernetes" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubauth" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/hubocm" + "github.com/ArdurAI/sith/internal/hubserver" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/tests/testutil/ocmlab" +) + +const m0RuntimeWorkspaceID tenancy.WorkspaceID = "workspace-m0" + +func TestHubRuntimeDirectClusterProxyM0(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) + defer cancel() + hubClient, err := kubernetes.NewForConfig(ocmlab.HubConfig(t)) + if err != nil { + t.Fatal("construct M0 hub client failed") + } + credentialReader, err := hubocm.NewManagedServiceAccountReader(hubClient.CoreV1()) + if err != nil { + t.Fatal("construct scoped MSA reader failed") + } + transport, err := hubocm.New(hubocm.Config{ + CredentialReader: credentialReader, + ProxyAddress: ocmlab.StartProxyPortForward(ctx, t), + ProxyTLSConfig: ocmlab.ProxyTLS(ctx, t, hubClient), + KubeAPIServerName: "kubernetes", + }) + if err != nil { + t.Fatal("construct direct OCM transport failed") + } + store := &m0RuntimeStore{ + spokes: []hubfleet.Spoke{ + {ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}, + {ID: "spoke-b", ManagedClusterRef: "ocm/spoke-b"}, + }, + snapshots: make(map[string]hubfleet.Snapshot), failures: make(map[string]hubfleet.FailureKind), + } + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, Auditor: pep.AuditFunc(func(context.Context, pep.AuditEvent) error { return nil }), + }) + if err != nil { + t.Fatal(err) + } + collector, err := hubfleet.NewCollector(hubfleet.CollectorConfig{Store: store, Transport: transport, PEP: enforcer}) + if err != nil { + t.Fatal(err) + } + imageSearcher, err := hubfleet.NewImageSearcher(hubfleet.ImageSearcherConfig{Querier: store, PEP: enforcer}) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + verifier, privateKey := m0RuntimeVerifier(t, now) + handler, err := hubserver.NewFleetHandler(hubserver.FleetHandlerConfig{ + Verifier: verifier, Collector: collector, Reader: store, ImageSearcher: imageSearcher, PEP: enforcer, + }) + if err != nil { + t.Fatal(err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + serverTLS, clientTLS := runtimeTestTLS(t) + server, err := NewServer(ServerConfig{Listener: listener, Handler: handler, TLSConfig: serverTLS}) + if err != nil { + t.Fatal(err) + } + serverCtx, stopServer := context.WithCancel(ctx) + defer stopServer() + serverDone := make(chan error, 1) + go func() { serverDone <- server.Run(serverCtx) }() + + client := &http.Client{Transport: &http.Transport{TLSClientConfig: clientTLS}, Timeout: 2 * time.Minute} + defer client.CloseIdleConnections() + endpoint := "https://" + listener.Addr().String() + "/v1/workspaces/workspace-m0" + token := m0RuntimeToken(t, privateKey, now) + refresh := m0RuntimeRequest(t, ctx, client, http.MethodPost, endpoint+"/fleet:refresh", token) + defer refresh.Body.Close() + if refresh.StatusCode != http.StatusOK { + t.Fatalf("runtime refresh status = %d", refresh.StatusCode) + } + var coverage fleet.Coverage + if err := json.NewDecoder(refresh.Body).Decode(&coverage); err != nil { + t.Fatal(err) + } + if coverage.Requested != 2 || coverage.Reachable != 2 || len(coverage.Unreachable) != 0 || len(coverage.Stale) != 0 { + t.Fatalf("runtime direct refresh coverage = %#v", coverage) + } + fleetResponse := m0RuntimeRequest(t, ctx, client, http.MethodGet, endpoint+"/fleet", token) + defer fleetResponse.Body.Close() + if fleetResponse.StatusCode != http.StatusOK { + t.Fatalf("runtime fleet status = %d", fleetResponse.StatusCode) + } + var result fleet.FleetResult + if err := json.NewDecoder(fleetResponse.Body).Decode(&result); err != nil { + t.Fatal(err) + } + if len(result.Clusters) != 2 || result.Coverage.Requested != 2 || result.Coverage.Reachable != 2 { + t.Fatalf("runtime direct fleet = %#v", result) + } + digest := m0RuntimeFixtureDigest(t, store) + imageResponse := m0RuntimeRequest(t, ctx, client, http.MethodGet, endpoint+"/fleet/images/"+digest, token) + defer imageResponse.Body.Close() + if imageResponse.StatusCode != http.StatusOK { + t.Fatalf("runtime image search status = %d", imageResponse.StatusCode) + } + var imageResult fleet.QueryResult + if err := json.NewDecoder(imageResponse.Body).Decode(&imageResult); err != nil { + t.Fatal(err) + } + if len(imageResult.Facts) != 2 || imageResult.Coverage.Requested != 2 || imageResult.Coverage.Reachable != 2 || + imageResult.Facts[0].Ref.Kind != "Pod" || imageResult.Facts[1].Ref.Kind != "Pod" || + !slices.Equal([]string{imageResult.Facts[0].Ref.Scope, imageResult.Facts[1].Ref.Scope}, []string{"spoke-a", "spoke-b"}) { + t.Fatalf("runtime exact image search = %#v", imageResult) + } + stopServer() + if err := <-serverDone; err != nil { + t.Fatal(err) + } +} + +type m0RuntimeStore struct { + mu sync.Mutex + spokes []hubfleet.Spoke + snapshots map[string]hubfleet.Snapshot + failures map[string]hubfleet.FailureKind +} + +func (store *m0RuntimeStore) RegisteredSpokes(_ context.Context, scope tenancy.Scope) ([]hubfleet.Spoke, error) { + if err := scope.RequireWorkspace(m0RuntimeWorkspaceID); err != nil { + return nil, err + } + store.mu.Lock() + defer store.mu.Unlock() + return append([]hubfleet.Spoke(nil), store.spokes...), nil +} + +func (store *m0RuntimeStore) ReplaceSnapshot( + _ context.Context, + scope tenancy.Scope, + spoke hubfleet.Spoke, + snapshot hubfleet.Snapshot, + _ time.Time, +) error { + if err := scope.RequireWorkspace(m0RuntimeWorkspaceID); err != nil { + return err + } + store.mu.Lock() + defer store.mu.Unlock() + store.snapshots[spoke.ID] = snapshot + delete(store.failures, spoke.ID) + return nil +} + +func (store *m0RuntimeStore) MarkSnapshotFailure( + _ context.Context, + scope tenancy.Scope, + spoke hubfleet.Spoke, + failure hubfleet.FailureKind, + _ time.Time, +) (bool, error) { + if err := scope.RequireWorkspace(m0RuntimeWorkspaceID); err != nil { + return false, err + } + store.mu.Lock() + defer store.mu.Unlock() + _, retained := store.snapshots[spoke.ID] + store.failures[spoke.ID] = failure + return retained, nil +} + +func (store *m0RuntimeStore) ReadFleet( + _ context.Context, + scope tenancy.Scope, + _ time.Duration, + _ time.Time, +) (fleet.FleetResult, error) { + if err := scope.RequireWorkspace(m0RuntimeWorkspaceID); err != nil { + return fleet.FleetResult{}, err + } + store.mu.Lock() + defer store.mu.Unlock() + result := fleet.FleetResult{Clusters: make([]fleet.Cluster, 0, len(store.spokes)), Coverage: fleet.Coverage{Requested: len(store.spokes)}} + for _, spoke := range store.spokes { + snapshot, exists := store.snapshots[spoke.ID] + failure := store.failures[spoke.ID] + reachable := exists && failure == "" + if reachable { + result.Coverage.Reachable++ + } else { + result.Coverage.Unreachable = append(result.Coverage.Unreachable, spoke.ID) + if exists { + result.Coverage.Stale = append(result.Coverage.Stale, spoke.ID) + } + } + result.Clusters = append(result.Clusters, fleet.Cluster{ + Name: spoke.ID, Context: spoke.ManagedClusterRef, SourceKind: hubfleet.SourceKind, Reachable: reachable, ObservedAt: snapshot.ObservedAt, + }) + } + return result, nil +} + +func (store *m0RuntimeStore) QueryFleet( + _ context.Context, + scope tenancy.Scope, + query fleet.Query, + freshness time.Duration, + now time.Time, +) (fleet.QueryResult, error) { + if err := scope.RequireWorkspace(m0RuntimeWorkspaceID); err != nil { + return fleet.QueryResult{}, err + } + if len(query.Kinds) != 1 || query.Kinds[0] != fleet.FactInventory || query.Selector.ResourceKind != "Pod" || + fleet.ValidateImageDigest(query.Selector.Image) != nil || freshness < time.Second || now.IsZero() { + return fleet.QueryResult{}, fmt.Errorf("M0 runtime store received an unsupported fleet query") + } + store.mu.Lock() + defer store.mu.Unlock() + result := fleet.QueryResult{Facts: []fleet.Fact{}, Coverage: fleet.Coverage{Requested: len(store.spokes)}} + for _, spoke := range store.spokes { + snapshot, exists := store.snapshots[spoke.ID] + failure := store.failures[spoke.ID] + if !exists || failure != "" { + result.Coverage.Unreachable = append(result.Coverage.Unreachable, spoke.ID) + if exists { + result.Coverage.Stale = append(result.Coverage.Stale, spoke.ID) + } + continue + } + result.Coverage.Reachable++ + stale := now.Sub(snapshot.ObservedAt) > freshness + if stale { + result.Coverage.Stale = append(result.Coverage.Stale, spoke.ID) + } + for _, evidence := range snapshot.Facts { + if evidence.Kind != fleet.FactInventory || evidence.Ref.Kind != "Pod" || !m0RuntimeFactHasDigest(evidence, query.Selector.Image) { + continue + } + result.Facts = append(result.Facts, fleet.Fact{Evidence: evidence, Workspace: string(scope.WorkspaceID()), Stale: stale}) + } + } + slices.Sort(result.Coverage.Unreachable) + slices.Sort(result.Coverage.Stale) + return result, nil +} + +func m0RuntimeFixtureDigest(t *testing.T, store *m0RuntimeStore) string { + t.Helper() + store.mu.Lock() + defer store.mu.Unlock() + var digest string + for _, spokeID := range []string{"spoke-a", "spoke-b"} { + snapshot, exists := store.snapshots[spokeID] + if !exists { + t.Fatalf("M0 spoke %s snapshot was not recorded", spokeID) + } + found := "" + for _, evidence := range snapshot.Facts { + if evidence.Kind == fleet.FactInventory && evidence.Ref.Kind == "Pod" && evidence.Ref.Namespace == "sith-demo" && evidence.Ref.Name != "" { + var observed struct { + ImageDigests []string `json:"image_digests"` + } + if err := json.Unmarshal(evidence.Observed, &observed); err != nil { + t.Fatal(err) + } + if len(observed.ImageDigests) == 1 { + found = observed.ImageDigests[0] + } + } + } + if found == "" { + t.Fatalf("M0 spoke %s did not retain one immutable fixture image digest", spokeID) + } + if digest != "" && digest != found { + t.Fatalf("M0 fixture digests differ: %q and %q", digest, found) + } + digest = found + } + return digest +} + +func m0RuntimeFactHasDigest(evidence fleet.Evidence, digest string) bool { + var observed struct { + ImageDigests []string `json:"image_digests"` + } + return json.Unmarshal(evidence.Observed, &observed) == nil && slices.Contains(observed.ImageDigests, digest) +} + +type m0RuntimeClaims struct { + Memberships map[string]tenancy.Role `json:"memberships"` + jwt.RegisteredClaims +} + +func m0RuntimeVerifier(t *testing.T, now time.Time) (*hubauth.JWTVerifier, ed25519.PrivateKey) { + t.Helper() + privateKey := ed25519.NewKeyFromSeed([]byte("01234567890123456789012345678901")) + publicKey := privateKey.Public().(ed25519.PublicKey) + verifier, err := hubauth.NewJWTVerifier(hubauth.JWTConfig{ + Issuer: "https://issuer.sith.test", Audience: "https://hub.sith.test", Keys: map[string]ed25519.PublicKey{"m0-session": publicKey}, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + return verifier, privateKey +} + +func m0RuntimeToken(t *testing.T, privateKey ed25519.PrivateKey, now time.Time) string { + t.Helper() + token := jwt.NewWithClaims(jwt.SigningMethodEdDSA, m0RuntimeClaims{ + Memberships: map[string]tenancy.Role{string(m0RuntimeWorkspaceID): tenancy.RoleReader}, + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: "https://issuer.sith.test", Subject: "user:m0", Audience: jwt.ClaimStrings{"https://hub.sith.test"}, + ExpiresAt: jwt.NewNumericDate(now.Add(time.Hour)), IssuedAt: jwt.NewNumericDate(now.Add(-time.Minute)), ID: "m0-session-1", + }, + }) + token.Header["typ"] = "sith-session+jwt" + token.Header["kid"] = "m0-session" + raw, err := token.SignedString(privateKey) + if err != nil { + t.Fatal(err) + } + return raw +} + +func m0RuntimeRequest(t *testing.T, ctx context.Context, client *http.Client, method, endpoint, token string) *http.Response { + t.Helper() + request, err := http.NewRequestWithContext(ctx, method, endpoint, nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+token) + response, err := client.Do(request) + if err != nil { + t.Fatal(err) + } + return response +} diff --git a/internal/hubruntime/runtime.go b/internal/hubruntime/runtime.go new file mode 100644 index 0000000..702e0ce --- /dev/null +++ b/internal/hubruntime/runtime.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubruntime + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "net/http" + "time" +) + +const defaultShutdownTimeout = 10 * time.Second + +// ServerConfig fixes the TLS listener and authenticated handler for one hub process. +type ServerConfig struct { + Listener net.Listener + Handler http.Handler + TLSConfig *tls.Config + ShutdownTimeout time.Duration +} + +// Server runs the hub's fixed authenticated HTTP surface over deployment-provided TLS. +type Server struct { + listener net.Listener + handler http.Handler + tlsConfig *tls.Config + shutdownTimeout time.Duration +} + +// NewServer constructs a server that refuses plaintext and dynamically supplied certificates. +func NewServer(config ServerConfig) (*Server, error) { + if config.Listener == nil || config.Handler == nil || config.TLSConfig == nil { + return nil, fmt.Errorf("construct hub server: listener, handler, and TLS configuration are required") + } + if config.TLSConfig.MinVersion < tls.VersionTLS12 || len(config.TLSConfig.Certificates) != 1 || + len(config.TLSConfig.Certificates[0].Certificate) == 0 || config.TLSConfig.Certificates[0].PrivateKey == nil || + config.TLSConfig.GetCertificate != nil || config.TLSConfig.GetConfigForClient != nil { + return nil, fmt.Errorf("construct hub server: TLS 1.2+ and one static server certificate are required") + } + if config.ShutdownTimeout == 0 { + config.ShutdownTimeout = defaultShutdownTimeout + } + if config.ShutdownTimeout < time.Second || config.ShutdownTimeout > time.Minute { + return nil, fmt.Errorf("construct hub server: shutdown timeout must be between 1s and 1m") + } + return &Server{ + listener: config.Listener, + handler: config.Handler, + tlsConfig: config.TLSConfig.Clone(), + shutdownTimeout: config.ShutdownTimeout, + }, nil +} + +// Run serves until the process context is canceled or a non-shutdown server error occurs. +func (server *Server) Run(ctx context.Context) error { + if server == nil || server.listener == nil || server.handler == nil || server.tlsConfig == nil || ctx == nil { + return fmt.Errorf("run hub server: server and context are required") + } + httpServer := &http.Server{ + Handler: server.handler, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: time.Minute, + MaxHeaderBytes: 16 * 1024, + } + serveDone := make(chan error, 1) + go func() { + serveDone <- httpServer.Serve(tls.NewListener(server.listener, server.tlsConfig.Clone())) + }() + + select { + case err := <-serveDone: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return fmt.Errorf("run hub server: %w", err) + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), server.shutdownTimeout) + defer cancel() + shutdownErr := httpServer.Shutdown(shutdownCtx) + serveErr := <-serveDone + if shutdownErr != nil { + return fmt.Errorf("stop hub server: %w", shutdownErr) + } + if !errors.Is(serveErr, http.ErrServerClosed) { + return fmt.Errorf("stop hub server: %w", serveErr) + } + return nil + } +} diff --git a/internal/hubruntime/runtime_test.go b/internal/hubruntime/runtime_test.go new file mode 100644 index 0000000..81d0395 --- /dev/null +++ b/internal/hubruntime/runtime_test.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubruntime + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "testing" + "time" +) + +func TestServerServesTLSAndStopsWithContext(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + serverTLS, clientTLS := runtimeTestTLS(t) + server, err := NewServer(ServerConfig{ + Listener: listener, + Handler: http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.TLS == nil { + t.Fatal("plaintext request reached hub handler") + } + response.WriteHeader(http.StatusNoContent) + }), + TLSConfig: serverTLS, + }) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- server.Run(ctx) }() + + client := &http.Client{Transport: &http.Transport{TLSClientConfig: clientTLS}, Timeout: time.Second} + defer client.CloseIdleConnections() + endpoint := "https://" + listener.Addr().String() + deadline := time.Now().Add(2 * time.Second) + for { + response, requestErr := client.Get(endpoint) + if requestErr == nil { + if response.StatusCode != http.StatusNoContent { + t.Fatalf("status = %d", response.StatusCode) + } + _ = response.Body.Close() + break + } + if time.Now().After(deadline) { + t.Fatalf("TLS request failed: %v", requestErr) + } + time.Sleep(10 * time.Millisecond) + } + cancel() + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestNewServerRejectsUnsafeConfiguration(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + serverTLS, _ := runtimeTestTLS(t) + for _, test := range []struct { + name string + mutate func(*ServerConfig) + }{ + {name: "missing handler", mutate: func(config *ServerConfig) { config.Handler = nil }}, + {name: "missing listener", mutate: func(config *ServerConfig) { config.Listener = nil }}, + {name: "weak TLS", mutate: func(config *ServerConfig) { config.TLSConfig.MinVersion = tls.VersionTLS11 }}, + {name: "dynamic certificate", mutate: func(config *ServerConfig) { + config.TLSConfig.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, nil } + }}, + {name: "bad timeout", mutate: func(config *ServerConfig) { config.ShutdownTimeout = 500 * time.Millisecond }}, + } { + t.Run(test.name, func(t *testing.T) { + config := ServerConfig{Listener: listener, Handler: http.NotFoundHandler(), TLSConfig: serverTLS.Clone()} + test.mutate(&config) + if _, err := NewServer(config); err == nil { + t.Fatal("NewServer accepted unsafe configuration") + } + }) + } +} + +func TestLoadDeploymentConfigRequiresEverySecurityInput(t *testing.T) { + config, err := loadDeploymentConfig(func(string) (string, bool) { return "", false }) + if err == nil || config != (deploymentConfig{}) { + t.Fatalf("config/error = %#v/%v", config, err) + } + values := map[string]string{ + "SITH_HUB_LISTEN_ADDR": "127.0.0.1:8443", + "SITH_HUB_DATABASE_URL": "postgres://sith@db/sith?sslmode=require", + "SITH_HUB_SESSION_ISSUER": "https://issuer.sith.test", + "SITH_HUB_SESSION_AUDIENCE": "https://hub.sith.test", + "SITH_HUB_SESSION_KEY_ID": "session-2026-07", + "SITH_HUB_SESSION_PUBLIC_KEY_FILE": "/mnt/session/public.pem", + "SITH_HUB_SERVER_TLS_CERT_FILE": "/mnt/server/tls.crt", + "SITH_HUB_SERVER_TLS_KEY_FILE": "/mnt/server/tls.key", + "SITH_HUB_PROXY_ADDRESS": "proxy.sith.test:8090", + "SITH_HUB_PROXY_SERVER_NAME": "proxy.sith.test", + "SITH_HUB_PROXY_CA_FILE": "/mnt/proxy/ca.crt", + "SITH_HUB_PROXY_CERT_FILE": "/mnt/proxy/tls.crt", + "SITH_HUB_PROXY_KEY_FILE": "/mnt/proxy/tls.key", + "SITH_HUB_KUBE_API_SERVER_NAME": "kubernetes", + } + config, err = loadDeploymentConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }) + if err != nil || config.listenAddress != "127.0.0.1:8443" || config.proxyAddress != "proxy.sith.test:8090" { + t.Fatalf("config/error = %#v/%v", config, err) + } + values["SITH_HUB_LISTEN_ADDR"] = ":8443" + if _, err := loadDeploymentConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }); err == nil { + t.Fatal("loadDeploymentConfig accepted an ambiguous listener") + } +} + +func TestReadMountedFileRequiresReadOnlyRegularFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "mounted.pem") + if err := os.WriteFile(path, []byte("mounted material"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := readMountedFile("test material", path, 1024); err == nil { + t.Fatal("readMountedFile accepted a writable file") + } + if err := os.Chmod(path, 0o400); err != nil { + t.Fatal(err) + } + contents, err := readMountedFile("test material", path, 1024) + if err != nil || string(contents) != "mounted material" { + t.Fatalf("contents/error = %q/%v", contents, err) + } + clear(contents) + if _, err := readMountedFile("test directory", filepath.Dir(path), 1024); err == nil { + t.Fatal("readMountedFile accepted a directory") + } +} + +func runtimeTestTLS(t *testing.T) (*tls.Config, *tls.Config) { + t.Helper() + publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.Now() + certificateDER, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{ + SerialNumber: big.NewInt(1), NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour), DNSNames: []string{"localhost"}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + }, &x509.Certificate{ + SerialNumber: big.NewInt(1), NotBefore: now.Add(-time.Minute), NotAfter: now.Add(time.Hour), DNSNames: []string{"localhost"}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + }, publicKey, privateKey) + if err != nil { + t.Fatal(err) + } + certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificateDER}) + privateDER, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + t.Fatal(err) + } + privatePEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateDER}) + certificate, err := tls.X509KeyPair(certificatePEM, privatePEM) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(&x509.Certificate{Raw: certificateDER}) + return &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{certificate}}, &tls.Config{RootCAs: pool, ServerName: "localhost", MinVersion: tls.VersionTLS12} +} diff --git a/internal/hubserver/auth.go b/internal/hubserver/auth.go index 130cae4..67d91e7 100644 --- a/internal/hubserver/auth.go +++ b/internal/hubserver/auth.go @@ -21,7 +21,16 @@ type Verifier interface { } // Authenticate constructs middleware that removes spoofable identity headers and requires a token. +// It uses no authentication observer; composition roots that need local security logging should +// use AuthenticateWithObserver. func Authenticate(verifier Verifier, next http.Handler) (http.Handler, error) { + return AuthenticateWithObserver(verifier, nil, next) +} + +// AuthenticateWithObserver constructs authentication middleware with one passive refusal +// observer. The observer is never given request metadata, credentials, verifier errors, or caller +// correlation values, and cannot alter the uniform unauthorized response. +func AuthenticateWithObserver(verifier Verifier, observer AuthObserver, next http.Handler) (http.Handler, error) { if verifier == nil { return nil, fmt.Errorf("construct authentication middleware: verifier is required") } @@ -32,15 +41,16 @@ func Authenticate(verifier Verifier, next http.Handler) (http.Handler, error) { cloned := request.Clone(request.Context()) cloned.Header = request.Header.Clone() stripUntrustedIdentityHeaders(cloned.Header) + stripUntrustedCorrelationHeaders(cloned.Header) rawToken, ok := bearerToken(cloned.Header.Values("Authorization")) if !ok { - writeUnauthorized(response) + refuseAuthentication(observer, response) return } cloned.Header.Del("Authorization") principal, err := verifier.Verify(cloned.Context(), rawToken) if err != nil { - writeUnauthorized(response) + refuseAuthentication(observer, response) return } ctx := context.WithValue(cloned.Context(), principalContextKey{}, principal) @@ -89,9 +99,25 @@ func stripUntrustedIdentityHeaders(headers http.Header) { } } +func stripUntrustedCorrelationHeaders(headers http.Header) { + for name := range headers { + normalized := strings.ToLower(name) + if normalized == "traceparent" || normalized == "tracestate" || normalized == "b3" || + normalized == "x-request-id" || normalized == "x-correlation-id" || normalized == "x-trace-id" || + strings.HasPrefix(normalized, "x-b3-") { + headers.Del(name) + } + } +} + func writeUnauthorized(response http.ResponseWriter) { response.Header().Set("Cache-Control", "no-store") response.Header().Set("Content-Type", "application/json") response.WriteHeader(http.StatusUnauthorized) _, _ = response.Write([]byte("{\"error\":\"unauthorized\"}\n")) } + +func refuseAuthentication(observer AuthObserver, response http.ResponseWriter) { + ObserveAuth(observer, AuthEvent{Outcome: AuthOutcomeRefused}) + writeUnauthorized(response) +} diff --git a/internal/hubserver/auth_observability.go b/internal/hubserver/auth_observability.go new file mode 100644 index 0000000..628afd9 --- /dev/null +++ b/internal/hubserver/auth_observability.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import "fmt" + +// AuthOutcome is the closed self-observability result of one pre-principal +// authentication attempt. It intentionally does not distinguish credential failure modes. +type AuthOutcome string + +// AuthOutcomeRefused is emitted for every request the bearer-token middleware rejects. +const AuthOutcomeRefused AuthOutcome = "refused" + +// AuthEvent is one passive, sanitized authentication observation. It deliberately has no +// request, credential, verifier-error, principal, path, network, or correlation fields: none are +// trusted before authentication succeeds. +type AuthEvent struct { + Outcome AuthOutcome +} + +// Validate rejects unsupported outcome values before an observer can emit them. +func (event AuthEvent) Validate() error { + if event.Outcome != AuthOutcomeRefused { + return fmt.Errorf("authentication event outcome is unsupported") + } + return nil +} + +// AuthObserver receives passive, already-sanitized authentication events. Implementations must +// never alter authentication behavior; ObserveAuth isolates faulty observers defensively. +type AuthObserver interface { + ObserveAuth(AuthEvent) +} + +// AuthObserverFunc adapts a function to AuthObserver. +type AuthObserverFunc func(AuthEvent) + +// ObserveAuth calls function. +func (function AuthObserverFunc) ObserveAuth(event AuthEvent) { + function(event) +} + +// ObserveAuth sends a valid event to a passive observer. Invalid events and observer panics are +// intentionally ignored so observability cannot alter the uniform unauthorized response. +func ObserveAuth(observer AuthObserver, event AuthEvent) { + if observer == nil || event.Validate() != nil { + return + } + defer func() { + _ = recover() + }() + observer.ObserveAuth(event) +} diff --git a/internal/hubserver/auth_observability_test.go b/internal/hubserver/auth_observability_test.go new file mode 100644 index 0000000..8264eb9 --- /dev/null +++ b/internal/hubserver/auth_observability_test.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "context" + "crypto/ed25519" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/hubauth" + "github.com/ArdurAI/sith/internal/tenancy" +) + +type authVerifierFunc func(context.Context, string) (tenancy.Principal, error) + +func (function authVerifierFunc) Verify(ctx context.Context, token string) (tenancy.Principal, error) { + return function(ctx, token) +} + +func TestAuthenticateWithObserverRecordsOnlyUniformRefusals(t *testing.T) { + var events []AuthEvent + observer := AuthObserverFunc(func(event AuthEvent) { events = append(events, event) }) + verifier := authVerifierFunc(func(context.Context, string) (tenancy.Principal, error) { + return tenancy.Principal{}, errors.New("verifier token=secret") + }) + handler, err := AuthenticateWithObserver(verifier, observer, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("unauthorized request reached handler") + })) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + name string + values []string + }{ + {name: "missing"}, + {name: "wrong scheme", values: []string{"Basic token=secret"}}, + {name: "ambiguous", values: []string{"Bearer one", "Bearer two"}}, + {name: "verifier rejection", values: []string{"Bearer token=secret"}}, + } { + t.Run(test.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/api?token=secret", nil) + request.Header.Set("X-Correlation-ID", "workspace-a/token=secret") + for _, value := range test.values { + request.Header.Add("Authorization", value) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusUnauthorized || response.Body.String() != "{\"error\":\"unauthorized\"}\n" { + t.Fatalf("status = %d, body = %q", response.Code, response.Body.String()) + } + if len(events) != 1 || events[0] != (AuthEvent{Outcome: AuthOutcomeRefused}) { + t.Fatalf("events = %#v, want one fixed refusal", events) + } + events = nil + }) + } +} + +func TestAuthenticateWithObserverIsSilentAfterValidAuthentication(t *testing.T) { + now := time.Date(2026, 7, 14, 13, 0, 0, 0, time.UTC) + publicKey, privateKey := hubTestKeyPair() + verifier, err := hubauth.NewJWTVerifier(hubauth.JWTConfig{ + Issuer: hubTestIssuer, Audience: hubTestAudience, Keys: map[string]ed25519.PublicKey{hubTestKeyID: publicKey}, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + var events []AuthEvent + handler, err := AuthenticateWithObserver(verifier, AuthObserverFunc(func(event AuthEvent) { events = append(events, event) }), http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusNoContent) + })) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/api", nil) + request.Header.Set("Authorization", "Bearer "+signHubTestToken(t, hubValidClaims(now), privateKey)) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusNoContent || len(events) != 0 { + t.Fatalf("status = %d events = %#v", response.Code, events) + } +} + +func TestObserveAuthRejectsUnsafeEventsAndContainsObserverPanics(t *testing.T) { + called := false + ObserveAuth(AuthObserverFunc(func(AuthEvent) { called = true }), AuthEvent{Outcome: "token=secret"}) + if called { + t.Fatal("unsafe authentication event reached observer") + } + + ObserveAuth(AuthObserverFunc(func(AuthEvent) { panic("observer fault") }), AuthEvent{Outcome: AuthOutcomeRefused}) + + handler, err := AuthenticateWithObserver(authVerifierFunc(func(context.Context, string) (tenancy.Principal, error) { + return tenancy.Principal{}, errors.New("invalid") + }), AuthObserverFunc(func(AuthEvent) { panic("observer fault") }), http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("unauthorized request reached handler") + })) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "https://hub.sith.test/api", nil)) + if response.Code != http.StatusUnauthorized || response.Body.String() != "{\"error\":\"unauthorized\"}\n" { + t.Fatalf("status = %d, body = %q", response.Code, response.Body.String()) + } +} diff --git a/internal/hubserver/auth_test.go b/internal/hubserver/auth_test.go index 468a420..bae9546 100644 --- a/internal/hubserver/auth_test.go +++ b/internal/hubserver/auth_test.go @@ -129,6 +129,38 @@ func TestAuthenticateRejectsMissingForgedAndAmbiguousBearerTokens(t *testing.T) } } +func TestAuthenticateStripsUntrustedCorrelationHeaders(t *testing.T) { + now := time.Date(2026, 7, 14, 12, 0, 0, 0, time.UTC) + publicKey, privateKey := hubTestKeyPair() + verifier, err := hubauth.NewJWTVerifier(hubauth.JWTConfig{ + Issuer: hubTestIssuer, Audience: hubTestAudience, Keys: map[string]ed25519.PublicKey{hubTestKeyID: publicKey}, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + handler, err := Authenticate(verifier, http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + for _, name := range []string{"Traceparent", "Tracestate", "B3", "X-B3-Traceid", "X-Request-ID", "X-Correlation-ID", "X-Trace-ID"} { + if value := request.Header.Get(name); value != "" { + t.Fatalf("untrusted correlation header %s reached handler: %q", name, value) + } + } + response.WriteHeader(http.StatusNoContent) + })) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/api", nil) + request.Header.Set("Authorization", "Bearer "+signHubTestToken(t, hubValidClaims(now), privateKey)) + for _, name := range []string{"Traceparent", "Tracestate", "B3", "X-B3-Traceid", "X-Request-ID", "X-Correlation-ID", "X-Trace-ID"} { + request.Header.Set(name, "workspace-a/token=secret") + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %q", response.Code, response.Body.String()) + } +} + func FuzzBearerTokenNeverAcceptsMetadata(f *testing.F) { f.Add("Bearer token") f.Add("bearer abc.def.ghi") diff --git a/internal/hubserver/fleet.go b/internal/hubserver/fleet.go new file mode 100644 index 0000000..9d2cdcc --- /dev/null +++ b/internal/hubserver/fleet.go @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +const fleetRoutePrefix = "/v1/workspaces/" + +// FleetRefresher performs one bounded refresh for the verified workspace scope. +type FleetRefresher interface { + Collect(context.Context, tenancy.Scope) (fleet.Coverage, error) +} + +// FleetImageSearcher resolves one exact immutable image digest in the signed workspace. +type FleetImageSearcher interface { + Search(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) +} + +// FleetHandlerConfig supplies the authenticated dependencies for the fixed hub fleet API. +type FleetHandlerConfig struct { + Verifier Verifier + AuthObserver AuthObserver + Collector FleetRefresher + Reader hubfleet.FleetReader + ImageSearcher FleetImageSearcher + PEP *pep.Enforcer +} + +// NewFleetHandler constructs the fixed, authenticated hub read surface. +// +// A caller can only address a workspace contained in its verified session. The handler accepts no +// user-supplied transport target, selector, credentials, or freshness override. +func NewFleetHandler(config FleetHandlerConfig) (http.Handler, error) { + if config.Verifier == nil || config.Collector == nil || config.Reader == nil || config.ImageSearcher == nil || config.PEP == nil { + return nil, fmt.Errorf("construct hub fleet handler: verifier, collector, reader, image searcher, and policy enforcer are required") + } + + handler := http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + setNoStore(response.Header()) + workspaceID, operation, imageDigest, ok := parseFleetRoute(request.URL) + if !ok { + writeFleetError(response, http.StatusNotFound, "not_found") + return + } + scope, err := ScopeFromContext(request.Context(), workspaceID) + if err != nil { + writeFleetError(response, http.StatusForbidden, "forbidden") + return + } + traceContext, _, err := tracing.Ensure(request.Context()) + if err != nil { + writeFleetError(response, http.StatusServiceUnavailable, "trace_unavailable") + return + } + request = request.WithContext(traceContext) + + switch operation { + case fleetOperationRead: + if request.Method != http.MethodGet { + response.Header().Set("Allow", http.MethodGet) + writeFleetError(response, http.StatusMethodNotAllowed, "method_not_allowed") + return + } + source, err := hubfleet.NewSource(hubfleet.SourceConfig{ + Reader: config.Reader, + Scope: scope, + PEP: config.PEP, + }) + if err != nil { + writeFleetError(response, http.StatusForbidden, "forbidden") + return + } + result, err := source.Fleet(request.Context()) + if err != nil { + writeFleetError(response, http.StatusServiceUnavailable, "fleet_unavailable") + return + } + writeFleetJSON(response, http.StatusOK, result) + case fleetOperationRefresh: + if request.Method != http.MethodPost { + response.Header().Set("Allow", http.MethodPost) + writeFleetError(response, http.StatusMethodNotAllowed, "method_not_allowed") + return + } + coverage, err := config.Collector.Collect(request.Context(), scope) + if err != nil { + writeFleetError(response, http.StatusServiceUnavailable, "refresh_unavailable") + return + } + writeFleetJSON(response, http.StatusOK, coverage) + case fleetOperationImageSearch: + if request.Method != http.MethodGet { + response.Header().Set("Allow", http.MethodGet) + writeFleetError(response, http.StatusMethodNotAllowed, "method_not_allowed") + return + } + result, err := config.ImageSearcher.Search(request.Context(), scope, hubfleet.ImageSearchRequest{Digest: imageDigest}) + if err != nil { + writeFleetError(response, http.StatusServiceUnavailable, "image_search_unavailable") + return + } + writeFleetJSON(response, http.StatusOK, result) + default: + writeFleetError(response, http.StatusNotFound, "not_found") + } + }) + + return AuthenticateWithObserver(config.Verifier, config.AuthObserver, handler) +} + +type fleetOperation uint8 + +const ( + fleetOperationRead fleetOperation = iota + 1 + fleetOperationRefresh + fleetOperationImageSearch +) + +func parseFleetRoute(requestURL *url.URL) (tenancy.WorkspaceID, fleetOperation, string, bool) { + if requestURL == nil || requestURL.RawQuery != "" { + return "", 0, "", false + } + escapedPath := requestURL.EscapedPath() + if !strings.HasPrefix(escapedPath, fleetRoutePrefix) { + return "", 0, "", false + } + workspaceSegment, resource, found := strings.Cut(strings.TrimPrefix(escapedPath, fleetRoutePrefix), "/") + if !found || workspaceSegment == "" { + return "", 0, "", false + } + workspace, err := url.PathUnescape(workspaceSegment) + if err != nil || url.PathEscape(workspace) != workspaceSegment { + return "", 0, "", false + } + workspaceID := tenancy.WorkspaceID(workspace) + if tenancy.ValidateWorkspaceID(workspaceID) != nil { + return "", 0, "", false + } + switch resource { + case "fleet": + return workspaceID, fleetOperationRead, "", true + case "fleet:refresh": + return workspaceID, fleetOperationRefresh, "", true + default: + digestSegment, found := strings.CutPrefix(resource, "fleet/images/") + if !found || digestSegment == "" || strings.Contains(digestSegment, "/") { + return "", 0, "", false + } + digest, err := url.PathUnescape(digestSegment) + if err != nil || url.PathEscape(digest) != digestSegment || fleet.ValidateImageDigest(digest) != nil { + return "", 0, "", false + } + return workspaceID, fleetOperationImageSearch, digest, true + } +} + +func writeFleetJSON(response http.ResponseWriter, status int, value any) { + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(status) + _ = json.NewEncoder(response).Encode(value) +} + +func writeFleetError(response http.ResponseWriter, status int, code string) { + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(status) + _ = json.NewEncoder(response).Encode(struct { + Error string `json:"error"` + }{Error: code}) +} diff --git a/internal/hubserver/fleet_test.go b/internal/hubserver/fleet_test.go new file mode 100644 index 0000000..3f15bc5 --- /dev/null +++ b/internal/hubserver/fleet_test.go @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubauth" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +type fleetRefresherFunc func(context.Context, tenancy.Scope) (fleet.Coverage, error) + +func (function fleetRefresherFunc) Collect(ctx context.Context, scope tenancy.Scope) (fleet.Coverage, error) { + return function(ctx, scope) +} + +type fleetReaderFunc func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) + +func (function fleetReaderFunc) ReadFleet( + ctx context.Context, + scope tenancy.Scope, + freshness time.Duration, + now time.Time, +) (fleet.FleetResult, error) { + return function(ctx, scope, freshness, now) +} + +type fleetImageSearcherFunc func(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) + +func (function fleetImageSearcherFunc) Search( + ctx context.Context, + scope tenancy.Scope, + request hubfleet.ImageSearchRequest, +) (fleet.QueryResult, error) { + return function(ctx, scope, request) +} + +var _ FleetRefresher = fleetRefresherFunc(nil) +var _ hubfleet.FleetReader = fleetReaderFunc(nil) +var _ FleetImageSearcher = fleetImageSearcherFunc(nil) + +func TestFleetHandlerRefreshUsesOnlySignedWorkspaceScope(t *testing.T) { + now := time.Date(2026, 7, 14, 7, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + called := false + handler, err := NewFleetHandler(FleetHandlerConfig{ + Verifier: verifier, + Collector: fleetRefresherFunc(func(_ context.Context, scope tenancy.Scope) (fleet.Coverage, error) { + called = true + if got, want := scope.WorkspaceID(), tenancy.WorkspaceID("workspace-a"); got != want { + t.Fatalf("workspace scope = %q, want %q", got, want) + } + if got, want := scope.Subject(), "user:alice"; got != want { + t.Fatalf("scope subject = %q, want %q", got, want) + } + return fleet.Coverage{Requested: 2, Reachable: 2}, nil + }), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + t.Fatal("refresh reached fleet reader") + return fleet.FleetResult{}, nil + }), + ImageSearcher: fleetImageSearcherFunc(func(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) { + t.Fatal("refresh reached image searcher") + return fleet.QueryResult{}, nil + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), + }) + if err != nil { + t.Fatal(err) + } + request := authenticatedFleetRequest(t, http.MethodPost, "/v1/workspaces/workspace-a/fleet:refresh", privateKey, now) + request.Header.Set("X-Workspace", "workspace-b") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("status = %d, body = %q", response.Code, response.Body.String()) + } + if !called { + t.Fatal("refresh collector was not called") + } + if response.Header().Get("Cache-Control") != "no-store" || response.Header().Get("Pragma") != "no-cache" { + t.Fatalf("refresh response caching headers = Cache-Control %q, Pragma %q", response.Header().Get("Cache-Control"), response.Header().Get("Pragma")) + } + var coverage fleet.Coverage + if err := json.NewDecoder(response.Body).Decode(&coverage); err != nil { + t.Fatal(err) + } + if coverage.Requested != 2 || coverage.Reachable != 2 || len(coverage.Unreachable) != 0 || len(coverage.Stale) != 0 { + t.Fatalf("coverage = %+v", coverage) + } +} + +func TestFleetHandlerMintsLocalTraceAfterSignedScope(t *testing.T) { + now := time.Date(2026, 7, 14, 7, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + var auditEvents []pep.AuditEvent + var traceEvents []tracing.Event + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, + Auditor: pep.AuditFunc(func(_ context.Context, event pep.AuditEvent) error { + auditEvents = append(auditEvents, event) + return nil + }), + TraceObserver: tracing.ObserverFunc(func(event tracing.Event) { traceEvents = append(traceEvents, event) }), + }) + if err != nil { + t.Fatal(err) + } + var collectorTrace tracing.ID + handler, err := NewFleetHandler(FleetHandlerConfig{ + Verifier: verifier, + Collector: fleetRefresherFunc(func(ctx context.Context, scope tenancy.Scope) (fleet.Coverage, error) { + var ok bool + collectorTrace, ok = tracing.FromContext(ctx) + if !ok { + t.Fatal("authenticated collector received no trace context") + } + if err := enforcer.AuthorizeRead(ctx, scope, pep.NewReadInput(pep.VerbSpokeSnapshotRefresh, nil)); err != nil { + t.Fatalf("AuthorizeRead() error = %v", err) + } + return fleet.Coverage{Requested: 1, Reachable: 1}, nil + }), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + t.Fatal("refresh reached fleet reader") + return fleet.FleetResult{}, nil + }), + ImageSearcher: fleetImageSearcherFunc(func(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) { + t.Fatal("refresh reached image searcher") + return fleet.QueryResult{}, nil + }), + PEP: enforcer, + }) + if err != nil { + t.Fatal(err) + } + request := authenticatedFleetRequest(t, http.MethodPost, "/v1/workspaces/workspace-a/fleet:refresh", privateKey, now) + request.Header.Set("Traceparent", "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01") + request.Header.Set("X-B3-Traceid", "0123456789abcdef0123456789abcdef") + request.Header.Set("X-Request-ID", "workspace-a/token=secret") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK || !collectorTrace.Valid() { + t.Fatalf("status = %d trace = %q body = %q", response.Code, collectorTrace, response.Body.String()) + } + if collectorTrace == "0123456789abcdef0123456789abcdef" || len(auditEvents) != 1 || auditEvents[0].TraceID != collectorTrace || + len(traceEvents) != 1 || traceEvents[0].TraceID != collectorTrace || traceEvents[0].Stage != tracing.StagePEPDecision { + t.Fatalf("trace propagation = collector %q audits %#v events %#v", collectorTrace, auditEvents, traceEvents) + } + for _, name := range []string{"Traceparent", "Tracestate", "B3", "X-B3-Traceid", "X-Request-ID", "X-Correlation-ID", "X-Trace-ID"} { + if value := response.Header().Get(name); value != "" { + t.Fatalf("response echoed untrusted trace header %s: %q", name, value) + } + } +} + +func TestFleetHandlerReadConstructsRequestScopedSource(t *testing.T) { + now := time.Date(2026, 7, 14, 7, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + called := false + handler, err := NewFleetHandler(FleetHandlerConfig{ + Verifier: verifier, + Collector: fleetRefresherFunc(func(context.Context, tenancy.Scope) (fleet.Coverage, error) { + t.Fatal("fleet read reached collector") + return fleet.Coverage{}, nil + }), + Reader: fleetReaderFunc(func(_ context.Context, scope tenancy.Scope, freshness time.Duration, gotNow time.Time) (fleet.FleetResult, error) { + called = true + if got, want := scope.WorkspaceID(), tenancy.WorkspaceID("workspace-a"); got != want { + t.Fatalf("workspace scope = %q, want %q", got, want) + } + if freshness != 5*time.Minute { + t.Fatalf("freshness = %s, want default 5m", freshness) + } + if gotNow.IsZero() { + t.Fatal("fleet source supplied a zero observation time") + } + return fleet.FleetResult{Clusters: []fleet.Cluster{{Name: "spoke-a", SourceKind: hubfleet.SourceKind, Reachable: true}}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}}, nil + }), + ImageSearcher: fleetImageSearcherFunc(func(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) { + t.Fatal("fleet read reached image searcher") + return fleet.QueryResult{}, nil + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), + }) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedFleetRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/fleet", privateKey, now)) + if response.Code != http.StatusOK { + t.Fatalf("status = %d, body = %q", response.Code, response.Body.String()) + } + if !called { + t.Fatal("fleet reader was not called") + } + var result fleet.FleetResult + if err := json.NewDecoder(response.Body).Decode(&result); err != nil { + t.Fatal(err) + } + if len(result.Clusters) != 1 || result.Clusters[0].Name != "spoke-a" || !result.Coverage.Complete() { + t.Fatalf("fleet result = %+v", result) + } +} + +func TestFleetHandlerRejectsForeignWorkspaceBeforeDependencies(t *testing.T) { + now := time.Date(2026, 7, 14, 7, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + handler, err := NewFleetHandler(FleetHandlerConfig{ + Verifier: verifier, + Collector: fleetRefresherFunc(func(context.Context, tenancy.Scope) (fleet.Coverage, error) { + t.Fatal("foreign workspace reached collector") + return fleet.Coverage{}, nil + }), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + t.Fatal("foreign workspace reached reader") + return fleet.FleetResult{}, nil + }), + ImageSearcher: fleetImageSearcherFunc(func(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) { + t.Fatal("foreign workspace reached image searcher") + return fleet.QueryResult{}, nil + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), + }) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedFleetRequest(t, http.MethodGet, "/v1/workspaces/workspace-b/fleet", privateKey, now)) + if response.Code != http.StatusForbidden || response.Body.String() != "{\"error\":\"forbidden\"}\n" { + t.Fatalf("status = %d, body = %q", response.Code, response.Body.String()) + } +} + +func TestFleetHandlerRejectsUnsupportedRoutesMethodsAndQueries(t *testing.T) { + now := time.Date(2026, 7, 14, 7, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + handler, err := NewFleetHandler(FleetHandlerConfig{ + Verifier: verifier, + Collector: fleetRefresherFunc(func(context.Context, tenancy.Scope) (fleet.Coverage, error) { + t.Fatal("invalid request reached collector") + return fleet.Coverage{}, nil + }), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + t.Fatal("invalid request reached reader") + return fleet.FleetResult{}, nil + }), + ImageSearcher: fleetImageSearcherFunc(func(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) { + t.Fatal("invalid request reached image searcher") + return fleet.QueryResult{}, nil + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), + }) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name string + method string + target string + wantCode int + wantBody string + wantAllow string + }{ + {name: "unknown route", method: http.MethodGet, target: "/v1/workspaces/workspace-a/fleet/extra", wantCode: http.StatusNotFound, wantBody: "{\"error\":\"not_found\"}\n"}, + {name: "query rejected", method: http.MethodGet, target: "/v1/workspaces/workspace-a/fleet?freshness=1s", wantCode: http.StatusNotFound, wantBody: "{\"error\":\"not_found\"}\n"}, + {name: "noncanonical image", method: http.MethodGet, target: "/v1/workspaces/workspace-a/fleet/images/registry.example%2Fpayments%3Alatest", wantCode: http.StatusNotFound, wantBody: "{\"error\":\"not_found\"}\n"}, + {name: "read method", method: http.MethodPost, target: "/v1/workspaces/workspace-a/fleet", wantCode: http.StatusMethodNotAllowed, wantBody: "{\"error\":\"method_not_allowed\"}\n", wantAllow: http.MethodGet}, + {name: "refresh method", method: http.MethodGet, target: "/v1/workspaces/workspace-a/fleet:refresh", wantCode: http.StatusMethodNotAllowed, wantBody: "{\"error\":\"method_not_allowed\"}\n", wantAllow: http.MethodPost}, + {name: "image method", method: http.MethodPost, target: "/v1/workspaces/workspace-a/fleet/images/sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", wantCode: http.StatusMethodNotAllowed, wantBody: "{\"error\":\"method_not_allowed\"}\n", wantAllow: http.MethodGet}, + } { + t.Run(test.name, func(t *testing.T) { + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedFleetRequest(t, test.method, test.target, privateKey, now)) + if response.Code != test.wantCode || response.Body.String() != test.wantBody || response.Header().Get("Allow") != test.wantAllow { + t.Fatalf("status = %d, body = %q, allow = %q", response.Code, response.Body.String(), response.Header().Get("Allow")) + } + }) + } +} + +func TestFleetHandlerHidesDependencyErrors(t *testing.T) { + now := time.Date(2026, 7, 14, 7, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + handler, err := NewFleetHandler(FleetHandlerConfig{ + Verifier: verifier, + Collector: fleetRefresherFunc(func(context.Context, tenancy.Scope) (fleet.Coverage, error) { + return fleet.Coverage{}, errors.New("secret token leaked") + }), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, errors.New("database topology leaked") + }), + ImageSearcher: fleetImageSearcherFunc(func(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) { + return fleet.QueryResult{}, errors.New("registry credential leaked") + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), + }) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + method string + target string + body string + }{ + {method: http.MethodPost, target: "/v1/workspaces/workspace-a/fleet:refresh", body: "{\"error\":\"refresh_unavailable\"}\n"}, + {method: http.MethodGet, target: "/v1/workspaces/workspace-a/fleet", body: "{\"error\":\"fleet_unavailable\"}\n"}, + {method: http.MethodGet, target: "/v1/workspaces/workspace-a/fleet/images/sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", body: "{\"error\":\"image_search_unavailable\"}\n"}, + } { + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedFleetRequest(t, test.method, test.target, privateKey, now)) + if response.Code != http.StatusServiceUnavailable || response.Body.String() != test.body || strings.Contains(response.Body.String(), "leaked") { + t.Fatalf("status = %d, body = %q", response.Code, response.Body.String()) + } + } +} + +func TestFleetHandlerSearchesOneExactImageDigestInSignedWorkspace(t *testing.T) { + now := time.Date(2026, 7, 14, 7, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + digest := "sha256:" + strings.Repeat("a", 64) + called := false + handler, err := NewFleetHandler(FleetHandlerConfig{ + Verifier: verifier, + Collector: fleetRefresherFunc(func(context.Context, tenancy.Scope) (fleet.Coverage, error) { + t.Fatal("image search reached collector") + return fleet.Coverage{}, nil + }), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + t.Fatal("image search reached fleet reader") + return fleet.FleetResult{}, nil + }), + ImageSearcher: fleetImageSearcherFunc(func(_ context.Context, scope tenancy.Scope, request hubfleet.ImageSearchRequest) (fleet.QueryResult, error) { + called = true + if scope.WorkspaceID() != "workspace-a" || request.Digest != digest || request.Limit != 0 { + t.Fatalf("image search scope/request = %#v/%#v", scope, request) + } + return fleet.QueryResult{Facts: []fleet.Fact{{Workspace: "workspace-a", Evidence: fleet.Evidence{Ref: fleet.ResourceRef{Scope: "spoke-a", Kind: "Pod", Name: "payments"}}}}, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}}, nil + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), + }) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedFleetRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/fleet/images/"+digest, privateKey, now)) + if response.Code != http.StatusOK || !called { + t.Fatalf("status = %d, called = %t, body = %q", response.Code, called, response.Body.String()) + } + var result fleet.QueryResult + if err := json.NewDecoder(response.Body).Decode(&result); err != nil || len(result.Facts) != 1 || result.Facts[0].Ref.Scope != "spoke-a" || !result.Coverage.Complete() { + t.Fatalf("image result = %#v, error = %v", result, err) + } +} + +func TestNewFleetHandlerRejectsMissingDependencies(t *testing.T) { + if _, err := NewFleetHandler(FleetHandlerConfig{}); err == nil { + t.Fatal("NewFleetHandler accepted missing dependencies") + } +} + +func TestFleetHandlerForwardsAuthRefusalsToConfiguredObserver(t *testing.T) { + now := time.Date(2026, 7, 14, 13, 0, 0, 0, time.UTC) + verifier, _ := fleetTestVerifier(t, now) + var events []AuthEvent + handler, err := NewFleetHandler(FleetHandlerConfig{ + Verifier: verifier, + AuthObserver: AuthObserverFunc(func(event AuthEvent) { events = append(events, event) }), + Collector: fleetRefresherFunc(func(context.Context, tenancy.Scope) (fleet.Coverage, error) { + t.Fatal("unauthenticated request reached collector") + return fleet.Coverage{}, nil + }), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + t.Fatal("unauthenticated request reached reader") + return fleet.FleetResult{}, nil + }), + ImageSearcher: fleetImageSearcherFunc(func(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) { + t.Fatal("unauthenticated request reached image searcher") + return fleet.QueryResult{}, nil + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), + }) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/fleet", nil)) + if response.Code != http.StatusUnauthorized || len(events) != 1 || events[0] != (AuthEvent{Outcome: AuthOutcomeRefused}) { + t.Fatalf("status = %d events = %#v", response.Code, events) + } +} + +func fleetTestVerifier(t *testing.T, now time.Time) (*hubauth.JWTVerifier, ed25519.PrivateKey) { + t.Helper() + publicKey, privateKey := hubTestKeyPair() + verifier, err := hubauth.NewJWTVerifier(hubauth.JWTConfig{ + Issuer: hubTestIssuer, Audience: hubTestAudience, Keys: map[string]ed25519.PublicKey{hubTestKeyID: publicKey}, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + return verifier, privateKey +} + +func fleetTestPEP(t *testing.T, hook pep.PolicyHook) *pep.Enforcer { + t.Helper() + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: hook, + Auditor: pep.AuditFunc(func(context.Context, pep.AuditEvent) error { + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + return enforcer +} + +func authenticatedFleetRequest(t *testing.T, method, target string, privateKey ed25519.PrivateKey, now time.Time) *http.Request { + t.Helper() + request := httptest.NewRequest(method, "https://hub.sith.test"+target, nil) + request.Header.Set("Authorization", "Bearer "+signHubTestToken(t, hubValidClaims(now), privateKey)) + return request +} diff --git a/internal/observability/auth.go b/internal/observability/auth.go new file mode 100644 index 0000000..7e15ff6 --- /dev/null +++ b/internal/observability/auth.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package observability + +import ( + "fmt" + "log/slog" + + "github.com/ArdurAI/sith/internal/hubserver" +) + +// NewSlogAuthObserver constructs the local structured authentication-refusal recorder used by +// the hub runtime. It serializes only the closed AuthEvent contract; it owns no listener, +// exporter, queue, persistence, telemetry backend, or network path. +func NewSlogAuthObserver(logger *slog.Logger) (hubserver.AuthObserver, error) { + if logger == nil { + return nil, fmt.Errorf("construct authentication observer: logger is required") + } + return slogAuthObserver{logger: logger}, nil +} + +type slogAuthObserver struct { + logger *slog.Logger +} + +func (observer slogAuthObserver) ObserveAuth(event hubserver.AuthEvent) { + if observer.logger == nil || event.Validate() != nil { + return + } + observer.logger.Warn( + "authentication refused", + "surface", "hub-auth", + "auth_outcome", event.Outcome, + ) +} diff --git a/internal/observability/auth_test.go b/internal/observability/auth_test.go new file mode 100644 index 0000000..f0a0814 --- /dev/null +++ b/internal/observability/auth_test.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 + +package observability + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + + "github.com/ArdurAI/sith/internal/hubserver" +) + +func TestSlogAuthObserverEmitsOnlyValidatedFixedFields(t *testing.T) { + var output bytes.Buffer + observer, err := NewSlogAuthObserver(slog.New(slog.NewJSONHandler(&output, nil))) + if err != nil { + t.Fatal(err) + } + observer.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeRefused}) + observer.ObserveAuth(hubserver.AuthEvent{Outcome: "token=secret"}) + + lines := strings.Split(strings.TrimSpace(output.String()), "\n") + if len(lines) != 1 { + t.Fatalf("authentication log lines = %q", output.String()) + } + var record map[string]any + if err := json.Unmarshal([]byte(lines[0]), &record); err != nil { + t.Fatal(err) + } + for field := range record { + if !map[string]bool{"time": true, "level": true, "msg": true, "surface": true, "auth_outcome": true}[field] { + t.Fatalf("authentication log contains unexpected field %q: %#v", field, record) + } + } + if record["level"] != "WARN" || record["msg"] != "authentication refused" || + record["surface"] != "hub-auth" || record["auth_outcome"] != string(hubserver.AuthOutcomeRefused) { + t.Fatalf("authentication record = %#v", record) + } + for _, forbidden := range []string{"token=secret", "workspace-a", "trace", "header", "principal", "verifier"} { + if strings.Contains(lines[0], forbidden) { + t.Fatalf("authentication record leaked %q: %s", forbidden, lines[0]) + } + } +} + +func TestSlogAuthObserverTextHandlerAndNilLogger(t *testing.T) { + var output bytes.Buffer + observer, err := NewSlogAuthObserver(slog.New(slog.NewTextHandler(&output, nil))) + if err != nil { + t.Fatal(err) + } + observer.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeRefused}) + line := output.String() + for _, field := range []string{"level=WARN", "msg=\"authentication refused\"", "surface=hub-auth", "auth_outcome=refused"} { + if !strings.Contains(line, field) { + t.Fatalf("text authentication log missing %q: %q", field, line) + } + } + for _, forbidden := range []string{"token", "workspace", "trace", "header", "principal", "verifier"} { + if strings.Contains(line, forbidden) { + t.Fatalf("text authentication record leaked %q: %q", forbidden, line) + } + } + if _, err := NewSlogAuthObserver(nil); err == nil { + t.Fatal("NewSlogAuthObserver() accepted nil logger") + } +} diff --git a/internal/observability/metrics.go b/internal/observability/metrics.go new file mode 100644 index 0000000..7833693 --- /dev/null +++ b/internal/observability/metrics.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package observability exposes bounded, pull-only metrics about Sith's own process behavior. +// It deliberately owns no listener, remote exporter, persistence, or external telemetry data. +package observability + +import ( + "fmt" + "net/http" + "regexp" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/pep" +) + +var ( + versionLabelPattern = regexp.MustCompile(`^v?[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$`) + commitLabelPattern = regexp.MustCompile(`^(?:none|unknown|[0-9a-f]{7,64})$`) +) + +// Config supplies non-sensitive build metadata and an optional isolated registry. A nil Registry +// creates a fresh pedantic registry; the Prometheus global registry is never used. +type Config struct { + Registry *prometheus.Registry + Version string + Commit string +} + +// Metrics records low-cardinality control-plane observations and exposes the matching handler. +// All label normalization occurs before a metric is created, so caller-controlled values cannot +// increase cardinality or cross the privacy boundary. +type Metrics struct { + gatherer prometheus.Gatherer + policyDecisions *prometheus.CounterVec + policyDuration *prometheus.HistogramVec + snapshotAttempts *prometheus.CounterVec + snapshotDuration *prometheus.HistogramVec +} + +var ( + _ pep.DecisionObserver = (*Metrics)(nil) + _ hubfleet.SnapshotObserver = (*Metrics)(nil) +) + +// New constructs metrics against a caller-owned or fresh isolated registry. Registration errors +// are returned rather than panicking, making duplicate or incompatible metrics a startup failure. +func New(config Config) (*Metrics, error) { + registry := config.Registry + if registry == nil { + registry = prometheus.NewPedanticRegistry() + } + if registry == prometheus.DefaultRegisterer || registry == prometheus.DefaultGatherer { + return nil, fmt.Errorf("construct metrics: Prometheus global registry is not allowed") + } + + metrics := &Metrics{ + gatherer: registry, + policyDecisions: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "policy", Name: "decisions_total", + Help: "Total completed Sith policy-read decisions by closed verb and outcome.", + }, []string{"verb", "outcome"}), + policyDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "sith", Subsystem: "policy", Name: "decision_duration_seconds", + Help: "Duration of completed Sith policy-read decisions by closed verb and outcome.", + }, []string{"verb", "outcome"}), + snapshotAttempts: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "federation", Name: "spoke_snapshot_attempts_total", + Help: "Total completed Sith federated spoke snapshot attempts by closed outcome.", + }, []string{"outcome"}), + snapshotDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "sith", Subsystem: "federation", Name: "spoke_snapshot_duration_seconds", + Help: "Duration of completed Sith federated spoke snapshot attempts by closed outcome.", + }, []string{"outcome"}), + } + buildInfo := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "sith", Name: "build_info", Help: "Sith build metadata with safe release identifiers only.", + ConstLabels: prometheus.Labels{ + "version": normalizedVersion(config.Version), + "commit": normalizedCommit(config.Commit), + }, + }) + buildInfo.Set(1) + + registered := make([]prometheus.Collector, 0, 7) + for _, collector := range []struct { + name string + collector prometheus.Collector + }{ + {name: "Go runtime", collector: collectors.NewGoCollector()}, + {name: "process", collector: collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})}, + {name: "build info", collector: buildInfo}, + {name: "policy decisions", collector: metrics.policyDecisions}, + {name: "policy duration", collector: metrics.policyDuration}, + {name: "snapshot attempts", collector: metrics.snapshotAttempts}, + {name: "snapshot duration", collector: metrics.snapshotDuration}, + } { + if err := registry.Register(collector.collector); err != nil { + for index := len(registered) - 1; index >= 0; index-- { + registry.Unregister(registered[index]) + } + return nil, fmt.Errorf("register %s metrics: %w", collector.name, err) + } + registered = append(registered, collector.collector) + } + + return metrics, nil +} + +// Handler returns an embeddable Prometheus exposition handler. It does not bind a port or make +// outbound calls; a future hub composition root owns listener, TLS, and scrape authorization. +func (metrics *Metrics) Handler() http.Handler { + if metrics == nil || metrics.gatherer == nil { + return http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + http.Error(writer, "metrics unavailable", http.StatusServiceUnavailable) + }) + } + return promhttp.HandlerFor(metrics.gatherer, promhttp.HandlerOpts{ErrorHandling: promhttp.HTTPErrorOnError}) +} + +// ObserveDecision records one completed policy decision using only fixed label vocabularies. +func (metrics *Metrics) ObserveDecision(verb pep.Verb, outcome pep.DecisionOutcome, duration time.Duration) { + if metrics == nil || metrics.policyDecisions == nil || metrics.policyDuration == nil { + return + } + verbLabel := normalizedVerb(verb) + outcomeLabel := normalizedDecisionOutcome(outcome) + metrics.policyDecisions.WithLabelValues(verbLabel, outcomeLabel).Inc() + metrics.policyDuration.WithLabelValues(verbLabel, outcomeLabel).Observe(normalizedDuration(duration)) +} + +// ObserveSpokeSnapshot records one completed federated snapshot attempt using only a fixed outcome +// vocabulary. It intentionally omits all tenant and spoke identifiers. +func (metrics *Metrics) ObserveSpokeSnapshot(outcome hubfleet.SnapshotOutcome, duration time.Duration) { + if metrics == nil || metrics.snapshotAttempts == nil || metrics.snapshotDuration == nil { + return + } + outcomeLabel := normalizedSnapshotOutcome(outcome) + metrics.snapshotAttempts.WithLabelValues(outcomeLabel).Inc() + metrics.snapshotDuration.WithLabelValues(outcomeLabel).Observe(normalizedDuration(duration)) +} + +func normalizedVersion(value string) string { + if value == "dev" || value == "unknown" || versionLabelPattern.MatchString(value) { + return value + } + return "unknown" +} + +func normalizedCommit(value string) string { + if commitLabelPattern.MatchString(value) { + return value + } + return "unknown" +} + +func normalizedVerb(verb pep.Verb) string { + if verb.Valid() { + return string(verb) + } + return "invalid" +} + +func normalizedDecisionOutcome(outcome pep.DecisionOutcome) string { + switch outcome { + case pep.DecisionOutcomeAllow, pep.DecisionOutcomeDeny, pep.DecisionOutcomeRequireApproval, pep.DecisionOutcomeError: + return string(outcome) + default: + return string(pep.DecisionOutcomeError) + } +} + +func normalizedSnapshotOutcome(outcome hubfleet.SnapshotOutcome) string { + switch outcome { + case hubfleet.SnapshotOutcomeSuccess, + hubfleet.SnapshotOutcomeTransport, + hubfleet.SnapshotOutcomeDeadline, + hubfleet.SnapshotOutcomeInvalidSnapshot, + hubfleet.SnapshotOutcomeStoreError, + hubfleet.SnapshotOutcomeCanceled: + return string(outcome) + default: + return string(hubfleet.SnapshotOutcomeStoreError) + } +} + +func normalizedDuration(duration time.Duration) float64 { + if duration < 0 { + return 0 + } + return duration.Seconds() +} diff --git a/internal/observability/metrics_test.go b/internal/observability/metrics_test.go new file mode 100644 index 0000000..7f5b860 --- /dev/null +++ b/internal/observability/metrics_test.go @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: Apache-2.0 + +package observability + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/pep" +) + +func TestMetricsExposeOnlyBoundedSelfObservability(t *testing.T) { + registry := prometheus.NewPedanticRegistry() + metrics, err := New(Config{Registry: registry, Version: "v1.2.3", Commit: "0123456789abcdef"}) + if err != nil { + t.Fatal(err) + } + metrics.ObserveDecision(pep.VerbFleetRead, pep.DecisionOutcomeAllow, 125*time.Millisecond) + metrics.ObserveDecision(pep.Verb("workspace-a/token=secret"), pep.DecisionOutcome("untrusted"), -time.Second) + metrics.ObserveSpokeSnapshot(hubfleet.SnapshotOutcomeSuccess, 25*time.Millisecond) + metrics.ObserveSpokeSnapshot(hubfleet.SnapshotOutcome("spoke-a/token=secret"), -time.Second) + + response := httptest.NewRecorder() + metrics.Handler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "http://metrics.invalid/metrics", nil)) + if response.Code != http.StatusOK { + t.Fatalf("metrics status = %d, body = %s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, metric := range []string{ + "sith_build_info", + "sith_policy_decisions_total", + "sith_policy_decision_duration_seconds", + "sith_federation_spoke_snapshot_attempts_total", + "sith_federation_spoke_snapshot_duration_seconds", + `verb="fleet.read"`, + `verb="invalid"`, + `outcome="allow"`, + `outcome="error"`, + `outcome="success"`, + `outcome="store-error"`, + } { + if !strings.Contains(body, metric) { + t.Fatalf("metrics output missing %q: %s", metric, body) + } + } + for _, forbidden := range []string{"workspace-a", "spoke-a", "token=secret", "untrusted"} { + if strings.Contains(body, forbidden) { + t.Fatalf("metrics output leaked %q: %s", forbidden, body) + } + } + assertSithMetricLabels(t, metrics) +} + +func TestMetricsUseIndependentRegistriesAndNormalizeBuildLabels(t *testing.T) { + first, err := New(Config{Version: "v9.9.9", Commit: "abcdef0"}) + if err != nil { + t.Fatal(err) + } + second, err := New(Config{Version: "token=secret", Commit: "workspace-a"}) + if err != nil { + t.Fatal(err) + } + first.ObserveDecision(pep.VerbFleetRead, pep.DecisionOutcomeAllow, time.Millisecond) + + response := httptest.NewRecorder() + second.Handler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "http://metrics.invalid/metrics", nil)) + if response.Code != http.StatusOK { + t.Fatalf("second registry status = %d", response.Code) + } + body := response.Body.String() + if strings.Contains(body, "sith_policy_decisions_total") || strings.Contains(body, "token=secret") || strings.Contains(body, "workspace-a") { + t.Fatalf("isolated metrics registry leaked another registry or unsafe metadata: %s", body) + } + if !strings.Contains(body, `sith_build_info{commit="unknown",version="unknown"} 1`) { + t.Fatalf("unsafe build metadata was not normalized: %s", body) + } +} + +func TestMetricsRejectDuplicateRegistrations(t *testing.T) { + registry := prometheus.NewPedanticRegistry() + conflicting := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "policy", Name: "decisions_total", + Help: "Total completed Sith policy-read decisions by closed verb and outcome.", + }, []string{"verb", "outcome"}) + if err := registry.Register(conflicting); err != nil { + t.Fatal(err) + } + if _, err := New(Config{Registry: registry}); err == nil { + t.Fatal("New() accepted a duplicate metric registration") + } + if !registry.Unregister(conflicting) { + t.Fatal("remove conflicting collector") + } + if _, err := New(Config{Registry: registry}); err != nil { + t.Fatalf("New() left partial registration state after failure: %v", err) + } +} + +func TestMetricsRejectPrometheusGlobalRegistry(t *testing.T) { + registry, ok := prometheus.DefaultRegisterer.(*prometheus.Registry) + if !ok { + t.Fatal("Prometheus default registerer is not a concrete registry") + } + if _, err := New(Config{Registry: registry}); err == nil { + t.Fatal("New() accepted Prometheus global registry") + } +} + +func assertSithMetricLabels(t *testing.T, metrics *Metrics) { + t.Helper() + families, err := metrics.gatherer.Gather() + if err != nil { + t.Fatal(err) + } + allowed := map[string]map[string]bool{ + "sith_build_info": {"commit": true, "version": true}, + "sith_policy_decisions_total": {"outcome": true, "verb": true}, + "sith_policy_decision_duration_seconds": {"outcome": true, "verb": true}, + "sith_federation_spoke_snapshot_attempts_total": {"outcome": true}, + "sith_federation_spoke_snapshot_duration_seconds": {"outcome": true}, + } + for _, family := range families { + labels, sithMetric := allowed[family.GetName()] + if !sithMetric { + continue + } + for _, metric := range family.Metric { + for _, label := range metric.Label { + if !labels[label.GetName()] { + t.Fatalf("metric %s exposed forbidden label %q", family.GetName(), label.GetName()) + } + } + } + } +} diff --git a/internal/observability/tracing.go b/internal/observability/tracing.go new file mode 100644 index 0000000..ddcb485 --- /dev/null +++ b/internal/observability/tracing.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package observability + +import ( + "fmt" + "log/slog" + + "github.com/ArdurAI/sith/internal/tracing" +) + +// NewSlogTraceObserver constructs the local structured trace recorder used by the hub runtime. +// It serializes only the already-validated trace contract; it owns no network, listener, queue, +// persistence, exporter, or SDK integration. +func NewSlogTraceObserver(logger *slog.Logger) (tracing.Observer, error) { + if logger == nil { + return nil, fmt.Errorf("construct trace observer: logger is required") + } + return slogTraceObserver{logger: logger}, nil +} + +type slogTraceObserver struct { + logger *slog.Logger +} + +func (observer slogTraceObserver) ObserveTrace(event tracing.Event) { + if observer.logger == nil || event.Validate() != nil { + return + } + observer.logger.Info( + "trace stage", + "trace_id", event.TraceID, + "trace_stage", event.Stage, + "trace_outcome", event.Outcome, + "duration_ms", event.Duration.Milliseconds(), + ) +} diff --git a/internal/observability/tracing_test.go b/internal/observability/tracing_test.go new file mode 100644 index 0000000..b98d70f --- /dev/null +++ b/internal/observability/tracing_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +package observability + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/tracing" +) + +func TestSlogTraceObserverEmitsOnlyValidatedTraceFields(t *testing.T) { + var output bytes.Buffer + observer, err := NewSlogTraceObserver(slog.New(slog.NewJSONHandler(&output, nil))) + if err != nil { + t.Fatal(err) + } + observer.ObserveTrace(tracing.Event{ + TraceID: "0123456789abcdef0123456789abcdef", Stage: tracing.StagePEPDecision, + Outcome: tracing.OutcomeSuccess, Duration: 125 * time.Millisecond, + }) + observer.ObserveTrace(tracing.Event{ + TraceID: "workspace-a/token=secret", Stage: tracing.StagePEPDecision, + Outcome: tracing.OutcomeSuccess, Duration: time.Millisecond, + }) + lines := strings.Split(strings.TrimSpace(output.String()), "\n") + if len(lines) != 1 { + t.Fatalf("trace lines = %q", output.String()) + } + var record map[string]any + if err := json.Unmarshal([]byte(lines[0]), &record); err != nil { + t.Fatal(err) + } + for field := range record { + if !map[string]bool{"time": true, "level": true, "msg": true, "trace_id": true, "trace_stage": true, "trace_outcome": true, "duration_ms": true}[field] { + t.Fatalf("trace log contains unexpected field %q: %#v", field, record) + } + } + if record["msg"] != "trace stage" || record["trace_id"] != "0123456789abcdef0123456789abcdef" || + record["trace_stage"] != string(tracing.StagePEPDecision) || record["trace_outcome"] != string(tracing.OutcomeSuccess) || + record["duration_ms"] != float64(125) { + t.Fatalf("trace record = %#v", record) + } + for _, forbidden := range []string{"workspace-a", "token=secret", "actor", "spoke", "endpoint", "arguments_digest"} { + if strings.Contains(lines[0], forbidden) { + t.Fatalf("trace log leaked %q: %s", forbidden, lines[0]) + } + } +} + +func TestNewSlogTraceObserverRejectsNilLogger(t *testing.T) { + if _, err := NewSlogTraceObserver(nil); err == nil { + t.Fatal("NewSlogTraceObserver() accepted nil logger") + } +} diff --git a/internal/pep/audit.go b/internal/pep/audit.go new file mode 100644 index 0000000..5a23a6f --- /dev/null +++ b/internal/pep/audit.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pep + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/ArdurAI/sith/internal/tenancy" +) + +// NewSlogAuditor constructs a structured hub policy-audit sink. A nil logger is rejected rather +// than silently discarding governance events. +func NewSlogAuditor(logger *slog.Logger) (Auditor, error) { + if logger == nil { + return nil, fmt.Errorf("construct policy slog auditor: logger is required") + } + return slogAuditor{logger: logger}, nil +} + +type slogAuditor struct { + logger *slog.Logger +} + +func (auditor slogAuditor) Record(ctx context.Context, event AuditEvent) error { + if auditor.logger == nil { + return fmt.Errorf("record policy audit: logger is required") + } + if err := event.Validate(); err != nil { + return fmt.Errorf("record policy audit: %w", err) + } + attributes := []any{ + "audit", true, + "surface", "hub-pep", + slog.Time("audit_at", event.At.UTC()), + "trace_id", event.TraceID, + "workspace", event.WorkspaceID, + "actor", event.Actor, + "role", event.Role, + "action", event.Action, + "verb", event.Verb, + "verdict", event.Verdict, + "reason_code", event.ReasonCode, + } + if event.Verdict == VerdictAllow { + auditor.logger.InfoContext(ctx, "policy decision", attributes...) + return nil + } + auditor.logger.WarnContext(ctx, "policy decision", attributes...) + return nil +} + +// Validate rejects malformed events before a logger can serialize them. +func (event AuditEvent) Validate() error { + if event.At.IsZero() || event.At.After(time.Now().Add(time.Minute)) { + return fmt.Errorf("policy audit time is invalid") + } + if !event.TraceID.Valid() { + return fmt.Errorf("policy audit trace identifier is invalid") + } + if err := tenancy.ValidateWorkspaceID(event.WorkspaceID); err != nil { + return fmt.Errorf("policy audit workspace: %w", err) + } + if err := validateSafeText("policy audit actor", event.Actor, maxActorBytes); err != nil { + return err + } + if !event.Role.Valid() || event.Action != tenancy.ActionRead || !event.Verb.Valid() { + return fmt.Errorf("policy audit has unsupported role, action, or verb") + } + return (Decision{Verdict: event.Verdict, ReasonCode: event.ReasonCode}).Validate() +} diff --git a/internal/pep/audit_test.go b/internal/pep/audit_test.go new file mode 100644 index 0000000..ff5ad0d --- /dev/null +++ b/internal/pep/audit_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pep + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +func TestSlogAuditorEmitsSanitizedStructuredPolicyEvents(t *testing.T) { + var output bytes.Buffer + auditor, err := NewSlogAuditor(slog.New(slog.NewJSONHandler(&output, nil))) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Add(-3 * time.Second).Truncate(time.Second) + for _, event := range []AuditEvent{ + policyAuditEvent(now, VerdictAllow, "phase-1-read"), + policyAuditEvent(now.Add(time.Second), VerdictDeny, "policy-deny"), + policyAuditEvent(now.Add(2*time.Second), VerdictRequireApproval, "approval-required"), + } { + if err := auditor.Record(context.Background(), event); err != nil { + t.Fatalf("Record() error = %v", err) + } + } + lines := strings.Split(strings.TrimSpace(output.String()), "\n") + if len(lines) != 3 { + t.Fatalf("log lines = %q", output.String()) + } + for index, line := range lines { + var record map[string]any + if err := json.Unmarshal([]byte(line), &record); err != nil { + t.Fatalf("decode log %d: %v", index, err) + } + if record["msg"] != "policy decision" || record["audit"] != true || record["surface"] != "hub-pep" || + record["trace_id"] != "0123456789abcdef0123456789abcdef" || record["workspace"] != "workspace-a" || + record["actor"] != "user:reader" || record["verb"] != string(VerbFleetRead) { + t.Fatalf("record %d = %#v", index, record) + } + if _, exists := record["arguments_digest"]; exists || strings.Contains(line, "payments") || strings.Contains(line, "token") { + t.Fatalf("record %d leaked non-audit data: %s", index, line) + } + } + if !strings.Contains(lines[0], "\"level\":\"INFO\"") || !strings.Contains(lines[1], "\"level\":\"WARN\"") || + !strings.Contains(lines[2], "\"level\":\"WARN\"") { + t.Fatalf("unexpected policy audit severities: %q", output.String()) + } +} + +func TestSlogAuditorRejectsUnsafeEventsWithoutEmission(t *testing.T) { + var output bytes.Buffer + auditor, err := NewSlogAuditor(slog.New(slog.NewTextHandler(&output, nil))) + if err != nil { + t.Fatal(err) + } + unsafe := policyAuditEvent(time.Now().UTC(), VerdictAllow, "phase-1-read") + unsafe.ReasonCode = "token=secret" + if err := auditor.Record(context.Background(), unsafe); err == nil { + t.Fatal("Record() accepted unsafe reason code") + } + if output.Len() != 0 { + t.Fatalf("unsafe event was emitted: %q", output.String()) + } + if _, err := NewSlogAuditor(nil); err == nil { + t.Fatal("NewSlogAuditor() accepted nil logger") + } +} + +func TestSlogAuditorRejectsUnsafeTraceIdentifierWithoutEmission(t *testing.T) { + var output bytes.Buffer + auditor, err := NewSlogAuditor(slog.New(slog.NewTextHandler(&output, nil))) + if err != nil { + t.Fatal(err) + } + unsafe := policyAuditEvent(time.Now().UTC(), VerdictAllow, "phase-1-read") + unsafe.TraceID = "workspace-a/token=secret" + if err := auditor.Record(context.Background(), unsafe); err == nil { + t.Fatal("Record() accepted unsafe trace identifier") + } + if output.Len() != 0 { + t.Fatalf("unsafe event was emitted: %q", output.String()) + } +} + +func TestSlogAuditorTextHandlerPreservesSafeFieldsAndSeverity(t *testing.T) { + var output bytes.Buffer + auditor, err := NewSlogAuditor(slog.New(slog.NewTextHandler(&output, nil))) + if err != nil { + t.Fatal(err) + } + if err := auditor.Record(context.Background(), policyAuditEvent(time.Now().UTC(), VerdictRequireApproval, "approval-required")); err != nil { + t.Fatalf("Record() error = %v", err) + } + line := output.String() + for _, field := range []string{"level=WARN", "msg=\"policy decision\"", "audit=true", "surface=hub-pep", "trace_id=0123456789abcdef0123456789abcdef", "workspace=workspace-a", "verdict=require-approval", "reason_code=approval-required"} { + if !strings.Contains(line, field) { + t.Fatalf("text audit log missing %q: %q", field, line) + } + } + if strings.Contains(line, "arguments_digest") || strings.Contains(line, "payments") || strings.Contains(line, "token") { + t.Fatalf("text audit log leaked non-audit data: %q", line) + } +} + +func policyAuditEvent(at time.Time, verdict Verdict, reason string) AuditEvent { + return AuditEvent{ + At: at, TraceID: tracing.ID("0123456789abcdef0123456789abcdef"), WorkspaceID: "workspace-a", Actor: "user:reader", Role: tenancy.RoleReader, + Action: tenancy.ActionRead, Verb: VerbFleetRead, Verdict: verdict, ReasonCode: reason, + } +} diff --git a/internal/pep/metrics.go b/internal/pep/metrics.go new file mode 100644 index 0000000..ca2a051 --- /dev/null +++ b/internal/pep/metrics.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pep + +import ( + "context" + "time" + + "github.com/ArdurAI/sith/internal/tracing" +) + +// DecisionOutcome is the bounded self-observability result of one policy read attempt. +// It intentionally carries no workspace, actor, selector, credential, or reason-code material. +type DecisionOutcome string + +// Closed policy-observability outcomes. +const ( + DecisionOutcomeAllow DecisionOutcome = "allow" + DecisionOutcomeDeny DecisionOutcome = "deny" + DecisionOutcomeRequireApproval DecisionOutcome = "require-approval" + DecisionOutcomeError DecisionOutcome = "error" +) + +// DecisionObserver receives passive, bounded measurements for policy reads. Implementations must +// not block or mutate the authorization path. The enforcer isolates observer panics defensively. +type DecisionObserver interface { + ObserveDecision(verb Verb, outcome DecisionOutcome, duration time.Duration) +} + +type noopDecisionObserver struct{} + +func (noopDecisionObserver) ObserveDecision(Verb, DecisionOutcome, time.Duration) {} + +func (enforcer *Enforcer) observeDecision(verb Verb, outcome DecisionOutcome, duration time.Duration) { + if enforcer == nil || enforcer.observer == nil { + return + } + defer func() { + _ = recover() + }() + enforcer.observer.ObserveDecision(normalizedObservedVerb(verb), outcome, duration) +} + +func (enforcer *Enforcer) observeTrace(ctx context.Context, outcome DecisionOutcome, duration time.Duration) { + if enforcer == nil || enforcer.tracer == nil { + return + } + traceID, ok := tracing.FromContext(ctx) + if !ok { + return + } + tracing.Observe(enforcer.tracer, tracing.Event{ + TraceID: traceID, Stage: tracing.StagePEPDecision, Outcome: traceOutcome(outcome), Duration: duration, + }) +} + +func traceOutcome(outcome DecisionOutcome) tracing.Outcome { + switch outcome { + case DecisionOutcomeAllow: + return tracing.OutcomeSuccess + case DecisionOutcomeDeny, DecisionOutcomeRequireApproval: + return tracing.OutcomeRefused + default: + return tracing.OutcomeFailure + } +} + +func normalizedObservedVerb(verb Verb) Verb { + if verb.Valid() { + return verb + } + return "invalid" +} diff --git a/internal/pep/metrics_test.go b/internal/pep/metrics_test.go new file mode 100644 index 0000000..b282594 --- /dev/null +++ b/internal/pep/metrics_test.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pep + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestEnforcerObservesClosedOutcomesWithoutChangingPolicyBehavior(t *testing.T) { + tests := []struct { + name string + hook PolicyHook + verb Verb + wantOutcome DecisionOutcome + wantError bool + wantObserved Verb + }{ + {name: "allow", hook: AllowReadHook{}, verb: VerbFleetRead, wantOutcome: DecisionOutcomeAllow, wantObserved: VerbFleetRead}, + {name: "deny", hook: HookFunc(func(context.Context, Request) (Decision, error) { + return Decision{Verdict: VerdictDeny, ReasonCode: "policy-deny"}, nil + }), verb: VerbFleetRead, wantOutcome: DecisionOutcomeDeny, wantError: true, wantObserved: VerbFleetRead}, + {name: "approval", hook: HookFunc(func(context.Context, Request) (Decision, error) { + return Decision{Verdict: VerdictRequireApproval, ReasonCode: "approval-required"}, nil + }), verb: VerbFleetRead, wantOutcome: DecisionOutcomeRequireApproval, wantError: true, wantObserved: VerbFleetRead}, + {name: "hook error", hook: HookFunc(func(context.Context, Request) (Decision, error) { + return Decision{}, errors.New("dependency unavailable") + }), verb: VerbFleetRead, wantOutcome: DecisionOutcomeError, wantError: true, wantObserved: VerbFleetRead}, + {name: "invalid verb", hook: AllowReadHook{}, verb: "caller-supplied", wantOutcome: DecisionOutcomeDeny, wantError: true, wantObserved: "invalid"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + observer := &recordingDecisionObserver{} + enforcer, err := NewEnforcer(Config{Hook: test.hook, Auditor: AuditFunc(func(context.Context, AuditEvent) error { return nil }), Observer: observer}) + if err != nil { + t.Fatal(err) + } + err = enforcer.AuthorizeRead(context.Background(), testScope(t), NewReadInput(test.verb, nil)) + if (err != nil) != test.wantError { + t.Fatalf("AuthorizeRead() error = %v, want error %t", err, test.wantError) + } + if len(observer.events) != 1 { + t.Fatalf("observations = %#v, want one", observer.events) + } + event := observer.events[0] + if event.verb != test.wantObserved || event.outcome != test.wantOutcome || event.duration < 0 { + t.Fatalf("observation = %#v", event) + } + }) + } +} + +func TestEnforcerRecoversFromPanickingObserver(t *testing.T) { + enforcer, err := NewEnforcer(Config{ + Hook: AllowReadHook{}, Auditor: AuditFunc(func(context.Context, AuditEvent) error { return nil }), + Observer: decisionObserverFunc(func(Verb, DecisionOutcome, time.Duration) { panic("metrics fault") }), + }) + if err != nil { + t.Fatal(err) + } + if err := enforcer.AuthorizeRead(context.Background(), testScope(t), NewReadInput(VerbFleetRead, nil)); err != nil { + t.Fatalf("AuthorizeRead() changed because observer panicked: %v", err) + } +} + +type decisionObservation struct { + verb Verb + outcome DecisionOutcome + duration time.Duration +} + +type recordingDecisionObserver struct { + events []decisionObservation +} + +func (observer *recordingDecisionObserver) ObserveDecision(verb Verb, outcome DecisionOutcome, duration time.Duration) { + observer.events = append(observer.events, decisionObservation{verb: verb, outcome: outcome, duration: duration}) +} + +type decisionObserverFunc func(Verb, DecisionOutcome, time.Duration) + +func (function decisionObserverFunc) ObserveDecision(verb Verb, outcome DecisionOutcome, duration time.Duration) { + function(verb, outcome, duration) +} diff --git a/internal/pep/pep.go b/internal/pep/pep.go index 63164d0..d7de486 100644 --- a/internal/pep/pep.go +++ b/internal/pep/pep.go @@ -12,6 +12,7 @@ import ( "unicode" "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" ) const ( @@ -26,13 +27,14 @@ type Verb string const ( VerbFleetRead Verb = "fleet.read" VerbFleetCorrelate Verb = "fleet.correlate" + VerbFleetImageSearch Verb = "fleet.image.search" VerbSpokeSnapshotRefresh Verb = "fleet.snapshot.refresh" ) // Valid reports whether a verb belongs to the currently supported closed vocabulary. func (verb Verb) Valid() bool { switch verb { - case VerbFleetRead, VerbFleetCorrelate, VerbSpokeSnapshotRefresh: + case VerbFleetRead, VerbFleetCorrelate, VerbFleetImageSearch, VerbSpokeSnapshotRefresh: return true default: return false @@ -99,6 +101,7 @@ func (function HookFunc) Decide(ctx context.Context, request Request) (Decision, // fixed operation metadata; it intentionally omits credentials, query arguments, targets, and data. type AuditEvent struct { At time.Time + TraceID tracing.ID WorkspaceID tenancy.WorkspaceID Actor string Role tenancy.Role @@ -123,16 +126,20 @@ func (function AuditFunc) Record(ctx context.Context, event AuditEvent) error { // Config constructs an enforcement point with mandatory policy and audit dependencies. type Config struct { - Hook PolicyHook - Auditor Auditor - Now func() time.Time + Hook PolicyHook + Auditor Auditor + Observer DecisionObserver + TraceObserver tracing.Observer + Now func() time.Time } // Enforcer applies the fixed Phase-1 read pipeline and creates one audit record for every decision. type Enforcer struct { - hook PolicyHook - auditor Auditor - now func() time.Time + hook PolicyHook + auditor Auditor + observer DecisionObserver + tracer tracing.Observer + now func() time.Time } // NewEnforcer constructs a fail-closed policy enforcement point. @@ -143,7 +150,13 @@ func NewEnforcer(config Config) (*Enforcer, error) { if config.Now == nil { config.Now = time.Now } - return &Enforcer{hook: config.Hook, auditor: config.Auditor, now: config.Now}, nil + if config.Observer == nil { + config.Observer = noopDecisionObserver{} + } + if config.TraceObserver == nil { + config.TraceObserver = tracing.NoopObserver() + } + return &Enforcer{hook: config.Hook, auditor: config.Auditor, observer: config.Observer, tracer: config.TraceObserver, now: config.Now}, nil } // AllowReadHook is the temporary Phase-1 policy implementation. It allows only the closed read @@ -165,25 +178,46 @@ func (enforcer *Enforcer) AuthorizeRead(ctx context.Context, scope tenancy.Scope if enforcer == nil || enforcer.hook == nil || enforcer.auditor == nil || ctx == nil { return fmt.Errorf("authorize read: enforcer and context are required") } + traceContext, _, err := tracing.Ensure(ctx) + if err != nil { + return fmt.Errorf("authorize read: establish trace context: %w", err) + } + ctx = traceContext + startedAt := time.Now() + outcome := DecisionOutcomeError + defer func() { + enforcer.observeDecision(input.Verb, outcome, time.Since(startedAt)) + enforcer.observeTrace(ctx, outcome, time.Since(startedAt)) + }() request := Request{ WorkspaceID: scope.WorkspaceID(), Actor: scope.Subject(), Role: scope.Role(), Action: tenancy.ActionRead, Verb: input.Verb, ArgumentsDigest: input.ArgumentsDigest, } if err := request.Validate(); err != nil { + outcome = DecisionOutcomeDeny return enforcer.refuse(ctx, request, VerdictDeny, "invalid-request", "authorize read: invalid policy request") } if err := scope.Authorize(tenancy.ActionRead); err != nil { + outcome = DecisionOutcomeDeny return enforcer.refuse(ctx, request, VerdictDeny, "role-denied", "authorize read: role does not permit read") } decision, err := enforcer.hook.Decide(ctx, request) if err != nil { + outcome = DecisionOutcomeError return enforcer.refuse(ctx, request, VerdictDeny, "hook-error", "authorize read: policy hook failed") } if err := decision.Validate(); err != nil { + outcome = DecisionOutcomeError return enforcer.refuse(ctx, request, VerdictDeny, "invalid-decision", "authorize read: policy hook returned an invalid decision") } if decision.Verdict != VerdictAllow { + if decision.Verdict == VerdictRequireApproval { + outcome = DecisionOutcomeRequireApproval + } else { + outcome = DecisionOutcomeDeny + } if err := enforcer.record(ctx, request, decision); err != nil { + outcome = DecisionOutcomeError return fmt.Errorf("authorize read: audit policy refusal: %w", err) } if decision.Verdict == VerdictRequireApproval { @@ -192,8 +226,10 @@ func (enforcer *Enforcer) AuthorizeRead(ctx context.Context, scope tenancy.Scope return fmt.Errorf("authorize read: policy denied request") } if err := enforcer.record(ctx, request, decision); err != nil { + outcome = DecisionOutcomeError return fmt.Errorf("authorize read: audit policy decision: %w", err) } + outcome = DecisionOutcomeAllow return nil } @@ -229,8 +265,12 @@ func (enforcer *Enforcer) refuse(ctx context.Context, request Request, verdict V } func (enforcer *Enforcer) record(ctx context.Context, request Request, decision Decision) error { + traceID, ok := tracing.FromContext(ctx) + if !ok { + return fmt.Errorf("record policy audit: trace context is required") + } return enforcer.auditor.Record(ctx, AuditEvent{ - At: enforcer.now().UTC(), WorkspaceID: request.WorkspaceID, Actor: request.Actor, Role: request.Role, + At: enforcer.now().UTC(), TraceID: traceID, WorkspaceID: request.WorkspaceID, Actor: request.Actor, Role: request.Role, Action: request.Action, Verb: request.Verb, Verdict: decision.Verdict, ReasonCode: decision.ReasonCode, }) } diff --git a/internal/pep/tracing_test.go b/internal/pep/tracing_test.go new file mode 100644 index 0000000..2d1dc32 --- /dev/null +++ b/internal/pep/tracing_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pep + +import ( + "context" + "testing" + + "github.com/ArdurAI/sith/internal/tracing" +) + +func TestEnforcerCorrelatesAuditAndTraceWithOneLocalID(t *testing.T) { + var audits []AuditEvent + var events []tracing.Event + enforcer, err := NewEnforcer(Config{ + Hook: AllowReadHook{}, + Auditor: AuditFunc(func(_ context.Context, event AuditEvent) error { + audits = append(audits, event) + return nil + }), + TraceObserver: tracing.ObserverFunc(func(event tracing.Event) { events = append(events, event) }), + }) + if err != nil { + t.Fatal(err) + } + if err := enforcer.AuthorizeRead(context.Background(), testScope(t), NewReadInput(VerbFleetRead, nil)); err != nil { + t.Fatalf("AuthorizeRead() error = %v", err) + } + if len(audits) != 1 || !audits[0].TraceID.Valid() { + t.Fatalf("audit events = %#v", audits) + } + if len(events) != 1 || events[0].TraceID != audits[0].TraceID || events[0].Stage != tracing.StagePEPDecision || + events[0].Outcome != tracing.OutcomeSuccess || events[0].Duration < 0 { + t.Fatalf("trace events = %#v", events) + } +} + +func TestEnforcerSurvivesPanickingTraceObserver(t *testing.T) { + enforcer, err := NewEnforcer(Config{ + Hook: AllowReadHook{}, Auditor: AuditFunc(func(context.Context, AuditEvent) error { return nil }), + TraceObserver: tracing.ObserverFunc(func(tracing.Event) { panic("trace recorder fault") }), + }) + if err != nil { + t.Fatal(err) + } + if err := enforcer.AuthorizeRead(context.Background(), testScope(t), NewReadInput(VerbFleetRead, nil)); err != nil { + t.Fatalf("AuthorizeRead() changed because tracing panicked: %v", err) + } +} diff --git a/internal/privacy/boundary_test.go b/internal/privacy/boundary_test.go index d984ff3..cdb3b9e 100644 --- a/internal/privacy/boundary_test.go +++ b/internal/privacy/boundary_test.go @@ -20,13 +20,30 @@ var approvedNetworkImports = map[string]map[string]bool{ "internal/connector/kubeconfig/local_streams.go": {"net/http": true, "net/url": true}, "internal/hubserver/auth.go": {"net/http": true}, "internal/hubserver/exchange.go": {"net": true, "net/http": true}, + "internal/hubserver/fleet.go": {"net/http": true, "net/url": true}, // AWS STS egress is endpoint-pinned, SigV4-profiled, redirect-disabled, and never used by local mode. - "internal/hubauth/aws_sts.go": {"net/http": true, "net/url": true}, - "internal/hubauth/oidc.go": {"net": true, "net/http": true, "net/netip": true, "net/url": true}, - "internal/hubdb/app.go": {"net/netip": true}, - "internal/mcpserver/server.go": {"net": true, "net/http": true, "net/url": true}, - "internal/webui/api.go": {"net/http": true}, - "internal/webui/server.go": {"net": true, "net/http": true, "net/url": true}, + "internal/hubauth/aws_sts.go": {"net/http": true, "net/url": true}, + "internal/hubauth/oidc.go": {"net": true, "net/http": true, "net/netip": true, "net/url": true}, + "internal/hubdb/app.go": {"net/netip": true}, + // The governed-only direct OCM adapter is the reviewed Phase-1 exception to local-mode + // client-go confinement. It pins one registered cluster, uses scoped MSA credentials, + // and is exercised by the real two-spoke M0 gate. + "internal/hubocm/credentials.go": {"k8s.io/client-go/kubernetes/typed/core/v1": true}, + "internal/hubocm/direct.go": { + "net": true, "net/http": true, + "k8s.io/client-go/dynamic": true, "k8s.io/client-go/kubernetes": true, "k8s.io/client-go/rest": true, + "google.golang.org/grpc": true, "google.golang.org/grpc/credentials": true, + }, + // The in-cluster hub composition root is the reviewed boundary for its fixed TLS listener and + // scoped Kubernetes client. It delegates every spoke credential read to the direct OCM adapter. + "internal/hubruntime/config.go": { + "net": true, "k8s.io/client-go/kubernetes": true, "k8s.io/client-go/rest": true, + }, + "internal/hubruntime/runtime.go": {"net": true, "net/http": true}, + "internal/mcpserver/server.go": {"net": true, "net/http": true, "net/url": true}, + "internal/observability/metrics.go": {"net/http": true}, + "internal/webui/api.go": {"net/http": true}, + "internal/webui/server.go": {"net": true, "net/http": true, "net/url": true}, } var approvedFilesystemWrites = map[string]map[string]bool{ @@ -115,13 +132,19 @@ func reviewProductionFile( } markBoundary(seenProcesses, relative, importPath) } - if lowLevelNetworkPackage(importPath) { + if lowLevelNetworkPackage(importPath) && !approvedNetworkImports[relative][importPath] { t.Errorf("%s imports unapproved low-level network package %q", relative, importPath) } + if lowLevelNetworkPackage(importPath) { + markBoundary(seenNetwork, relative, importPath) + } if strings.HasPrefix(importPath, "k8s.io/client-go/") && - !strings.HasPrefix(relative, "internal/connector/kubeconfig/") { + !strings.HasPrefix(relative, "internal/connector/kubeconfig/") && !approvedNetworkImports[relative][importPath] { t.Errorf("%s imports Kubernetes transport outside the local source adapter: %q", relative, importPath) } + if strings.HasPrefix(importPath, "k8s.io/client-go/") { + markBoundary(seenNetwork, relative, importPath) + } if strings.HasPrefix(relative, "internal/keychain/") && filesystemPackage(importPath) { t.Errorf("%s imports filesystem package %q; keychain custody must not have a file fallback", relative, importPath) } diff --git a/internal/tracing/context.go b/internal/tracing/context.go new file mode 100644 index 0000000..852a7e3 --- /dev/null +++ b/internal/tracing/context.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package tracing provides a local, privacy-preserving trace context for governed hub work. +// It deliberately provides no wire propagation, listener, exporter, persistence, or telemetry +// SDK integration. Those boundaries require a separate review once a typed action protocol exists. +package tracing + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" +) + +const idBytes = 16 + +type contextKey struct{} + +// ID is one opaque, locally minted trace correlation identifier. It is not an intent identifier: +// the future E4/E6 typed-intent schema owns that binding. +type ID string + +// Valid reports whether an identifier has the fixed lower-case hexadecimal representation minted +// by NewID. Restricting the vocabulary prevents trace records from accepting caller text. +func (id ID) Valid() bool { + if len(id) != idBytes*2 { + return false + } + decoded := make([]byte, idBytes) + if _, err := hex.Decode(decoded, []byte(id)); err != nil { + return false + } + return string(id) == hex.EncodeToString(decoded) +} + +// NewID mints an opaque identifier from the operating system's cryptographic random source. +func NewID() (ID, error) { + raw := make([]byte, idBytes) + if _, err := rand.Read(raw); err != nil { + return "", fmt.Errorf("mint trace identifier: %w", err) + } + return ID(hex.EncodeToString(raw)), nil +} + +// FromContext returns the locally minted trace identifier, if one is present and valid. +func FromContext(ctx context.Context) (ID, bool) { + if ctx == nil { + return "", false + } + id, ok := ctx.Value(contextKey{}).(ID) + return id, ok && id.Valid() +} + +// Ensure returns a context with one locally minted trace identifier. It preserves an existing +// valid identifier and never reads request metadata, so hostile correlation headers are ignored. +func Ensure(ctx context.Context) (context.Context, ID, error) { + if ctx == nil { + return nil, "", fmt.Errorf("establish trace context: context is required") + } + if id, ok := FromContext(ctx); ok { + return ctx, id, nil + } + id, err := NewID() + if err != nil { + return nil, "", err + } + return context.WithValue(ctx, contextKey{}, id), id, nil +} diff --git a/internal/tracing/event.go b/internal/tracing/event.go new file mode 100644 index 0000000..a987f43 --- /dev/null +++ b/internal/tracing/event.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +package tracing + +import ( + "fmt" + "time" +) + +const maxDuration = time.Hour + +// Stage is the closed local-trace stage vocabulary. It intentionally names system behavior, not +// a tenant, resource, target, actor, or spoke. +type Stage string + +// Supported local trace stages. +const ( + StagePEPDecision Stage = "pep.decision" + StageSpokeSnapshot Stage = "spoke.snapshot" +) + +// Outcome is the closed result vocabulary for a trace stage. +type Outcome string + +// Supported trace outcomes. +const ( + OutcomeSuccess Outcome = "success" + OutcomeRefused Outcome = "refused" + OutcomeFailure Outcome = "failure" + OutcomeCanceled Outcome = "canceled" +) + +// Event is one passive, local trace-stage observation. It intentionally has no generic attribute +// map: adding request or response data requires a reviewed schema change instead of an accidental +// logging path. +type Event struct { + TraceID ID + Stage Stage + Outcome Outcome + Duration time.Duration +} + +// Validate rejects unbounded or caller-controlled trace data before an observer can emit it. +func (event Event) Validate() error { + if !event.TraceID.Valid() { + return fmt.Errorf("trace event identifier is invalid") + } + switch event.Stage { + case StagePEPDecision, StageSpokeSnapshot: + default: + return fmt.Errorf("trace event stage is unsupported") + } + switch event.Outcome { + case OutcomeSuccess, OutcomeRefused, OutcomeFailure, OutcomeCanceled: + default: + return fmt.Errorf("trace event outcome is unsupported") + } + if event.Duration < 0 || event.Duration > maxDuration { + return fmt.Errorf("trace event duration is outside the accepted bound") + } + return nil +} + +// Observer receives passive, already-sanitized trace events. Implementations must never alter the +// governed operation; Observe isolates a faulty observer defensively. +type Observer interface { + ObserveTrace(Event) +} + +// ObserverFunc adapts a function to Observer. +type ObserverFunc func(Event) + +// ObserveTrace calls function. +func (function ObserverFunc) ObserveTrace(event Event) { + function(event) +} + +type noopObserver struct{} + +func (noopObserver) ObserveTrace(Event) {} + +// NoopObserver returns the safe no-op observer used when a composition root does not attach a +// local trace recorder. +func NoopObserver() Observer { return noopObserver{} } + +// Observe sends a validated event to a passive observer. Invalid events and observer panics are +// intentionally ignored so observability cannot mutate authorization or collection behavior. +func Observe(observer Observer, event Event) { + if observer == nil || event.Validate() != nil { + return + } + defer func() { + _ = recover() + }() + observer.ObserveTrace(event) +} diff --git a/internal/tracing/tracing_test.go b/internal/tracing/tracing_test.go new file mode 100644 index 0000000..2264c7a --- /dev/null +++ b/internal/tracing/tracing_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +package tracing + +import ( + "context" + "testing" + "time" +) + +func TestEnsureMintsAndPreservesOneOpaqueID(t *testing.T) { + ctx, first, err := Ensure(context.Background()) + if err != nil { + t.Fatal(err) + } + if !first.Valid() { + t.Fatalf("minted trace identifier is invalid: %q", first) + } + secondContext, second, err := Ensure(ctx) + if err != nil { + t.Fatal(err) + } + if first != second || secondContext != ctx { + t.Fatalf("Ensure() changed an existing trace context: %q/%q", first, second) + } + var nilContext context.Context + if _, _, err := Ensure(nilContext); err == nil { + t.Fatal("Ensure() accepted a nil context") + } +} + +func TestEventAndObserverRejectUnsafeTraceDataWithoutAffectingCallers(t *testing.T) { + const traceID ID = "0123456789abcdef0123456789abcdef" + valid := Event{TraceID: traceID, Stage: StagePEPDecision, Outcome: OutcomeSuccess, Duration: time.Millisecond} + if err := valid.Validate(); err != nil { + t.Fatalf("valid trace event rejected: %v", err) + } + for _, event := range []Event{ + {TraceID: "trace=secret", Stage: StagePEPDecision, Outcome: OutcomeSuccess, Duration: time.Millisecond}, + {TraceID: traceID, Stage: "workspace-a", Outcome: OutcomeSuccess, Duration: time.Millisecond}, + {TraceID: traceID, Stage: StagePEPDecision, Outcome: "token=secret", Duration: time.Millisecond}, + {TraceID: traceID, Stage: StagePEPDecision, Outcome: OutcomeSuccess, Duration: -time.Millisecond}, + {TraceID: traceID, Stage: StagePEPDecision, Outcome: OutcomeSuccess, Duration: time.Hour + time.Nanosecond}, + } { + if err := event.Validate(); err == nil { + t.Fatalf("Validate() accepted unsafe event %#v", event) + } + } + called := false + Observe(ObserverFunc(func(Event) { called = true; panic("observer fault") }), valid) + if !called { + t.Fatal("Observe() did not call a valid observer") + } + called = false + Observe(ObserverFunc(func(Event) { called = true }), Event{TraceID: "token=secret"}) + if called { + t.Fatal("Observe() delivered an invalid event") + } +} diff --git a/sessions/2026-07-12-p1-observability-metrics.md b/sessions/2026-07-12-p1-observability-metrics.md new file mode 100644 index 0000000..5fdd024 --- /dev/null +++ b/sessions/2026-07-12-p1-observability-metrics.md @@ -0,0 +1,78 @@ +# Session — 2026-07-12 — P1 observability metrics + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/p1-observability-metrics` +**Slice:** [#119](https://github.com/ArdurAI/sith/issues/119), E10 [#28](https://github.com/ArdurAI/sith/issues/28) · **Status:** in progress + +--- + +## [G] Goal + +Add the first F10.1 self-observability boundary: bounded Prometheus exposition for Sith's own +policy and federated-snapshot behavior. It must be embeddable by the future hub composition root, +without starting a listener, exporting remotely, persisting telemetry, or creating a telemetry +lake. + +## [D] Design + +- `internal/observability` will own a caller-supplied, non-global Prometheus registry and expose an + `http.Handler` only. The future hub owns bind address, scrape authorization, readiness, and TLS. +- The PEP and snapshot collector will depend on tiny passive observer interfaces defined at their + existing package boundaries. A no-op observer is the default, so metrics cannot alter + authorization, auditing, transport, or persistence behavior. +- Labels are closed vocabulary only: policy verb/outcome and snapshot outcome. Workspace, actor, + role, cluster, spoke, resource, selector, endpoint, credential, raw error, request body, and + policy argument digest are forbidden from exposition. +- Registry setup uses explicit error-returning registration rather than global state or + `MustRegister`, so duplicate or inconsistent metric definitions fail construction predictably. + +## [R] Primary sources and constraints + +- The Prometheus Go client supports custom registries and `promhttp.HandlerFor`, avoiding default + global state and enabling isolated tests: . +- Prometheus recommends low-cardinality labels and specifically rejects user IDs and other unbounded + sets as labels: . +- `docs/EPICS.md` F10.1 requires metrics about Sith itself only; it explicitly excludes retaining + other systems' series. `internal/privacy/boundary_test.go` forbids telemetry SDKs and makes any + new HTTP surface an explicit, reviewed boundary. + +## [T] Planned evidence + +- Unit and race tests for policy allow/refuse/error and snapshot success/closed-failure outcomes. +- A real `httptest` scrape proving the declared exposition, fixed labels, no global registry + cross-test leakage, and hostile-input non-leakage. +- Repository CI, forced-RLS isolation, release reproducibility, and the real two-cluster kind gate + after implementation. + +## [S] Out of scope + +No hub listener, bind address, readiness/health claim, database probe, remote write/exporter, +OpenTelemetry tracing SDK, alert rule, persistent metric store, or external telemetry data. + +## [V] Validation evidence + +- Focused race tests cover the new observability adapter, PEP observer seam, snapshot observer seam, + and privacy boundary. The full `go test -race -count=1 ./...`, `go mod verify`, and + `govulncheck ./...` pass; `internal/observability` coverage is 91.8% in `make ci`. +- `make ci` passes format, lint, vet, module verification, vulnerability scanning, and the complete + race suite. `make e2e-isolation` passes the forced PostgreSQL RLS suite and its fixed 50,000-case + cross-workspace fuzz campaign. `make release-check` passes two reproducible four-platform builds, + SPDX SBOM verification, distribution validation, and Homebrew formula rendering. +- The real two-cluster `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` gate passes: + `TestKindFleetFanout` completed successfully in 79.815 seconds. A first attempt used a nonexistent + `$GOPATH/bin/kind` path and never reached the test; it is a local tool-path setup failure, not a + product signal. +- Manual red-team review verified closed-label normalization, no global Prometheus registry, no + listener/exporter/persistence, and panic isolation. It found a partial-registration rollback risk; + construction now unregisters only its own prior collectors on an error and a regression proves a + later retry succeeds. CodeRabbit CLI is not installed, so no external diff was sent. +- Final cleanup leaves zero kind clusters. A non-volume Docker prune removed only unused test + artifacts and reclaimed 1.217 GB; the two pre-existing active user containers remained running. + +## [C] Commit readiness + +- `README.md` was reviewed after implementation. It correctly describes `sith hub` as a staged + runtime and no user-reachable command, listener, or installation behavior changed, so no README + edit is warranted for this embeddable-only metrics slice. +- Final GitHub security queues are Dependabot `0`, code-scanning `0`, and secret-scanning `0`. + The separate ClusterGateway authorization fix remains clean and mergeable upstream, but is not + merged or released; #103/#104 stay open until that exact release can pass the real Sith M0 test. diff --git a/sessions/2026-07-12-p1-policy-audit-logging.md b/sessions/2026-07-12-p1-policy-audit-logging.md new file mode 100644 index 0000000..671071e --- /dev/null +++ b/sessions/2026-07-12-p1-policy-audit-logging.md @@ -0,0 +1,49 @@ +# Session — 2026-07-12 — P1 policy audit logging + +**Builder:** Gnani Rahul · **Branch:** gnanirahulnutakki/feat/p1-policy-audit-logging +**Slice:** [#115](https://github.com/ArdurAI/sith/issues/115), E10 [#28](https://github.com/ArdurAI/sith/issues/28) · **Status:** ready for review + +--- + +## [G] Goal + +Provide a structured, sanitized `slog` audit sink for the shipped hub-read PEP so operators can +distinguish permits from policy denials and approval requirements without logging fleet data, +selectors, credentials, or policy argument digests. + +## [D] Design + +- The sink accepts only the PEP's normalized `AuditEvent` and validates it again immediately before + emission. A nil logger or malformed event fails the synchronous auditor contract rather than + silently discarding the event. +- The emitted envelope contains only timestamp, workspace, actor, role, action, closed verb, + verdict, and bounded reason code. It intentionally omits raw arguments, the digest used only for + PDP binding, targets, facts, endpoints, and credentials. +- Allow is `INFO`; deny and require-approval are `WARN`, creating an alertable security distinction + without treating structured logging as the later E6 decision ledger. + +## [T] Evidence + +- Focused PEP race tests and lint pass. The full race suite, formatting, vet, golangci-lint, + `govulncheck`, M0 safety assertions, standard e2e smoke, and binary build pass. +- `make e2e-kind` passed with two real local clusters. `make e2e-isolation` passed the forced + PostgreSQL RLS suite plus the fixed 50,000x cross-workspace fuzz campaign. +- `make release-check` completed two reproducible four-platform snapshots; the final distribution + verifier and generated Homebrew formula pass independently. +- Manual red-team review confirms validation immediately before emission, JSON/text severity parity, + absence of raw selector/digest/credential terms, nil-logger failure, and no direct connector or + network path. CodeRabbit CLI is unavailable in this environment; no external diff was submitted. +- Post-gate cleanup confirmed zero kind clusters; Docker prune reclaimed **5.257 GB**. GitHub + Dependabot, code-scanning, and secret-scanning queues were each **0** open alerts. Pending hosted + CI and exact post-merge verification. + +## [S] Scope and safety + +No hub listener, endpoint, telemetry backend, persistence, metric/tracing exporter, source connector, +credential, query selector, or write path is added. This is local process logging of Sith's own +sanitized policy decisions only. + +## [N] Next + +Run all required gates, verify security queues and cleanup, check README, then create the signed/DCO/ +GSTACK checkpoint and one narrow PR into `dev`. diff --git a/sessions/2026-07-12-release-branch-safety.md b/sessions/2026-07-12-release-branch-safety.md new file mode 100644 index 0000000..2e7f66a --- /dev/null +++ b/sessions/2026-07-12-release-branch-safety.md @@ -0,0 +1,46 @@ +# Session — 2026-07-12 — Release branch safety + +**Builder:** Gnani Rahul · **Branch:** gnanirahulnutakki/chore/release-branch-safety +**Slice:** [#117](https://github.com/ArdurAI/sith/issues/117) · **Status:** ready for review + +--- + +## [G] Goal + +Prevent a `dev → main` release PR cleanup from deleting or rewriting Sith's durable integration or +release branch. + +## [D] Design + +- GitHub branch protection is intentionally minimal: both `dev` and `main` reject deletion and + force-pushes, without changing the existing required-check, review, or merge workflow. +- The release procedure now treats `dev` as a durable source branch. `--delete-branch` is permitted + only for a merged feature branch, never a release PR headed by `dev`. + +## [T] Evidence + +- Before correction, both branch-protection reads returned `404 Branch not protected`; the repository + had no rulesets. +- The GitHub branch-protection read-back now reports `allow_deletions=false` and + `allow_force_pushes=false` for both `dev` and `main`, with required status checks and review + policy unchanged (`null`). +- The exact restored `dev` history was preserved; this slice makes no history rewrite, runtime, or + dependency change. +- `make ci` passed formatting, lint, vulnerability scanning, race tests, the M0 safety suite, + latency, and standard e2e coverage. `make release-check` passed two reproducible four-platform + snapshots, distribution verification, and Homebrew formula generation. +- Manual red-team review confirmed the protection payload narrows only destructive operations, the + release wording cannot be mistaken for feature-branch cleanup, and no secret, runtime, or + telemetry path changed. GitHub Dependabot, code-scanning, and secret-scanning queues were each + zero open alerts before this documentation-only change. + +## [S] Scope and safety + +This is a release-integrity correction only. It does not add a review quorum, change CI status +requirements, broaden repository access, alter a feature path, or emit telemetry. + +## [N] Next + +Check README, create the signed/DCO/GSTACK checkpoint, then open a narrow PR into `dev`. Merge only +after hosted CI is green, verify the exact post-merge `dev` CI and GitHub security queues, and close +the corrective issue. diff --git a/sessions/2026-07-13-e2-direct-konnectivity-transport.md b/sessions/2026-07-13-e2-direct-konnectivity-transport.md new file mode 100644 index 0000000..e408a58 --- /dev/null +++ b/sessions/2026-07-13-e2-direct-konnectivity-transport.md @@ -0,0 +1,58 @@ +# Session — 2026-07-13 — e2-direct-konnectivity-transport + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e2-direct-konnectivity-transport` +**Slice(s):** E2 / [#123](https://github.com/ArdurAI/sith/issues/123) · **Status:** ready-to-commit + +--- + +[G] Goal: deliver #123, a safe Phase-1 alternative to the blocked ClusterGateway adapter, while +preserving OCM registration, ClusterProxy reverse tunnels, and managed-serviceaccount identity. +No caller authorization header, endpoint, kubeconfig, token, CA, or raw Kubernetes object may +cross the `hubfleet.Transport` seam. + +[S] Scope: `internal/hubocm`, the M0 experiment and its safety suite, narrowly reviewed privacy +boundary exceptions, dependencies, and operator-facing documentation. The ClusterGateway-specific +#103/#104 route remains out of scope and blocked pending an official upstream release. + +[A] Action: chose the released ClusterProxy `0.10.0`-matched Konnectivity client `v0.31.2`, rather +than an unreleased ClusterGateway fix or a custom agent/tunnel. The adapter treats +`ocm/` as a pinned identity, reads only the exact rotating +`Secret//sith-reader` projection per snapshot, validates exactly `token` and +`ca.crt`, and never caches or logs credential material. + +[A] Action: implemented proxy mTLS with explicit CA/server name, TLS 1.2 minimum, and one client +certificate. The spoke connection requires its CA and a Kubernetes server name; insecure TLS is +rejected. Each TCP dial gets one direct Konnectivity tunnel and closes its private HTTP transport +after the bounded snapshot. The adapter normalizes only Pods, Deployments, and optional Rollouts; +the pagination/resource boundary fails rather than silently returning a partial oversized result. + +[A] Action: added `make e2e-ocm` and its direct M0 integration test. The M0 reader RBAC permits +only cluster-wide `list` on Pods, Deployments, and Rollouts; Secrets and Nodes remain denied. The +M0 harness now waits for asynchronously placed addon objects to exist before waiting for +`Available=True`; its safety suite simulates initial `NotFound` responses. + +[T] Test: focused race-safe adapter tests cover exact Secret `get`, unsafe projection/configuration +rejection, TLS pinning, credential replacement, no-detail failures, fixed target dialing, tunnel +close, credential-buffer clearing, and normalized snapshot validation. The direct M0 gate passed +repeatedly; its final 152-second run reached both scopes, proved Secrets/Nodes denial and +outbound-only controls, took TLS-verified snapshots, and exercised MSA replacement. The target +removed all kind clusters and its owned scratch directory afterward. + +[T] Test: baseline gates passed: `go mod verify`, `govulncheck ./...`, repository-wide +`go test -race`, `make e2e-isolation` (including the fixed 50,000x cross-workspace fuzz campaign), +and `make e2e-kind`. The exact final tree passed `make ci` (including all 16 M0 safety assertions), +`make release-check`, and the direct M0 gate after the addon-race hardening. GitHub Dependabot, +code-scanning, and secret-scanning queues each report zero open alerts. + +[R] Review: manual red-team review found and closed two substantive risks before final validation: +the resource limit now fails before another list can be issued, and the addon lifecycle now handles +asynchronous creation. The narrow governed-only privacy allowlist is exercised by the real M0 gate; +no raw credentials or response bodies are present in tests, logs, or this journal. + +[C] Checkpoint #1: record the direct ClusterProxy alternative and evidence; next: create the +SSH-signed DCO commit with `GSTACK-Checkpoint: 2026-07-13/e2-direct-konnectivity-transport#1`, then +open a small PR into `dev`. + +--- + +**Session close:** direct alternative ready for review and merge · **Open questions touched:** none diff --git a/sessions/2026-07-13-security-ocm-upstream-monitor.md b/sessions/2026-07-13-security-ocm-upstream-monitor.md new file mode 100644 index 0000000..98bcf18 --- /dev/null +++ b/sessions/2026-07-13-security-ocm-upstream-monitor.md @@ -0,0 +1,46 @@ +# Security and OCM upstream monitor — 2026-07-13 + +[G] Goal: independently verify the ArdurAI/sith security-alert queues and the upstream +ClusterGateway authorization-isolation blocker before allowing any Phase-1 transport work. + +[S] Scope: documentation-only security gate on `dev`; no Sith transport code, RBAC policy, or +credential handling changes while the required upstream remediation has not shipped in an official +release. + +[A] Action: verified `origin/dev` at `cdb84600e2fd91d54d66368c1ce37b41318d42bd`; Dependabot, +code-scanning, and secret-scanning each report zero open alerts. Upstream +`oam-dev/cluster-gateway` master remains at `2b04dd452b7a9da6e75de105cd00499a1c6369bd` and the +latest official release is `v1.9.1`; neither is a safe consumption point for the authorization +fix. Upstream PR #171 (`5bd9423337eee6bdf10fc29041dcb0e77a7eae21`) removes inbound +`Authorization` before the managed-service-account transport and has green unit, gateway, +OCM-addon, and DCO checks, but remains open. + +[T] Test plan: run `make ci`, the real two-cluster kind integration gate, and `make release-check` +from the isolated worktree. The documentation change must leave the working tree clean except for +the intended roadmap and session evidence. + +[A] Action: the required `make ci` gate exposed 14 reachable Go standard-library advisories in the +locally installed Go 1.26.0 runtime. `govulncheck` identifies Go 1.26.5 as the minimum fixed patch +level for every finding, so `go.mod` now requires `toolchain go1.26.5`; no vulnerability suppression +was added. + +[A] Action: red-team review found that `make release-check` continued after a failed prerequisite +because its shell recipe did not enable fail-fast behavior. Added `set -e` so module verification, +reproducible archive creation, SBOM verification, and digest comparison each gate publication. The +initial local `go mod verify` failure was traced to a host GOPATH containing an unrelated `go.mod`; +the same command passes with an isolated GOPATH and the repository module cache. + +[T] Test: `make ci` PASS with Go 1.26.5: formatting, lint, vet, reachable-vulnerability scan, +race suite, M0 harness safety assertions, warm-view performance guard, generic e2e, and build all +passed. `make e2e-kind` PASS with the pinned kind v0.32.0 binary: two real clusters completed in +84.530 seconds and cleaned up. `make release-check` PASS with an isolated GOPATH: module +verification, two reproducible multi-platform archive/SBOM builds, distribution validation, and +digest comparison all completed after fail-fast hardening. + +[R] Review: manual red-team diff review found no change to transport implementation, MSA/RBAC +policy, credential logging, or response-body handling. The upstream candidate remains open and no +new official ClusterGateway release exists, so the roadmap preserves the #103/#104 block. The +optional CodeRabbit CLI is not installed locally; no external review payload was sent. + +[C] Checkpoint #1: record the release-consumption gate and the minimum secure Go toolchain, then +publish only after the required quality, two-cluster, and release checks pass. diff --git a/sessions/2026-07-14-e10-auth-refusal-logs.md b/sessions/2026-07-14-e10-auth-refusal-logs.md new file mode 100644 index 0000000..733fade --- /dev/null +++ b/sessions/2026-07-14-e10-auth-refusal-logs.md @@ -0,0 +1,73 @@ +# E10 F10.3a sanitized hub authentication-refusal logs + +Issue: [#139](https://github.com/ArdurAI/sith/issues/139) + +Branch: `gnanirahulnutakki/feat/e10-auth-refusal-logs` + +Base: `origin/dev` at `1ecfb56be314e7d01a9c2d2eec38dad2ddc56496` + +## [G] Goal + +Deliver the smallest independently shippable F10.3 foundation for a security-relevant hub +authentication signal: one sanitized local structured log on pre-principal credential refusal. +It must create no request logging, authentication oracle, telemetry product, or remote data path. + +## [S] Scope + +- Add a closed `hubserver` authentication event with exactly one outcome (`refused`) and a passive, + panic-isolated observer seam. +- Emit the same event for every missing, malformed, ambiguous, or invalid bearer credential before + a verified principal exists. Valid authentication emits no event. +- Add the `slog` adapter only at the hub runtime composition root. It emits WARN with the fixed + `hub-auth` surface and `refused` outcome. +- Keep all request metadata, token/header text, URL/path/query, remote address, workspace, + principal, verifier error, caller correlation carrier, trace ID, metric, listener, exporter, + persistence, rate limiter, and audit-ledger change out of scope. + +## [A] Analysis and red-team checks + +- A rejected request has not established a signed workspace scope. The implementation deliberately + neither accepts nor mints a trace/correlation ID there; F10.2a trace context remains strictly + post-authentication. +- The event has no generic attributes, error string, or metadata field. Its validator accepts only + the fixed outcome, preventing a future caller-controlled logging path through this seam. +- The event does not disclose credential failure mode. Missing credentials, wrong scheme, + ambiguity, and verifier failure each produce the existing identical HTTP 401 response plus the + same one-value observer event, preventing a logging-based authentication oracle. +- The observer is passive: invalid events are dropped and observer panics are recovered before the + existing uniform response is written. The hub runtime owns the sink; collectors and business + handlers receive no logging dependency. +- CodeRabbit reviewed the complete staged ten-file diff. It suggested a server-generated receipt + correlation ID and a bounded asynchronous delivery queue. The receipt conflicts with the + explicit pre-signed-scope no-correlation rule and was rejected. The queue would add an out-of- + scope worker, queue, capacity, and shutdown contract; it is recorded separately as #140 rather + than being hidden inside this narrow slice. + +## [T] Tests and evidence + +- Focused suites: PASS — `go test ./internal/hubserver ./internal/observability ./internal/hubruntime` + and `go test -race -count=1 ./internal/hubserver ./internal/observability ./internal/hubruntime`. + They prove uniform refusal events, valid-session silence, hostile token-like input exclusion, + malformed-event rejection, observer panic isolation, JSON/text allowlisted log fields, and + handler configuration propagation. +- Repository race suite: PASS — `go test -race -count=1 ./...`. +- Supply-chain checks: PASS — `go mod verify` and `govulncheck ./...` (no vulnerabilities found). +- `make ci`: PASS — gofmt, golangci-lint (0 issues), vet, vulnerability scan, full race/coverage, + M0 safety checks, UI latency guard, and tagged e2e. Relevant coverage: hubserver 90.1% and + observability 93.4%. +- `make e2e-isolation`: PASS — race-enabled pinned PostgreSQL/RLS suites (hubauth 85.2%, hubserver + 90.1%, fleetcache 87.0%, hubdb 73.5%) and fixed 50,000-execution workspace fuzz campaign. +- `make release-check`: PASS — reproducible Darwin/Linux amd64/arm64 snapshot archives, SPDX SBOMs, + checksums, and release distribution validation. +- `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind`: PASS — the controlled rerun exited + 0 in 156.669s for real two-cluster fan-out and OCI image contract gates. `kind get clusters` is + empty after cleanup; only unrelated `elated_antonelli` and `ardur-191-baseline` containers remain. +- GitHub security queues at slice start: Dependabot 0, code-scanning 0, secret-scanning 0. +- CodeRabbit staged-diff review: completed with two suggestions. Both were independently reviewed; + no in-scope defect remains. The bounded nonblocking-delivery follow-up is [#140](https://github.com/ArdurAI/sith/issues/140). + +## [C] Checkpoint #1 + +- Source, README, tests, local/red-team review, and full validation are ready for the signed + `2026-07-14/e10-auth-refusal-logs#1` commit. External CodeRabbit review is in progress and must + be resolved before PR creation. diff --git a/sessions/2026-07-14-e10-trace-context.md b/sessions/2026-07-14-e10-trace-context.md new file mode 100644 index 0000000..b903eac --- /dev/null +++ b/sessions/2026-07-14-e10-trace-context.md @@ -0,0 +1,78 @@ +# E10 F10.2a sanitized local trace context + +Issue: [#137](https://github.com/ArdurAI/sith/issues/137) + +Branch: `gnanirahulnutakki/feat/e10-trace-context` + +Base: `origin/dev` at `710a428dba4939a749c1a78f39bb97eb3cb4231c` + +## [G] Goal + +Deliver the smallest independently shippable F10.2 foundation for governed reads: a locally minted, +privacy-preserving trace context across the authenticated hub, PEP audit, and present snapshot +transport. It must create no telemetry product or remote data path. + +## [S] Scope + +- Add an internal trace contract with a cryptographically minted opaque 128-bit ID, two closed + stages (`pep.decision`, `spoke.snapshot`), four closed outcomes, and a one-hour duration bound. +- Mint the root only after the hub has verified the session and derived its signed workspace scope; + direct governed callers also receive a root before PEP audit. +- Strip `traceparent`, `tracestate`, B3, `X-B3-*`, and common request/correlation headers before + the authenticated downstream handler. Do not accept, echo, or forward caller correlation data. +- Carry the ID into the PEP audit event and structured local trace recorder. The recorder emits only + trace ID, stage, outcome, and duration milliseconds; malformed events do not emit and observer + faults cannot affect authorization or collection. +- Carry the same context into every bounded current OCM snapshot transport call. Do not record a + workspace, actor, role, verb, spoke identifier, endpoint, resource, selector, digest, credential, + raw error, returned data, or arbitrary attributes. + +## [A] Analysis and red-team checks + +- The repository privacy boundary explicitly rejects OpenTelemetry and other telemetry SDK imports. + The implementation therefore uses an internal typed contract, not an SDK-shaped façade; it owns no + network exporter, listener, queue, persistence, trace store, background worker, sampling sink, or + wire propagation. +- The full F10.2 requirement says `trace_id == intent_id`, but Phase 1 has no typed action intent or + E6 decision-ledger schema. This issue is explicitly F10.2a and does not fabricate that equality; + E4/E6 must define it when proposal/approval/dispatch semantics exist. +- Potential carrier injection is handled twice: authentication removes known carrier headers and + tracing never reads HTTP headers. Trace events cannot accept a generic attribute map, blocking + accidental leakage through a future caller-supplied label. +- PEP audit remains fail-closed. Trace observation is passive and panic-isolated, so a logger or + observer fault cannot widen access or stop independent spoke collection. + +## [T] Tests and evidence + +- Focused package suite: PASS — `go test ./internal/tracing ./internal/observability ./internal/pep ./internal/hubfleet ./internal/hubserver ./internal/hubruntime`. + It proves ID validation and preservation, event vocabulary/duration rejection, observer panic + isolation, local slog emission field allowlisting, audit correlation, hostile-header stripping, + authenticated hub root minting, and PEP-to-real-collector-to-transport propagation. +- Static supply-chain checks: PASS — `go vet ./...`, `go mod verify`, and `govulncheck ./...`. +- `git diff --check`: PASS at the first review checkpoint. + +## [C] Checkpoint #1 + +- CodeRabbit reviewed the complete uncommitted diff against `710a428` after a credential-pattern + scan and returned zero findings across all 22 changed files. Its explicit review scope covered + carrier injection, trace privacy, observer fault isolation, and accidental `intent_id` claims. +- `make ci`: PASS after one test-only staticcheck correction. It ran gofmt, golangci-lint (0 issues), + `govulncheck` (no vulnerabilities), the full race/coverage suite, static source boundaries, + binary E2E, latency check, and reproducible build. Relevant final coverage: tracing 86.8%, + observability 92.7%, PEP 83.9%, hubfleet 69.6%, and hubserver 89.5%. +- `make e2e-isolation`: PASS. It ran race-enabled pinned PostgreSQL isolation/destructive suites + (hubauth 85.2%, hubserver 89.5%, fleetcache 87.0%, hubdb 73.5%) and the fixed 50,000-execution + workspace selector fuzz campaign. +- `make release-check`: PASS. GoReleaser reproduced Darwin/Linux amd64/arm64 snapshot archives, + generated SPDX SBOMs/checksums twice, and generated `dist/sith.rb`. +- `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind`: PASS (165.233s) against temporary + real two-cluster fan-out and OCI image contract gates. `kind get clusters` is empty after test + cleanup. +- Final local cleanup: `docker system prune -af` reclaimed 2.226 GB of unused test/build artifacts; + active unrelated `elated_antonelli` and `ardur-191-baseline` containers remain running. +- Final GitHub security queues before commit: Dependabot 0, code-scanning 0, secret-scanning 0. + +## [C] Checkpoint #2 + +- Source is ready for README recheck, signed/DCO/GSTACK commit `2026-07-14/e10-trace-context#1`, + PR, review/CI, exact post-merge CI, and issue/roadmap updates. diff --git a/sessions/2026-07-14-e2-hub-direct-runtime.md b/sessions/2026-07-14-e2-hub-direct-runtime.md new file mode 100644 index 0000000..18584a9 --- /dev/null +++ b/sessions/2026-07-14-e2-hub-direct-runtime.md @@ -0,0 +1,62 @@ +# Session — 2026-07-14 — e2-hub-direct-runtime + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e2-hub-direct-runtime` +**Slice(s):** E2 / [#125](https://github.com/ArdurAI/sith/issues/125) · **Status:** ready-to-commit + +--- + +[G] Goal: deliver the Phase-1 hub runtime for the direct OCM transport: mount only authenticated, +workspace-scoped fleet refresh/read routes, use the existing `hubfleet.Transport` seam, and deploy +only from in-cluster identity and fixed read-only mounts. + +[S] Scope: authenticated fleet HTTP routes, runtime composition, direct OCM M0 integration, shared +test fixture, CLI wiring, privacy declarations, release documentation, and the gRPC security update. +Exchange routes and arbitrary caller-supplied cluster targets remain intentionally unmounted. + +[A] Action: runtime configuration fails closed unless every listener, database, session verifier, +hub TLS mount, proxy mTLS mount, and Kubernetes API name is explicitly supplied. It accepts only +read-only regular mounted files, uses `rest.InClusterConfig` without kubeconfig fallback, and +constructs the managed-service-account reader, RLS database store, PEP audit logger, direct OCM +adapter, collector, and authenticated handler as one bounded TLS server. + +[A] Action: the fleet API derives a fresh server-side `tenancy.Scope` from the verified Sith +session on each request. Fixed paths and methods reject query parameters, encoded path ambiguity, +foreign workspaces, unauthenticated requests, and dependency detail leakage. No credential, +transport target, selector, or cluster endpoint crosses the public boundary. + +[A] Action: updated `google.golang.org/grpc` to 1.79.3 for CVE-2026-33186 and verified +`govulncheck` reports no vulnerabilities. The current Konnectivity client, including v0.36.0, +still exposes only a legacy `grpc.DialContext`-based tunnel constructor. A one-expression +`staticcheck` waiver retains `grpc.WithBlock` because removing it allowed an already-cancelled +creation context to attempt a proxy connection. The adjacent regression test proves the bounded +fail-closed behavior; no project-wide lint rule is changed. + +[T] Test: targeted `go test -race -count=1 ./internal/hubocm ./internal/hubruntime` and +`govulncheck ./...` passed. The final guarded `make e2e-ocm` run passed in 215 seconds: it registered a +real hub plus two spokes, used scoped managed-service-account identity, actively denied hub node +and pod ingress, observed zero hub-initiated flows, exercised both direct adapter and runtime TLS +tests, emitted both required test-pass markers, and deleted every `sith-m0-*` Kind cluster afterward. + +[T] Test: `make ci` passed format, vet, lint, vulnerability scan, repository-wide race/coverage, +17 M0 harness safety assertions, latency, binary E2E, and build. `make release-check` passed +snapshot archives for Darwin/Linux on amd64/arm64, SPDX SBOM generation, checksums, and Homebrew +formula rendering. `kind get clusters` reported none after validation. + +[R] Review: red-team checks exercised path encoding, query rejection, workspace spoofing, token +and dependency-error redaction, unsafe mount rejection, and caller cancellation. The local port +8090 collision belongs to an unrelated active user process; the M0 harness defers only its +`clusteradm proxy health` check to the mandatory direct runtime gate, which passed. No user +process or unrelated active Docker container was stopped. CodeRabbit’s first pass found two valid +test-quality gaps, both fixed and rerun. Its follow-up PEP concern was rejected after source +inspection: `hubfleet.Collector.Collect` already authorizes and audits +`VerbSpokeSnapshotRefresh` before any spoke read, so handler duplication would create two policy +decisions. Its provenance documentation concern was fixed in the README. + +[C] Checkpoint #1: SSH-signed DCO/GSTACK implementation commit created with +`GSTACK-Checkpoint: 2026-07-14/e2-hub-direct-runtime#1`; next: push, open the reviewed PR into +`dev`, and verify exact post-merge CI. + +--- + +**Session close:** runtime ready for independent review and landing · **Open questions touched:** +none diff --git a/sessions/2026-07-14-e2-image-digest-search.md b/sessions/2026-07-14-e2-image-digest-search.md new file mode 100644 index 0000000..521ace1 --- /dev/null +++ b/sessions/2026-07-14-e2-image-digest-search.md @@ -0,0 +1,57 @@ +# Session — 2026-07-14 — e2-image-digest-search + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e2-image-digest-search` +**Slice(s):** E2 / [#127](https://github.com/ArdurAI/sith/issues/127) · **Status:** ready-to-commit + +--- + +[G] Goal: deliver the immutable image-evidence foundation for F2.4: read the runtime-resolved +digest from ordinary Pod container status, persist only a bounded canonical projection, and answer +one exact workspace-scoped fleet lookup with honest coverage. + +[S] Scope: exact lowercase SHA-256 digests from `Pod.Status.ContainerStatuses[].ImageID`, direct +OCM snapshot normalization, forced-RLS persistence and GIN lookup, one signed-session hub route, +and two-spoke evidence. Registry calls, image pulls, SBOM/CVE/feed retrieval, tags/prefixes, +arbitrary selectors, writes, credentials, init containers, and ephemeral containers are excluded. + +[A] Action: accepted only the known `containerd`, `docker-pullable`, `cri-o`, and `docker` runtime +prefixes (or a bare canonical digest), then retained only `sha256:<64 lowercase hex>`. Unknown, +mutable, malformed, ambiguous, init, and ephemeral values abstain. Inventory validates a sorted, +unique, at-most-64 digest list only for Pod facts; no raw Pod/status payload crosses the store seam. + +[A] Action: added a closed `fleet.image.search` PEP verb and a narrow `ImageSearcher`. It authorizes +the signed tenancy scope before its query port, hashes canonical validated arguments for audit, and +issues only `FactInventory + Pod + exact digest` queries. PostgreSQL uses parameterized JSONB array +membership behind forced RLS with a partial GIN expression index; labels, names, prefixes, health, +and CVE selectors remain rejected in this shape. + +[A] Action: mounted only `GET /v1/workspaces/{workspace}/fleet/images/{sha256:<64-lowercase-hex>}`. +The parser rejects queries, encoded ambiguity, noncanonical values, foreign workspace scopes, and +wrong methods. The route exposes no target, selector, freshness override, transport, or credential; +errors remain generic and responses are non-cacheable. + +[T] Test: focused race suites for `fleet`, `hubocm`, `hubfleet`, `hubdb`, `hubserver`, and +`hubruntime` passed. `make e2e-postgres` passed with 71.5% package coverage, including the new +migration, exact two-spoke result, and cross-workspace negative control. `make e2e-ocm` passed in +the real hub-plus-two-spoke M0 lab: direct adapter and TLS runtime tests observed the fixture's +actual runtime digest on both spokes and the harness deleted every temporary Kind cluster. + +[T] Test: final `go mod verify`, `govulncheck ./...`, `make ci`, and `make release-check` passed. +CI covered format, vet, lint, full race suites, safety harness, E2E, build, and a dual Darwin/Linux +amd64/arm64 reproducibility run with SPDX SBOMs. GitHub queues were checked immediately before +publication: Dependabot 0, code scanning 0, secret scanning 0. + +[R] Review: manual red-team review covered hostile runtime schemes, mutable strings, untrusted +JSON, parameterized exact lookup, RLS/cross-workspace isolation, policy refusal before query, +fixed-path/query/escape handling, safe errors, and no raw credential/status persistence. CodeRabbit +CLI v0.6.5 was authenticated and submitted the final uncommitted diff twice; each remote review +entered `reviewing` but emitted no finding before continuing to heartbeat, so both runs were stopped +after bounded waits. This external review-service delay is recorded, not treated as a Sith blocker. + +[C] Checkpoint #1: next: SSH-signed DCO/GSTACK implementation commit, push, PR into `dev`, exact +post-merge CI, issue/roadmap updates, and the next ordered unblocked backlog slice. + +--- + +**Session close:** exact image evidence ready for independent review and landing · **Open questions +touched:** none diff --git a/sessions/2026-07-14-e9-chart-profiles.md b/sessions/2026-07-14-e9-chart-profiles.md new file mode 100644 index 0000000..4f42696 --- /dev/null +++ b/sessions/2026-07-14-e9-chart-profiles.md @@ -0,0 +1,76 @@ +# E9 F9.3a fail-closed hub resource profiles + +Issue: [#135](https://github.com/ArdurAI/sith/issues/135) + +Branch: `gnanirahulnutakki/feat/e9-chart-profiles` + +Base: `origin/dev` at `741a5e5c26a5fe9374d26d66f82ef904fdfdb76a` + +## [G] Goal + +Add the first independently shippable F9.3 sub-slice: two bounded hub-chart resource envelopes +that preserve every existing admission, credential, and workload-security invariant. This is not a +claim that the still-unpublished chart already supplies an in-chart database, high availability, or +cloud-KMS topology. + +## [S] Scope + +- Version the chart `0.2.0` and add exactly `light` and `heavy` profile choices. +- Apply fixed CPU/memory requests and limits to both the long-running hub and its short-lived + migration hook. Light requests 100m CPU/128Mi and caps at 500m/512Mi; heavy requests 500m/512Mi + and caps at 2 CPU/2Gi. +- Reject unknown profiles and the removed arbitrary `resources` escape hatch through both the + value schema and template logic, including when Helm schema validation is bypassed. +- Assert real Helm renders for both profiles. After removing only container resource blocks from + deep-copied rendered objects, require every manifest to be structurally identical. +- Document the resource and cost envelope truthfully. Do not render an in-chart database, KMS + materializer, HA replica topology, public image, broad egress policy, or a spoke-agent addon. + +## [A] Analysis and red-team checks + +- The parent F9.3 eventual design describes light/minimal-Postgres and heavy/HA/external-Postgres/ + cloud-KMS topology. Those dependencies do not exist yet: KMS custody belongs to E3 and a safe HA + claim needs its own operational evidence. Issue #135 was therefore explicitly scoped and renamed + F9.3a, preventing the profile labels from overstating current behavior. +- The profile is a closed choice, not a free-form resource object. Resource overrides create + unreviewed scheduling and cost behavior, so a top-level `resources` value fails even under + `--skip-schema-validation` rather than being silently ignored. +- Heavy reserves five times the requested CPU and four times the requested memory. This is a + bounded scheduling/cost envelope, not a capacity recommendation; measured deployment sizing is + deferred. +- The first peer review correctly flagged the communication risk that the docs could be read as + implementing the parent F9.3 topology. The implementation was not broadened to make unproven + claims. Instead, the issue and three operator-facing documents now label this as F9.3a and state + the deferred topology/custody work. The final independent review found zero issues. + +## [T] Tests and evidence + +- Test-first check: the new real-Helm contract initially failed against the prior chart because + `profile` was an unknown value. It passed after the selector and render assertions were added. +- Final focused command: `make e2e-helm HELM=/Volumes/EXTENDED/MacData/tools/bin/helm-v4.2.2` + PASS (2.130s) after the documentation clarification. It lints/renders both profiles, asserts + exact resources and non-resource equality, and rejects mutable images, malformed/missing image + digests, blank runtime Secret names, unknown profiles, image-pull-secret input, and resource + overrides; mutable images, unknown profiles, and resource overrides also fail with + `--skip-schema-validation`. +- `make ci`: PASS (format, vet, golangci-lint, govulncheck with no vulnerabilities, race/coverage + tests, source-boundary and operator-script safety checks, binary E2E, latency check, and + reproducible build). +- `make e2e-isolation`: PASS, including digest-pinned PostgreSQL RLS/destructive isolation and the + fixed 50,000-execution selector fuzz campaign. Final coverage: hubauth 85.2%, hubserver 90.0%, + hubdb destructive suite 73.5%. +- `make release-check`: PASS; GoReleaser reproduced Darwin/Linux amd64/arm64 snapshot archives, + generated SPDX SBOMs and checksums, and rendered the Homebrew formula twice. +- `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind`: PASS (161.993s) against real + temporary two-cluster fan-out; test cleanup removed both clusters. +- Final CodeRabbit uncommitted-diff review against `741a5e5`: PASS, zero findings. The first + review's one documentation-scope finding was resolved by the explicit F9.3a boundary above. +- `git diff --check`: PASS. Final GitHub queues before commit: Dependabot 0, code scanning 0, + secret scanning 0. `kind get clusters` reports none. Docker prune reclaimed 1.364 GB of only + disposable Kind/build artifacts while preserving active `elated_antonelli` and + `ardur-191-baseline` containers. + +## [C] Checkpoint #1 + +- Final source is ready for signed/DCO commit + `2026-07-14/e9-chart-profiles#1`; PR, merge, and exact post-merge `dev` evidence remain pending. diff --git a/sessions/2026-07-14-e9-helm-contract.md b/sessions/2026-07-14-e9-helm-contract.md new file mode 100644 index 0000000..4f136b0 --- /dev/null +++ b/sessions/2026-07-14-e9-helm-contract.md @@ -0,0 +1,78 @@ +# E9 fail-closed Helm hub chart contract + +Issue: [#133](https://github.com/ArdurAI/sith/issues/133) + +Branch: `gnanirahulnutakki/feat/e9-helm-contract` + +Base: `origin/dev` at `401431bcbeb571f8440dcc6f0fff556a021df773` + +## [G] Goal + +Provide a reviewable, install-time admission boundary for the unpublished Sith hub: an operator +must provide an immutable hub image and references to already-provisioned runtime and migration +Secrets before Helm can render the release. + +## [S] Scope + +- Add a versioned Helm v2 chart that accepts only an explicit `repository@sha256:<64 lowercase + hex>` hub image reference; reject tags and digest-less references even if Helm schema validation + is bypassed. +- Render only references to existing runtime and migration Secrets. The chart never creates a + Secret or renders `data`/`stringData` material. +- Run `sith hub migrate` as a short-lived pre-install/pre-upgrade hook with the owner database URL + and no Kubernetes API token; run the long-lived hub with the non-owner runtime configuration. +- Deploy the hub with a non-root identity, a read-only root filesystem, dropped Linux capabilities, + `privileged: false`, no privilege escalation, and `RuntimeDefault` seccomp. +- Grant the hub only `get` on core `secrets` named `sith-reader` through a ClusterRole. No list, + watch, wildcard resource, or migration ServiceAccount access is rendered. +- Pin Helm v4.2.2 by verified upstream SHA-256 in CI and inspect a real Helm render under the race + detector. This slice does not publish an image, provision a KMS/external-secret materializer, or + impose a broad egress NetworkPolicy; cluster operators retain network-policy ownership. + +## [A] Analysis and red-team checks + +- `values.schema.json` rejects unknown inputs and invalid required values, while the image template + independently calls `required`, `regexMatch`, and `fail`. A caller using + `--skip-schema-validation` therefore cannot bypass the immutable-image admission rule. +- The Deployment service selector is tested against the Pod template labels, and the + ClusterRoleBinding subject is tested against the Deployment service-account name, preventing + future name/selector drift from silently disconnecting the hub or its constrained permission. +- The runtime Secret volume has only the required public/session and TLS key files. The database + URL is read directly from the existing runtime Secret and no secret value crosses the rendered + manifest or test output. +- Helm hooks are deliberately bounded (`backoffLimit: 0`, 300-second deadline, one-hour TTL) and + include the hook deletion policy. A failed migration blocks install/upgrade before the non-owner + hub deployment is accepted. +- Initial peer review identified four hardening gaps: an unverified binding subject, unchecked + Service selector/ports, implicit `privileged`, and permissive float-to-int test conversion. + All were corrected. The final independent review against `401431b` returned zero findings. + +## [T] Tests and evidence + +- Official Helm v4.2.2 was downloaded over HTTPS and verified against the upstream darwin/arm64 + SHA-256 before local contract testing. +- `make e2e-helm HELM=/Volumes/EXTENDED/MacData/tools/bin/helm-v4.2.2`: PASS (2.057s) after the + final review corrections. It runs `helm lint`, renders valid values, rejects mutable and + digest-less images, blank Secret references, and unknown keys, and proves the mutable image is + still rejected with `--skip-schema-validation`. +- `make ci`: PASS (format, vet, golangci-lint, govulncheck with no vulnerabilities, race/coverage + tests, source-boundary and operator-script checks, binary E2E, latency check, and reproducible + build). +- `make e2e-isolation`: PASS, including PostgreSQL RLS/destructive isolation and the fixed + 50,000-execution selector fuzz campaign. Final coverage: hubauth 85.2%, hubserver 90.0%, hubdb + destructive suite 73.5%. +- `make release-check`: PASS, including reproducible Darwin/Linux amd64/arm64 snapshot archives, + SPDX SBOM generation, checksums, and repeated Homebrew formula rendering. +- `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind`: PASS (159.994s) against the real + two-cluster contract; cleanup removed both temporary clusters. +- Final CodeRabbit uncommitted-diff review against `401431b`: PASS, zero findings across all chart, + CI, documentation, and Helm-test files. +- `git diff --check`: PASS. Final GitHub queues before commit: Dependabot 0, code scanning 0, + secret scanning 0. `kind get clusters` reports none; Docker prune reclaimed 1.364 GB of only + unused kind/build artifacts while preserving the active `elated_antonelli` and + `ardur-191-baseline` containers. + +## [C] Checkpoint #1 + +- Final source is ready for signed/DCO commit + `2026-07-14/e9-helm-contract#1`; PR, merge, and exact post-merge `dev` evidence remain pending. diff --git a/sessions/2026-07-14-e9-hub-migrate.md b/sessions/2026-07-14-e9-hub-migrate.md new file mode 100644 index 0000000..fc43298 --- /dev/null +++ b/sessions/2026-07-14-e9-hub-migrate.md @@ -0,0 +1,59 @@ +# Session — 2026-07-14 — e9-hub-migrate + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e9-hub-migrate` +**Slice(s):** E9 / [#129](https://github.com/ArdurAI/sith/issues/129) · **Status:** ready-to-commit + +--- + +[G] Goal: add the isolated schema-migration entry point needed before the E9 Helm deployment +chart can safely run `sith hub` with only its non-owner application database role. + +[S] Scope: `sith hub migrate`, environment loading for exactly one owner database URL and one +application role, existing checksum-ledger/RLS migration enforcement, focused documentation, and +real PostgreSQL and multi-cluster evidence. Helm templates, chart values, database provisioning, +the hub listener, Kubernetes clients, collection, and any upstream ClusterGateway release change +remain out of scope. + +[A] Action: added a one-shot `hubdb.Migrate` boundary that rejects missing/ambiguous URLs, +same-owner/application roles, and remote plaintext transport before dialing. It opens one owner +connection, applies existing serializable checksum-locked migrations and forced-RLS audit, then +performs a bounded five-second best-effort close. Transaction commit is the success boundary, so a +post-commit close acknowledgement cannot misclassify a completed migration Job as failed. + +[A] Action: added `sith hub migrate` and a separate runtime environment loader. The subcommand +does not construct the hub TLS listener, in-cluster Kubernetes identity/client, OCM transport, PEP, +collector, or application pool. Documentation explicitly keeps the owner credential out of the hub +Deployment, chart values, and logs. + +[T] Test: final focused race suites passed. `make e2e-postgres` passed with 71.3% `hubdb` +coverage, including first-run and idempotent migration via the new public seam. `make e2e-isolation` +passed (`hubdb` 73.5%) and completed the fixed 50,002-execution cross-workspace fuzz campaign. +Final `make ci` passed formatting, vet, lint, `govulncheck`, full race tests, safety harness, E2E, +and build. + +[T] Test: final `make e2e-kind` passed in 88.8 seconds. Final `make e2e-ocm` passed the real +hub-plus-two-spoke M0 lab: scoped managed-serviceaccount identity, active hub-to-node/pod deny, +and direct proxy/runtime tests all passed; the harness deleted all temporary Kind clusters. Local +port 8090 was occupied by an unrelated service, so only the auxiliary `clusteradm` proxy probe was +deferred; the mandatory direct E2E gate passed. Final `make release-check` passed module +verification, reproducible Darwin/Linux amd64/arm64 artifacts, SPDX SBOMs, and checksums. + +[R] Review: manual red-team review covered role separation, remote plaintext rejection, no owner +credential path into the long-running hub, cancellation-safe bounded cleanup, RLS preservation, +and documentation secrecy. CodeRabbit CLI v0.6.5 reviewed the uncommitted diff three times. It +first identified post-commit close status and unbounded cleanup; both were fixed and revalidated. +The final review completed with zero findings. + +[S] Security: immediately before publication, GitHub queues were Dependabot 0, code scanning 0, +and secret scanning 0. Docker prune reclaimed 1.217 GB of only unused resources; no Kind clusters +remain and the two pre-existing unrelated containers remain running. + +[C] Checkpoint #1: next: read README once more, create SSH-signed DCO/GSTACK commit as +`gnani.nutakki@gmail.com`, push, open PR into `dev`, force-merge only after green review/CI, verify +the exact post-merge `dev` CI and security queues, close #129, update #27/#39, then take the next +unblocked backlog slice. + +--- + +**Session close:** migration-command prerequisite ready for independent review and landing · +**Open questions touched:** none diff --git a/sessions/2026-07-14-e9-oci-image.md b/sessions/2026-07-14-e9-oci-image.md new file mode 100644 index 0000000..f5d64e2 --- /dev/null +++ b/sessions/2026-07-14-e9-oci-image.md @@ -0,0 +1,66 @@ +# Session — 2026-07-14 — e9-oci-image + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e9-oci-image` +**Slice(s):** E9 / [#131](https://github.com/ArdurAI/sith/issues/131) · **Status:** ready-to-commit + +--- + +[G] Goal: provide the immutable, least-privilege OCI artifact contract the E9 Helm deployment +slice needs, without publishing an image or expanding the hub runtime's authority. + +[S] Scope: a pinned distroless static runtime, static Linux binary-only build context, non-root +entrypoint, local linux/amd64 and linux/arm64 inspection, hardened real-Kind execution, CI +coverage, and deployment/release documentation. Helm templates, registry publishing, signatures, +attestations, image tags, secrets, service-account credentials, and chart values remain out of +scope. + +[A] Action: added `Containerfile` with the verified multi-platform digest for +`gcr.io/distroless/static-debian12`, copying only the existing static target-architecture Sith +binary as UID/GID `65532`. The contract permits exactly one `FROM`, `ARG TARGETARCH`, `COPY`, +`USER`, and `ENTRYPOINT`, and rejects package/fetch or unrecognized instructions. It has no shell, +package manager, default configuration, Kubernetes credential, database URL, or secret. + +[A] Action: added local OCI integration coverage for both target architectures. The native image +runs as non-root with a read-only filesystem, no network, all capabilities dropped, and +`no-new-privileges`; an attempted shell override must fail. The native image is also loaded and +executed as a tokenless, non-root, read-only, RuntimeDefault-seccomp Job on each of two real Kind +clusters. CI now runs the dual-architecture OCI contract before Kind tests. + +[A] Action: documented that no OCI image is currently published and that a future Helm chart must +require `repository@sha256:...`, never a mutable tag. The no-network constraint is limited to the +isolated verification path; a real hub deployment needs narrowly allowlisted egress only to +configured runtime dependencies such as the database and, if enabled, pinned OIDC discovery/JWKS +endpoints. + +[T] Test: final `make e2e-oci` passed in 10.620 seconds. It covered both `linux/amd64` and +`linux/arm64`, the native hardened execution, no-shell negative case, and instruction-contract +regressions for comments, duplicate `FROM`/`USER`/`ENTRYPOINT`, and unrecognized directives. +Final `make e2e-kind` passed in 159.641 seconds, including its two hardened real-cluster Jobs. + +[T] Test: final `make ci` passed formatting, vet, lint (0 issues), `govulncheck` (no +vulnerabilities), full race suites, the M0 safety harness (17 assertions), binary E2E, and build. +Final `make e2e-isolation` passed destructive PostgreSQL coverage (`hubauth` 85.2%, `hubserver` +90.0%, `hubdb` 73.5%) and the fixed 50,031-execution cross-workspace fuzz campaign. Final +`make release-check` passed module verification and two reproducible Darwin/Linux amd64/arm64 +archives with SPDX SBOMs, checksums, and the Homebrew formula. + +[R] Review: manual red-team review checked digest pinning, static-only contents, non-root identity, +read-only/no-capability/no-token Kind execution, absence of registry publication, no-shell +enforcement, and runtime egress clarity. CodeRabbit CLI v0.6.5 first identified ambiguous runtime +egress wording, overly loose instruction matching, and an incomplete Kind API-error path. All +three were corrected with regression coverage and the final review completed with zero findings. + +[S] Security: immediately before publication, GitHub queues were Dependabot 0, code scanning 0, +and secret scanning 0. Docker prune reclaimed 1.453 GB of unused resources; no Kind clusters +remain and the two pre-existing unrelated containers remain running. + +[C] Checkpoint #1: next: read README once more, create SSH-signed DCO/GSTACK commit as +`gnani.nutakki@gmail.com`, push, open PR into `dev`, merge only after green review/CI, verify the +exact post-merge `dev` CI and security queues, close #131, update #27/#39, then take the next +unblocked E9 chart slice. + +--- + +**Session close:** OCI deployment artifact contract ready for independent review and landing · +**Open questions touched:** public registry/release-bound signing and attestations intentionally +remain a later release-boundary slice. diff --git a/tests/e2e/helm_chart_test.go b/tests/e2e/helm_chart_test.go new file mode 100644 index 0000000..ff5c344 --- /dev/null +++ b/tests/e2e/helm_chart_test.go @@ -0,0 +1,504 @@ +// SPDX-License-Identifier: Apache-2.0 +//go:build e2e && helm + +package e2e_test + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" +) + +const ( + helmContractVersion = "v4.2.2" + validHubImage = "registry.example.invalid/sith/hub@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +) + +type helmProfileResources struct { + requests map[string]string + limits map[string]string +} + +var hubProfileResources = map[string]helmProfileResources{ + "light": { + requests: map[string]string{"cpu": "100m", "memory": "128Mi"}, + limits: map[string]string{"cpu": "500m", "memory": "512Mi"}, + }, + "heavy": { + requests: map[string]string{"cpu": "500m", "memory": "512Mi"}, + limits: map[string]string{"cpu": "2", "memory": "2Gi"}, + }, +} + +func TestHelmHubChartContract(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) + defer cancel() + root := repositoryRoot(t) + helm := os.Getenv("HELM_BIN") + if helm == "" { + helm = "helm" + } + if _, err := exec.LookPath(helm); err != nil { + t.Fatalf("find Helm binary %q: %v", helm, err) + } + if output, err := runHelm(ctx, t, helm, root, "version", "--short"); err != nil || !strings.HasPrefix(strings.TrimSpace(output), helmContractVersion) { + t.Fatalf("Helm version/output = %q / %v, want %s", output, err, helmContractVersion) + } + + chart := filepath.Join(root, "charts", "sith-hub") + lightValues := writeHelmValues(t, validHubValues()) + if output, err := runHelm(ctx, t, helm, root, "lint", chart, "--values", lightValues); err != nil { + t.Fatalf("helm lint valid chart: %v\n%s", err, output) + } + lightRendered, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--values", lightValues) + if err != nil { + t.Fatalf("helm template light profile: %v\n%s", err, lightRendered) + } + lightObjects := assertHelmHubRender(t, lightRendered, "light") + heavyRendered, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--values", writeHelmValues(t, profileHubValues("heavy"))) + if err != nil { + t.Fatalf("helm template heavy profile: %v\n%s", err, heavyRendered) + } + heavyObjects := assertHelmHubRender(t, heavyRendered, "heavy") + assertProfileOnlyChangesResources(t, lightObjects, heavyObjects) + + for name, invalid := range map[string]string{ + "mutable tag": strings.Replace(validHubValues(), validHubImage, "registry.example.invalid/sith/hub:latest", 1), + "missing digest": strings.Replace(validHubValues(), validHubImage, "registry.example.invalid/sith/hub", 1), + "blank runtime secret": strings.Replace(validHubValues(), "existingSecret: sith-runtime", "existingSecret: \"\"", 1), + "unknown profile": strings.Replace(validHubValues(), "profile: light", "profile: unbounded", 1), + "unexpected image pull secret": validHubValues() + "\nimagePullSecrets:\n password: must-not-render\n", + "unexpected resources": validHubValues() + "\nresources:\n requests:\n cpu: 999\n", + } { + t.Run(name, func(t *testing.T) { + if output, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--values", writeHelmValues(t, invalid)); err == nil { + t.Fatalf("helm template accepted %s values:\n%s", name, output) + } + }) + } + + for name, invalid := range map[string]string{ + "mutable image": strings.Replace(validHubValues(), validHubImage, "registry.example.invalid/sith/hub:latest", 1), + "unknown profile": strings.Replace(validHubValues(), "profile: light", "profile: unbounded", 1), + "unexpected resources": validHubValues() + "\nresources:\n requests:\n cpu: 999\n", + } { + t.Run("skip schema validation "+name, func(t *testing.T) { + if output, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--skip-schema-validation", "--values", writeHelmValues(t, invalid)); err == nil { + t.Fatalf("helm template --skip-schema-validation accepted %s:\n%s", name, output) + } + }) + } +} + +func validHubValues() string { + return profileHubValues("light") +} + +func profileHubValues(profile string) string { + return fmt.Sprintf(`profile: %s +image: + reference: %s +runtime: + existingSecret: sith-runtime + sessionIssuer: https://issuer.sith.example + sessionAudience: https://hub.sith.example + sessionKeyID: session-2026-07 + proxyAddress: cluster-proxy.open-cluster-management.svc:8090 + proxyServerName: cluster-proxy.open-cluster-management.svc + kubeAPIServerName: kubernetes +migration: + existingSecret: sith-migration + applicationRole: sith_app +`, profile, validHubImage) +} + +func writeHelmValues(t *testing.T, contents string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "values.yaml") + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatalf("write Helm values: %v", err) + } + return path +} + +func runHelm(ctx context.Context, t *testing.T, helm, root string, args ...string) (string, error) { + t.Helper() + command := exec.CommandContext(ctx, helm, args...) + command.Dir = root + scratch := t.TempDir() + command.Env = append(os.Environ(), + "HELM_CACHE_HOME="+filepath.Join(scratch, "cache"), + "HELM_CONFIG_HOME="+filepath.Join(scratch, "config"), + "HELM_DATA_HOME="+filepath.Join(scratch, "data"), + "HELM_PLUGINS="+filepath.Join(scratch, "plugins"), + ) + output, err := command.CombinedOutput() + return string(output), err +} + +func assertHelmHubRender(t *testing.T, rendered, profile string) []*unstructured.Unstructured { + t.Helper() + if strings.Contains(rendered, "kind: Secret") || strings.Contains(rendered, "stringData:") || strings.Contains(rendered, "\ndata:") { + t.Fatal("rendered chart created or embedded secret data") + } + objects := decodeHelmObjects(t, rendered) + if len(objects) != 6 { + t.Fatalf("rendered object count = %d, want 6", len(objects)) + } + for _, object := range objects { + if object.GetNamespace() != "" { + t.Fatalf("%s unexpectedly sets namespace %q", object.GetKind(), object.GetNamespace()) + } + } + + serviceAccountName, selectorLabels := assertHubDeployment(t, requiredHelmObject(t, objects, "Deployment"), profile) + assertHubRBAC(t, requiredHelmObject(t, objects, "ClusterRole"), requiredHelmObject(t, objects, "ClusterRoleBinding"), serviceAccountName) + assertMigrationJob(t, requiredHelmObject(t, objects, "Job"), profile) + assertHubService(t, requiredHelmObject(t, objects, "Service"), selectorLabels) + serviceAccount := requiredHelmObject(t, objects, "ServiceAccount") + if value, found, _ := unstructured.NestedBool(serviceAccount.Object, "automountServiceAccountToken"); !found || !value { + t.Fatal("hub ServiceAccount must explicitly mount its in-cluster token") + } + return objects +} + +func decodeHelmObjects(t *testing.T, rendered string) []*unstructured.Unstructured { + t.Helper() + decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewBufferString(rendered), 4096) + var objects []*unstructured.Unstructured + for { + var object map[string]any + err := decoder.Decode(&object) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("decode Helm output: %v", err) + } + if len(object) != 0 { + objects = append(objects, &unstructured.Unstructured{Object: object}) + } + } + return objects +} + +func requiredHelmObject(t *testing.T, objects []*unstructured.Unstructured, kind string) *unstructured.Unstructured { + t.Helper() + var found *unstructured.Unstructured + for _, object := range objects { + if object.GetKind() != kind { + continue + } + if found != nil { + t.Fatalf("rendered more than one %s", kind) + } + found = object + } + if found == nil { + t.Fatalf("rendered no %s", kind) + } + return found +} + +func assertHubDeployment(t *testing.T, deployment *unstructured.Unstructured, profile string) (string, map[string]any) { + t.Helper() + podSpec := nestedHelmMap(t, deployment.Object, "spec", "template", "spec") + if value, found, _ := unstructured.NestedBool(podSpec, "automountServiceAccountToken"); !found || !value { + t.Fatal("hub Deployment must deliberately mount its in-cluster service-account token") + } + assertHelmPodSecurity(t, podSpec, true) + container := onlyHelmContainer(t, podSpec) + if container["image"] != validHubImage || container["imagePullPolicy"] != "IfNotPresent" { + t.Fatalf("hub image contract = %#v", container) + } + assertHelmContainerSecurity(t, container) + assertHelmProfileResources(t, container, profile) + environment := helmEnvironment(t, container) + if len(environment) != 14 || environment["SITH_HUB_DATABASE_URL"] != "secret:sith-runtime/database-url" { + t.Fatalf("hub environment = %#v", environment) + } + for _, name := range []string{"SITH_HUB_SESSION_PUBLIC_KEY_FILE", "SITH_HUB_SERVER_TLS_CERT_FILE", "SITH_HUB_SERVER_TLS_KEY_FILE", "SITH_HUB_PROXY_CA_FILE", "SITH_HUB_PROXY_CERT_FILE", "SITH_HUB_PROXY_KEY_FILE"} { + if !strings.HasPrefix(environment[name], "/var/run/sith/runtime/") { + t.Fatalf("hub mounted path %s = %q", name, environment[name]) + } + } + volumes, found, err := unstructured.NestedSlice(podSpec, "volumes") + if err != nil || !found || len(volumes) != 1 { + t.Fatalf("hub volumes = %#v / %v", volumes, err) + } + volume, ok := volumes[0].(map[string]any) + if !ok { + t.Fatalf("hub volume = %#v", volumes[0]) + } + secret := nestedHelmMap(t, volume, "secret") + if volume["name"] != "runtime" || secret["secretName"] != "sith-runtime" || helmInt(t, secret["defaultMode"]) != 288 { + t.Fatalf("hub runtime Secret volume = %#v", volume) + } + serviceAccountName, ok := podSpec["serviceAccountName"].(string) + if !ok || serviceAccountName == "" { + t.Fatalf("hub service account = %#v", podSpec["serviceAccountName"]) + } + return serviceAccountName, nestedHelmMap(t, deployment.Object, "spec", "template", "metadata", "labels") +} + +func assertHubService(t *testing.T, service *unstructured.Unstructured, wantSelector map[string]any) { + t.Helper() + spec := nestedHelmMap(t, service.Object, "spec") + if spec["type"] != "ClusterIP" { + t.Fatalf("service spec = %#v", spec) + } + ports, found, err := unstructured.NestedSlice(spec, "ports") + if err != nil || !found || len(ports) == 0 { + t.Fatalf("service ports = %#v / %v", ports, err) + } + if !reflect.DeepEqual(nestedHelmMap(t, spec, "selector"), wantSelector) { + t.Fatalf("service selector = %#v, want pod labels %#v", spec["selector"], wantSelector) + } +} + +func assertHubRBAC(t *testing.T, role, binding *unstructured.Unstructured, serviceAccountName string) { + t.Helper() + rules, found, err := unstructured.NestedSlice(role.Object, "rules") + if err != nil || !found || len(rules) != 1 { + t.Fatalf("hub ClusterRole rules = %#v / %v", rules, err) + } + rule, ok := rules[0].(map[string]any) + if !ok || !reflect.DeepEqual(stringSlice(t, rule["apiGroups"]), []string{""}) || !reflect.DeepEqual(stringSlice(t, rule["resources"]), []string{"secrets"}) || !reflect.DeepEqual(stringSlice(t, rule["resourceNames"]), []string{"sith-reader"}) || !reflect.DeepEqual(stringSlice(t, rule["verbs"]), []string{"get"}) { + t.Fatalf("hub ClusterRole is broader than the fixed reader Secret: %#v", rule) + } + roleRef := nestedHelmMap(t, binding.Object, "roleRef") + if roleRef["kind"] != "ClusterRole" || roleRef["name"] != role.GetName() { + t.Fatalf("ClusterRoleBinding roleRef = %#v", roleRef) + } + subjects, found, err := unstructured.NestedSlice(binding.Object, "subjects") + if err != nil || !found || len(subjects) != 1 { + t.Fatalf("ClusterRoleBinding subjects = %#v / %v", subjects, err) + } + subject, ok := subjects[0].(map[string]any) + if !ok || subject["kind"] != "ServiceAccount" || subject["name"] != serviceAccountName || subject["namespace"] != "sith-system" { + t.Fatalf("ClusterRoleBinding subject = %#v", subject) + } +} + +func assertMigrationJob(t *testing.T, job *unstructured.Unstructured, profile string) { + t.Helper() + annotations := job.GetAnnotations() + if annotations["helm.sh/hook"] != "pre-install,pre-upgrade" || annotations["helm.sh/hook-delete-policy"] != "before-hook-creation,hook-succeeded" || annotations["helm.sh/hook-weight"] != "-10" { + t.Fatalf("migration hook annotations = %#v", annotations) + } + spec := nestedHelmMap(t, job.Object, "spec") + if helmInt(t, spec["backoffLimit"]) != 0 || helmInt(t, spec["activeDeadlineSeconds"]) != 300 || helmInt(t, spec["ttlSecondsAfterFinished"]) != 3600 { + t.Fatalf("migration Job lifecycle = %#v", spec) + } + podSpec := nestedHelmMap(t, job.Object, "spec", "template", "spec") + if value, found, _ := unstructured.NestedBool(podSpec, "automountServiceAccountToken"); !found || value { + t.Fatal("migration Job must not mount a Kubernetes service-account token") + } + if _, found := podSpec["serviceAccountName"]; found { + t.Fatalf("migration Job unexpectedly selects a service account: %#v", podSpec) + } + assertHelmPodSecurity(t, podSpec, false) + container := onlyHelmContainer(t, podSpec) + if !reflect.DeepEqual(stringSlice(t, container["args"]), []string{"hub", "migrate"}) { + t.Fatalf("migration arguments = %#v", container["args"]) + } + assertHelmContainerSecurity(t, container) + assertHelmProfileResources(t, container, profile) + environment := helmEnvironment(t, container) + if len(environment) != 2 || environment["SITH_HUB_MIGRATION_OWNER_DATABASE_URL"] != "secret:sith-migration/owner-database-url" || environment["SITH_HUB_APPLICATION_DATABASE_ROLE"] != "sith_app" { + t.Fatalf("migration environment = %#v", environment) + } +} + +func assertProfileOnlyChangesResources(t *testing.T, light, heavy []*unstructured.Unstructured) { + t.Helper() + if len(light) != len(heavy) { + t.Fatalf("light/heavy rendered object counts = %d/%d", len(light), len(heavy)) + } + heavyByKind := make(map[string]*unstructured.Unstructured, len(heavy)) + for _, object := range heavy { + heavyByKind[object.GetKind()] = object + } + for _, lightObject := range light { + heavyObject, found := heavyByKind[lightObject.GetKind()] + if !found { + t.Fatalf("heavy profile did not render %s", lightObject.GetKind()) + } + lightCopy := lightObject.DeepCopy() + heavyCopy := heavyObject.DeepCopy() + removeHelmProfileResources(t, lightCopy) + removeHelmProfileResources(t, heavyCopy) + if !reflect.DeepEqual(lightCopy.Object, heavyCopy.Object) { + t.Fatalf("profile changed non-resource %s manifest", lightObject.GetKind()) + } + } +} + +func removeHelmProfileResources(t *testing.T, object *unstructured.Unstructured) { + t.Helper() + if object.GetKind() != "Deployment" && object.GetKind() != "Job" { + return + } + spec, ok := object.Object["spec"].(map[string]any) + if !ok { + t.Fatalf("%s spec = %#v", object.GetKind(), object.Object["spec"]) + } + template, ok := spec["template"].(map[string]any) + if !ok { + t.Fatalf("%s template = %#v", object.GetKind(), spec["template"]) + } + podSpec, ok := template["spec"].(map[string]any) + if !ok { + t.Fatalf("%s Pod spec = %#v", object.GetKind(), template["spec"]) + } + containers, ok := podSpec["containers"].([]any) + if !ok || len(containers) != 1 { + t.Fatalf("%s containers = %#v", object.GetKind(), podSpec["containers"]) + } + container, ok := containers[0].(map[string]any) + if !ok { + t.Fatalf("%s container = %#v", object.GetKind(), containers[0]) + } + delete(container, "resources") +} + +func assertHelmPodSecurity(t *testing.T, podSpec map[string]any, requireFSGroup bool) { + t.Helper() + security := nestedHelmMap(t, podSpec, "securityContext") + if security["runAsNonRoot"] != true || helmInt(t, security["runAsUser"]) != 65532 || helmInt(t, security["runAsGroup"]) != 65532 { + t.Fatalf("pod security context = %#v", security) + } + if requireFSGroup && helmInt(t, security["fsGroup"]) != 65532 { + t.Fatalf("pod fsGroup = %#v", security) + } + if nestedHelmMap(t, security, "seccompProfile")["type"] != "RuntimeDefault" { + t.Fatalf("pod seccomp profile = %#v", security) + } +} + +func onlyHelmContainer(t *testing.T, podSpec map[string]any) map[string]any { + t.Helper() + containers, found, err := unstructured.NestedSlice(podSpec, "containers") + if err != nil || !found || len(containers) != 1 { + t.Fatalf("containers = %#v / %v", containers, err) + } + container, ok := containers[0].(map[string]any) + if !ok { + t.Fatalf("container = %#v", containers[0]) + } + return container +} + +func assertHelmContainerSecurity(t *testing.T, container map[string]any) { + t.Helper() + security := nestedHelmMap(t, container, "securityContext") + if security["privileged"] != false || security["allowPrivilegeEscalation"] != false || security["readOnlyRootFilesystem"] != true { + t.Fatalf("container security context = %#v", security) + } + if !reflect.DeepEqual(stringSlice(t, nestedHelmMap(t, security, "capabilities")["drop"]), []string{"ALL"}) { + t.Fatalf("container capabilities = %#v", security["capabilities"]) + } +} + +func assertHelmProfileResources(t *testing.T, container map[string]any, profile string) { + t.Helper() + want, found := hubProfileResources[profile] + if !found { + t.Fatalf("unknown expected profile %q", profile) + } + resources := nestedHelmMap(t, container, "resources") + if len(resources) != 2 || !reflect.DeepEqual(stringMap(t, nestedHelmMap(t, resources, "requests")), want.requests) || !reflect.DeepEqual(stringMap(t, nestedHelmMap(t, resources, "limits")), want.limits) { + t.Fatalf("%s profile resources = %#v", profile, resources) + } +} + +func helmEnvironment(t *testing.T, container map[string]any) map[string]string { + t.Helper() + entries, found, err := unstructured.NestedSlice(container, "env") + if err != nil || !found { + t.Fatalf("container environment = %#v / %v", entries, err) + } + environment := make(map[string]string, len(entries)) + for _, entry := range entries { + value, ok := entry.(map[string]any) + if !ok { + t.Fatalf("environment entry = %#v", entry) + } + name, _ := value["name"].(string) + if literal, found := value["value"].(string); found { + environment[name] = literal + continue + } + secret := nestedHelmMap(t, value, "valueFrom", "secretKeyRef") + environment[name] = "secret:" + fmt.Sprint(secret["name"]) + "/" + fmt.Sprint(secret["key"]) + } + return environment +} + +func nestedHelmMap(t *testing.T, object map[string]any, fields ...string) map[string]any { + t.Helper() + value, found, err := unstructured.NestedMap(object, fields...) + if err != nil || !found { + t.Fatalf("missing map %s: %v", strings.Join(fields, "."), err) + } + return value +} + +func stringSlice(t *testing.T, value any) []string { + t.Helper() + items, ok := value.([]any) + if !ok { + t.Fatalf("string slice = %#v", value) + } + result := make([]string, len(items)) + for index, item := range items { + stringValue, ok := item.(string) + if !ok { + t.Fatalf("string slice item = %#v", item) + } + result[index] = stringValue + } + return result +} + +func stringMap(t *testing.T, values map[string]any) map[string]string { + t.Helper() + result := make(map[string]string, len(values)) + for key, value := range values { + stringValue, ok := value.(string) + if !ok { + t.Fatalf("string map %s = %#v", key, value) + } + result[key] = stringValue + } + return result +} + +func helmInt(t *testing.T, value any) int64 { + t.Helper() + switch number := value.(type) { + case int64: + return number + case int: + return int64(number) + case float64: + if float64(int64(number)) != number { + t.Fatalf("non-integer number = %#v", value) + } + return int64(number) + default: + t.Fatalf("integer = %#v", value) + return 0 + } +} diff --git a/tests/e2e/oci_image_helpers_test.go b/tests/e2e/oci_image_helpers_test.go new file mode 100644 index 0000000..fde0031 --- /dev/null +++ b/tests/e2e/oci_image_helpers_test.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 +//go:build e2e && (oci || kind) + +package e2e_test + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +const distrolessRuntimeImage = "gcr.io/distroless/static-debian12@sha256:b7bb25d9f7c31d2bdd1982feb4dafcaf137703c7075dbe2febb41c24212b946f" + +var forbiddenContainerfileInstruction = regexp.MustCompile(`(?im)^[\t ]*(?:run|add)\b`) + +func buildOCIImage(ctx context.Context, t *testing.T, root, architecture string) string { + t.Helper() + if architecture != "amd64" && architecture != "arm64" { + t.Fatalf("unsupported OCI architecture %q", architecture) + } + if _, err := exec.LookPath("docker"); err != nil { + t.Fatalf("find docker: %v", err) + } + + contextDir := t.TempDir() + binary := filepath.Join(contextDir, "bin", "linux", architecture, "sith") + if err := os.MkdirAll(filepath.Dir(binary), 0o755); err != nil { + t.Fatalf("create OCI build context: %v", err) + } + build := exec.CommandContext(ctx, "go", "build", "-trimpath", "-buildvcs=false", "-mod=readonly", "-o", binary, "./cmd/sith") + build.Dir = root + build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", "GOARCH="+architecture) + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build linux/%s Sith binary: %v\n%s", architecture, err, output) + } + + tag := fmt.Sprintf("sith-oci-test:%s-%d", architecture, time.Now().UnixNano()) + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _ = exec.CommandContext(cleanupCtx, "docker", "image", "rm", "--force", tag).Run() + }) + buildImage := exec.CommandContext( + ctx, + "docker", "buildx", "build", "--platform", "linux/"+architecture, "--load", "--tag", tag, + "--file", filepath.Join(root, "Containerfile"), contextDir, + ) + if output, err := buildImage.CombinedOutput(); err != nil { + t.Fatalf("build linux/%s OCI image: %v\n%s", architecture, err, output) + } + return tag +} + +func assertContainerfileContract(t *testing.T, root string) { + t.Helper() + contents, err := os.ReadFile(filepath.Join(root, "Containerfile")) + if err != nil { + t.Fatalf("read Containerfile: %v", err) + } + if err := validateContainerfileContract(string(contents)); err != nil { + t.Fatal(err) + } +} + +func validateContainerfileContract(contract string) error { + if forbiddenContainerfileInstruction.MatchString(contract) { + return fmt.Errorf("Containerfile must not install packages or fetch build inputs") + } + expected := map[string]string{ + "ARG": "ARG TARGETARCH", + "COPY": "COPY --chown=65532:65532 --chmod=0555 bin/linux/${TARGETARCH}/sith /usr/local/bin/sith", + "ENTRYPOINT": "ENTRYPOINT [\"/usr/local/bin/sith\"]", + "FROM": "FROM " + distrolessRuntimeImage, + "USER": "USER 65532:65532", + } + found := make(map[string][]string, len(expected)) + for lineNumber, line := range strings.Split(contract, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + instruction := strings.Join(strings.Fields(line), " ") + directive := strings.ToUpper(strings.Fields(instruction)[0]) + if _, permitted := expected[directive]; !permitted { + return fmt.Errorf("Containerfile line %d uses forbidden instruction %q", lineNumber+1, directive) + } + found[directive] = append(found[directive], instruction) + } + for directive, required := range expected { + if actual := found[directive]; len(actual) != 1 || actual[0] != required { + return fmt.Errorf("Containerfile must contain exactly %q, got %q", required, actual) + } + } + return nil +} + +func TestContainerfileInstructionGuard(t *testing.T) { + t.Parallel() + for _, instruction := range []string{"RUN true", "run true", "RuN true", "ADD https://example.invalid/input /input"} { + if !forbiddenContainerfileInstruction.MatchString(instruction) { + t.Fatalf("instruction guard accepted %q", instruction) + } + } + if forbiddenContainerfileInstruction.MatchString("# RUN is forbidden, but this comment is not an instruction") { + t.Fatal("instruction guard rejected a comment") + } + valid := strings.Join([]string{ + "FROM " + distrolessRuntimeImage, + "ARG TARGETARCH", + "COPY --chown=65532:65532 --chmod=0555 bin/linux/${TARGETARCH}/sith /usr/local/bin/sith", + "USER 65532:65532", + "ENTRYPOINT [\"/usr/local/bin/sith\"]", + }, "\n") + if err := validateContainerfileContract(valid); err != nil { + t.Fatalf("valid Containerfile rejected: %v", err) + } + for name, invalid := range map[string]string{ + "comment cannot satisfy requirement": strings.Replace(valid, "USER 65532:65532", "# USER 65532:65532", 1), + "second from": valid + "\nFROM scratch", + "second user": valid + "\nUSER root", + "second entrypoint": valid + "\nENTRYPOINT [\"/bin/sh\"]", + "unrecognized": valid + "\nENV PATH=/tmp", + } { + if err := validateContainerfileContract(invalid); err == nil { + t.Fatalf("%s Containerfile accepted", name) + } + } +} diff --git a/tests/e2e/oci_image_inspect_test.go b/tests/e2e/oci_image_inspect_test.go new file mode 100644 index 0000000..222e9fa --- /dev/null +++ b/tests/e2e/oci_image_inspect_test.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 +//go:build e2e && oci + +package e2e_test + +import ( + "context" + "encoding/json" + "os/exec" + "testing" +) + +type ociImageInspection struct { + Architecture string `json:"Architecture"` + OS string `json:"Os"` + Config struct { + Entrypoint []string `json:"Entrypoint"` + User string `json:"User"` + } `json:"Config"` +} + +func inspectOCIImage(ctx context.Context, t *testing.T, tag string) ociImageInspection { + t.Helper() + inspect := exec.CommandContext(ctx, "docker", "image", "inspect", tag) + output, err := inspect.Output() + if err != nil { + t.Fatalf("inspect OCI image %s: %v", tag, err) + } + var images []ociImageInspection + if err := json.Unmarshal(output, &images); err != nil || len(images) != 1 { + t.Fatalf("decode OCI image %s inspection: %#v / %v", tag, images, err) + } + return images[0] +} + +func assertOCIImageContract(t *testing.T, image ociImageInspection, architecture string) { + t.Helper() + if image.OS != "linux" || image.Architecture != architecture { + t.Fatalf("OCI image platform = %s/%s, want linux/%s", image.OS, image.Architecture, architecture) + } + if image.Config.User != "65532:65532" { + t.Fatalf("OCI image user = %q, want 65532:65532", image.Config.User) + } + if len(image.Config.Entrypoint) != 1 || image.Config.Entrypoint[0] != "/usr/local/bin/sith" { + t.Fatalf("OCI image entrypoint = %#v, want Sith binary", image.Config.Entrypoint) + } +} diff --git a/tests/e2e/oci_image_kind_test.go b/tests/e2e/oci_image_kind_test.go new file mode 100644 index 0000000..61ec039 --- /dev/null +++ b/tests/e2e/oci_image_kind_test.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +//go:build e2e && kind + +package e2e_test + +import ( + "context" + "encoding/json" + "fmt" + "os/exec" + "runtime" + "strings" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +func TestKindOCIImageContract(t *testing.T) { + kindBinary := environmentOr("KIND_BIN", "kind") + if _, err := exec.LookPath(kindBinary); err != nil { + t.Fatalf("find kind binary %q: %v", kindBinary, err) + } + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + root := repositoryRoot(t) + assertContainerfileContract(t, root) + image := buildOCIImage(ctx, t, root, runtime.GOARCH) + + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + clusters := []string{"sith-oci-a-" + suffix, "sith-oci-b-" + suffix} + created := make([]string, 0, len(clusters)) + t.Cleanup(func() { + for _, name := range created { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Minute) + _ = exec.CommandContext(cleanupCtx, kindBinary, "delete", "cluster", "--name", name).Run() + cleanupCancel() + } + }) + + for _, name := range clusters { + created = append(created, name) + runCommand(ctx, t, "", kindBinary, "create", "cluster", "--name", name, "--image", defaultKindNodeImage, "--wait", "180s") + runCommand(ctx, t, "", kindBinary, "load", "docker-image", image, "--name", name) + assertOCIImageJob(ctx, t, kindBinary, name, image) + } +} + +func assertOCIImageJob(ctx context.Context, t *testing.T, kindBinary, clusterName, image string) { + t.Helper() + kubeconfig := runCommand(ctx, t, "", kindBinary, "get", "kubeconfig", "--name", clusterName) + configuration, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeconfig)) + if err != nil { + t.Fatalf("parse kind kubeconfig for %s: %v", clusterName, err) + } + client, err := kubernetes.NewForConfig(configuration) + if err != nil { + t.Fatalf("construct kind client for %s: %v", clusterName, err) + } + nonRoot, readOnly, noPrivilegeEscalation, automountToken := true, true, false, false + runAsUser := int64(65532) + backoffLimit := int32(0) + job, err := client.BatchV1().Jobs("default").Create(ctx, &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "sith-oci-contract"}, + Spec: batchv1.JobSpec{ + BackoffLimit: &backoffLimit, + Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + AutomountServiceAccountToken: &automountToken, + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: &nonRoot, RunAsUser: &runAsUser, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{{ + Name: "sith", Image: image, ImagePullPolicy: corev1.PullNever, + Args: []string{"version", "--output", "json"}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &noPrivilegeEscalation, ReadOnlyRootFilesystem: &readOnly, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + }}, + }}, + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create hardened OCI Job on %s: %v", clusterName, err) + } + + if err := wait.PollUntilContextTimeout(ctx, time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + current, err := client.BatchV1().Jobs("default").Get(ctx, job.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if current.Status.Failed > 0 { + return false, fmt.Errorf("OCI Job failed on %s", clusterName) + } + return current.Status.Succeeded == 1, nil + }); err != nil { + t.Fatalf("wait for hardened OCI Job on %s: %v", clusterName, err) + } + pods, err := client.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + job.Name}) + if err != nil { + t.Fatalf("list OCI Job Pods on %s: %v", clusterName, err) + } + if len(pods.Items) != 1 { + t.Fatalf("find OCI Job Pod on %s: %#v", clusterName, pods.Items) + } + output, err := client.CoreV1().Pods("default").GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{}).Do(ctx).Raw() + if err != nil || !json.Valid(output) || !strings.Contains(string(output), "\"version\"") { + t.Fatalf("OCI Job output on %s = %q / %v", clusterName, output, err) + } +} diff --git a/tests/e2e/oci_image_test.go b/tests/e2e/oci_image_test.go new file mode 100644 index 0000000..69d5290 --- /dev/null +++ b/tests/e2e/oci_image_test.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +//go:build e2e && oci + +package e2e_test + +import ( + "context" + "encoding/json" + "os/exec" + "runtime" + "testing" + "time" +) + +func TestOCIImageCrossPlatformContract(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + root := repositoryRoot(t) + assertContainerfileContract(t, root) + + for _, architecture := range []string{"amd64", "arm64"} { + architecture := architecture + t.Run(architecture, func(t *testing.T) { + tag := buildOCIImage(ctx, t, root, architecture) + assertOCIImageContract(t, inspectOCIImage(ctx, t, tag), architecture) + if architecture != runtime.GOARCH { + return + } + version := exec.CommandContext( + ctx, + "docker", "run", "--rm", "--read-only", "--network", "none", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", "--user", "65532:65532", tag, "version", "--output", "json", + ) + output, err := version.CombinedOutput() + if err != nil || !json.Valid(output) { + t.Fatalf("run hardened native OCI image: %v\n%s", err, output) + } + + shell := exec.CommandContext(ctx, "docker", "run", "--rm", "--entrypoint", "/bin/sh", tag) + if output, err := shell.CombinedOutput(); err == nil { + t.Fatalf("distroless OCI image unexpectedly started a shell: %s", output) + } + }) + } +} diff --git a/tests/e2e/smoke_test.go b/tests/e2e/smoke_test.go index 0a0501b..32b537e 100644 --- a/tests/e2e/smoke_test.go +++ b/tests/e2e/smoke_test.go @@ -58,7 +58,7 @@ func TestBinarySmoke(t *testing.T) { {name: "search no egress", args: []string{"search", "status:Running", "-o", "json"}, contains: "no kubeconfig contexts discovered", wantError: true}, {name: "correlate no egress", args: []string{"correlate", "deploy/payments", "status!=Healthy", "-o", "json"}, contains: "no kubeconfig contexts discovered", wantError: true}, {name: "investigate no egress", args: []string{"investigate", "-o", "json"}, contains: "no kubeconfig contexts discovered", wantError: true}, - {name: "hub stub", args: []string{"hub"}, contains: "phase-1+"}, + {name: "hub requires secure configuration", args: []string{"hub"}, contains: "SITH_HUB_LISTEN_ADDR is required", wantError: true}, {name: "no arguments", contains: "Usage:"}, {name: "help", args: []string{"--help"}, contains: "Usage:"}, } @@ -270,6 +270,11 @@ func (guard *egressGuard) environment(configRoot, kubeconfig string) []string { "ALL_PROXY": guard.server.URL, "HTTP_PROXY": guard.server.URL, "HTTPS_PROXY": guard.server.URL, "NO_PROXY": "", "all_proxy": guard.server.URL, "http_proxy": guard.server.URL, "https_proxy": guard.server.URL, "no_proxy": "", "XDG_CONFIG_HOME": configRoot, "KUBECONFIG": kubeconfig, + "SITH_HUB_LISTEN_ADDR": "", "SITH_HUB_DATABASE_URL": "", "SITH_HUB_SESSION_ISSUER": "", + "SITH_HUB_SESSION_AUDIENCE": "", "SITH_HUB_SESSION_KEY_ID": "", "SITH_HUB_SESSION_PUBLIC_KEY_FILE": "", + "SITH_HUB_SERVER_TLS_CERT_FILE": "", "SITH_HUB_SERVER_TLS_KEY_FILE": "", "SITH_HUB_PROXY_ADDRESS": "", + "SITH_HUB_PROXY_SERVER_NAME": "", "SITH_HUB_PROXY_CA_FILE": "", "SITH_HUB_PROXY_CERT_FILE": "", + "SITH_HUB_PROXY_KEY_FILE": "", "SITH_HUB_KUBE_API_SERVER_NAME": "", } environment := make([]string, 0, len(os.Environ())+len(overrides)) for _, entry := range os.Environ() { diff --git a/tests/scripts/m0_ocm_falsification_safety_test.sh b/tests/scripts/m0_ocm_falsification_safety_test.sh index 3070353..d07d8ba 100644 --- a/tests/scripts/m0_ocm_falsification_safety_test.sh +++ b/tests/scripts/m0_ocm_falsification_safety_test.sh @@ -107,6 +107,28 @@ expect_failure "Docker validation rejects a remote endpoint override" \ env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/docker-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ DOCKER_HOST="tcp://127.0.0.1:2375" bash -c 'source "$1"; validate_local_docker_endpoint' _ "${SCRIPT}" +health_fallback_marker="${TEST_ROOT}/health-fallback" +health_fallback_output="$(env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/health-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ + HEALTH_FALLBACK_MARKER="${health_fallback_marker}" bash -c ' + source "$1" + proxy_health_port_available() { return 1; } + verify_cluster_registration() { :; } + verify_scoped_proxy() { : >"${HEALTH_FALLBACK_MARKER}"; } + verify_spoke_ingress_boundary() { :; } + verify_outbound_only() { :; } + verify_lab + ' _ "${SCRIPT}")" +[[ ! -e "${health_fallback_marker}" ]] +[[ "${health_fallback_output}" == *"deferring clusteradm proxy checks to the mandatory direct e2e gate"* ]] || { + printf '[m0-safety] FAIL: occupied port did not log the direct-e2e fallback\n' >&2 + exit 1 +} +[[ "${health_fallback_output}" == *"transport=direct-e2e-required"* ]] || { + printf '[m0-safety] FAIL: occupied port did not select direct-e2e-required transport\n' >&2 + exit 1 +} +pass "occupied ClusterProxy port defers clusteradm checks to direct e2e" + cleanup_marker="${TEST_ROOT}/forced-cleanup" expect_failure "retained run fails closed when bootstrap rotation is unproven" \ env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/rotation-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ @@ -147,6 +169,31 @@ expect_failure "malformed token output still requires invalidation" \ [[ "$(cat "${token_flag_marker}")" == "1" ]] pass "token acquisition boundary is conservative" +addon_wait_marker="${TEST_ROOT}/addon-wait" +env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/addon-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ + KUBECTL_BIN=fake_kubectl ADDON_WAIT_MARKER="${addon_wait_marker}" bash -c ' + attempts=0 + fake_kubectl() { + for argument in "$@"; do + if [[ "${argument}" == "get" ]]; then + attempts=$((attempts + 1)) + [[ "${attempts}" -ge 3 ]] + return + fi + if [[ "${argument}" == "wait" ]]; then + printf "%s\n" "${attempts}" >"${ADDON_WAIT_MARKER}" + return 0 + fi + done + return 1 + } + sleep() { :; } + source "$1" + wait_for_addon_creation spoke-a cluster-proxy + ' _ "${SCRIPT}" +[[ "$(cat "${addon_wait_marker}")" == "3" ]] +pass "addon wait tolerates asynchronous creation before availability" + expect_failure "unrelated hub exec failure cannot satisfy the active deny" \ env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/probe-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ DOCKER_BIN=fake_docker bash -c ' diff --git a/tests/testutil/ocmlab/ocmlab.go b/tests/testutil/ocmlab/ocmlab.go new file mode 100644 index 0000000..7add0b6 --- /dev/null +++ b/tests/testutil/ocmlab/ocmlab.go @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package ocmlab provides the retained disposable M0 lab connection fixture for integration tests. +package ocmlab + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net" + "os" + "os/exec" + "strconv" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + // ProxyNamespace is the fixed OCM ClusterProxy namespace created by the M0 harness. + ProxyNamespace = "open-cluster-management-addon" + // ProxyService is the fixed ClusterProxy service exposed by the M0 harness. + ProxyService = "proxy-entrypoint" + // ProxyRemotePort is the fixed ClusterProxy service port used by the M0 harness. + ProxyRemotePort = 8090 +) + +// HubConfig loads the isolated M0 hub kubeconfig supplied by the falsification harness. +func HubConfig(t testing.TB) *rest.Config { + t.Helper() + loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: RequiredEnv(t, "SITH_OCM_HUB_KUBECONFIG")}, + &clientcmd.ConfigOverrides{CurrentContext: RequiredEnv(t, "SITH_OCM_HUB_CONTEXT")}, + ) + config, err := loader.ClientConfig() + if err != nil { + t.Fatal("load isolated M0 hub kubeconfig failed") + } + return config +} + +// ProxyTLS reads the lab's proxy credentials, mirroring the mounted material used by production. +func ProxyTLS(ctx context.Context, t testing.TB, client kubernetes.Interface) *tls.Config { + t.Helper() + caSecret, err := client.CoreV1().Secrets(ProxyNamespace).Get(ctx, "proxy-server-ca", metav1.GetOptions{}) + if err != nil { + t.Fatal("read M0 proxy CA fixture failed") + } + clientSecret, err := client.CoreV1().Secrets(ProxyNamespace).Get(ctx, "proxy-client", metav1.GetOptions{}) + if err != nil { + t.Fatal("read M0 proxy client fixture failed") + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caSecret.Data["ca.crt"]) { + t.Fatal("M0 proxy CA fixture was invalid") + } + certificate, err := tls.X509KeyPair(clientSecret.Data["tls.crt"], clientSecret.Data["tls.key"]) + if err != nil { + t.Fatal("M0 proxy client fixture was invalid") + } + return &tls.Config{ + RootCAs: pool, MinVersion: tls.VersionTLS12, ServerName: "localhost", Certificates: []tls.Certificate{certificate}, + } +} + +// StartProxyPortForward opens one loopback-only path to the M0 ClusterProxy service. +func StartProxyPortForward(ctx context.Context, t testing.TB) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal("reserve loopback port for M0 proxy test failed") + } + port := listener.Addr().(*net.TCPAddr).Port + if err := listener.Close(); err != nil { + t.Fatal("release reserved loopback port failed") + } + kubectl, err := exec.LookPath(RequiredEnv(t, "KUBECTL_BIN")) + if err != nil { + t.Fatal("M0 kubectl binary was unavailable") + } + // #nosec G204 -- kubectl is resolved from the harness path and all arguments are fixed M0 fixture inputs. + command := exec.CommandContext(ctx, kubectl, + "--kubeconfig", RequiredEnv(t, "SITH_OCM_HUB_KUBECONFIG"), + "--context", RequiredEnv(t, "SITH_OCM_HUB_CONTEXT"), + "-n", ProxyNamespace, + "port-forward", "--address", "127.0.0.1", "service/"+ProxyService, + strconv.Itoa(port)+":"+strconv.Itoa(ProxyRemotePort), + ) + command.Stdout = nil + command.Stderr = nil + if err := command.Start(); err != nil { + t.Fatal("start M0 proxy port-forward failed") + } + t.Cleanup(func() { + if command.Process != nil { + _ = command.Process.Kill() + } + _ = command.Wait() + }) + address := net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) + deadline := time.NewTimer(30 * time.Second) + defer deadline.Stop() + for { + connection, err := net.DialTimeout("tcp", address, 500*time.Millisecond) + if err == nil { + _ = connection.Close() + return address + } + select { + case <-ctx.Done(): + t.Fatal("M0 proxy port-forward did not become reachable") + case <-deadline.C: + t.Fatal("M0 proxy port-forward did not become reachable") + case <-time.After(100 * time.Millisecond): + } + } +} + +// RequiredEnv returns one mandatory harness input without echoing its value. +func RequiredEnv(t testing.TB, name string) string { + t.Helper() + value := os.Getenv(name) + if value == "" { + t.Fatalf("required M0 test environment %s is unset", name) + } + return value +}