diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 280d816d..ea667b8f 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -56,8 +56,9 @@ jobs: # can only stamp one image across N paths, which can't express "three # different images, four different paths" at all. # - # compute-mcp is stamped with the manager's image and the manager's tag: one - # image carries both binaries, so the two paths cannot name different builds. + # compute-mcp and the shell agent are stamped with the manager's image and + # the manager's tag: one image carries all three binaries, so the paths + # cannot name different builds. publish-kustomize-bundles: needs: [ @@ -78,4 +79,5 @@ jobs: - { path: config/components/consumer-ui-plugin, name: ghcr.io/datum-cloud/compute-consumer-ui-plugin, tag: "${{ needs.publish-consumer-ui-plugin-image.outputs.tag }}" } - { path: config/components/provider-ui-plugin, name: ghcr.io/datum-cloud/compute-provider-ui-plugin, tag: "${{ needs.publish-provider-ui-plugin-image.outputs.tag }}" } - { path: config/components/compute-mcp, name: ghcr.io/datum-cloud/compute, tag: "${{ needs.publish-container-image.outputs.tag }}" } + - { path: config/components/shell-agent, name: ghcr.io/datum-cloud/compute, tag: "${{ needs.publish-container-image.outputs.tag }}" } secrets: inherit diff --git a/Dockerfile b/Dockerfile index ea349bff..5ffd405e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,7 @@ RUN go mod download # package whose embed.FS carries the knowledge and runbooks compute-mcp serves. COPY cmd/main.go cmd/main.go COPY cmd/compute-mcp/ cmd/compute-mcp/ +COPY cmd/shell-agent/ cmd/shell-agent/ COPY api/ api/ COPY internal/ internal/ COPY pkg/ pkg/ @@ -61,6 +62,16 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \ -ldflags "-s -w" \ -o compute-mcp ./cmd/compute-mcp +# shell-agent runs in each cell beside its tunnel endpoint. It shares the +# session API and the connection protocol package with the manager, so it +# ships in the same image and cannot skew from the controllers that deliver +# its sessions. +RUN --mount=type=cache,target=/go/pkg/mod/ \ + --mount=type=cache,target="/root/.cache/go-build" \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ + -ldflags "-s -w" \ + -o shell-agent ./cmd/shell-agent + # Use distroless as minimal base image to package the binaries # Refer to https://github.com/GoogleContainerTools/distroless for more details # @@ -74,6 +85,7 @@ FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/manager . COPY --from=builder /workspace/compute-mcp . +COPY --from=builder /workspace/shell-agent . USER 65532:65532 # The manager keeps the entrypoint it has always had, so nothing that runs this diff --git a/cmd/shell-agent/main.go b/cmd/shell-agent/main.go new file mode 100644 index 00000000..c2e6a1a3 --- /dev/null +++ b/cmd/shell-agent/main.go @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Command shell-agent serves instance shell sessions in a compute cell. Each +// replica pairs by ordinal with a tunnel endpoint, which is the only way a +// client reaches it: the endpoint proxies one TCP target, this agent's exec +// port. The agent claims sessions delivered to the cell, runs their commands +// through the cell apiserver, and writes their status on the cell's copies, +// which Karmada reflects back to the hub. It needs no hub credential. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/features" + "go.datum.net/compute/internal/shellagent" +) + +var setupLog = ctrl.Log.WithName("setup") + +type options struct { + listen string + probeAddr string + target string + relayURLs string + managedBy string + endpointPodPrefix string + featureGates string + drainTimeout time.Duration + killGrace time.Duration + pingInterval time.Duration + pongTimeout time.Duration +} + +func main() { + defaults := shellagent.DefaultConfig() + var opts options + flag.StringVar(&opts.listen, "listen", ":7777", "Address to serve session connections on.") + flag.StringVar(&opts.probeAddr, "health-probe-bind-address", ":8081", "Address the probe endpoint binds to.") + flag.StringVar(&opts.target, "target", "", + "host:port the paired tunnel endpoint proxies to this agent, published as the session's target.") + flag.StringVar(&opts.relayURLs, "relay-urls", os.Getenv("DATUM_CONNECT_RELAY_URLS"), + "Comma-separated relays the paired tunnel endpoint uses, nearest first.") + flag.StringVar(&opts.managedBy, "managed-by", strings.Join(defaults.ManagedBy, ","), + "Comma-separated managed-by label values of instance pods that may take sessions.") + flag.StringVar(&opts.endpointPodPrefix, "endpoint-pod-prefix", "exec-endpoint", + "Name of the tunnel endpoint StatefulSet; the paired pod is -.") + flag.StringVar(&opts.featureGates, "feature-gates", "", + "Feature gates, for example InstanceConsoleSessions=true.") + flag.DurationVar(&opts.drainTimeout, "drain-timeout", defaults.DrainTimeout, + "How long connected sessions may continue after the agent is asked to stop.") + flag.DurationVar(&opts.killGrace, "kill-grace", defaults.KillGrace, + "Time between SIGHUP and SIGKILL when a session's processes are stopped.") + flag.DurationVar(&opts.pingInterval, "ping-interval", defaults.PingInterval, + "How often the agent pings a connected client.") + flag.DurationVar(&opts.pongTimeout, "pong-timeout", defaults.PongTimeout, + "How long a connected client may send nothing, not even a pong, before its session ends as Disconnected.") + zapOpts := zap.Options{} + zapOpts.BindFlags(flag.CommandLine) + flag.Parse() + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOpts))) + + if err := run(opts, defaults); err != nil { + setupLog.Error(err, "shell agent failed") + os.Exit(1) + } +} + +func run(opts options, cfg shellagent.Config) error { + if opts.featureGates != "" { + if err := features.MutableFeatureGate.Set(opts.featureGates); err != nil { + return fmt.Errorf("parse feature gates: %w", err) + } + } + signals := make(chan os.Signal, 2) + signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT) + + if !features.FeatureGate.Enabled(features.InstanceConsoleSessions) { + setupLog.Info("the InstanceConsoleSessions feature gate is off; not serving sessions") + <-signals + return nil + } + + podName, namespace := os.Getenv("POD_NAME"), os.Getenv("POD_NAMESPACE") + ordinal, err := ordinalOf(podName) + if err != nil || namespace == "" { + return fmt.Errorf("POD_NAME must be a StatefulSet pod name and POD_NAMESPACE must be set: %w", err) + } + cfg.Namespace = namespace + cfg.Ordinal = ordinal + cfg.EndpointPodName = fmt.Sprintf("%s-%d", opts.endpointPodPrefix, ordinal) + cfg.Target = opts.target + cfg.RelayURLs = splitList(opts.relayURLs) + cfg.ManagedBy = splitList(opts.managedBy) + cfg.DrainTimeout = opts.drainTimeout + cfg.KillGrace = opts.killGrace + cfg.PingInterval = opts.pingInterval + cfg.PongTimeout = opts.pongTimeout + + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(computev1alpha.AddToScheme(scheme)) + + cellConfig := ctrl.GetConfigOrDie() + + mgr, err := ctrl.NewManager(cellConfig, ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: opts.probeAddr, + }) + if err != nil { + return fmt.Errorf("create manager: %w", err) + } + cellClient, err := client.New(cellConfig, client.Options{Scheme: scheme}) + if err != nil { + return err + } + executor, err := shellagent.NewAPIServer(cellConfig) + if err != nil { + return fmt.Errorf("configure pod exec: %w", err) + } + + incarnation := podName + "/" + strconv.FormatInt(time.Now().UnixNano(), 36) + agent, err := shellagent.New(cfg, mgr.GetClient(), cellClient, executor, incarnation) + if err != nil { + return err + } + ctx, cancel := context.WithCancel(ctrl.LoggerInto(context.Background(), ctrl.Log.WithName("shell-agent"))) + defer cancel() + if err := agent.EnsureIdentity(ctx); err != nil { + return err + } + if err := agent.SetupWithManager(mgr); err != nil { + return fmt.Errorf("set up controller: %w", err) + } + + server := &http.Server{Addr: opts.listen, Handler: agent.Handler(ctx), ReadHeaderTimeout: 10 * time.Second} + for _, r := range []manager.RunnableFunc{ + func(ctx context.Context) error { + go func() { + <-ctx.Done() + _ = server.Close() + }() + if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + }, + agent.RunLiveness, + agent.RunSweeper, + agent.RunKeyRotation, + } { + if err := mgr.Add(r); err != nil { + return err + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return err + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return err + } + + go func() { + <-signals + setupLog.Info("draining", "timeout", cfg.DrainTimeout) + agent.Drain(ctx) + setupLog.Info("drained") + cancel() + <-signals + os.Exit(1) + }() + + setupLog.Info("starting shell agent", "endpointID", agent.EndpointID(), "target", cfg.Target) + return mgr.Start(ctx) +} + +func ordinalOf(podName string) (int, error) { + i := strings.LastIndex(podName, "-") + if i < 0 { + return 0, fmt.Errorf("pod name %q has no ordinal", podName) + } + return strconv.Atoi(podName[i+1:]) +} + +func splitList(s string) []string { + var out []string + for _, v := range strings.Split(s, ",") { + if v = strings.TrimSpace(v); v != "" { + out = append(out, v) + } + } + return out +} diff --git a/config/components/shell-agent-cilium/kustomization.yaml b/config/components/shell-agent-cilium/kustomization.yaml new file mode 100644 index 00000000..9b7bc752 --- /dev/null +++ b/config/components/shell-agent-cilium/kustomization.yaml @@ -0,0 +1,10 @@ +# Cilium policies for the shell agent and its tunnel endpoints. Include beside +# config/components/shell-agent on cells that run Cilium. Unlike the portable +# policies they police traffic to the node's own apiserver, kubelet and etcd, +# so the endpoint, which holds no credential, cannot reach them at all. The +# agent never leaves the cell: it talks only to the cell apiserver. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - policy.yaml diff --git a/config/components/shell-agent-cilium/policy.yaml b/config/components/shell-agent-cilium/policy.yaml new file mode 100644 index 00000000..75d4f35e --- /dev/null +++ b/config/components/shell-agent-cilium/policy.yaml @@ -0,0 +1,85 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: exec-agent + namespace: compute-shell-system +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: exec-agent + enableDefaultDeny: + ingress: true + egress: true + ingress: + - fromEndpoints: + - matchLabels: + app.kubernetes.io/name: exec-endpoint + toPorts: + - ports: + - port: "7777" + protocol: TCP + - fromEntities: + - host + toPorts: + - ports: + - port: "8081" + protocol: TCP + egress: + - toEntities: + - kube-apiserver + toPorts: + - ports: + - port: "443" + protocol: TCP + - port: "6443" + protocol: TCP + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: exec-endpoint + namespace: compute-shell-system +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + enableDefaultDeny: + ingress: true + egress: true + egress: + - toEndpoints: + - matchLabels: + app.kubernetes.io/name: exec-agent + toPorts: + - ports: + - port: "7777" + protocol: TCP + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP + # Relays only: world excludes the cluster's own nodes and apiserver. + - toEntities: + - world + toPorts: + - ports: + - port: "443" + protocol: TCP + - port: "7842" + protocol: UDP diff --git a/config/components/shell-agent/agent.yaml b/config/components/shell-agent/agent.yaml new file mode 100644 index 00000000..883af530 --- /dev/null +++ b/config/components/shell-agent/agent.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: exec-agent + namespace: compute-shell-system + labels: + app.kubernetes.io/name: exec-agent + app.kubernetes.io/part-of: compute +spec: + replicas: 2 + serviceName: exec-agent + podManagementPolicy: Parallel + selector: + matchLabels: + app.kubernetes.io/name: exec-agent + template: + metadata: + labels: + app.kubernetes.io/name: exec-agent + app.kubernetes.io/part-of: compute + spec: + serviceAccountName: exec-agent + # The drain window, then time to stop the processes of sessions still + # open at its end. + terminationGracePeriodSeconds: 75 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: exec-agent + containers: + - name: agent + # The manager's image, which carries this binary too. + image: ghcr.io/datum-cloud/compute:latest + imagePullPolicy: IfNotPresent + command: + - /shell-agent + args: + - --listen=:7777 + - --health-probe-bind-address=:8081 + - --target=$(POD_NAME).exec-agent.$(POD_NAMESPACE).svc.cluster.local:7777 + - --managed-by=kata-provider + - --endpoint-pod-prefix=exec-endpoint + - --drain-timeout=30s + - --feature-gates=$(FEATURE_GATES) + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: FEATURE_GATES + value: InstanceConsoleSessions=true + envFrom: + - configMapRef: + name: exec-relays + ports: + - name: exec + containerPort: 7777 + protocol: TCP + - name: probes + containerPort: 8081 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: probes + initialDelaySeconds: 10 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: probes + initialDelaySeconds: 5 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 256Mi diff --git a/config/components/shell-agent/endpoint.yaml b/config/components/shell-agent/endpoint.yaml new file mode 100644 index 00000000..33efac7e --- /dev/null +++ b/config/components/shell-agent/endpoint.yaml @@ -0,0 +1,105 @@ +# Datum Connect tunnel endpoints. Ordinal n serves the identity key in +# endpoint-key-, which its paired agent creates and rotates, and proxies to +# that agent only. Both keys are projected and the ordinal picks one, because a +# StatefulSet cannot name a different Secret per pod. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: exec-endpoint + namespace: compute-shell-system + labels: + app.kubernetes.io/name: exec-endpoint + app.kubernetes.io/part-of: compute +spec: + replicas: 2 + serviceName: exec-endpoint + podManagementPolicy: Parallel + selector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + template: + metadata: + labels: + app.kubernetes.io/name: exec-endpoint + app.kubernetes.io/part-of: compute + spec: + serviceAccountName: exec-endpoint + automountServiceAccountToken: false + enableServiceLinks: false + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + containers: + - name: endpoint + image: ghcr.io/datum-cloud/datum-connect + imagePullPolicy: IfNotPresent + args: + - serve + - --listen-key-file=/var/run/datum-connect/keys/$(POD_INDEX) + - --tcp-proxy=exec-agent-$(POD_INDEX).exec-agent.$(POD_NAMESPACE).svc.cluster.local:7777 + env: + - name: POD_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: RUST_LOG + value: info + envFrom: + - configMapRef: + name: exec-relays + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 256Mi + volumeMounts: + - name: keys + mountPath: /var/run/datum-connect/keys + readOnly: true + - name: state + mountPath: /var/lib/datum-connect + volumes: + - name: keys + projected: + defaultMode: 0440 + sources: + - secret: + name: endpoint-key-0 + optional: true + items: + - key: key + path: "0" + - secret: + name: endpoint-key-1 + optional: true + items: + - key: key + path: "1" + - name: state + emptyDir: {} diff --git a/config/components/shell-agent/exec-admission-policy.yaml b/config/components/shell-agent/exec-admission-policy.yaml new file mode 100644 index 00000000..2e73526c --- /dev/null +++ b/config/components/shell-agent/exec-admission-policy.yaml @@ -0,0 +1,42 @@ +# The agent's ClusterRole grants pods/exec in every namespace, because tenant +# namespaces are created per project. This policy narrows what the apiserver +# lets the agent exec into to tenant namespaces, so a compromised agent cannot +# exec into system workloads, including this component's own. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: compute-shell-agent-exec +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CONNECT + resources: + - pods/exec + - pods/attach + - pods/portforward + matchConditions: + - name: shell-agent + expression: request.userInfo.username == 'system:serviceaccount:compute-shell-system:exec-agent' + validations: + - expression: request.resource.resource == 'pods' && request.subResource == 'exec' + message: The shell agent may only exec into pods. + reason: Forbidden + - expression: request.namespace.startsWith('ns-') + messageExpression: >- + 'the shell agent may only exec into tenant namespaces, not ' + request.namespace + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: compute-shell-agent-exec +spec: + policyName: compute-shell-agent-exec + validationActions: + - Deny diff --git a/config/components/shell-agent/kustomization.yaml b/config/components/shell-agent/kustomization.yaml new file mode 100644 index 00000000..e216d89a --- /dev/null +++ b/config/components/shell-agent/kustomization.yaml @@ -0,0 +1,29 @@ +# Serves instance shell sessions in a cell: two shell agents, each paired by +# ordinal with a Datum Connect tunnel endpoint that is the agent's only way in. +# +# Everything lives in compute-shell-system, and the session target clients dial +# names that namespace, so include this component from a kustomization that does +# not rename namespaces (see config/overlays/cell-shell-agent). The deployment +# supplies ConfigMap exec-relays: DATUM_CONNECT_RELAY_URLS, the relays the +# endpoints register with and the agents publish, nearest first. +# +# The agent needs no Karmada hub credential. It writes status only on the +# session copies in its cell, and Karmada reflects it back to the hub with the +# InstanceConsoleSession interpreter in config/components/federation. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - namespace.yaml + - rbac.yaml + - exec-admission-policy.yaml + - service.yaml + - agent.yaml + - endpoint.yaml + - networkpolicy.yaml + +# datum-connect's CLI image is first published by datum-cloud/app#192. Until +# that release exists this tag is a placeholder; pin the released tag here. +images: + - name: ghcr.io/datum-cloud/datum-connect + newTag: v0.0.0-unreleased diff --git a/config/components/shell-agent/namespace.yaml b/config/components/shell-agent/namespace.yaml new file mode 100644 index 00000000..fe40cdb7 --- /dev/null +++ b/config/components/shell-agent/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: compute-shell-system + labels: + pod-security.kubernetes.io/enforce: restricted diff --git a/config/components/shell-agent/networkpolicy.yaml b/config/components/shell-agent/networkpolicy.yaml new file mode 100644 index 00000000..792cc84e --- /dev/null +++ b/config/components/shell-agent/networkpolicy.yaml @@ -0,0 +1,82 @@ +# Portable policies. Most CNIs do not police a pod's traffic to its own node, +# so they cannot keep the endpoint off the node's apiserver; the Cilium +# policies in config/components/shell-agent-cilium do. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: exec-agent + namespace: compute-shell-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: exec-agent + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + ports: + - protocol: TCP + port: 7777 + egress: + # The cell apiserver. + - ports: + - protocol: TCP + port: 443 + - protocol: TCP + port: 6443 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: exec-endpoint + namespace: compute-shell-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + policyTypes: + - Ingress + - Egress + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: exec-agent + ports: + - protocol: TCP + port: 7777 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Relays, and QUIC address discovery. Traffic always goes through a relay; + # direct paths would need UDP egress to any address. + - ports: + - protocol: TCP + port: 443 + - protocol: UDP + port: 7842 diff --git a/config/components/shell-agent/rbac.yaml b/config/components/shell-agent/rbac.yaml new file mode 100644 index 00000000..0db0ba13 --- /dev/null +++ b/config/components/shell-agent/rbac.yaml @@ -0,0 +1,135 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exec-agent + namespace: compute-shell-system +--- +# The tunnel endpoint needs no Kubernetes API access at all. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exec-endpoint + namespace: compute-shell-system +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: compute-shell-agent +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + # Tenant namespaces are created per project, so exec is granted cluster-wide + # and narrowed to tenant namespaces by the compute-shell-agent-exec + # ValidatingAdmissionPolicy. + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - get + - apiGroups: + - compute.datumapis.com + resources: + - instances + verbs: + - get + - list + - watch + # The agent claims and ends sessions on their cell copies, whose status + # Karmada reflects back to the hub, and holds each copy with its finalizer + # until the session's processes are stopped. + - apiGroups: + - compute.datumapis.com + resources: + - instanceconsolesessions + verbs: + - get + - list + - watch + - patch + - apiGroups: + - compute.datumapis.com + resources: + - instanceconsolesessions/status + verbs: + - get + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: compute-shell-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: compute-shell-agent +subjects: + - kind: ServiceAccount + name: exec-agent + namespace: compute-shell-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: exec-agent + namespace: compute-shell-system +rules: + # Liveness, per-instance slots and the key rotation lock. + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - create + - update + - delete + # Create cannot be limited by name; reading and rotating can. + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - endpoint-key-0 + - endpoint-key-1 + verbs: + - get + - update + # Restarting its paired endpoint after a key rotation. + - apiGroups: + - "" + resources: + - pods + resourceNames: + - exec-endpoint-0 + - exec-endpoint-1 + verbs: + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: exec-agent + namespace: compute-shell-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: exec-agent +subjects: + - kind: ServiceAccount + name: exec-agent + namespace: compute-shell-system diff --git a/config/components/shell-agent/service.yaml b/config/components/shell-agent/service.yaml new file mode 100644 index 00000000..6ee1e833 --- /dev/null +++ b/config/components/shell-agent/service.yaml @@ -0,0 +1,18 @@ +# Gives each agent the stable name its paired endpoint proxies to, +# exec-agent-.exec-agent.compute-shell-system.svc.cluster.local, which is +# also the target clients name. A draining agent keeps its name so clients get +# its refusal rather than a lookup failure. +apiVersion: v1 +kind: Service +metadata: + name: exec-agent + namespace: compute-shell-system +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app.kubernetes.io/name: exec-agent + ports: + - name: exec + port: 7777 + targetPort: exec diff --git a/config/overlays/cell-shell-agent/kustomization.yaml b/config/overlays/cell-shell-agent/kustomization.yaml new file mode 100644 index 00000000..4a7274a5 --- /dev/null +++ b/config/overlays/cell-shell-agent/kustomization.yaml @@ -0,0 +1,10 @@ +# Instance shell sessions for a cell, opt-in and separate from the cell +# overlay: that overlay moves everything into compute-system, and the shell +# service must stay in compute-shell-system, whose name is part of every +# session's connection target. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../components/shell-agent + - ../../components/shell-agent-cilium diff --git a/go.mod b/go.mod index 7a20b8bd..e69d4e52 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( require ( github.com/distribution/reference v0.6.0 + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/modelcontextprotocol/go-sdk v1.7.0 ) @@ -70,6 +71,7 @@ require ( github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/signal v0.7.1 // indirect github.com/morikuni/aec v1.1.0 // indirect diff --git a/go.sum b/go.sum index 1214b6b6..f8a77325 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9 github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -180,6 +182,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -222,6 +226,8 @@ github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/policy-helpers v0.0.0-20260612073044-d5411a945cfc h1:dvhPFj1niuMP3CBCjhiZWJQr//+w1LOA8cHclFJnNe0= github.com/moby/policy-helpers v0.0.0-20260612073044-d5411a945cfc/go.mod h1:frGYJTxenVCGPa9doaqZSU9FqzT7bt+1dFeVAaCFoyQ= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= diff --git a/internal/cmd/compute/build/analysis_test.go b/internal/cmd/compute/build/analysis_test.go index 9a4be064..dbe51cf6 100644 --- a/internal/cmd/compute/build/analysis_test.go +++ b/internal/cmd/compute/build/analysis_test.go @@ -39,6 +39,7 @@ ENTRYPOINT ["/usr/local/bin/app"] node, sourceStage := df.findEntrypointProducer("/usr/local/bin/app", toolchain{Kind: toolchainGo}) if node == nil { t.Fatal("expected producer") + return } if node.StartLine != 3 { t.Fatalf("expected line 3, got %d", node.StartLine) @@ -62,6 +63,7 @@ ENTRYPOINT ["/server"] node := df.findEntrypointCopy("/server") if node == nil { t.Fatal("expected entrypoint copy") + return } if node.StartLine != 4 { t.Fatalf("expected line 4, got %d", node.StartLine) @@ -399,6 +401,7 @@ func TestCheckMissingLibrariesReportsDynamicLoader(t *testing.T) { _, elfFile := detectToolchain(data) if elfFile == nil { t.Fatal("expected ELF") + return } if interp := elfInterp(elfFile); interp == "" { t.Skip("test toolchain produced no PT_INTERP") diff --git a/internal/shellagent/agent.go b/internal/shellagent/agent.go new file mode 100644 index 00000000..91f7e6c3 --- /dev/null +++ b/internal/shellagent/agent.go @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package shellagent is the cell side of instance shell sessions. An agent +// claims InstanceConsoleSessions delivered to its cell, serves each one's +// single connection through its paired tunnel endpoint, runs the session's +// command in the instance through the cell apiserver, and stops that command +// when the session ends for any reason. +// +// Every agent reads and writes only the session copies in its cell. Karmada +// reflects their status back to the hub, and the management plane copies it +// on to the project. The agent holds no hub credential. +package shellagent + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +const managedByLabel = "managed-by" + +// Config is an agent's configuration. +type Config struct { + // Namespace holds the agent's Leases and its endpoint's key Secret. + Namespace string + // Ordinal pairs the agent with the tunnel endpoint of the same ordinal. + Ordinal int + // EndpointPodName is the paired tunnel endpoint's pod, restarted when its + // key rotates. + EndpointPodName string + // Target is the host:port the paired endpoint proxies to this agent. + Target string + // RelayURLs are the relays the paired endpoint is reachable through, + // nearest first. + RelayURLs []string + // ManagedBy lists the managed-by label values of pods that may take + // sessions. + ManagedBy []string + + ConnectTimeout time.Duration + SlotsPerInstance int + MaxOpenSessions int + DrainTimeout time.Duration + KillGrace time.Duration + KeyRotationPeriod time.Duration + SweepInterval time.Duration + // PingInterval is how often the agent pings a connected client. + PingInterval time.Duration + // PongTimeout is how long a connected client may send nothing, not even a + // pong, before the agent ends the session as Disconnected. It must be + // longer than PingInterval. + PongTimeout time.Duration + // ExecTimeout bounds each command the agent runs in an instance to probe, + // list or stop session processes, so a hung exec cannot hold a reconcile + // worker or the sweep. Zero means KillGrace plus 30 seconds. + ExecTimeout time.Duration +} + +// DefaultConfig returns the contract's limits. +func DefaultConfig() Config { + return Config{ + ManagedBy: []string{"kata-provider"}, + ConnectTimeout: 60 * time.Second, + SlotsPerInstance: 3, + MaxOpenSessions: 200, + DrainTimeout: 30 * time.Second, + KillGrace: 3 * time.Second, + KeyRotationPeriod: 30 * 24 * time.Hour, + SweepInterval: time.Minute, + PingInterval: 10 * time.Second, + PongTimeout: 20 * time.Second, + } +} + +// Agent serves shell sessions in one cell. +type Agent struct { + cfg Config + + sessions client.Reader + cell client.Client + exec Executor + now func() time.Time + + incarnation string + identity atomic.Pointer[identity] + draining atomic.Bool + rotating atomic.Bool + + mu sync.Mutex + live map[string]*liveSession + open map[string]types.NamespacedName + + probes sync.Map +} + +// New returns an agent. sessions reads session copies and Instances from the +// cell, typically through a cache; cell reaches the cell apiserver directly. +func New(cfg Config, sessions client.Reader, cell client.Client, exec Executor, incarnation string) (*Agent, error) { + if cfg.Namespace == "" || cfg.Target == "" || len(cfg.RelayURLs) == 0 || len(cfg.ManagedBy) == 0 { + return nil, errors.New("namespace, target, relay URLs and managed-by values are required") + } + if cfg.ExecTimeout <= 0 { + cfg.ExecTimeout = cfg.KillGrace + 30*time.Second + } + if cfg.PingInterval <= 0 || cfg.PongTimeout <= cfg.PingInterval { + return nil, errors.New("the ping interval must be positive and shorter than the pong timeout") + } + return &Agent{ + cfg: cfg, + sessions: sessions, + cell: cell, + exec: exec, + now: time.Now, + incarnation: incarnation, + live: map[string]*liveSession{}, + open: map[string]types.NamespacedName{}, + }, nil +} + +// EndpointID is the paired tunnel endpoint's current iroh endpoint ID. +func (a *Agent) EndpointID() string { + if id := a.identity.Load(); id != nil { + return id.endpointID + } + return "" +} + +func (a *Agent) claiming() bool { + return !a.draining.Load() && !a.rotating.Load() && a.identity.Load() != nil +} + +type liveSession struct { + cancel context.CancelFunc + done chan struct{} + stream *clientStream + tty bool + reason string +} + +// register records a connection. It returns false when the session already +// has one. +func (a *Agent) register(uid string, cancel context.CancelFunc, tty bool) (*liveSession, bool) { + a.mu.Lock() + defer a.mu.Unlock() + if _, ok := a.live[uid]; ok { + return nil, false + } + l := &liveSession{cancel: cancel, done: make(chan struct{}), tty: tty} + a.live[uid] = l + return l, true +} + +func (a *Agent) attach(l *liveSession, s *clientStream) { + a.mu.Lock() + l.stream = s + a.mu.Unlock() +} + +func (a *Agent) unregister(uid string) { + a.mu.Lock() + delete(a.live, uid) + a.mu.Unlock() +} + +func (a *Agent) isLive(uid string) bool { + a.mu.Lock() + defer a.mu.Unlock() + _, ok := a.live[uid] + return ok +} + +func (a *Agent) stopReason(uid string) string { + a.mu.Lock() + defer a.mu.Unlock() + if l := a.live[uid]; l != nil { + return l.reason + } + return "" +} + +func (a *Agent) liveSessions() map[string]*liveSession { + a.mu.Lock() + defer a.mu.Unlock() + out := make(map[string]*liveSession, len(a.live)) + for uid, l := range a.live { + out[uid] = l + } + return out +} + +// stop ends a connected session for reason and waits until its processes +// have been stopped and its end recorded. +func (a *Agent) stop(uid, reason string) { + a.mu.Lock() + l := a.live[uid] + if l != nil && l.reason == "" { + l.reason = reason + } + a.mu.Unlock() + if l == nil { + return + } + l.cancel() + select { + case <-l.done: + case <-time.After(a.cfg.KillGrace + 30*time.Second): + } +} + +// track counts a session claimed by this agent toward its open sessions. +func (a *Agent) track(uid string, key types.NamespacedName) { + a.mu.Lock() + a.open[uid] = key + a.mu.Unlock() +} + +func (a *Agent) forget(uid string) { + a.mu.Lock() + delete(a.open, uid) + a.mu.Unlock() +} + +func (a *Agent) openSessions() map[string]types.NamespacedName { + a.mu.Lock() + defer a.mu.Unlock() + out := make(map[string]types.NamespacedName, len(a.open)) + for uid, key := range a.open { + out[uid] = key + } + return out +} + +func (a *Agent) openCount() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.open) +} + +func sessionUID(s *computev1alpha.InstanceConsoleSession) string { + return s.Labels[computev1alpha.InstanceConsoleSessionUIDLabel] +} + +func endpointOf(s *computev1alpha.InstanceConsoleSession) string { + if s.Status.Connection == nil { + return "" + } + return s.Status.Connection.EndpointID +} + +func sessionTTL(s *computev1alpha.InstanceConsoleSession) time.Duration { + if s.Spec.TTL == nil || s.Spec.TTL.Duration <= 0 { + return 15 * time.Minute + } + return s.Spec.TTL.Duration +} diff --git a/internal/shellagent/checks.go b/internal/shellagent/checks.go new file mode 100644 index 00000000..1e641a90 --- /dev/null +++ b/internal/shellagent/checks.go @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "sync" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +// errNotInCell means the cell does not run the session's workload +// deployment, so the session is another cell's to claim. +var errNotInCell = errors.New("the session's instance does not run in this cell") + +// rejection ends a session before it is claimed. +type rejection struct { + reason string + message string +} + +func reject(reason, format string, args ...any) *rejection { + return &rejection{reason: reason, message: fmt.Sprintf(format, args...)} +} + +// target is where a session's command runs. +type target struct { + instance *computev1alpha.Instance + pod *corev1.Pod + container string + containerID string + markerDir string +} + +// check runs the checks that come before a claim: the cell Instance, its pod, +// the container, and a shell and the requested executable in it. +func (a *Agent) check(ctx context.Context, s *computev1alpha.InstanceConsoleSession) (*target, *rejection, error) { + inst, rej, err := a.findInstance(ctx, s) + if rej != nil || err != nil { + return nil, rej, err + } + pod, containerID, rej, err := a.findPod(ctx, inst, s.Spec.ContainerName) + if rej != nil || err != nil { + return nil, rej, err + } + dir, rej, err := a.probe(ctx, pod, s.Spec.ContainerName, s.Spec.Command[0]) + if rej != nil || err != nil { + return nil, rej, err + } + return &target{ + instance: inst, + pod: pod, + container: s.Spec.ContainerName, + containerID: containerID, + markerDir: dir, + }, nil, nil +} + +// findInstance finds the cell Instance a session copy names. A same-name +// Instance from another workload deployment is a replacement, not the +// Instance the session was created for. +func (a *Agent) findInstance(ctx context.Context, s *computev1alpha.InstanceConsoleSession) (*computev1alpha.Instance, *rejection, error) { + name := s.Labels[computev1alpha.InstanceConsoleSessionInstanceNameLabel] + deploymentUID := s.Labels[computev1alpha.WorkloadDeploymentUIDLabel] + if name == "" || deploymentUID == "" || len(s.Spec.Command) == 0 { + return nil, reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + "The session was delivered without the instance it runs in."), nil + } + var inst computev1alpha.Instance + err := a.sessions.Get(ctx, client.ObjectKey{Namespace: s.Namespace, Name: name}, &inst) + if err != nil && !apierrors.IsNotFound(err) { + return nil, nil, err + } + if err == nil && inst.Labels[computev1alpha.WorkloadDeploymentUIDLabel] == deploymentUID { + if !inst.DeletionTimestamp.IsZero() { + return nil, reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + "The instance is being deleted."), nil + } + return &inst, nil, nil + } + var siblings computev1alpha.InstanceList + if err := a.sessions.List(ctx, &siblings, client.InNamespace(s.Namespace), + client.MatchingLabels{computev1alpha.WorkloadDeploymentUIDLabel: deploymentUID}); err != nil { + return nil, nil, err + } + if len(siblings.Items) == 0 { + return nil, nil, errNotInCell + } + return nil, reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotFound, + "The instance no longer exists or has been replaced."), nil +} + +// findPod finds the running pod that realizes a cell Instance, and the ID of +// the requested container in it. +func (a *Agent) findPod(ctx context.Context, inst *computev1alpha.Instance, container string) (*corev1.Pod, string, *rejection, error) { + notRunning := reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, "The instance is not running.") + var pod corev1.Pod + err := a.cell.Get(ctx, client.ObjectKeyFromObject(inst), &pod) + if apierrors.IsNotFound(err) { + return nil, "", notRunning, nil + } + if err != nil { + return nil, "", nil, err + } + if owner := metav1.GetControllerOf(&pod); owner == nil || owner.UID != inst.UID { + return nil, "", notRunning, nil + } + if !slices.Contains(a.cfg.ManagedBy, pod.Labels[managedByLabel]) { + return nil, "", reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + "Sessions are not available for this instance's runtime."), nil + } + if !slices.ContainsFunc(pod.Spec.Containers, func(c corev1.Container) bool { return c.Name == container }) { + return nil, "", reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + "The instance has no container named %q.", container), nil + } + if pod.Status.Phase != corev1.PodRunning || !pod.DeletionTimestamp.IsZero() { + return nil, "", notRunning, nil + } + status := containerStatus(&pod, container) + if status == nil || status.State.Running == nil { + return nil, "", reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + "The container %q is not running.", container), nil + } + return &pod, status.ContainerID, nil, nil +} + +func containerStatus(pod *corev1.Pod, name string) *corev1.ContainerStatus { + for i := range pod.Status.ContainerStatuses { + if pod.Status.ContainerStatuses[i].Name == name { + return &pod.Status.ContainerStatuses[i] + } + } + return nil +} + +type probeResult struct { + mu sync.Mutex + dir string + executables map[string]bool +} + +const maxProbeCacheEntries = 4096 + +// probe checks that a container has a shell, a writable directory for the +// marker file, and the requested executable. What it finds is cached per pod +// UID and container, so each check execs into an instance once. +func (a *Agent) probe(ctx context.Context, pod *corev1.Pod, container, executable string) (string, *rejection, error) { + key := string(pod.UID) + "/" + container + if v, ok := a.probes.Load(key); ok { + r := v.(*probeResult) + r.mu.Lock() + dir, known := r.dir, r.executables[executable] + r.mu.Unlock() + if known { + return dir, nil, nil + } + } + + out, code, err := a.run(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, + container, probeCommand(executable)) + var failed *ExecFailedError + switch { + case errors.As(err, &failed): + return "", reject(computev1alpha.InstanceConsoleSessionReasonNoShell, + "The container has no shell (sh), which the platform needs to manage the command."), nil + case err != nil: + return "", nil, err + case code == probeExitCommandMissing: + return "", reject(computev1alpha.InstanceConsoleSessionReasonCommandUnavailable, + "The container has no executable named %q.", executable), nil + case code == probeExitNoWritableDir: + return "", reject(computev1alpha.InstanceConsoleSessionReasonNoShell, + "The container has no writable temporary directory, which the platform needs to manage the command."), nil + case code != 0: + return "", reject(computev1alpha.InstanceConsoleSessionReasonNoShell, + "The container's shell (sh) failed to run."), nil + } + + lines := strings.Fields(out) + if len(lines) == 0 { + return "", reject(computev1alpha.InstanceConsoleSessionReasonNoShell, + "The container's shell (sh) failed to run."), nil + } + dir := lines[len(lines)-1] + if a.probeCacheSize() >= maxProbeCacheEntries { + a.probes.Clear() + } + v, _ := a.probes.LoadOrStore(key, &probeResult{executables: map[string]bool{}}) + r := v.(*probeResult) + r.mu.Lock() + r.dir = dir + r.executables[executable] = true + r.mu.Unlock() + return dir, nil, nil +} + +func (a *Agent) probeCacheSize() int { + n := 0 + a.probes.Range(func(_, _ any) bool { + n++ + return true + }) + return n +} diff --git a/internal/shellagent/drain.go b/internal/shellagent/drain.go new file mode 100644 index 00000000..b8ce72ed --- /dev/null +++ b/internal/shellagent/drain.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "fmt" + "sync" + "time" + + "sigs.k8s.io/controller-runtime/pkg/log" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +// Drain stops the agent claiming and connecting sessions, ends the ones it +// claimed that no client connected to, warns connected sessions that have a +// terminal, and waits up to the drain timeout for connected sessions to end +// before ending the rest with AgentShutdown. +func (a *Agent) Drain(ctx context.Context) { + a.draining.Store(true) + logger := log.FromContext(ctx) + deadline := time.Now().Add(a.cfg.DrainTimeout) + + for uid, key := range a.openSessions() { + if a.isLive(uid) { + continue + } + var session computev1alpha.InstanceConsoleSession + if err := a.cell.Get(ctx, key, &session); err != nil { + continue + } + if err := a.endUnconnected(ctx, &session, computev1alpha.InstanceConsoleSessionReasonAgentShutdown); err != nil { + logger.Error(err, "end unconnected session", "session", uid) + } + } + + notice := fmt.Sprintf("\r\n*** Maintenance: this session will close by %s. Start a new session to continue. ***\r\n", + deadline.UTC().Format("15:04:05 MST")) + for _, l := range a.liveSessions() { + a.mu.Lock() + stream, tty := l.stream, l.tty + a.mu.Unlock() + if tty && stream != nil { + _ = stream.Notify(notice) + } + } + + for time.Now().Before(deadline) && len(a.liveSessions()) > 0 { + select { + case <-ctx.Done(): + return + case <-time.After(250 * time.Millisecond): + } + } + + var wg sync.WaitGroup + for uid := range a.liveSessions() { + wg.Go(func() { + a.stop(uid, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + }) + } + wg.Wait() + a.releaseLease(ctx, a.EndpointID()) +} diff --git a/internal/shellagent/exec.go b/internal/shellagent/exec.go new file mode 100644 index 00000000..8cbf99ac --- /dev/null +++ b/internal/shellagent/exec.go @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "context" + "crypto/rand" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + + "go.datum.net/compute/internal/consolesession" +) + +// ExecOptions is one pods/exec request. +type ExecOptions struct { + Container string + Command []string + Stdin bool + TTY bool +} + +func (o ExecOptions) query() url.Values { + q := url.Values{} + for _, c := range o.Command { + q.Add("command", c) + } + q.Set("container", o.Container) + q.Set("stdout", "true") + if o.Stdin { + q.Set("stdin", "true") + } + if o.TTY { + q.Set("tty", "true") + } else { + q.Set("stderr", "true") + } + return q +} + +// Executor opens exec streams into pods. +type Executor interface { + // Open upgrades a pods/exec request to a v5.channel.k8s.io WebSocket and + // returns the apiserver's 101 response with the stream. handshake carries + // the client's WebSocket key when the stream is relayed to a client, so + // the apiserver's accept header matches it; a nil handshake gets a fresh + // key. + Open(ctx context.Context, pod types.NamespacedName, opts ExecOptions, handshake http.Header) (*http.Response, net.Conn, *bufio.Reader, error) +} + +// ExecRefusedError is an exec the apiserver answered with something other +// than an upgrade. +type ExecRefusedError struct { + StatusCode int + Message string +} + +func (e *ExecRefusedError) Error() string { + return fmt.Sprintf("exec refused with HTTP %d: %s", e.StatusCode, e.Message) +} + +// APIServer opens exec streams against the cell apiserver. +type APIServer struct { + base *url.URL + tlsConfig *tls.Config + transport http.RoundTripper +} + +// NewAPIServer returns an Executor that authenticates to the apiserver the +// way cfg does. +func NewAPIServer(cfg *rest.Config) (*APIServer, error) { + base, _, err := rest.DefaultServerUrlFor(cfg) + if err != nil { + return nil, err + } + tlsConfig, err := rest.TLSConfigFor(cfg) + if err != nil { + return nil, err + } + if tlsConfig == nil { + tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } + tlsConfig = tlsConfig.Clone() + tlsConfig.NextProtos = []string{"http/1.1"} + transport, err := rest.HTTPWrappersForConfig(cfg, &capture{}) + if err != nil { + return nil, err + } + return &APIServer{base: base, tlsConfig: tlsConfig, transport: transport}, nil +} + +// capture ends a round trip at the transport, after the client-go wrappers +// have added credentials, so the authorized request can be written onto a +// connection the agent owns. +type capture struct{} + +type capturedRequest struct { + req *http.Request +} + +func (e *capturedRequest) Error() string { return "request captured" } + +func (capture) RoundTrip(req *http.Request) (*http.Response, error) { + return nil, &capturedRequest{req: req} +} + +func (a *APIServer) authorize(req *http.Request) (*http.Request, error) { + _, err := a.transport.RoundTrip(req) + var captured *capturedRequest + if errors.As(err, &captured) { + return captured.req, nil + } + if err == nil { + err = errors.New("credentials transport returned a response") + } + return nil, err +} + +// Open implements Executor. +func (a *APIServer) Open(ctx context.Context, pod types.NamespacedName, opts ExecOptions, handshake http.Header) (*http.Response, net.Conn, *bufio.Reader, error) { + target := *a.base + target.Path = strings.TrimSuffix(target.Path, "/") + + fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/exec", pod.Namespace, pod.Name) + target.RawQuery = opts.query().Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil) + if err != nil { + return nil, nil, nil, err + } + setHandshake(req.Header, handshake) + req, err = a.authorize(req) + if err != nil { + return nil, nil, nil, fmt.Errorf("authorize exec request: %w", err) + } + + host := target.Host + if target.Port() == "" { + port := "443" + if target.Scheme == "http" { + port = "80" + } + host = net.JoinHostPort(target.Hostname(), port) + } + var conn net.Conn + if target.Scheme == "http" { + conn, err = (&net.Dialer{}).DialContext(ctx, "tcp", host) + } else { + conn, err = (&tls.Dialer{Config: a.tlsConfig}).DialContext(ctx, "tcp", host) + } + if err != nil { + return nil, nil, nil, fmt.Errorf("dial apiserver: %w", err) + } + return upgrade(conn, req) +} + +func setHandshake(h, from http.Header) { + key := from.Get("Sec-WebSocket-Key") + if key == "" { + raw := make([]byte, 16) + _, _ = rand.Read(raw) + key = base64.StdEncoding.EncodeToString(raw) + } + h.Set("Connection", "Upgrade") + h.Set("Upgrade", "websocket") + h.Set("Sec-WebSocket-Version", "13") + h.Set("Sec-WebSocket-Key", key) + h.Set("Sec-WebSocket-Protocol", consolesession.SubProtocol) +} + +func upgrade(conn net.Conn, req *http.Request) (*http.Response, net.Conn, *bufio.Reader, error) { + if err := req.Write(conn); err != nil { + _ = conn.Close() + return nil, nil, nil, fmt.Errorf("send exec request: %w", err) + } + reader := bufio.NewReader(conn) + resp, err := http.ReadResponse(reader, req) + if err != nil { + _ = conn.Close() + return nil, nil, nil, fmt.Errorf("read exec response: %w", err) + } + if resp.StatusCode != http.StatusSwitchingProtocols { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + _ = resp.Body.Close() + _ = conn.Close() + return nil, nil, nil, &ExecRefusedError{StatusCode: resp.StatusCode, Message: strings.TrimSpace(string(body))} + } + return resp, conn, reader, nil +} + +// ExecFailedError is a command the apiserver could not run, as opposed to a +// command that ran and exited. +type ExecFailedError struct { + Message string +} + +func (e *ExecFailedError) Error() string { + return "exec failed: " + e.Message +} + +// run runs a command in a pod to completion, giving up after the agent's exec +// timeout. +func (a *Agent) run(ctx context.Context, pod types.NamespacedName, container string, command []string) (string, int, error) { + ctx, cancel := context.WithTimeout(ctx, a.cfg.ExecTimeout) + defer cancel() + return run(ctx, a.exec, pod, container, command) +} + +// run runs a command to completion without stdin and returns its combined +// output and exit code. +func run(ctx context.Context, exec Executor, pod types.NamespacedName, container string, command []string) (string, int, error) { + _, conn, reader, err := exec.Open(ctx, pod, ExecOptions{Container: container, Command: command}, nil) + if err != nil { + return "", 0, err + } + defer func() { _ = conn.Close() }() + stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + defer stop() + + var out strings.Builder + var status []byte + var channel byte + for { + f, err := readFrame(reader) + if err != nil { + if ctx.Err() != nil { + return out.String(), 0, ctx.Err() + } + return out.String(), 0, fmt.Errorf("exec stream ended without a status: %w", err) + } + switch { + case f.opcode == opClose: + return out.String(), 0, errors.New("exec stream closed without a status") + case f.control(): + continue + case f.opcode != opContinuation: + if len(f.payload) == 0 { + continue + } + channel = f.payload[0] + f.payload = f.payload[1:] + } + switch channel { + case channelStdout, channelStderr: + out.Write(f.payload) + case channelStatus: + status = append(status, f.payload...) + } + if channel != channelStatus || !f.fin { + continue + } + code, message, ok := exitCodeFromStatus(status) + if !ok { + return out.String(), 0, &ExecFailedError{Message: message} + } + return out.String(), int(code), nil + } +} diff --git a/internal/shellagent/exec_test.go b/internal/shellagent/exec_test.go new file mode 100644 index 00000000..4643bafa --- /dev/null +++ b/internal/shellagent/exec_test.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "encoding/pem" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + + "go.datum.net/compute/internal/consolesession" +) + +func TestAPIServerOpensAuthorizedExecStream(t *testing.T) { + var got *http.Request + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.Clone(r.Context()) + conn, _, err := http.NewResponseController(w).Hijack() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + _, _ = conn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + + "Sec-WebSocket-Protocol: " + consolesession.SubProtocol + "\r\n\r\n")) + writeServerFrame(conn, channelStdout, []byte("/tmp\n")) + writeExitStatus(conn, 0) + })) + defer srv.Close() + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) + exec, err := NewAPIServer(&rest.Config{ + Host: srv.URL, + BearerToken: "agent-token", + TLSClientConfig: rest.TLSClientConfig{CAData: caPEM}, + }) + if err != nil { + t.Fatal(err) + } + + pod := types.NamespacedName{Namespace: "ns-1", Name: "web-0"} + out, code, err := run(t.Context(), exec, pod, "app", probeCommand("bash")) + + if err != nil || code != 0 || out != "/tmp\n" { + t.Fatalf("run() = %q, %d, %v", out, code, err) + } + if got.URL.Path != "/api/v1/namespaces/ns-1/pods/web-0/exec" { + t.Fatalf("path = %s", got.URL.Path) + } + q := got.URL.Query() + if !reflect.DeepEqual(q["command"], probeCommand("bash")) || q.Get("container") != "app" || + q.Get("stdout") != "true" || q.Get("stderr") != "true" || q.Has("stdin") || q.Has("tty") { + t.Fatalf("query = %v", q) + } + if got.Header.Get("Authorization") != "Bearer agent-token" || + got.Header.Get("Sec-WebSocket-Protocol") != consolesession.SubProtocol || + got.Header.Get("Sec-WebSocket-Key") == "" { + t.Fatalf("headers = %v", got.Header) + } +} + +func TestWrappedCommandArguments(t *testing.T) { + tests := []struct { + name string + stdin, tty bool + mode string + }{ + {name: "stdin", stdin: true, mode: "-"}, + {name: "terminal", tty: true, mode: "-"}, + {name: "stdin and terminal", stdin: true, tty: true, mode: "-"}, + {name: "neither", mode: "null"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := wrappedCommand("/tmp", "uid-1", []string{"ls", "-l"}, tt.stdin, tt.tty) + want := []string{"sh", "-c", wrapperScript, "sh", wrapperScript, "/tmp/.datum-exec-uid-1", tt.mode, "0", "ls", "-l"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("wrappedCommand() = %q, want %q", got, want) + } + }) + } +} diff --git a/internal/shellagent/frames.go b/internal/shellagent/frames.go new file mode 100644 index 00000000..3d8379a5 --- /dev/null +++ b/internal/shellagent/frames.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "crypto/rand" + "encoding/binary" + "fmt" + "io" +) + +// MaxFrameSize is the largest WebSocket frame the agent relays. A larger frame +// ends the session. +const MaxFrameSize = 16 << 20 + +const ( + opContinuation = 0x0 + opText = 0x1 + opBinary = 0x2 + opClose = 0x8 + opPing = 0x9 + opPong = 0xa + + closeNormal = 1000 + + channelStdin = 0 + channelStdout = 1 + channelStderr = 2 + channelStatus = 3 +) + +// frame is one WebSocket frame. raw holds the frame exactly as it arrived, so +// it can be relayed untouched; payload is its unmasked payload. +type frame struct { + fin bool + opcode byte + payload []byte + raw []byte +} + +func (f *frame) control() bool { + return f.opcode >= opClose +} + +// errFrameTooLarge reports a frame over MaxFrameSize. +type errFrameTooLarge struct { + size uint64 +} + +func (e errFrameTooLarge) Error() string { + return fmt.Sprintf("frame of %d bytes exceeds the %d byte limit", e.size, MaxFrameSize) +} + +// here be dragons +func readFrame(r *bufio.Reader) (*frame, error) { + head := make([]byte, 2, 14) + if _, err := io.ReadFull(r, head); err != nil { + return nil, err + } + length := uint64(head[1] & 0x7f) + switch length { + case 126: + ext := make([]byte, 2) + if _, err := io.ReadFull(r, ext); err != nil { + return nil, err + } + head = append(head, ext...) + length = uint64(binary.BigEndian.Uint16(ext)) + case 127: + ext := make([]byte, 8) + if _, err := io.ReadFull(r, ext); err != nil { + return nil, err + } + head = append(head, ext...) + length = binary.BigEndian.Uint64(ext) + } + var mask []byte + if head[1]&0x80 != 0 { + mask = make([]byte, 4) + if _, err := io.ReadFull(r, mask); err != nil { + return nil, err + } + head = append(head, mask...) + } + if length > MaxFrameSize { + return nil, errFrameTooLarge{size: length} + } + raw := make([]byte, len(head)+int(length)) + copy(raw, head) + if _, err := io.ReadFull(r, raw[len(head):]); err != nil { + return nil, err + } + payload := raw[len(head):] + if mask != nil { + payload = make([]byte, length) + for i := range payload { + payload[i] = raw[len(head)+i] ^ mask[i%4] + } + } + return &frame{fin: head[0]&0x80 != 0, opcode: head[0] & 0x0f, payload: payload, raw: raw}, nil +} + +// encodeFrame builds a single final frame. Frames a client sends must be +// masked; frames a server sends must not. +func encodeFrame(opcode byte, payload []byte, masked bool) []byte { + out := []byte{0x80 | opcode} + maskBit := byte(0) + if masked { + maskBit = 0x80 + } + switch n := len(payload); { + case n < 126: + out = append(out, maskBit|byte(n)) + case n <= 0xffff: + out = append(out, maskBit|126) + out = binary.BigEndian.AppendUint16(out, uint16(n)) + default: + out = append(out, maskBit|127) + out = binary.BigEndian.AppendUint64(out, uint64(n)) + } + if !masked { + return append(out, payload...) + } + mask := make([]byte, 4) + _, _ = rand.Read(mask) + out = append(out, mask...) + for i, b := range payload { + out = append(out, b^mask[i%4]) + } + return out +} + +func closePayload(code uint16) []byte { + return binary.BigEndian.AppendUint16(nil, code) +} diff --git a/internal/shellagent/harness_test.go b/internal/shellagent/harness_test.go new file mode 100644 index 00000000..9d87c7a4 --- /dev/null +++ b/internal/shellagent/harness_test.go @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha1" //nolint:gosec // RFC 6455 derives the accept key with SHA-1 + "encoding/base64" + "encoding/json" + "io" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/consolesession" +) + +const ( + testAgentNamespace = "compute-shell-system" + testNamespace = "ns-project-uid" + testInstance = "web-0" + testInstanceUID = types.UID("cell-instance-uid") + testDeploymentUID = "cell-wd-uid" + testContainer = "app" + testSession = "s1" + testUID = "uid-1" +) + +// clock is a settable time source shared by the agents in a test. +type clock struct { + mu sync.Mutex + t time.Time +} + +func (c *clock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *clock) advance(d time.Duration) { + c.mu.Lock() + c.t = c.t.Add(d) + c.mu.Unlock() +} + +type harness struct { + t *testing.T + ctx context.Context + cell client.Client + exec *fakeAPIServer + clock *clock +} + +func newScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(computev1alpha.AddToScheme(scheme)) + return scheme +} + +func newHarness(t *testing.T) *harness { + t.Helper() + scheme := newScheme() + owner := metav1.OwnerReference{ + APIVersion: computev1alpha.GroupVersion.String(), + Kind: "Instance", + Name: testInstance, + UID: testInstanceUID, + Controller: ptr.To(true), + } + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{ + Name: testInstance, + Namespace: testNamespace, + UID: testInstanceUID, + Labels: map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID}, + }} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: testInstance, + Namespace: testNamespace, + UID: "pod-uid", + Labels: map[string]string{managedByLabel: "kata-provider"}, + OwnerReferences: []metav1.OwnerReference{owner}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: testContainer}}}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: testContainer, + ContainerID: "containerd://abc", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }}, + }, + } + session := &computev1alpha.InstanceConsoleSession{} + return &harness{ + t: t, + ctx: ctrl.LoggerInto(context.Background(), ctrl.Log), + cell: fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(session). + WithObjects(instance, pod).Build(), + exec: newFakeAPIServer(), + clock: &clock{t: time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC)}, + } +} + +// agent returns an agent whose endpoint ID derives from a fresh key and whose +// liveness Lease is current. +func (h *harness) agent(mutate ...func(*Config)) *Agent { + h.t.Helper() + cfg := DefaultConfig() + cfg.Namespace = testAgentNamespace + cfg.Target = "exec-agent-0.exec-agent.compute-shell-system.svc.cluster.local:7777" + cfg.RelayURLs = []string{"https://relay.example.test"} + cfg.KillGrace = 0 + for _, m := range mutate { + m(&cfg) + } + a, err := New(cfg, h.cell, h.cell, h.exec, "test") + if err != nil { + h.t.Fatal(err) + } + a.now = h.clock.now + seed := make([]byte, ed25519.SeedSize) + _, _ = rand.Read(seed) + a.identity.Store(&identity{endpointID: EndpointID(seed), created: h.clock.now()}) + if err := a.renewLease(h.ctx); err != nil { + h.t.Fatal(err) + } + return a +} + +// session creates a session's cell copy and returns the client's key. +func (h *harness) session(name, uid string, mutate ...func(*computev1alpha.InstanceConsoleSession)) ed25519.PrivateKey { + h.t.Helper() + _, key, _ := ed25519.GenerateKey(rand.Reader) + s := &computev1alpha.InstanceConsoleSession{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + CreationTimestamp: metav1.NewTime(h.clock.now()), + Labels: map[string]string{ + computev1alpha.InstanceConsoleSessionUIDLabel: uid, + computev1alpha.InstanceConsoleSessionInstanceNameLabel: testInstance, + computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID, + }, + }, + Spec: computev1alpha.InstanceConsoleSessionSpec{ + InstanceRef: computev1alpha.InstanceConsoleSessionInstanceRef{Name: testInstance, UID: "project-instance-uid"}, + ContainerName: testContainer, + Command: []string{"sh"}, + Stdin: true, + Terminal: true, + ClientPublicKey: consolesession.PublicKey(key), + TTL: &metav1.Duration{Duration: 15 * time.Minute}, + }, + } + for _, m := range mutate { + m(s) + } + if err := h.cell.Create(h.ctx, s); err != nil { + h.t.Fatal(err) + } + return key +} + +func (h *harness) cellSession(name string) *computev1alpha.InstanceConsoleSession { + h.t.Helper() + var s computev1alpha.InstanceConsoleSession + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testNamespace, Name: name}, &s); err != nil { + h.t.Fatal(err) + } + return &s +} + +func (h *harness) reconcile(a *Agent, name string) ctrl.Result { + h.t.Helper() + res, err := a.Reconcile(h.ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: name}}) + if err != nil { + h.t.Fatalf("reconcile %s: %v", name, err) + } + return res +} + +func (h *harness) slots() []coordinationv1.Lease { + h.t.Helper() + var leases coordinationv1.LeaseList + if err := h.cell.List(h.ctx, &leases, client.MatchingLabels{componentLabel: slotComponent}); err != nil { + h.t.Fatal(err) + } + return leases.Items +} + +func requireReason(t *testing.T, s *computev1alpha.InstanceConsoleSession, want string) { + t.Helper() + if got := readyReason(s); got != want { + c := readyCondition(s) + t.Fatalf("Ready reason = %q, want %q (condition %+v)", got, want, c) + } +} + +// fakeAPIServer plays the apiserver's pods/exec endpoint. Commands the agent +// runs to completion answer from scripts; session streams are handed to +// onSession. +type fakeAPIServer struct { + mu sync.Mutex + commands [][]string + probeExit int + probeDir string + markers string + killExit int + killHangs bool + onSession func(conn net.Conn, r *bufio.Reader) +} + +func newFakeAPIServer() *fakeAPIServer { + return &fakeAPIServer{probeDir: "/tmp"} +} + +func (f *fakeAPIServer) ran(script string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, c := range f.commands { + if len(c) > 2 && c[2] == script { + n++ + } + } + return n +} + +func (f *fakeAPIServer) Open(_ context.Context, _ types.NamespacedName, opts ExecOptions, handshake http.Header) (*http.Response, net.Conn, *bufio.Reader, error) { + f.mu.Lock() + f.commands = append(f.commands, opts.Command) + onSession := f.onSession + f.mu.Unlock() + + agentSide, serverSide := net.Pipe() + resp := &http.Response{ + StatusCode: http.StatusSwitchingProtocols, + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{ + "Upgrade": []string{"websocket"}, + "Connection": []string{"Upgrade"}, + "Sec-Websocket-Protocol": []string{consolesession.SubProtocol}, + "Sec-Websocket-Accept": []string{acceptKey(handshake.Get("Sec-WebSocket-Key"))}, + }, + } + serverReader := bufio.NewReader(serverSide) + go func() { + defer func() { _ = serverSide.Close() }() + if handshake != nil && onSession != nil { + onSession(serverSide, serverReader) + return + } + f.answer(serverSide, opts.Command) + }() + return resp, agentSide, bufio.NewReader(agentSide), nil +} + +func acceptKey(key string) string { + sum := sha1.Sum([]byte(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11")) + return base64.StdEncoding.EncodeToString(sum[:]) +} + +func (f *fakeAPIServer) answer(conn net.Conn, command []string) { + f.mu.Lock() + probeExit, probeDir, markers, killExit, killHangs := f.probeExit, f.probeDir, f.markers, f.killExit, f.killHangs + f.mu.Unlock() + switch command[2] { + case probeScript: + if probeExit == 0 { + writeServerFrame(conn, channelStdout, []byte(probeDir+"\n")) + } + writeExitStatus(conn, probeExit) + case listMarkersScript: + writeServerFrame(conn, channelStdout, []byte(markers)) + writeExitStatus(conn, 0) + case killScript: + if killHangs { + _, _ = io.Copy(io.Discard, conn) + return + } + writeExitStatus(conn, killExit) + default: + writeExitStatus(conn, 0) + } +} + +func writeServerFrame(conn net.Conn, channel byte, payload []byte) { + _, _ = conn.Write(encodeFrame(opBinary, append([]byte{channel}, payload...), false)) +} + +func writeExitStatus(conn net.Conn, code int) { + var status metav1.Status + if code == 0 { + status = metav1.Status{Status: metav1.StatusSuccess} + } else { + c := int32(code) + status = closingStatus(computev1alpha.InstanceConsoleSessionReasonCompleted, &c, "") + } + body, _ := json.Marshal(status) + writeServerFrame(conn, channelStatus, body) + _, _ = conn.Write(encodeFrame(opClose, closePayload(closeNormal), false)) +} + +// wsClient is a minimal client side of the connection protocol. +type wsClient struct { + conn net.Conn + reader *bufio.Reader + status int + body string +} + +func dialSession(t *testing.T, addr, uid string, key ed25519.PrivateKey, at time.Time) *wsClient { + t.Helper() + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + req, _ := http.NewRequest(http.MethodGet, "http://"+addr+consolesession.ExecPath(uid), nil) + req.Header.Set("Connection", "Upgrade") + req.Header.Set("Upgrade", "websocket") + req.Header.Set("Sec-WebSocket-Version", "13") + req.Header.Set("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==") + req.Header.Set("Sec-WebSocket-Protocol", consolesession.SubProtocol) + if key != nil { + ts, sig := consolesession.Sign(key, uid, at) + req.Header.Set(consolesession.HeaderTimestamp, ts) + req.Header.Set(consolesession.HeaderSignature, sig) + } + if err := req.Write(conn); err != nil { + t.Fatal(err) + } + reader := bufio.NewReader(conn) + resp, err := http.ReadResponse(reader, req) + if err != nil { + t.Fatal(err) + } + c := &wsClient{conn: conn, reader: reader, status: resp.StatusCode} + if resp.StatusCode != http.StatusSwitchingProtocols { + var b strings.Builder + buf := make([]byte, 512) + n, _ := resp.Body.Read(buf) + b.Write(buf[:n]) + c.body = b.String() + } + return c +} + +// readUntilClose returns every data frame by channel, the status message and +// the close code. +func (c *wsClient) readUntilClose(t *testing.T) (map[byte]string, *metav1.Status, int) { + t.Helper() + _ = c.conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + out := map[byte]string{} + var status *metav1.Status + for { + f, err := readFrame(c.reader) + if err != nil { + t.Fatalf("read frame: %v", err) + } + if f.opcode == opClose { + code := 0 + if len(f.payload) >= 2 { + code = int(f.payload[0])<<8 | int(f.payload[1]) + } + return out, status, code + } + if f.control() || len(f.payload) == 0 { + continue + } + if f.payload[0] == channelStatus { + if status != nil { + t.Fatal("received more than one status message") + } + status = &metav1.Status{} + if err := json.Unmarshal(f.payload[1:], status); err != nil { + t.Fatal(err) + } + continue + } + out[f.payload[0]] += string(f.payload[1:]) + } +} + +func (c *wsClient) send(t *testing.T, opcode byte, payload []byte) { + t.Helper() + if _, err := c.conn.Write(encodeFrame(opcode, payload, true)); err != nil { + t.Fatal(err) + } +} diff --git a/internal/shellagent/identity.go b/internal/shellagent/identity.go new file mode 100644 index 00000000..21d92e0c --- /dev/null +++ b/internal/shellagent/identity.go @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "fmt" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // KeySecretKey is the key in an endpoint's Secret that holds its raw + // 32-byte identity key. + KeySecretKey = "key" + + annotationKeyCreated = annotationPrefix + "key-created-at" + rotationLockName = "endpoint-key-rotation" + rotationLockDuration = 2 * time.Hour + endpointRestartWait = 5 * time.Minute +) + +var rotationPollInterval = 5 * time.Second + +type identity struct { + endpointID string + created time.Time +} + +// KeySecretName is the Secret holding the identity key of the tunnel endpoint +// with the given ordinal. +func KeySecretName(ordinal int) string { + return fmt.Sprintf("endpoint-key-%d", ordinal) +} + +// EndpointID returns the iroh endpoint ID of a raw ed25519 identity key. +func EndpointID(seed []byte) string { + return hex.EncodeToString(ed25519.NewKeyFromSeed(seed).Public().(ed25519.PublicKey)) +} + +func newSeed() ([]byte, error) { + seed := make([]byte, ed25519.SeedSize) + if _, err := rand.Read(seed); err != nil { + return nil, err + } + return seed, nil +} + +// EnsureIdentity creates the paired endpoint's key Secret if it is missing and +// loads the endpoint ID the agent publishes. +func (a *Agent) EnsureIdentity(ctx context.Context) error { + key := client.ObjectKey{Namespace: a.cfg.Namespace, Name: KeySecretName(a.cfg.Ordinal)} + var secret corev1.Secret + err := a.cell.Get(ctx, key, &secret) + if apierrors.IsNotFound(err) { + var seed []byte + if seed, err = newSeed(); err != nil { + return err + } + secret = corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + Labels: map[string]string{componentLabel: "shell-endpoint-key"}, + Annotations: map[string]string{annotationKeyCreated: a.now().UTC().Format(time.RFC3339)}, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{KeySecretKey: seed}, + } + err = a.cell.Create(ctx, &secret) + if apierrors.IsAlreadyExists(err) { + err = a.cell.Get(ctx, key, &secret) + } + } + if err != nil { + return fmt.Errorf("load endpoint key %s: %w", key.Name, err) + } + return a.loadIdentity(&secret) +} + +func (a *Agent) loadIdentity(secret *corev1.Secret) error { + seed := secret.Data[KeySecretKey] + if len(seed) != ed25519.SeedSize { + return fmt.Errorf("endpoint key %s holds %d bytes, want %d", secret.Name, len(seed), ed25519.SeedSize) + } + created := secret.CreationTimestamp.Time + if at, err := time.Parse(time.RFC3339, secret.Annotations[annotationKeyCreated]); err == nil { + created = at + } + a.identity.Store(&identity{endpointID: EndpointID(seed), created: created}) + return nil +} + +// RunKeyRotation replaces the endpoint's identity key once it is older than +// the rotation period, checking hourly until ctx ends. +func (a *Agent) RunKeyRotation(ctx context.Context) error { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + if id := a.identity.Load(); id != nil && a.now().Sub(id.created) >= a.cfg.KeyRotationPeriod { + if err := a.rotate(ctx); err != nil && ctx.Err() == nil { + log.FromContext(ctx).Error(err, "rotate endpoint key") + } + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +// rotate stops claiming, waits for open sessions to end or expire, replaces +// the key, restarts the endpoint so it serves the new key, and resumes. One +// agent in the cell rotates at a time, so the cell keeps taking sessions. +func (a *Agent) rotate(ctx context.Context) error { + locked, err := a.lockRotation(ctx) + if err != nil || !locked { + return err + } + defer a.unlockRotation(ctx) + a.rotating.Store(true) + defer a.rotating.Store(false) + logger := log.FromContext(ctx) + logger.Info("rotating endpoint key; waiting for open sessions to end") + + idle := 0 + for idle < 2 { + if a.draining.Load() { + return nil + } + if a.openCount() == 0 && len(a.liveSessions()) == 0 { + idle++ + } else { + idle = 0 + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(rotationPollInterval): + } + } + + previous := a.EndpointID() + key := client.ObjectKey{Namespace: a.cfg.Namespace, Name: KeySecretName(a.cfg.Ordinal)} + var secret corev1.Secret + if err := a.cell.Get(ctx, key, &secret); err != nil { + return err + } + seed, err := newSeed() + if err != nil { + return err + } + secret.Data = map[string][]byte{KeySecretKey: seed} + if secret.Annotations == nil { + secret.Annotations = map[string]string{} + } + secret.Annotations[annotationKeyCreated] = a.now().UTC().Format(time.RFC3339) + if err := a.cell.Update(ctx, &secret); err != nil { + return err + } + if err := a.loadIdentity(&secret); err != nil { + return err + } + a.releaseLease(ctx, previous) + logger.Info("replaced endpoint key", "endpointID", a.EndpointID()) + return a.restartEndpoint(ctx) +} + +// restartEndpoint deletes the paired endpoint's pod and waits for its +// replacement to be ready with the new key. +func (a *Agent) restartEndpoint(ctx context.Context) error { + if a.cfg.EndpointPodName == "" { + return nil + } + key := client.ObjectKey{Namespace: a.cfg.Namespace, Name: a.cfg.EndpointPodName} + var pod corev1.Pod + if err := a.cell.Get(ctx, key, &pod); client.IgnoreNotFound(err) != nil { + return err + } + previous := pod.UID + if previous != "" { + if err := a.cell.Delete(ctx, &pod, client.Preconditions{UID: &previous}); client.IgnoreNotFound(err) != nil { + return err + } + } + waitCtx, cancel := context.WithTimeout(ctx, endpointRestartWait) + defer cancel() + for { + var next corev1.Pod + err := a.cell.Get(waitCtx, key, &next) + if err == nil && next.UID != previous && podReady(&next) { + return nil + } + select { + case <-waitCtx.Done(): + return fmt.Errorf("endpoint %s did not become ready after its key rotated", key.Name) + case <-time.After(2 * time.Second): + } + } +} + +// endpointReady reports whether the paired endpoint can carry a connection. A +// session claimed while it cannot fails at the client's dial. +func (a *Agent) endpointReady(ctx context.Context) (bool, error) { + if a.cfg.EndpointPodName == "" { + return true, nil + } + var pod corev1.Pod + err := a.cell.Get(ctx, client.ObjectKey{Namespace: a.cfg.Namespace, Name: a.cfg.EndpointPodName}, &pod) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + return podReady(&pod), nil +} + +func podReady(pod *corev1.Pod) bool { + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +func (a *Agent) lockRotation(ctx context.Context) (bool, error) { + now := metav1.NewMicroTime(a.now()) + lock := &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: rotationLockName, Namespace: a.cfg.Namespace}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: ptr.To(a.incarnation), + LeaseDurationSeconds: ptr.To(int32(rotationLockDuration.Seconds())), + AcquireTime: &now, + RenewTime: &now, + }, + } + err := a.cell.Create(ctx, lock) + if err == nil { + return true, nil + } + if !apierrors.IsAlreadyExists(err) { + return false, err + } + var held coordinationv1.Lease + if err := a.cell.Get(ctx, client.ObjectKeyFromObject(lock), &held); err != nil { + return false, client.IgnoreNotFound(err) + } + if held.Spec.RenewTime != nil && a.now().Before(held.Spec.RenewTime.Add(rotationLockDuration)) { + return false, nil + } + _ = a.cell.Delete(ctx, &held, client.Preconditions{UID: &held.UID}) + return false, nil +} + +func (a *Agent) unlockRotation(ctx context.Context) { + err := a.cell.Delete(ctx, &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{ + Name: rotationLockName, + Namespace: a.cfg.Namespace, + }}) + if client.IgnoreNotFound(err) != nil { + log.FromContext(ctx).Error(err, "release key rotation lock") + } +} diff --git a/internal/shellagent/liveness.go b/internal/shellagent/liveness.go new file mode 100644 index 00000000..fd88d5ed --- /dev/null +++ b/internal/shellagent/liveness.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // LeaseDuration is how long an agent's liveness Lease holds without a + // renewal before other agents take over its sessions. + LeaseDuration = 15 * time.Second + // LeaseRenewInterval is how often an agent renews its liveness Lease. + LeaseRenewInterval = 5 * time.Second + + livenessComponent = "shell-agent-liveness" +) + +// agentLeaseName names an agent's liveness Lease by the full SHA-256 of its +// endpoint ID. Lease names are DNS subdomains, which fit all 64 hex digits, so +// two agents never share a Lease. +func agentLeaseName(endpointID string) string { + sum := sha256.Sum256([]byte(endpointID)) + return "agent-" + hex.EncodeToString(sum[:]) +} + +// RunLiveness renews the agent's liveness Lease until ctx ends. +func (a *Agent) RunLiveness(ctx context.Context) error { + ticker := time.NewTicker(LeaseRenewInterval) + defer ticker.Stop() + for { + if err := a.renewLease(ctx); err != nil && ctx.Err() == nil { + log.FromContext(ctx).Error(err, "renew liveness lease") + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +func (a *Agent) renewLease(ctx context.Context) error { + id := a.EndpointID() + if id == "" { + return nil + } + now := metav1.NewMicroTime(a.now()) + key := client.ObjectKey{Namespace: a.cfg.Namespace, Name: agentLeaseName(id)} + var lease coordinationv1.Lease + err := a.cell.Get(ctx, key, &lease) + if apierrors.IsNotFound(err) { + return a.cell.Create(ctx, &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + Labels: map[string]string{componentLabel: livenessComponent}, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: ptr.To(a.incarnation), + LeaseDurationSeconds: ptr.To(int32(LeaseDuration.Seconds())), + AcquireTime: &now, + RenewTime: &now, + }, + }) + } + if err != nil { + return err + } + lease.Spec.HolderIdentity = ptr.To(a.incarnation) + lease.Spec.LeaseDurationSeconds = ptr.To(int32(LeaseDuration.Seconds())) + lease.Spec.RenewTime = &now + return a.cell.Update(ctx, &lease) +} + +// releaseLease deletes a liveness Lease once its agent no longer serves +// sessions under that endpoint ID. +func (a *Agent) releaseLease(ctx context.Context, endpointID string) { + err := a.cell.Delete(ctx, &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{ + Name: agentLeaseName(endpointID), + Namespace: a.cfg.Namespace, + }}) + if client.IgnoreNotFound(err) != nil { + log.FromContext(ctx).Error(err, "release liveness lease") + } +} + +// agentAlive reports whether the agent serving endpointID renewed its +// liveness Lease within the Lease's duration. +func (a *Agent) agentAlive(ctx context.Context, endpointID string) (bool, error) { + if endpointID == a.EndpointID() { + return true, nil + } + var lease coordinationv1.Lease + err := a.cell.Get(ctx, client.ObjectKey{Namespace: a.cfg.Namespace, Name: agentLeaseName(endpointID)}, &lease) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + if lease.Spec.RenewTime == nil || lease.Spec.LeaseDurationSeconds == nil { + return false, nil + } + expires := lease.Spec.RenewTime.Add(time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second) + return a.now().Before(expires), nil +} diff --git a/internal/shellagent/reconciler.go b/internal/shellagent/reconciler.go new file mode 100644 index 00000000..5f71bb2a --- /dev/null +++ b/internal/shellagent/reconciler.go @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "errors" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +const ( + claimRetryInterval = 5 * time.Second + cleanupRetryInterval = 5 * time.Second +) + +// SetupWithManager reconciles session copies delivered to the cell. The agent +// writes status only on these copies; Karmada carries it back to the hub. +func (a *Agent) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + Named("shell-agent"). + For(&computev1alpha.InstanceConsoleSession{}). + WithOptions(controller.Options{MaxConcurrentReconciles: 16}). + Complete(a) +} + +// Reconcile drives one session: claiming it, ending it at its deadlines or +// when it is revoked, taking it over from a lost agent, and stopping its +// processes before its finalizer is released. +func (a *Agent) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var session computev1alpha.InstanceConsoleSession + if err := a.sessions.Get(ctx, req.NamespacedName, &session); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("session", sessionUID(&session))) + return a.reconcileSession(ctx, &session) +} + +func (a *Agent) reconcileSession(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(session) + if !session.DeletionTimestamp.IsZero() { + return a.cleanUp(ctx, session) + } + if uid == "" { + return ctrl.Result{}, a.rejectUnclaimed(ctx, session, reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + "The session was delivered without its session UID.")) + } + if revoked(session) && !isTerminal(session) { + return a.revoke(ctx, session) + } + if isTerminal(session) { + a.forget(uid) + return a.settle(ctx, session) + } + + owner := endpointOf(session) + switch { + case owner == "": + return a.claim(ctx, session) + case owner != a.EndpointID(): + return a.watchOwner(ctx, session, owner) + } + + a.track(uid, client.ObjectKeyFromObject(session)) + now := a.now() + switch readyReason(session) { + case computev1alpha.InstanceConsoleSessionReasonSessionReady: + if a.draining.Load() { + return ctrl.Result{}, a.endUnconnected(ctx, session, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + } + if session.Status.ConnectBefore != nil && now.Before(session.Status.ConnectBefore.Time) { + return ctrl.Result{RequeueAfter: session.Status.ConnectBefore.Sub(now)}, nil + } + log.FromContext(ctx).Info("session was not connected in time") + return ctrl.Result{}, a.endUnconnected(ctx, session, computev1alpha.InstanceConsoleSessionReasonNotConnected) + case computev1alpha.InstanceConsoleSessionReasonConnected: + if !a.isLive(uid) { + log.FromContext(ctx).Info("ending a session this agent's previous run served") + return ctrl.Result{}, a.abandon(ctx, session, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) + } + if session.Status.ExpiresAt != nil && now.Before(session.Status.ExpiresAt.Time) { + return ctrl.Result{RequeueAfter: session.Status.ExpiresAt.Sub(now)}, nil + } + log.FromContext(ctx).Info("session expired") + a.stop(uid, computev1alpha.InstanceConsoleSessionReasonExpired) + } + return ctrl.Result{}, nil +} + +// claim checks a session and, if it can run, takes a slot and publishes where +// to connect. The cell's agents race on the cell copy with optimistic +// concurrency, so exactly one claim lands. A cell that does not run the +// session's instance leaves it to the cell that does, or to the control plane. +func (a *Agent) claim(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + if !a.claiming() || a.openCount() >= a.cfg.MaxOpenSessions { + return ctrl.Result{RequeueAfter: claimRetryInterval}, nil + } + ready, err := a.endpointReady(ctx) + if err != nil { + return ctrl.Result{}, err + } + if !ready { + return ctrl.Result{RequeueAfter: claimRetryInterval}, nil + } + target, rej, err := a.check(ctx, session) + if errors.Is(err, errNotInCell) { + return ctrl.Result{RequeueAfter: claimRetryInterval}, nil + } + if err != nil { + return ctrl.Result{}, err + } + if rej != nil { + log.FromContext(ctx).Info("refusing session", "reason", rej.reason, "message", rej.message) + return ctrl.Result{}, a.rejectUnclaimed(ctx, session, rej) + } + + uid := sessionUID(session) + held, err := a.acquireSlot(ctx, target.instance.UID, &slot{ + sessionUID: uid, + session: client.ObjectKeyFromObject(session), + pod: client.ObjectKeyFromObject(target.pod), + podUID: target.pod.UID, + container: target.container, + containerID: target.containerID, + markerDir: target.markerDir, + endpointID: a.EndpointID(), + }) + if errors.Is(err, errClaimInProgress) { + return ctrl.Result{RequeueAfter: claimRetryInterval}, nil + } + if err != nil { + return ctrl.Result{}, err + } + if held == nil { + return ctrl.Result{}, a.rejectUnclaimed(ctx, session, reject( + computev1alpha.InstanceConsoleSessionReasonTooManySessions, + "The instance already has %d open sessions, the most it allows.", a.cfg.SlotsPerInstance)) + } + + claimed := session.DeepCopy() + if err := addFinalizer(ctx, a.cell, claimed); err != nil { + return a.abortClaim(ctx, client.ObjectKeyFromObject(session), held, err) + } + now := a.now() + connectBefore := metav1.NewTime(now.Add(a.cfg.ConnectTimeout)) + claimed.Status.Connection = &computev1alpha.InstanceConsoleSessionConnection{ + EndpointID: a.EndpointID(), + RelayURLs: a.cfg.RelayURLs, + Target: a.cfg.Target, + } + claimed.Status.ConnectBefore = &connectBefore + setReady(claimed, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonSessionReady, + "Connect before the connection deadline.") + if err := a.cell.Status().Update(ctx, claimed); err != nil { + return a.abortClaim(ctx, client.ObjectKeyFromObject(session), held, err) + } + a.track(uid, client.ObjectKeyFromObject(session)) + log.FromContext(ctx).Info("claimed session", "pod", target.pod.Name, "container", target.container) + return ctrl.Result{RequeueAfter: a.cfg.ConnectTimeout}, nil +} + +// abortClaim frees the slot of a claim that did not land. A conflict means +// the session changed, most likely because another agent claimed it, and the +// retry sees the change. The cache can also lag this agent's own claim, and +// the slot of a claim that already stands is kept. +func (a *Agent) abortClaim(ctx context.Context, key types.NamespacedName, held *slot, err error) (ctrl.Result, error) { + if apierrors.IsConflict(err) { + var current computev1alpha.InstanceConsoleSession + if getErr := a.cell.Get(ctx, key, ¤t); getErr != nil { + return ctrl.Result{}, client.IgnoreNotFound(getErr) + } + if endpointOf(¤t) == a.EndpointID() && !isTerminal(¤t) { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + } + if relErr := a.releaseSlot(ctx, held); relErr != nil { + return ctrl.Result{}, relErr + } + if apierrors.IsConflict(err) { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return ctrl.Result{}, err +} + +// watchOwner takes over a session whose agent stopped renewing its liveness +// Lease. A session with no slot has nothing left to take over. +func (a *Agent) watchOwner(ctx context.Context, session *computev1alpha.InstanceConsoleSession, owner string) (ctrl.Result, error) { + held, err := a.slotFor(ctx, sessionUID(session)) + if err != nil || held == nil { + return ctrl.Result{}, err + } + alive, err := a.agentAlive(ctx, owner) + if err != nil { + return ctrl.Result{}, err + } + if alive { + return ctrl.Result{RequeueAfter: LeaseDuration}, nil + } + log.FromContext(ctx).Info("taking over a session from a lost agent", "agent", owner) + return ctrl.Result{}, a.abandon(ctx, session, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) +} + +// abandon ends a session no agent is serving: it stops the session's process, +// which a dropped stream leaves running, and records why. The slot is kept if +// the process could not be confirmed stopped, for the sweep to retry. +func (a *Agent) abandon(ctx context.Context, session *computev1alpha.InstanceConsoleSession, owner, reason string) error { + uid := sessionUID(session) + held, err := a.slotFor(ctx, uid) + if err != nil { + return err + } + stopped := true + if held != nil { + if stopped, err = a.stopProcesses(ctx, held); err != nil { + return err + } + } + if _, err := a.end(ctx, client.ObjectKeyFromObject(session), reason, endMessage(reason), nil, stopped, + func(s *computev1alpha.InstanceConsoleSession) bool { return endpointOf(s) == owner }); err != nil { + return err + } + a.forget(uid) + if stopped { + return a.releaseSlot(ctx, held) + } + return nil +} + +// endUnconnected ends a session this agent claimed but no client connected +// to. No process was started, so its slot is freed at once. +func (a *Agent) endUnconnected(ctx context.Context, session *computev1alpha.InstanceConsoleSession, reason string) error { + uid := sessionUID(session) + me := a.EndpointID() + ended, err := a.end(ctx, client.ObjectKeyFromObject(session), reason, endMessage(reason), nil, true, + func(s *computev1alpha.InstanceConsoleSession) bool { + return endpointOf(s) == me && readyReason(s) == computev1alpha.InstanceConsoleSessionReasonSessionReady + }) + if err != nil || !ended { + return err + } + a.forget(uid) + held, err := a.slotFor(ctx, uid) + if err != nil { + return err + } + return a.releaseSlot(ctx, held) +} + +func (a *Agent) rejectUnclaimed(ctx context.Context, session *computev1alpha.InstanceConsoleSession, rej *rejection) error { + _, err := a.end(ctx, client.ObjectKeyFromObject(session), rej.reason, rej.message, nil, true, + func(s *computev1alpha.InstanceConsoleSession) bool { return s.Status.Connection == nil }) + return err +} + +// settle frees the slot of an ended session once its process is stopped, and +// then records when it was, unless a live agent is still ending it. +func (a *Agent) settle(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(session) + if a.isLive(uid) { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, nil + } + held, err := a.slotFor(ctx, uid) + if err != nil { + return ctrl.Result{}, err + } + if held == nil && session.Status.EndedAt != nil { + return ctrl.Result{}, nil + } + if owner := endpointOf(session); owner != "" && owner != a.EndpointID() { + if alive, err := a.agentAlive(ctx, owner); err != nil || alive { + return ctrl.Result{RequeueAfter: LeaseDuration}, err + } + } + if held != nil { + reaped, err := a.reap(ctx, held) + if err != nil || !reaped { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, err + } + } + return ctrl.Result{}, a.markStopped(ctx, client.ObjectKeyFromObject(session), uid) +} + +// revoke ends a session that is still open because the control plane revoked +// it or its copy is being deleted. The end is recorded only once the +// session's processes have been stopped, so the control plane can take it as +// confirmation that the command is gone. +func (a *Agent) revoke(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(session) + reason := computev1alpha.InstanceConsoleSessionReasonRevoked + switch owner := endpointOf(session); { + case owner == a.EndpointID() && a.isLive(uid): + log.FromContext(ctx).Info("revoking session") + a.stop(uid, reason) + case owner == a.EndpointID(): + if err := a.endUnconnected(ctx, session, reason); err != nil { + return ctrl.Result{}, err + } + case owner == "": + if err := a.rejectUnclaimed(ctx, session, reject(reason, "%s", endMessage(reason))); err != nil { + return ctrl.Result{}, err + } + default: + held, err := a.slotFor(ctx, uid) + if err != nil || held == nil { + return ctrl.Result{}, err + } + alive, err := a.agentAlive(ctx, owner) + if err != nil { + return ctrl.Result{}, err + } + if alive { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, nil + } + if err := a.abandon(ctx, session, owner, reason); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil +} + +// cleanUp handles a session whose cell copy is being deleted. The session is +// revoked if it is still open, and the finalizer is released only once its +// process is stopped. +func (a *Agent) cleanUp(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(session) + if !isTerminal(session) { + if res, err := a.revoke(ctx, session); err != nil || res.RequeueAfter > 0 { + return res, err + } + } + if a.isLive(uid) { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + held, err := a.slotFor(ctx, uid) + if err != nil { + return ctrl.Result{}, err + } + if held != nil { + reaped, err := a.reap(ctx, held) + if err != nil || !reaped { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, err + } + } + a.forget(uid) + if err := a.markStopped(ctx, client.ObjectKeyFromObject(session), uid); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, removeFinalizer(ctx, a.cell, client.ObjectKeyFromObject(session)) +} + +// end records a terminal reason on the cell copy if the session has not ended +// yet and guard still holds for its latest state. It reports whether this +// call ended the session. endedAt is set only when stopped says nothing the +// session started is still running, because the control plane takes it as +// confirmation of cleanup; otherwise markStopped sets it once the processes +// are stopped. +func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, message string, exitCode *int32, + stopped bool, guard func(*computev1alpha.InstanceConsoleSession) bool) (bool, error) { + ended := false + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var s computev1alpha.InstanceConsoleSession + if err := a.cell.Get(ctx, key, &s); err != nil { + return err + } + if isTerminal(&s) || guard != nil && !guard(&s) { + return nil + } + if stopped { + now := metav1.NewTime(a.now()) + s.Status.EndedAt = &now + } + s.Status.ExitCode = exitCode + setReady(&s, metav1.ConditionFalse, reason, message) + if err := a.cell.Status().Update(ctx, &s); err != nil { + return err + } + ended = true + return nil + }) + return ended, client.IgnoreNotFound(err) +} + +// markStopped records endedAt on an ended session whose processes are now +// known to be stopped. A copy with the same name that belongs to another +// session is left alone. +func (a *Agent) markStopped(ctx context.Context, key types.NamespacedName, uid string) error { + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var s computev1alpha.InstanceConsoleSession + if err := a.cell.Get(ctx, key, &s); err != nil { + return err + } + if sessionUID(&s) != uid || !isTerminal(&s) || s.Status.EndedAt != nil { + return nil + } + now := metav1.NewTime(a.now()) + s.Status.EndedAt = &now + return a.cell.Status().Update(ctx, &s) + }) + return client.IgnoreNotFound(err) +} + +// revoked reports whether the control plane asked the cell to end the +// session. +func revoked(s *computev1alpha.InstanceConsoleSession) bool { + _, ok := s.Annotations[computev1alpha.InstanceConsoleSessionRevokeAnnotation] + return ok +} + +func addFinalizer(ctx context.Context, c client.Client, s *computev1alpha.InstanceConsoleSession) error { + if controllerutil.ContainsFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { + return nil + } + base := s.DeepCopy() + controllerutil.AddFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) + return c.Patch(ctx, s, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) +} + +func removeFinalizer(ctx context.Context, c client.Client, key types.NamespacedName) error { + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var s computev1alpha.InstanceConsoleSession + if err := c.Get(ctx, key, &s); err != nil { + return err + } + if !controllerutil.ContainsFinalizer(&s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { + return nil + } + base := s.DeepCopy() + controllerutil.RemoveFinalizer(&s, computev1alpha.InstanceConsoleSessionAgentFinalizer) + return c.Patch(ctx, &s, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) + }) + return client.IgnoreNotFound(err) +} diff --git a/internal/shellagent/reconciler_test.go b/internal/shellagent/reconciler_test.go new file mode 100644 index 00000000..6275becf --- /dev/null +++ b/internal/shellagent/reconciler_test.go @@ -0,0 +1,894 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "fmt" + "net" + "strings" + "sync" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +func TestClaimPublishesConnection(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + + res := h.reconcile(a, testSession) + + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonSessionReady) + if s.Status.Connection == nil || s.Status.Connection.EndpointID != a.EndpointID() || + s.Status.Connection.Target != a.cfg.Target || len(s.Status.Connection.RelayURLs) != 1 { + t.Fatalf("connection = %+v", s.Status.Connection) + } + if want := h.clock.now().Add(time.Minute); !s.Status.ConnectBefore.Time.Equal(want) { + t.Fatalf("connectBefore = %v, want %v", s.Status.ConnectBefore, want) + } + if res.RequeueAfter != time.Minute { + t.Fatalf("requeue after %v, want the connect timeout", res.RequeueAfter) + } + if !controllerutil.ContainsFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { + t.Fatal("cell copy has no finalizer") + } + if n := len(h.slots()); n != 1 { + t.Fatalf("slots = %d, want 1", n) + } +} + +func TestClaimRaceHasOneWinner(t *testing.T) { + h := newHarness(t) + agents := []*Agent{h.agent(), h.agent()} + h.session(testSession, testUID) + + var wg sync.WaitGroup + for _, a := range agents { + wg.Go(func() { + for range 3 { + _, _ = a.Reconcile(h.ctx, reqFor(testSession)) + } + }) + } + wg.Wait() + + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonSessionReady) + owners := 0 + for _, a := range agents { + if s.Status.Connection.EndpointID == a.EndpointID() { + owners++ + } + } + if owners != 1 { + t.Fatalf("session claimed by %d agents", owners) + } + if n := len(h.slots()); n != 1 { + t.Fatalf("slots = %d, want 1", n) + } +} + +func TestClaimWithStaleCopyLosesToWinner(t *testing.T) { + h := newHarness(t) + winner, loser := h.agent(), h.agent() + h.session(testSession, testUID) + stale := h.cellSession(testSession) + + h.reconcile(winner, testSession) + if _, err := loser.claim(h.ctx, stale); err != nil { + t.Logf("losing claim returned %v", err) + } + + s := h.cellSession(testSession) + if s.Status.Connection.EndpointID != winner.EndpointID() { + t.Fatal("a stale claim overwrote the winner's connection") + } + slots := h.slots() + if len(slots) != 1 || slots[0].Annotations[annotationEndpoint] != winner.EndpointID() { + t.Fatalf("slots = %+v, want only the winner's", slots) + } +} + +func TestStaleRetryKeepsTheAgentsOwnClaim(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + stale := h.cellSession(testSession) + h.reconcile(a, testSession) + + if _, err := a.claim(h.ctx, stale); err != nil { + t.Fatalf("retried claim returned %v", err) + } + + s := h.cellSession(testSession) + if s.Status.Connection == nil || s.Status.Connection.EndpointID != a.EndpointID() { + t.Fatalf("connection = %+v", s.Status.Connection) + } + if n := len(h.slots()); n != 1 { + t.Fatalf("slots = %d, want the claim's slot kept", n) + } +} + +func TestSlotCapIsExactlyThree(t *testing.T) { + h := newHarness(t) + agents := []*Agent{h.agent(), h.agent()} + for i := range 6 { + h.session(fmt.Sprintf("s%d", i), fmt.Sprintf("uid-%d", i)) + } + + var wg sync.WaitGroup + for i := range 6 { + a := agents[i%2] + wg.Go(func() { + for range 3 { + _, _ = a.Reconcile(h.ctx, reqFor(fmt.Sprintf("s%d", i))) + } + }) + } + wg.Wait() + + counts := map[string]int{} + for i := range 6 { + counts[readyReason(h.cellSession(fmt.Sprintf("s%d", i)))]++ + } + if counts[computev1alpha.InstanceConsoleSessionReasonSessionReady] != 3 || + counts[computev1alpha.InstanceConsoleSessionReasonTooManySessions] != 3 { + t.Fatalf("outcomes = %v, want 3 ready and 3 refused", counts) + } + if n := len(h.slots()); n != 3 { + t.Fatalf("slots = %d, want 3", n) + } +} + +func TestSlotFreedByEndedSessionIsReused(t *testing.T) { + h := newHarness(t) + a := h.agent() + for i := range 4 { + h.session(fmt.Sprintf("s%d", i), fmt.Sprintf("uid-%d", i)) + } + for i := range 3 { + h.reconcile(a, fmt.Sprintf("s%d", i)) + } + s0 := h.cellSession("s0") + setReady(s0, metav1.ConditionFalse, computev1alpha.InstanceConsoleSessionReasonCompleted, "done") + if err := h.cell.Status().Update(h.ctx, s0); err != nil { + t.Fatal(err) + } + + h.reconcile(a, "s3") + + requireReason(t, h.cellSession("s3"), computev1alpha.InstanceConsoleSessionReasonSessionReady) +} + +func TestChecksBeforeClaim(t *testing.T) { + cases := []struct { + name string + mutate func(h *harness) + want string + }{ + {"no shell", func(h *harness) { h.exec.probeExit = 127 }, computev1alpha.InstanceConsoleSessionReasonNoShell}, + {"no writable directory", func(h *harness) { h.exec.probeExit = probeExitNoWritableDir }, computev1alpha.InstanceConsoleSessionReasonNoShell}, + {"missing executable", func(h *harness) { h.exec.probeExit = probeExitCommandMissing }, computev1alpha.InstanceConsoleSessionReasonCommandUnavailable}, + {"replaced instance", func(h *harness) { + patchInstance(h, func(i *computev1alpha.Instance) { + i.Labels[computev1alpha.WorkloadDeploymentUIDLabel] = "newer" + }) + if err := h.cell.Create(h.ctx, &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{ + Name: "web-1", + Namespace: testNamespace, + Labels: map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID}, + }}); err != nil { + h.t.Fatal(err) + } + }, computev1alpha.InstanceConsoleSessionReasonInstanceNotFound}, + {"pod not running", func(h *harness) { + patchPod(h, func(p *corev1.Pod) { p.Status.Phase = "Pending" }) + }, computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning}, + {"pod from another instance", func(h *harness) { + patchPod(h, func(p *corev1.Pod) { p.OwnerReferences[0].UID = "other" }) + }, computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning}, + {"other runtime", func(h *harness) { + patchPod(h, func(p *corev1.Pod) { p.Labels[managedByLabel] = "unikraft-provider" }) + }, computev1alpha.InstanceConsoleSessionReasonInvalid}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + a := h.agent() + tc.mutate(h) + h.session(testSession, testUID) + + h.reconcile(a, testSession) + + s := h.cellSession(testSession) + requireReason(t, s, tc.want) + if s.Status.Connection != nil || len(h.slots()) != 0 { + t.Fatal("a refused session was claimed") + } + }) + } +} + +func TestSessionForAnotherCellIsIgnored(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID, func(s *computev1alpha.InstanceConsoleSession) { + s.Labels[computev1alpha.WorkloadDeploymentUIDLabel] = "elsewhere" + }) + + for range 3 { + if res := h.reconcile(a, testSession); res.RequeueAfter <= 0 { + t.Fatal("expected a retry while the session is unclaimed") + } + h.clock.advance(10 * time.Minute) + } + + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) + if len(h.slots()) != 0 { + t.Fatal("a cell without the instance took a slot") + } +} + +// revokeSession annotates the cell copy the way Karmada does once the control +// plane revokes the session's hub copy. +func revokeSession(t *testing.T, h *harness) { + t.Helper() + s := h.cellSession(testSession) + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + s.Annotations[computev1alpha.InstanceConsoleSessionRevokeAnnotation] = h.clock.now().Format(time.RFC3339) + if err := h.cell.Update(h.ctx, s); err != nil { + t.Fatal(err) + } +} + +func requireRevokedUnclaimed(t *testing.T, h *harness) { + t.Helper() + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonRevoked) + if s.Status.Connection != nil || len(h.slots()) != 0 || + controllerutil.ContainsFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { + t.Fatalf("a revoked session was claimed: %+v", s) + } +} + +func TestRevokedSessionIsNotClaimed(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + revokeSession(t, h) + + h.reconcile(a, testSession) + + requireRevokedUnclaimed(t, h) +} + +func TestClaimLosesToRevoke(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + stale := h.cellSession(testSession) + revokeSession(t, h) + + if _, err := a.claim(h.ctx, stale); err != nil { + t.Logf("losing claim returned %v", err) + } + s := h.cellSession(testSession) + if s.Status.Connection != nil || len(h.slots()) != 0 { + t.Fatalf("a claim read before the revoke landed: %+v", s) + } + + h.reconcile(a, testSession) + requireRevokedUnclaimed(t, h) +} + +func TestProbeIsCachedPerPodAndContainer(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.session("s2", "uid-2") + + h.reconcile(a, testSession) + h.reconcile(a, "s2") + + if n := h.exec.ran(probeScript); n != 1 { + t.Fatalf("probe ran %d times, want 1", n) + } +} + +func TestNotConnectedDeadline(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + + h.clock.advance(time.Minute) + h.reconcile(a, testSession) + + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonNotConnected) + if s.Status.EndedAt == nil || s.Status.ExitCode != nil { + t.Fatalf("status = %+v", s.Status) + } + if n := len(h.slots()); n != 0 { + t.Fatalf("slots = %d, want 0", n) + } + if a.openCount() != 0 { + t.Fatal("ended session still counts as open") + } +} + +func TestExpiryStopsConnectedSession(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Minute)) + stopped := fakeLive(a, testUID, false) + + if res := h.reconcile(a, testSession); res.RequeueAfter != time.Minute { + t.Fatalf("requeue after %v, want the time left", res.RequeueAfter) + } + h.clock.advance(time.Minute) + h.reconcile(a, testSession) + + if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonExpired { + t.Fatalf("stop reason = %q, want Expired", got) + } +} + +func TestLivenessTakeover(t *testing.T) { + h := newHarness(t) + lost, survivor := h.agent(), h.agent() + h.session(testSession, testUID) + h.reconcile(lost, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + + if res := h.reconcile(survivor, testSession); res.RequeueAfter != LeaseDuration { + t.Fatalf("requeue after %v while the owner is alive, want %v", res.RequeueAfter, LeaseDuration) + } + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) + + h.clock.advance(LeaseDuration + time.Second) + if err := survivor.renewLease(h.ctx); err != nil { + t.Fatal(err) + } + h.reconcile(survivor, testSession) + + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonAgentLost) + if h.exec.ran(killScript) != 1 { + t.Fatal("the lost session's processes were not stopped") + } + if n := len(h.slots()); n != 0 { + t.Fatalf("slots = %d, want 0", n) + } +} + +func TestRestartedAgentEndsItsPreviousSessions(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + + h.reconcile(a, testSession) + + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonAgentLost) + if h.exec.ran(killScript) != 1 { + t.Fatal("the previous run's processes were not stopped") + } +} + +func TestSessionOwnedByAnotherAgentWithoutSlotIsLeftAlone(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + s := h.cellSession(testSession) + s.Status.Connection = &computev1alpha.InstanceConsoleSessionConnection{EndpointID: "another-agents-endpoint"} + setReady(s, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonSessionReady, "ready") + if err := h.cell.Status().Update(h.ctx, s); err != nil { + t.Fatal(err) + } + + h.reconcile(a, testSession) + + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) +} + +func TestRevokeStopsConnectedSession(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + stopped := fakeLive(a, testUID, false) + revokeSession(t, h) + + h.reconcile(a, testSession) + + if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonRevoked { + t.Fatalf("stop reason = %q, want Revoked", got) + } + if !controllerutil.ContainsFinalizer(h.cellSession(testSession), computev1alpha.InstanceConsoleSessionAgentFinalizer) { + t.Fatal("the finalizer holds the copy until the control plane deletes it") + } +} + +func TestRevokeEndsUnconnectedSessionAndFreesItsSlot(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + revokeSession(t, h) + + h.reconcile(a, testSession) + + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonRevoked) + if s.Status.EndedAt == nil { + t.Fatal("the end has no time") + } + if len(h.slots()) != 0 { + t.Fatal("slot kept after the revoke") + } +} + +func TestRevokeOfLostAgentsSessionStopsProcessesFirst(t *testing.T) { + h := newHarness(t) + lost, survivor := h.agent(), h.agent() + h.session(testSession, testUID) + h.reconcile(lost, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + lost.releaseLease(h.ctx, lost.EndpointID()) + revokeSession(t, h) + + h.reconcile(survivor, testSession) + + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonRevoked) + if h.exec.ran(killScript) == 0 || len(h.slots()) != 0 { + t.Fatal("the end was reported before the lost agent's processes were stopped") + } +} + +func TestDeletionReleasesFinalizerAfterProcessesStop(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + stopped := fakeLive(a, testUID, false) + + if err := h.cell.Delete(h.ctx, h.cellSession(testSession)); err != nil { + t.Fatal(err) + } + h.reconcile(a, testSession) + if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonRevoked { + t.Fatalf("stop reason = %q, want Revoked", got) + } + + var cell computev1alpha.InstanceConsoleSession + if err := h.cell.Get(h.ctx, reqFor(testSession).NamespacedName, &cell); !apierrors.IsNotFound(err) { + t.Fatalf("cell copy still exists with finalizers %v: %v", cell.Finalizers, err) + } + if h.exec.ran(killScript) == 0 || len(h.slots()) != 0 { + t.Fatal("processes were not stopped before the finalizer was released") + } +} + +func TestFinalizerHeldWhileProcessesCannotBeStopped(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + endAs(h, computev1alpha.InstanceConsoleSessionReasonAgentLost) + h.exec.killExit = 1 + + if err := h.cell.Delete(h.ctx, h.cellSession(testSession)); err != nil { + t.Fatal(err) + } + if res := h.reconcile(a, testSession); res.RequeueAfter == 0 { + t.Fatal("expected a retry while processes may be running") + } + if !controllerutil.ContainsFinalizer(h.cellSession(testSession), computev1alpha.InstanceConsoleSessionAgentFinalizer) { + t.Fatal("finalizer released before processes were stopped") + } + + h.exec.killExit = 0 + h.reconcile(a, testSession) + var cell computev1alpha.InstanceConsoleSession + if err := h.cell.Get(h.ctx, reqFor(testSession).NamespacedName, &cell); !apierrors.IsNotFound(err) { + t.Fatalf("cell copy kept after processes stopped: %v", err) + } +} + +func TestUnclaimedDeletionIsRevoked(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID, func(s *computev1alpha.InstanceConsoleSession) { + s.Finalizers = []string{"test/hold"} + }) + if err := h.cell.Delete(h.ctx, h.cellSession(testSession)); err != nil { + t.Fatal(err) + } + + h.reconcile(a, testSession) + + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonRevoked) +} + +func TestDrain(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.DrainTimeout = 100 * time.Millisecond }) + h.session("waiting", "uid-w") + h.session("tty", "uid-t") + h.session("pipe", "uid-p", func(s *computev1alpha.InstanceConsoleSession) { s.Spec.Terminal = false }) + for _, name := range []string{"waiting", "tty", "pipe"} { + h.reconcile(a, name) + } + markConnected(h, "tty", h.clock.now().Add(time.Hour)) + markConnected(h, "pipe", h.clock.now().Add(time.Hour)) + ttyClient, ttyStopped := liveWithStream(a, "uid-t", true) + pipeClient, pipeStopped := liveWithStream(a, "uid-p", false) + + a.Drain(h.ctx) + + requireReason(t, h.cellSession("waiting"), computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + for _, stopped := range []chan string{ttyStopped, pipeStopped} { + if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonAgentShutdown { + t.Fatalf("stop reason = %q, want AgentShutdown", got) + } + } + if got := <-ttyClient; got == "" { + t.Fatal("terminal session got no maintenance notice") + } + if got := <-pipeClient; got != "" { + t.Fatalf("session without a terminal got injected bytes %q", got) + } + if !a.draining.Load() || a.claiming() { + t.Fatal("a drained agent still claims") + } + h.session("late", "uid-l") + h.reconcile(a, "late") + requireReason(t, h.cellSession("late"), computev1alpha.InstanceConsoleSessionReasonPending) +} + +func TestConcurrencyCapDefersClaims(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.MaxOpenSessions = 1 }) + h.session(testSession, testUID) + h.session("s2", "uid-2") + + h.reconcile(a, testSession) + res := h.reconcile(a, "s2") + + requireReason(t, h.cellSession("s2"), computev1alpha.InstanceConsoleSessionReasonPending) + if res.RequeueAfter == 0 { + t.Fatal("a deferred claim must be retried") + } +} + +func TestClaimWaitsForReadyEndpoint(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.EndpointPodName = "exec-endpoint-0" }) + h.session(testSession, testUID) + + res := h.reconcile(a, testSession) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) + if res.RequeueAfter == 0 { + t.Fatal("a claim deferred for a missing endpoint must be retried") + } + + endpoint := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: testAgentNamespace, Name: "exec-endpoint-0"}} + if err := h.cell.Create(h.ctx, endpoint); err != nil { + t.Fatal(err) + } + h.reconcile(a, testSession) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) + + endpoint.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if err := h.cell.Status().Update(h.ctx, endpoint); err != nil { + t.Fatal(err) + } + h.reconcile(a, testSession) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) +} + +func TestSweepStopsOrphanedProcesses(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, "uid-live") + h.reconcile(a, testSession) + orphan := &slot{ + sessionUID: "uid-gone", + session: types.NamespacedName{Namespace: testNamespace, Name: "gone"}, + pod: types.NamespacedName{Namespace: testNamespace, Name: testInstance}, + podUID: "pod-uid", + container: testContainer, + markerDir: "/tmp", + } + if err := h.cell.Create(h.ctx, orphan.leaseFor(testAgentNamespace, "slot-orphan-0")); err != nil { + t.Fatal(err) + } + h.exec.markers = "uid-live\nuid-gone\nuid-older\n" + + if err := a.sweep(h.ctx); err != nil { + t.Fatal(err) + } + + killed := map[string]bool{} + for _, c := range h.exec.commands { + if len(c) > 4 && c[2] == killScript { + killed[c[4]] = true + } + } + for _, uid := range []string{"uid-gone", "uid-older"} { + if !killed[markerPath("/tmp", uid)] { + t.Fatalf("orphan %s was not stopped (killed %v)", uid, killed) + } + } + if killed[markerPath("/tmp", "uid-live")] { + t.Fatal("an open session's process was stopped") + } + slots := h.slots() + if len(slots) != 1 || ptr.Deref(slots[0].Spec.HolderIdentity, "") != "uid-live" { + t.Fatalf("slots = %v, want only the open session's", slots) + } +} + +func TestEnsureIdentityCreatesKey(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.Ordinal = 1 }) + a.identity.Store(nil) + + if err := a.EnsureIdentity(h.ctx); err != nil { + t.Fatal(err) + } + var secret corev1.Secret + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testAgentNamespace, Name: "endpoint-key-1"}, &secret); err != nil { + t.Fatal(err) + } + if len(secret.Data[KeySecretKey]) != 32 || a.EndpointID() != EndpointID(secret.Data[KeySecretKey]) { + t.Fatal("endpoint ID does not match the stored key") + } + first := a.EndpointID() + if err := a.EnsureIdentity(h.ctx); err != nil || a.EndpointID() != first { + t.Fatal("an existing key was replaced") + } +} + +func TestRotationWaitsForOpenSessions(t *testing.T) { + rotationPollInterval = 10 * time.Millisecond + h := newHarness(t) + a := h.agent() + a.identity.Store(nil) + if err := a.EnsureIdentity(h.ctx); err != nil { + t.Fatal(err) + } + before := a.EndpointID() + a.track("uid-open", types.NamespacedName{Name: "open"}) + + done := make(chan error, 1) + go func() { done <- a.rotate(h.ctx) }() + time.Sleep(50 * time.Millisecond) + if a.EndpointID() != before || !a.rotating.Load() || a.claiming() { + t.Fatal("rotation must stop claiming and wait for open sessions") + } + a.forget("uid-open") + if err := <-done; err != nil { + t.Fatal(err) + } + if a.EndpointID() == before || a.rotating.Load() { + t.Fatal("key was not rotated") + } + var lease coordinationv1.Lease + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testAgentNamespace, Name: agentLeaseName(before)}, &lease); err == nil { + t.Fatal("the old endpoint's liveness lease was kept") + } +} + +func reqFor(name string) ctrl.Request { + return ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: name}} +} + +func markConnected(h *harness, name string, expires time.Time) { + h.t.Helper() + s := h.cellSession(name) + started := metav1.NewTime(h.clock.now()) + s.Status.StartedAt = &started + s.Status.ExpiresAt = &metav1.Time{Time: expires} + setReady(s, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonConnected, "connected") + if err := h.cell.Status().Update(h.ctx, s); err != nil { + h.t.Fatal(err) + } +} + +func endAs(h *harness, reason string) { + h.t.Helper() + s := h.cellSession(testSession) + setReady(s, metav1.ConditionFalse, reason, "ended") + if err := h.cell.Status().Update(h.ctx, s); err != nil { + h.t.Fatal(err) + } +} + +// fakeLive registers a connection that reports the reason it was stopped for. +func fakeLive(a *Agent, uid string, tty bool) chan string { + stopped := make(chan string, 1) + var l *liveSession + l, _ = a.register(uid, func() { + go func() { + stopped <- a.stopReason(uid) + a.unregister(uid) + close(l.done) + }() + }, tty) + return stopped +} + +// liveWithStream registers a connection with a client stream, and reports +// what the client read before the stream was stopped. +func liveWithStream(a *Agent, uid string, tty bool) (chan string, chan string) { + clientSide, agentSide := net.Pipe() + read := make(chan string, 1) + go func() { + buf := make([]byte, 4096) + _ = clientSide.SetReadDeadline(time.Now().Add(time.Second)) + n, _ := clientSide.Read(buf) + read <- string(buf[:n]) + }() + stopped := fakeLive(a, uid, tty) + a.mu.Lock() + a.live[uid].stream = &clientStream{conn: agentSide} + a.mu.Unlock() + return read, stopped +} + +func patchInstance(h *harness, mutate func(*computev1alpha.Instance)) { + h.t.Helper() + var inst computev1alpha.Instance + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testNamespace, Name: testInstance}, &inst); err != nil { + h.t.Fatal(err) + } + mutate(&inst) + if err := h.cell.Update(h.ctx, &inst); err != nil { + h.t.Fatal(err) + } +} + +func patchPod(h *harness, mutate func(*corev1.Pod)) { + h.t.Helper() + var pod corev1.Pod + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testNamespace, Name: testInstance}, &pod); err != nil { + h.t.Fatal(err) + } + mutate(&pod) + status := pod.Status + if err := h.cell.Update(h.ctx, &pod); err != nil { + h.t.Fatal(err) + } + pod.Status = status + if err := h.cell.Status().Update(h.ctx, &pod); err != nil { + h.t.Fatal(err) + } +} + +func TestTerminalSessionIsNotRequeued(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + endAs(h, computev1alpha.InstanceConsoleSessionReasonNoShell) + + if res := h.reconcile(a, testSession); res.RequeueAfter != 0 { + t.Fatalf("requeue after %v for an ended session this agent never held", res.RequeueAfter) + } +} + +func TestHungExecIsBounded(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.ExecTimeout = 200 * time.Millisecond }) + h.session(testSession, testUID) + h.reconcile(a, testSession) + held, err := a.slotFor(h.ctx, testUID) + if err != nil || held == nil { + t.Fatalf("slot = %v, %v", held, err) + } + h.exec.mu.Lock() + h.exec.killHangs = true + h.exec.mu.Unlock() + + start := time.Now() + stopped, err := a.stopProcesses(h.ctx, held) + + if err != nil || stopped { + t.Fatalf("stopProcesses() = %v, %v; want not stopped, no error", stopped, err) + } + if took := time.Since(start); took > 5*time.Second { + t.Fatalf("a hung exec held the caller for %v", took) + } +} + +func TestAgentLeaseNamesAreDistinctAndValid(t *testing.T) { + a := agentLeaseName(strings.Repeat("a", 16) + strings.Repeat("0", 48)) + b := agentLeaseName(strings.Repeat("a", 16) + strings.Repeat("1", 48)) + if a == b { + t.Fatal("endpoint IDs sharing a prefix share a liveness Lease") + } + if errs := validation.IsDNS1123Subdomain(a); len(errs) != 0 { + t.Fatalf("lease name %q is invalid: %v", a, errs) + } +} + +func TestEndIsConfirmedOnlyOnceProcessesStop(t *testing.T) { + h := newHarness(t) + lost, survivor := h.agent(), h.agent() + h.session(testSession, testUID) + h.reconcile(lost, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + lost.releaseLease(h.ctx, lost.EndpointID()) + h.exec.killExit = 1 + + h.reconcile(survivor, testSession) + + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonAgentLost) + if s.Status.EndedAt != nil { + t.Fatal("endedAt set while the session's processes may still run") + } + if len(h.slots()) != 1 { + t.Fatal("slot released before the processes were stopped") + } + + if res := h.reconcile(survivor, testSession); res.RequeueAfter == 0 { + t.Fatal("expected a retry while processes may be running") + } + if h.cellSession(testSession).Status.EndedAt != nil { + t.Fatal("endedAt set after another failed stop") + } + + h.exec.killExit = 0 + h.reconcile(survivor, testSession) + s = h.cellSession(testSession) + if s.Status.EndedAt == nil { + t.Fatal("endedAt not set once the processes stopped") + } + if len(h.slots()) != 0 { + t.Fatal("slot kept after the processes stopped") + } +} + +func TestSweepConfirmsEndOnceProcessesStop(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + endAs(h, computev1alpha.InstanceConsoleSessionReasonDisconnected) + + if err := a.sweep(h.ctx); err != nil { + t.Fatal(err) + } + + if h.cellSession(testSession).Status.EndedAt == nil { + t.Fatal("the sweep stopped the processes but did not record it") + } + if len(h.slots()) != 0 { + t.Fatal("slot kept after the sweep stopped the processes") + } +} diff --git a/internal/shellagent/scripts.go b/internal/shellagent/scripts.go new file mode 100644 index 00000000..66a4ec4f --- /dev/null +++ b/internal/shellagent/scripts.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "strconv" + "strings" +) + +const markerPrefix = ".datum-exec-" + +// The probe exits 4 when the requested executable is missing and 3 when the +// container has no writable directory for the marker file. A container +// without sh cannot run it at all. +const probeScript = `command -v "$1" >/dev/null 2>&1 || exit 4 +for d in /tmp /dev/shm /run /var/tmp; do + if [ -d "$d" ] && [ -w "$d" ]; then echo "$d"; exit 0; fi +done +exit 3` + +const ( + probeExitCommandMissing = 4 + probeExitNoWritableDir = 3 +) + +// procStatFunction reads /proc//stat into Z (state), G (process group), +// S (session) and T (start time). The command name may hold spaces and +// parentheses, so the fields are taken after its last ")". +const procStatFunction = `st() { Z=; G=; S=; T=; read -r l < "$P/$1/stat" 2>/dev/null || return 1; set -- ${l##*)}; Z=$1 G=$3 S=$4 T=${20}; } +` + +// The wrapper records who to stop when the session ends, then replaces itself +// with the command. Job control moves background jobs to process groups of +// their own but never out of the session, so the wrapper records its session +// when it leads one, as every exec does under Kata. Elsewhere it starts a new +// session with setsid where it can, and falls back to its process group. The +// record carries the wrapper's start time so a reused process ID is never +// mistaken for it. Without a terminal or stdin the command reads /dev/null, +// because Kata leaves an exec's stdin open when none was asked for. +const wrapperScript = `w=$1 m=$2 mode=$3 again=$4; shift 4; P=/proc +` + procStatFunction + `if st $$ && [ "$S" != "$$" ] && [ "$G" != "$$" ] && [ "$again" = 0 ] && command -v setsid >/dev/null 2>&1; then + exec setsid sh -c "$w" sh "$w" "$m" "$mode" 1 "$@" +fi +if ! st $$; then echo "$$" > "$m" +elif [ "$S" = "$$" ]; then echo "$$ $T session" > "$m" +else echo "$$ $T group" > "$m"; fi +[ "$mode" = null ] && exec "$@" /dev/null && M="-$p" + kill -0 "$p" 2>/dev/null && M="$M $p" + return 0 + fi + for d in "$P"/[0-9]*; do + q=${d##*/} + [ "$q" = "$$" ] && continue + st "$q" || continue + [ "$Z" = Z ] && continue + [ "$T" -ge "$t" ] 2>/dev/null || continue + if [ "$S" = "$p" ] && [ "$kind" = session ] || [ "$G" = "$p" ]; then M="$M $q"; fi + done +} +members +[ -z "$M" ] && { rm -f "$f"; exit 0; } +kill -HUP $M 2>/dev/null; kill -CONT $M 2>/dev/null +i=0 +while [ "$i" -lt "$grace" ]; do + sleep 1; i=$((i+1)) + members + [ -z "$M" ] && break +done +i=0 +while :; do + members + [ -z "$M" ] && { rm -f "$f"; exit 0; } + [ "$i" -ge 5 ] && exit 5 + kill -KILL $M 2>/dev/null + sleep 1; i=$((i+1)) +done` + +const listMarkersScript = `for f in "$1"/` + markerPrefix + `*; do + [ -f "$f" ] && echo "${f##*/` + markerPrefix + `}" +done +true` + +func markerPath(dir, sessionUID string) string { + return strings.TrimSuffix(dir, "/") + "/" + markerPrefix + sessionUID +} + +func probeCommand(executable string) []string { + return []string{"sh", "-c", probeScript, "sh", executable} +} + +func wrappedCommand(dir, sessionUID string, command []string, stdin, tty bool) []string { + mode := "-" + if !stdin && !tty { + mode = "null" + } + return append([]string{"sh", "-c", wrapperScript, "sh", wrapperScript, markerPath(dir, sessionUID), mode, "0"}, command...) +} + +func killCommand(dir, sessionUID string, graceSeconds int) []string { + return []string{"sh", "-c", killScript, "sh", markerPath(dir, sessionUID), strconv.Itoa(graceSeconds), "/proc"} +} + +func listMarkersCommand(dir string) []string { + return []string{"sh", "-c", listMarkersScript, "sh", dir} +} diff --git a/internal/shellagent/scripts_test.go b/internal/shellagent/scripts_test.go new file mode 100644 index 00000000..741b8926 --- /dev/null +++ b/internal/shellagent/scripts_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "testing" +) + +type fakeProcess struct { + pid, pgrp, sid, start int + comm, state string + ignoresHUP bool + unkillable bool +} + +// fakeKill replaces kill and sleep in the kill script: a signalled process +// dies unless it ignores SIGHUP or cannot be killed, and every call is logged. +const fakeKill = `kill() { + sig=$1; shift + echo "$sig $*" >> "$LOG" + for q in "$@"; do + case "$sig" in + -HUP) grep -qx "$q" "$NOHUP" || rm -rf "$P/$q" ;; + -KILL) grep -qx "$q" "$UNKILLABLE" || rm -rf "$P/$q" ;; + esac + done +} +sleep() { :; } +` + +type killRun struct { + exit int + signalled map[string][]int + survivors []int + markerLeft bool +} + +func runKillScript(t *testing.T, marker string, procs []fakeProcess) killRun { + t.Helper() + dir := t.TempDir() + proc := filepath.Join(dir, "proc") + var nohup, unkillable []string + for _, p := range procs { + comm, state := p.comm, p.state + if comm == "" { + comm = "sh" + } + if state == "" { + state = "S" + } + stat := fmt.Sprintf("%d (%s) %s 1 %d %d 34816 %d 0 0 0 0 0 0 0 0 0 20 0 1 0 %d 4210688 489\n", + p.pid, comm, state, p.pgrp, p.sid, p.pgrp, p.start) + if err := os.MkdirAll(filepath.Join(proc, strconv.Itoa(p.pid)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proc, strconv.Itoa(p.pid), "stat"), []byte(stat), 0o644); err != nil { + t.Fatal(err) + } + if p.ignoresHUP { + nohup = append(nohup, strconv.Itoa(p.pid)) + } + if p.unkillable { + unkillable = append(unkillable, strconv.Itoa(p.pid)) + } + } + markerFile := filepath.Join(dir, ".datum-exec-uid") + if err := os.WriteFile(markerFile, []byte(marker+"\n"), 0o644); err != nil { + t.Fatal(err) + } + logFile := filepath.Join(dir, "log") + write := func(name string, lines []string) string { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + return path + } + cmd := exec.Command("sh", "-c", fakeKill+killScript, "sh", markerFile, "3", proc) + cmd.Env = append(os.Environ(), "LOG="+logFile, "NOHUP="+write("nohup", nohup), + "UNKILLABLE="+write("unkillable", unkillable)) + out, err := cmd.CombinedOutput() + res := killRun{signalled: map[string][]int{}} + if exitErr, ok := err.(*exec.ExitError); ok { + res.exit = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("kill script: %v: %s", err, out) + } + log, _ := os.ReadFile(logFile) + for _, line := range strings.Split(strings.TrimSpace(string(log)), "\n") { + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + for _, f := range fields[1:] { + n, _ := strconv.Atoi(f) + res.signalled[fields[0]] = append(res.signalled[fields[0]], n) + } + } + for sig := range res.signalled { + sort.Ints(res.signalled[sig]) + res.signalled[sig] = uniq(res.signalled[sig]) + } + entries, _ := os.ReadDir(proc) + for _, e := range entries { + n, _ := strconv.Atoi(e.Name()) + res.survivors = append(res.survivors, n) + } + sort.Ints(res.survivors) + _, statErr := os.Stat(markerFile) + res.markerLeft = statErr == nil + return res +} + +func uniq(in []int) []int { + var out []int + for i, v := range in { + if i == 0 || v != in[i-1] { + out = append(out, v) + } + } + return out +} + +func TestKillScriptStopsTheWholeSession(t *testing.T) { + res := runKillScript(t, "40 1000 session", []fakeProcess{ + {pid: 1, pgrp: 1, sid: 1, start: 10, comm: "init"}, + {pid: 40, pgrp: 40, sid: 40, start: 1000}, + {pid: 41, pgrp: 41, sid: 40, start: 1100, comm: "sleep"}, + {pid: 42, pgrp: 42, sid: 40, start: 1200, comm: "nohup sleep", ignoresHUP: true}, + {pid: 43, pgrp: 43, sid: 40, start: 1300, comm: "odd) name (x", state: "Z"}, + {pid: 44, pgrp: 44, sid: 44, start: 1400, comm: "other session"}, + {pid: 45, pgrp: 45, sid: 40, start: 900, comm: "older"}, + }) + + if res.exit != 0 || res.markerLeft { + t.Fatalf("exit %d, marker kept %v", res.exit, res.markerLeft) + } + if got := fmt.Sprint(res.signalled["-HUP"]); got != "[40 41 42]" { + t.Fatalf("SIGHUP sent to %s", got) + } + if got := fmt.Sprint(res.signalled["-KILL"]); got != "[42]" { + t.Fatalf("SIGKILL sent to %s, want only the process that ignored SIGHUP", got) + } + if got := fmt.Sprint(res.survivors); got != "[1 43 44 45]" { + t.Fatalf("survivors = %s", got) + } +} + +func TestKillScriptKeepsMarkerWhileProcessesSurvive(t *testing.T) { + res := runKillScript(t, "40 1000 session", []fakeProcess{ + {pid: 40, pgrp: 40, sid: 40, start: 1000}, + {pid: 41, pgrp: 41, sid: 40, start: 1100, ignoresHUP: true, unkillable: true}, + }) + + if res.exit != 5 || !res.markerLeft { + t.Fatalf("exit %d, marker kept %v; want 5 and the marker kept for the sweep", res.exit, res.markerLeft) + } +} + +func TestKillScriptLeavesAReusedProcessIDAlone(t *testing.T) { + res := runKillScript(t, "40 1000 session", []fakeProcess{ + {pid: 40, pgrp: 40, sid: 40, start: 5000}, + {pid: 41, pgrp: 41, sid: 40, start: 5100}, + }) + + if res.exit != 0 || res.markerLeft || len(res.signalled) != 0 { + t.Fatalf("exit %d, marker kept %v, signalled %v", res.exit, res.markerLeft, res.signalled) + } +} + +func TestKillScriptStopsOnlyTheGroupWithoutASession(t *testing.T) { + res := runKillScript(t, "40 1000 group", []fakeProcess{ + {pid: 30, pgrp: 30, sid: 30, start: 900}, + {pid: 40, pgrp: 40, sid: 30, start: 1000}, + {pid: 41, pgrp: 40, sid: 30, start: 1100}, + {pid: 42, pgrp: 42, sid: 30, start: 1200}, + }) + + if res.exit != 0 || res.markerLeft { + t.Fatalf("exit %d, marker kept %v", res.exit, res.markerLeft) + } + if got := fmt.Sprint(res.survivors); got != "[30 42]" { + t.Fatalf("survivors = %s, want the rest of the shared session", got) + } +} + +func TestKillScriptWithoutRecordDoesNothing(t *testing.T) { + res := runKillScript(t, "not-a-pid", []fakeProcess{{pid: 40, pgrp: 40, sid: 40, start: 1000}}) + + if res.exit != 0 || res.markerLeft || len(res.signalled) != 0 { + t.Fatalf("exit %d, marker kept %v, signalled %v", res.exit, res.markerLeft, res.signalled) + } +} + +// The wrapper runs for real where there is a /proc: it records a session it +// leads, or starts one with setsid, and gives a command without stdin or a +// terminal an empty input. +func TestWrapperRecordsItsSession(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("needs /proc") + } + dir := t.TempDir() + marker := filepath.Join(dir, ".datum-exec-uid") + args := wrappedCommand(dir, "uid", []string{"sh", "-c", `cat; read -r l < /proc/$$/stat; set -- ${l##*)}; echo "$4"`}, false, false) + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdin = strings.NewReader("must not be read\n") + out, err := cmd.Output() + if err != nil { + t.Fatalf("wrapper: %v", err) + } + record, err := os.ReadFile(marker) + if err != nil { + t.Fatal(err) + } + fields := strings.Fields(string(record)) + if len(fields) != 3 { + t.Fatalf("marker = %q", record) + } + if strings.Contains(string(out), "must not be read") { + t.Fatal("the command read stdin the session did not ask for") + } + if _, err := exec.LookPath("setsid"); err == nil && + (fields[2] != "session" || strings.TrimSpace(string(out)) != fields[0]) { + t.Fatalf("marker = %q, command session %q; want a session the wrapper leads", record, out) + } +} diff --git a/internal/shellagent/server.go b/internal/shellagent/server.go new file mode 100644 index 00000000..2cdb1bc8 --- /dev/null +++ b/internal/shellagent/server.go @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "net/http" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/consolesession" +) + +// Handler serves the connection protocol's exec endpoint. +func (a *Agent) Handler(ctx context.Context) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET "+consolesession.ExecPath("{uid}"), func(w http.ResponseWriter, r *http.Request) { + a.serveExec(log.IntoContext(r.Context(), log.FromContext(ctx)), w, r) + }) + return mux +} + +func wantsExecStream(r *http.Request) bool { + if !strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + return false + } + for _, value := range r.Header.Values("Sec-WebSocket-Protocol") { + for _, protocol := range strings.Split(value, ",") { + if strings.TrimSpace(protocol) == consolesession.SubProtocol { + return true + } + } + } + return false +} + +// findSession returns the cell copy of the session with a project UID. +func (a *Agent) findSession(ctx context.Context, uid string) (*computev1alpha.InstanceConsoleSession, error) { + var sessions computev1alpha.InstanceConsoleSessionList + if err := a.sessions.List(ctx, &sessions, client.MatchingLabels{computev1alpha.InstanceConsoleSessionUIDLabel: uid}); err != nil { + return nil, err + } + if len(sessions.Items) == 0 { + return nil, nil + } + return &sessions.Items[0], nil +} + +//nolint:gocyclo // each refusal is one branch of the protocol, in its order +func (a *Agent) serveExec(ctx context.Context, w http.ResponseWriter, r *http.Request) { + uid := r.PathValue("uid") + logger := log.FromContext(ctx).WithValues("session", uid) + + if a.draining.Load() { + http.Error(w, "The platform is restarting this cell's shell service. Create a new session.", http.StatusServiceUnavailable) + return + } + if !wantsExecStream(r) { + http.Error(w, "Only "+consolesession.SubProtocol+" WebSocket upgrades are supported.", http.StatusBadRequest) + return + } + cell, err := a.findSession(ctx, uid) + if err != nil { + logger.Error(err, "look up session") + http.Error(w, "The session could not be looked up.", http.StatusInternalServerError) + return + } + if cell == nil { + http.Error(w, "Session not found.", http.StatusNotFound) + return + } + now := a.now() + if err := consolesession.Verify(cell.Spec.ClientPublicKey, uid, r.Header.Get(consolesession.HeaderTimestamp), + r.Header.Get(consolesession.HeaderSignature), now, consolesession.ClockSkew); err != nil { + logger.Info("refused connection", "reason", err.Error()) + http.Error(w, "Not authorized for this session.", http.StatusForbidden) + return + } + + var current computev1alpha.InstanceConsoleSession + err = a.cell.Get(ctx, client.ObjectKeyFromObject(cell), ¤t) + if err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "read session") + http.Error(w, "The session could not be read.", http.StatusInternalServerError) + return + } + if err != nil || sessionUID(¤t) != uid || endpointOf(¤t) != "" && endpointOf(¤t) != a.EndpointID() { + http.Error(w, "Session not found.", http.StatusNotFound) + return + } + if !current.DeletionTimestamp.IsZero() || revoked(¤t) { + http.Error(w, endMessage(computev1alpha.InstanceConsoleSessionReasonRevoked), http.StatusGone) + return + } + if readyReason(¤t) != computev1alpha.InstanceConsoleSessionReasonSessionReady || isTerminal(¤t) { + code, msg := refusal(¤t) + http.Error(w, msg, code) + return + } + if current.Status.ConnectBefore != nil && !now.Before(current.Status.ConnectBefore.Time) { + http.Error(w, endMessage(computev1alpha.InstanceConsoleSessionReasonNotConnected), http.StatusGone) + return + } + held, err := a.slotFor(ctx, uid) + if err != nil || held == nil { + http.Error(w, "The session is not ready for a connection.", http.StatusConflict) + return + } + + started := metav1.NewTime(now) + expires := metav1.NewTime(now.Add(sessionTTL(¤t))) + runCtx, cancel := context.WithDeadline(context.Background(), expires.Time) + defer cancel() + live, ok := a.register(uid, cancel, cell.Spec.Terminal) + if !ok { + http.Error(w, "The session already has a connection.", http.StatusConflict) + return + } + defer func() { + a.unregister(uid) + close(live.done) + }() + + current.Status.StartedAt = &started + current.Status.ExpiresAt = &expires + setReady(¤t, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonConnected, "A client is connected.") + if err := a.cell.Status().Update(ctx, ¤t); err != nil { + if apierrors.IsConflict(err) { + http.Error(w, "The session already has a connection or has ended.", http.StatusConflict) + return + } + logger.Error(err, "record connection") + http.Error(w, "The session could not be started.", http.StatusInternalServerError) + return + } + logger.Info("session connected", "pod", held.pod.String(), "container", held.container) + + result := a.runSession(runCtx, w, r, ¤t, held, live) + a.finishSession(uid, client.ObjectKeyFromObject(¤t), held, result) + logger.Info("session ended", "reason", result.reason, "duration", a.now().Sub(started.Time).Round(time.Millisecond)) +} + +// runSession runs the session's command and relays its stream until it ends, +// then sends the client the session's closing status. +func (a *Agent) runSession(ctx context.Context, w http.ResponseWriter, r *http.Request, + session *computev1alpha.InstanceConsoleSession, held *slot, live *liveSession) outcome { + uid := sessionUID(session) + opts := ExecOptions{ + Container: held.container, + Command: wrappedCommand(held.markerDir, uid, session.Spec.Command, session.Spec.Stdin, session.Spec.Terminal), + Stdin: session.Spec.Stdin, + TTY: session.Spec.Terminal, + } + resp, backend, fromBackend, err := a.exec.Open(ctx, held.pod, opts, r.Header) + if err != nil { + log.FromContext(ctx).Error(err, "open exec stream", "session", uid) + http.Error(w, "The instance could not start the command.", http.StatusBadGateway) + return outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + message: "The instance could not start the command.", + } + } + defer func() { _ = backend.Close() }() + + conn, rw, err := http.NewResponseController(w).Hijack() + if err != nil { + return outcomeClientGone + } + if err := resp.Write(conn); err != nil { + _ = conn.Close() + return outcomeClientGone + } + stream := &clientStream{conn: conn} + a.attach(live, stream) + + result := relay(ctx, stream, rw.Reader, backend, fromBackend, a.cfg.PingInterval, a.cfg.PongTimeout) + if reason := a.stopReason(uid); reason != "" { + result = &outcome{reason: reason, message: endMessage(reason)} + } else if result == nil { + reason := computev1alpha.InstanceConsoleSessionReasonExpired + result = &outcome{reason: reason, message: endMessage(reason)} + } + stream.End(closingStatus(result.reason, result.exitCode, result.message)) + return *result +} + +// finishSession stops the session's processes, records its end, and frees its +// slot once the processes are known to be stopped. +func (a *Agent) finishSession(uid string, key client.ObjectKey, held *slot, result outcome) { + ctx, cancel := context.WithTimeout(context.Background(), a.cfg.KillGrace+30*time.Second) + defer cancel() + logger := log.FromContext(ctx).WithValues("session", uid) + + stopped, err := a.stopProcesses(ctx, held) + if err != nil { + logger.Error(err, "stop session processes") + } + message := result.message + if message == "" { + message = "The command exited." + } + me := a.EndpointID() + if _, err := a.end(ctx, key, result.reason, message, result.exitCode, stopped, + func(s *computev1alpha.InstanceConsoleSession) bool { return endpointOf(s) == me }); err != nil { + logger.Error(err, "record session end") + } + a.forget(uid) + if stopped { + if err := a.releaseSlot(ctx, held); err != nil { + logger.Error(err, "release session slot") + } + } +} diff --git a/internal/shellagent/server_test.go b/internal/shellagent/server_test.go new file mode 100644 index 00000000..18bf52fa --- /dev/null +++ b/internal/shellagent/server_test.go @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "crypto/ed25519" + "crypto/rand" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/consolesession" +) + +// serve starts the agent's exec endpoint on real time, so the connection +// deadline and the signature window follow the wall clock. +func serve(t *testing.T, h *harness, a *Agent) string { + t.Helper() + a.now = time.Now + h.clock.t = time.Now() + srv := httptest.NewServer(a.Handler(h.ctx)) + t.Cleanup(srv.Close) + return strings.TrimPrefix(srv.URL, "http://") +} + +// readySession creates a session and has the agent claim it. +func readySession(t *testing.T, h *harness, a *Agent, mutate ...func(*computev1alpha.InstanceConsoleSession)) ed25519.PrivateKey { + t.Helper() + key := h.session(testSession, testUID, mutate...) + h.reconcile(a, testSession) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) + return key +} + +func TestRefusals(t *testing.T) { + _, otherKey, _ := ed25519.GenerateKey(rand.Reader) + cases := []struct { + name string + setup func(t *testing.T, h *harness, a *Agent) (uid string, key ed25519.PrivateKey, at time.Time) + want int + }{ + {"unknown session", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + return "no-such-uid", otherKey, time.Now() + }, http.StatusNotFound}, + {"wrong key", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + readySession(t, h, a) + return testUID, otherKey, time.Now() + }, http.StatusForbidden}, + {"stale signature", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + return testUID, key, time.Now().Add(-consolesession.ClockSkew - 5*time.Second) + }, http.StatusForbidden}, + {"no signature", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + readySession(t, h, a) + return testUID, nil, time.Now() + }, http.StatusForbidden}, + {"claimed by another agent", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, h.agent()) + return testUID, key, time.Now() + }, http.StatusNotFound}, + {"not yet ready", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := h.session(testSession, testUID) + return testUID, key, time.Now() + }, http.StatusConflict}, + {"already connected", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + markConnected(h, testSession, time.Now().Add(time.Hour)) + return testUID, key, time.Now() + }, http.StatusConflict}, + {"completed", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + endAs(h, computev1alpha.InstanceConsoleSessionReasonCompleted) + return testUID, key, time.Now() + }, http.StatusConflict}, + {"revoked", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + endAs(h, computev1alpha.InstanceConsoleSessionReasonRevoked) + return testUID, key, time.Now() + }, http.StatusGone}, + {"revoke requested", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + revokeSession(t, h) + return testUID, key, time.Now() + }, http.StatusGone}, + {"being deleted", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + if err := h.cell.Delete(h.ctx, h.cellSession(testSession)); err != nil { + t.Fatal(err) + } + return testUID, key, time.Now() + }, http.StatusGone}, + {"expired", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + endAs(h, computev1alpha.InstanceConsoleSessionReasonExpired) + return testUID, key, time.Now() + }, http.StatusGone}, + {"past connectBefore", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + s := h.cellSession(testSession) + s.Status.ConnectBefore = &metav1.Time{Time: time.Now().Add(-time.Second)} + if err := h.cell.Status().Update(h.ctx, s); err != nil { + t.Fatal(err) + } + return testUID, key, time.Now() + }, http.StatusGone}, + {"draining", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + a.draining.Store(true) + return testUID, key, time.Now() + }, http.StatusServiceUnavailable}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + uid, key, at := tc.setup(t, h, a) + + c := dialSession(t, addr, uid, key, at) + + if c.status != tc.want { + t.Fatalf("status = %d (%s), want %d", c.status, c.body, tc.want) + } + }) + } +} + +func TestRefusesOtherSubprotocols(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + ts, sig := consolesession.Sign(key, testUID, time.Now()) + + req, _ := http.NewRequest(http.MethodGet, "http://"+addr+consolesession.ExecPath(testUID), nil) + req.Header.Set("Connection", "Upgrade") + req.Header.Set("Upgrade", "websocket") + req.Header.Set("Sec-WebSocket-Protocol", "v4.channel.k8s.io") + req.Header.Set(consolesession.HeaderTimestamp, ts) + req.Header.Set(consolesession.HeaderSignature, sig) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", resp.StatusCode) + } +} + +// scriptedExit has the fake apiserver echo stdin and exit with code. +func scriptedExit(code int) func(conn net.Conn, r *bufio.Reader) { + return func(conn net.Conn, r *bufio.Reader) { + writeServerFrame(conn, channelStdout, []byte("hello")) + f, err := readFrame(r) + if err != nil { + return + } + writeServerFrame(conn, channelStdout, f.payload[1:]) + writeExitStatus(conn, code) + } +} + +func TestClosingStatus(t *testing.T) { + cases := []struct { + name string + exit int + wantState string + wantCause string + }{ + {"normal exit", 0, metav1.StatusSuccess, ""}, + {"non-zero exit", 3, metav1.StatusFailure, "3"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = scriptedExit(tc.exit) + + c := dialSession(t, addr, testUID, key, time.Now()) + if c.status != http.StatusSwitchingProtocols { + t.Fatalf("status = %d (%s)", c.status, c.body) + } + c.send(t, opBinary, append([]byte{channelStdin}, " world"...)) + out, status, code := c.readUntilClose(t) + + if out[channelStdout] != "hello world" { + t.Fatalf("stdout = %q", out[channelStdout]) + } + if status == nil || status.Status != tc.wantState || code != closeNormal { + t.Fatalf("status = %+v, close code %d", status, code) + } + if tc.wantCause != "" { + if status.Reason != ReasonNonZeroExitCode || status.Details == nil || + status.Details.Causes[0].Type != "ExitCode" || status.Details.Causes[0].Message != tc.wantCause { + t.Fatalf("status = %+v, want NonZeroExitCode with cause %s", status, tc.wantCause) + } + } + s := waitForEnd(t, h) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonCompleted) + if s.Status.ExitCode == nil || int(*s.Status.ExitCode) != tc.exit || s.Status.StartedAt == nil || + s.Status.ExpiresAt == nil { + t.Fatalf("status = %+v", s.Status) + } + if h.exec.ran(killScript) != 1 || len(h.slots()) != 0 { + t.Fatal("process group was not stopped, or the slot was kept") + } + }) + } +} + +func TestPlatformEndings(t *testing.T) { + cases := []struct { + name string + ttl time.Duration + end func(a *Agent) + reason string + }{ + {"revoked", time.Hour, func(a *Agent) { + a.stop(testUID, computev1alpha.InstanceConsoleSessionReasonRevoked) + }, computev1alpha.InstanceConsoleSessionReasonRevoked}, + {"shutdown", time.Hour, func(a *Agent) { + a.stop(testUID, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + }, computev1alpha.InstanceConsoleSessionReasonAgentShutdown}, + {"expired", 500 * time.Millisecond, func(*Agent) {}, computev1alpha.InstanceConsoleSessionReasonExpired}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a, func(s *computev1alpha.InstanceConsoleSession) { + s.Spec.TTL = &metav1.Duration{Duration: tc.ttl} + }) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + writeServerFrame(conn, channelStdout, []byte("$ ")) + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + _ = c.conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + first, err := readFrame(c.reader) + if err != nil || len(first.payload) == 0 || first.payload[0] != channelStdout || string(first.payload[1:]) != "$ " { + t.Fatalf("first frame = %+v, %v", first, err) + } + go tc.end(a) + out, status, code := c.readUntilClose(t) + + if out[channelStdout] != "" { + t.Fatalf("stdout after the end = %q", out[channelStdout]) + } + if status == nil || status.Status != metav1.StatusFailure || string(status.Reason) != tc.reason || + status.Message == "" || code != closeNormal { + t.Fatalf("status = %+v, close code %d", status, code) + } + s := waitForEnd(t, h) + requireReason(t, s, tc.reason) + if s.Status.ExitCode != nil || h.exec.ran(killScript) != 1 { + t.Fatalf("exit code %v, kills %d", s.Status.ExitCode, h.exec.ran(killScript)) + } + }) + } +} + +func TestClientDisconnectStopsCommand(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + _ = c.conn.Close() + + s := waitForEnd(t, h) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonDisconnected) + if s.Status.ExitCode != nil || h.exec.ran(killScript) != 1 { + t.Fatal("a dropped connection must stop the command") + } +} + +func TestFailedStopLeavesEndUnconfirmed(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + h.exec.mu.Lock() + h.exec.killExit = 1 + h.exec.mu.Unlock() + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + _ = c.conn.Close() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && !isTerminal(h.cellSession(testSession)) { + time.Sleep(10 * time.Millisecond) + } + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonDisconnected) + if s.Status.EndedAt != nil { + t.Fatal("endedAt set although the command could not be confirmed stopped") + } + if len(h.slots()) != 1 { + t.Fatal("slot released although the command could not be confirmed stopped") + } +} + +func TestOversizedFrameEndsSession(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + + c := dialSession(t, addr, testUID, key, time.Now()) + head := []byte{0x80 | opBinary, 0x80 | 127, 0, 0, 0, 0, 0x01, 0, 0, 1, 0, 0, 0, 0} + if _, err := c.conn.Write(head); err != nil { + t.Fatal(err) + } + _, status, _ := c.readUntilClose(t) + + if status == nil || string(status.Reason) != computev1alpha.InstanceConsoleSessionReasonInvalid { + t.Fatalf("status = %+v", status) + } + requireReason(t, waitForEnd(t, h), computev1alpha.InstanceConsoleSessionReasonInvalid) +} + +func TestSecondConnectionIsRefused(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + + first := dialSession(t, addr, testUID, key, time.Now()) + second := dialSession(t, addr, testUID, key, time.Now()) + + if first.status != http.StatusSwitchingProtocols || second.status != http.StatusConflict { + t.Fatalf("statuses = %d, %d; want 101, 409", first.status, second.status) + } +} + +func waitForConnected(t *testing.T, h *harness) { + t.Helper() + reason := computev1alpha.InstanceConsoleSessionReasonConnected + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if readyReason(h.cellSession(testSession)) == reason { + return + } + time.Sleep(10 * time.Millisecond) + } + requireReason(t, h.cellSession(testSession), reason) +} + +func waitForEnd(t *testing.T, h *harness) *computev1alpha.InstanceConsoleSession { + name := testSession + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if s := h.cellSession(name); isTerminal(s) && len(h.slots()) == 0 { + return s + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("session %s did not end: %+v", name, h.cellSession(name).Status) + return nil +} diff --git a/internal/shellagent/slots.go b/internal/shellagent/slots.go new file mode 100644 index 00000000..c3fcaa69 --- /dev/null +++ b/internal/shellagent/slots.go @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +const ( + componentLabel = "app.kubernetes.io/component" + slotComponent = "shell-session-slot" + annotationPrefix = "shell-agent.compute.datumapis.com/" + annotationSession = annotationPrefix + "session" + annotationPod = annotationPrefix + "pod" + annotationPodUID = annotationPrefix + "pod-uid" + annotationContainer = annotationPrefix + "container" + annotationContainerID = annotationPrefix + "container-id" + annotationMarkerDir = annotationPrefix + "marker-dir" + annotationEndpoint = annotationPrefix + "endpoint" +) + +// A slot is a Lease held by one session of one Instance. Creating a Lease is +// atomic, so agents claiming at once cannot take more slots than exist. A +// slot also records where the session's command runs, and outlives the +// session until that command is known to be stopped: a held slot is the +// cell's record that a process may still need stopping. +type slot struct { + lease *coordinationv1.Lease + sessionUID string + session types.NamespacedName + pod types.NamespacedName + podUID types.UID + container string + containerID string + markerDir string + endpointID string +} + +func slotName(instanceUID types.UID, n int) string { + sum := sha256.Sum256([]byte(instanceUID)) + return fmt.Sprintf("slot-%s-%d", hex.EncodeToString(sum[:8]), n) +} + +func slotFromLease(l *coordinationv1.Lease) *slot { + s := &slot{ + lease: l, + sessionUID: ptr.Deref(l.Spec.HolderIdentity, ""), + podUID: types.UID(l.Annotations[annotationPodUID]), + container: l.Annotations[annotationContainer], + containerID: l.Annotations[annotationContainerID], + markerDir: l.Annotations[annotationMarkerDir], + endpointID: l.Annotations[annotationEndpoint], + } + s.session = splitKey(l.Annotations[annotationSession]) + s.pod = splitKey(l.Annotations[annotationPod]) + return s +} + +func splitKey(key string) types.NamespacedName { + for i := 0; i < len(key); i++ { + if key[i] == '/' { + return types.NamespacedName{Namespace: key[:i], Name: key[i+1:]} + } + } + return types.NamespacedName{Name: key} +} + +func (s *slot) leaseFor(namespace, name string) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + componentLabel: slotComponent, + computev1alpha.InstanceConsoleSessionUIDLabel: s.sessionUID, + }, + Annotations: map[string]string{ + annotationSession: s.session.String(), + annotationPod: s.pod.String(), + annotationPodUID: string(s.podUID), + annotationContainer: s.container, + annotationContainerID: s.containerID, + annotationMarkerDir: s.markerDir, + annotationEndpoint: s.endpointID, + }, + }, + Spec: coordinationv1.LeaseSpec{HolderIdentity: ptr.To(s.sessionUID)}, + } +} + +// errClaimInProgress means another agent in the cell is claiming the same +// session. +var errClaimInProgress = errors.New("another agent is claiming the session") + +// acquireSlot takes one of the Instance's slots for want. It returns nil when +// every slot is held by a session that may still be running. A slot already +// held by the same session is returned as is, so a retried claim is +// idempotent. +func (a *Agent) acquireSlot(ctx context.Context, instanceUID types.UID, want *slot) (*slot, error) { + for n := range a.cfg.SlotsPerInstance { + name := slotName(instanceUID, n) + for range 2 { + lease := want.leaseFor(a.cfg.Namespace, name) + err := a.cell.Create(ctx, lease) + if err == nil { + return slotFromLease(lease), nil + } + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + var held coordinationv1.Lease + err = a.cell.Get(ctx, client.ObjectKey{Namespace: a.cfg.Namespace, Name: name}, &held) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return nil, err + } + holder := slotFromLease(&held) + if holder.sessionUID == want.sessionUID { + if holder.endpointID == want.endpointID { + return holder, nil + } + alive, err := a.agentAlive(ctx, holder.endpointID) + if err != nil { + return nil, err + } + if alive { + return nil, errClaimInProgress + } + if err := a.releaseSlot(ctx, holder); err != nil { + return nil, err + } + continue + } + if a.isLive(holder.sessionUID) { + break + } + state, err := a.sessionState(ctx, holder.sessionUID) + if err != nil { + return nil, err + } + if state != sessionEnded { + break + } + if reaped, err := a.reap(ctx, holder); err != nil || !reaped { + break + } + } + } + return nil, nil +} + +// slotFor returns the slot a session holds in this cell, if any. +func (a *Agent) slotFor(ctx context.Context, sessionUID string) (*slot, error) { + var leases coordinationv1.LeaseList + if err := a.cell.List(ctx, &leases, client.InNamespace(a.cfg.Namespace), client.MatchingLabels{ + componentLabel: slotComponent, + computev1alpha.InstanceConsoleSessionUIDLabel: sessionUID, + }); err != nil { + return nil, err + } + for i := range leases.Items { + if ptr.Deref(leases.Items[i].Spec.HolderIdentity, "") == sessionUID { + return slotFromLease(&leases.Items[i]), nil + } + } + return nil, nil +} + +// releaseSlot frees a slot whose session never started a process or whose +// process is known to be stopped. +func (a *Agent) releaseSlot(ctx context.Context, s *slot) error { + if s == nil { + return nil + } + var opts []client.DeleteOption + if s.lease.UID != "" { + opts = append(opts, client.Preconditions{UID: &s.lease.UID}) + } + err := a.cell.Delete(ctx, s.lease, opts...) + if apierrors.IsNotFound(err) || apierrors.IsConflict(err) { + return nil + } + return err +} + +// reap stops the process a slot records, then frees the slot. It reports +// false, keeping the slot, when the process could not be confirmed stopped. +func (a *Agent) reap(ctx context.Context, s *slot) (bool, error) { + stopped, err := a.stopProcesses(ctx, s) + if err != nil || !stopped { + return false, err + } + return true, a.releaseSlot(ctx, s) +} + +// stopProcesses stops a session's process group in the pod its slot records. +// A pod or container that stopped, or was replaced, took the process with it. +func (a *Agent) stopProcesses(ctx context.Context, s *slot) (bool, error) { + if s.markerDir == "" || s.pod.Name == "" { + return true, nil + } + var pod corev1.Pod + err := a.cell.Get(ctx, s.pod, &pod) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + if pod.UID != s.podUID { + return true, nil + } + if status := containerStatus(&pod, s.container); status == nil || status.State.Running == nil || + s.containerID != "" && status.ContainerID != s.containerID { + return true, nil + } + grace := int(a.cfg.KillGrace.Seconds()) + out, code, err := a.run(ctx, s.pod, s.container, killCommand(s.markerDir, s.sessionUID, grace)) + if err != nil || code != 0 { + log.FromContext(ctx).Info("could not stop session processes", "session", s.sessionUID, + "pod", s.pod.String(), "exitCode", code, "output", out, "error", err) + return false, nil + } + return true, nil +} diff --git a/internal/shellagent/status.go b/internal/shellagent/status.go new file mode 100644 index 00000000..291e88bd --- /dev/null +++ b/internal/shellagent/status.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "encoding/json" + "fmt" + "strconv" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +// ReasonNonZeroExitCode is the reason Kubernetes reports on the exec status +// channel for a command that exited with a code other than zero. +const ReasonNonZeroExitCode = "NonZeroExitCode" + +const causeTypeExitCode = "ExitCode" + +var endMessages = map[string]string{ + computev1alpha.InstanceConsoleSessionReasonExpired: "The session reached its time limit and the command was stopped.", + computev1alpha.InstanceConsoleSessionReasonRevoked: "The session was deleted and the command was stopped.", + computev1alpha.InstanceConsoleSessionReasonNotConnected: "No client connected before the connection deadline.", + computev1alpha.InstanceConsoleSessionReasonAgentShutdown: "The platform stopped the session for maintenance. Start a new session to continue.", + computev1alpha.InstanceConsoleSessionReasonAgentLost: "The platform lost the session unexpectedly. Start a new session to continue.", + computev1alpha.InstanceConsoleSessionReasonTooManySessions: "The instance already has as many open sessions as it allows.", + computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning: "The instance is not running.", + computev1alpha.InstanceConsoleSessionReasonInstanceNotFound: "The instance no longer exists or has been replaced.", + computev1alpha.InstanceConsoleSessionReasonDisconnected: "The client disconnected and the command was stopped.", +} + +func endMessage(reason string) string { + if msg, ok := endMessages[reason]; ok { + return msg + } + return "The session ended." +} + +// terminalReason reports whether reason ends a session for good. +func terminalReason(reason string) bool { + switch reason { + case computev1alpha.InstanceConsoleSessionReasonCompleted, + computev1alpha.InstanceConsoleSessionReasonExpired, + computev1alpha.InstanceConsoleSessionReasonRevoked, + computev1alpha.InstanceConsoleSessionReasonNotConnected, + computev1alpha.InstanceConsoleSessionReasonAgentShutdown, + computev1alpha.InstanceConsoleSessionReasonAgentLost, + computev1alpha.InstanceConsoleSessionReasonTooManySessions, + computev1alpha.InstanceConsoleSessionReasonNoShell, + computev1alpha.InstanceConsoleSessionReasonCommandUnavailable, + computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + computev1alpha.InstanceConsoleSessionReasonInstanceNotFound, + computev1alpha.InstanceConsoleSessionReasonInvalid, + computev1alpha.InstanceConsoleSessionReasonUnavailable, + computev1alpha.InstanceConsoleSessionReasonDisconnected: + return true + } + return false +} + +func readyCondition(s *computev1alpha.InstanceConsoleSession) *metav1.Condition { + return meta.FindStatusCondition(s.Status.Conditions, computev1alpha.InstanceConsoleSessionReady) +} + +func readyReason(s *computev1alpha.InstanceConsoleSession) string { + if c := readyCondition(s); c != nil { + return c.Reason + } + return computev1alpha.InstanceConsoleSessionReasonPending +} + +func isTerminal(s *computev1alpha.InstanceConsoleSession) bool { + c := readyCondition(s) + return c != nil && c.Status == metav1.ConditionFalse && terminalReason(c.Reason) +} + +func setReady(s *computev1alpha.InstanceConsoleSession, status metav1.ConditionStatus, reason, message string) { + meta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{ + Type: computev1alpha.InstanceConsoleSessionReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: s.Generation, + }) +} + +// closingStatus is the one status message the agent sends on the status +// channel before it closes a session's stream. A command that exited reports +// what Kubernetes would; any other ending names its terminal reason. +func closingStatus(reason string, exitCode *int32, message string) metav1.Status { + status := metav1.Status{ + TypeMeta: metav1.TypeMeta{Kind: "Status", APIVersion: "v1"}, + Status: metav1.StatusFailure, + } + switch { + case reason == computev1alpha.InstanceConsoleSessionReasonCompleted && exitCode != nil && *exitCode == 0: + status.Status = metav1.StatusSuccess + case reason == computev1alpha.InstanceConsoleSessionReasonCompleted && exitCode != nil: + status.Reason = ReasonNonZeroExitCode + status.Message = fmt.Sprintf("command terminated with non-zero exit code: %d", *exitCode) + status.Details = &metav1.StatusDetails{Causes: []metav1.StatusCause{{ + Type: causeTypeExitCode, + Message: strconv.Itoa(int(*exitCode)), + }}} + default: + status.Reason = metav1.StatusReason(reason) + status.Message = message + } + return status +} + +// exitCodeFromStatus reads the command's exit code from the status message the +// apiserver sends when the command exits. ok is false when the status reports +// an error other than the command's exit. +func exitCodeFromStatus(raw []byte) (code int32, message string, ok bool) { + var status metav1.Status + if err := json.Unmarshal(raw, &status); err != nil { + return 0, "the platform could not read the command's result", false + } + if status.Status == metav1.StatusSuccess { + return 0, "", true + } + if status.Reason == ReasonNonZeroExitCode && status.Details != nil { + for _, cause := range status.Details.Causes { + if cause.Type != causeTypeExitCode { + continue + } + n, err := strconv.ParseInt(cause.Message, 10, 32) + if err == nil { + return int32(n), "", true + } + } + } + return 0, status.Message, false +} + +// refusal maps a session that cannot take a connection to the protocol's +// refusal code. +func refusal(s *computev1alpha.InstanceConsoleSession) (int, string) { + switch reason := readyReason(s); { + case reason == computev1alpha.InstanceConsoleSessionReasonRevoked, + reason == computev1alpha.InstanceConsoleSessionReasonExpired, + reason == computev1alpha.InstanceConsoleSessionReasonNotConnected: + return 410, endMessage(reason) + case isTerminal(s): + return 409, "the session has already ended" + case reason == computev1alpha.InstanceConsoleSessionReasonConnected: + return 409, "the session already has a connection" + default: + return 409, "the session is not ready for a connection" + } +} diff --git a/internal/shellagent/stream.go b/internal/shellagent/stream.go new file mode 100644 index 00000000..0a48d9a6 --- /dev/null +++ b/internal/shellagent/stream.go @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "net" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +var errStreamEnded = errors.New("stream ended") + +// clientStream owns writes to a client's exec WebSocket. Frames from the +// apiserver are relayed whole, so the agent can write its own messages between +// them without splitting a fragmented message. +type clientStream struct { + conn net.Conn + + mu sync.Mutex + inFragment bool + ended bool + endOnce sync.Once +} + +func (c *clientStream) relayData(raw []byte, fin bool) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.ended { + return errStreamEnded + } + c.inFragment = !fin + _, err := c.conn.Write(raw) + return err +} + +func (c *clientStream) relayControl(raw []byte) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.ended { + return errStreamEnded + } + _, err := c.conn.Write(raw) + return err +} + +// writeBetweenMessages writes frames once no relayed message is half sent, +// waiting at most two seconds for one to finish. +func (c *clientStream) writeBetweenMessages(frames ...[]byte) error { + deadline := time.Now().Add(2 * time.Second) + for { + c.mu.Lock() + if c.ended { + c.mu.Unlock() + return errStreamEnded + } + if !c.inFragment || time.Now().After(deadline) { + var err error + for _, f := range frames { + if _, err = c.conn.Write(f); err != nil { + break + } + } + c.mu.Unlock() + return err + } + c.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } +} + +// Notify writes a notice to the session's standard output. Only sessions with +// a terminal get notices. +func (c *clientStream) Notify(text string) error { + return c.writeBetweenMessages(encodeFrame(opBinary, append([]byte{channelStdout}, text...), false)) +} + +// End sends the session's one status message and closes the stream with +// code 1000. +func (c *clientStream) End(status metav1.Status) { + c.endOnce.Do(func() { + body, _ := json.Marshal(status) + _ = c.writeBetweenMessages( + encodeFrame(opBinary, append([]byte{channelStatus}, body...), false), + encodeFrame(opClose, closePayload(closeNormal), false), + ) + c.mu.Lock() + c.ended = true + c.mu.Unlock() + _ = c.conn.Close() + }) +} + +// outcome is how a connected session ended. +type outcome struct { + reason string + exitCode *int32 + message string +} + +var ( + outcomeClientGone = outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonDisconnected, + message: endMessage(computev1alpha.InstanceConsoleSessionReasonDisconnected), + } + outcomeStreamLost = outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + message: "The instance stopped the command's stream.", + } + outcomeFrameTooLarge = outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInvalid, + message: "The client sent a message larger than 16 MiB.", + } + outcomeMalformed = outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInvalid, + message: "The client sent a malformed WebSocket message.", + } +) + +// pingPayload marks the agent's own pings, so the client's pongs to them are +// not relayed to the apiserver. +var pingPayload = []byte("shell-agent") + +// relay copies frames both ways until the command exits, either side goes +// away, or ctx ends. It returns nil when ctx ended first. The client is pinged +// every pingInterval and counts as gone once it has sent nothing, not even a +// pong, for pongTimeout. +func relay(ctx context.Context, c *clientStream, fromClient *bufio.Reader, backend net.Conn, fromBackend *bufio.Reader, + pingInterval, pongTimeout time.Duration) *outcome { + results := make(chan outcome, 2) + done := make(chan struct{}) + defer close(done) + go func() { results <- c.pump(fromBackend) }() + go func() { results <- c.forward(fromClient, backend, pongTimeout) }() + go c.ping(pingInterval, done) + select { + case o := <-results: + return &o + case <-ctx.Done(): + return nil + } +} + +func (c *clientStream) ping(interval time.Duration, done <-chan struct{}) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + if err := c.relayControl(encodeFrame(opPing, pingPayload, false)); err != nil { + return + } + } + } +} + +// pump relays the apiserver's frames to the client, holding back the status +// message so the agent sends the stream's only one. +func (c *clientStream) pump(r *bufio.Reader) outcome { + var status []byte + intercept := false + for { + f, err := readFrame(r) + if err != nil || f.opcode == opClose { + return outcomeStreamLost + } + if f.control() { + if err := c.relayControl(f.raw); err != nil { + return outcomeClientGone + } + continue + } + if f.opcode != opContinuation { + intercept = len(f.payload) > 0 && f.payload[0] == channelStatus + if intercept { + status = append(status[:0], f.payload[1:]...) + } + } else if intercept { + status = append(status, f.payload...) + } + if !intercept { + if err := c.relayData(f.raw, f.fin); err != nil { + return outcomeClientGone + } + continue + } + if !f.fin { + continue + } + code, message, ok := exitCodeFromStatus(status) + if !ok { + return outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + message: "The instance could not run the command: " + message, + } + } + return outcome{reason: computev1alpha.InstanceConsoleSessionReasonCompleted, exitCode: &code} + } +} + +// forward relays the client's frames to the apiserver, except pongs to the +// agent's own pings. The apiserver reads only the first frame of a fragmented +// message, so fragmented messages are reassembled and sent as one frame. Any +// frame, including an unsolicited pong, shows the client is still there. +func (c *clientStream) forward(r *bufio.Reader, backend net.Conn, idle time.Duration) outcome { + var message []byte + var messageOp byte + fragmented := false + for { + if err := c.conn.SetReadDeadline(time.Now().Add(idle)); err != nil { + return outcomeClientGone + } + f, err := readFrame(r) + var tooLarge errFrameTooLarge + switch { + case errors.As(err, &tooLarge): + return outcomeFrameTooLarge + case err != nil, f.opcode == opClose: + return outcomeClientGone + case f.opcode == opPong && bytes.Equal(f.payload, pingPayload): + continue + } + out := f.raw + switch { + case f.control(): + case f.opcode == opContinuation: + if !fragmented { + return outcomeMalformed + } + if len(message)+len(f.payload) > MaxFrameSize { + return outcomeFrameTooLarge + } + message = append(message, f.payload...) + if !f.fin { + continue + } + out = encodeFrame(messageOp, message, true) + message, fragmented = nil, false + case fragmented: + return outcomeMalformed + case !f.fin: + message, messageOp, fragmented = append([]byte(nil), f.payload...), f.opcode, true + continue + } + if _, err := backend.Write(out); err != nil { + return outcomeStreamLost + } + } +} diff --git a/internal/shellagent/stream_test.go b/internal/shellagent/stream_test.go new file mode 100644 index 00000000..77959646 --- /dev/null +++ b/internal/shellagent/stream_test.go @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "bytes" + "context" + "crypto/ed25519" + "io" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/consolesession" +) + +func fastPings(c *Config) { + c.PingInterval = 50 * time.Millisecond + c.PongTimeout = 300 * time.Millisecond +} + +// backendRecorder is a fake command that runs until the stream closes and +// records every frame the agent relays to it. +type backendRecorder struct { + mu sync.Mutex + frames []*frame +} + +func (b *backendRecorder) run(conn net.Conn, r *bufio.Reader) { + writeServerFrame(conn, channelStdout, []byte("$ ")) + for { + f, err := readFrame(r) + if err != nil { + return + } + b.mu.Lock() + b.frames = append(b.frames, f) + b.mu.Unlock() + } +} + +func (b *backendRecorder) pongs() int { + b.mu.Lock() + defer b.mu.Unlock() + n := 0 + for _, f := range b.frames { + if f.opcode == opPong { + n++ + } + } + return n +} + +func TestSilentClientIsDisconnected(t *testing.T) { + h := newHarness(t) + a := h.agent(fastPings) + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = (&backendRecorder{}).run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + pinged := false + _ = c.conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + for !pinged { + f, err := readFrame(c.reader) + if err != nil { + t.Fatalf("read frame: %v", err) + } + pinged = f.opcode == opPing + } + + s := waitForEnd(t, h) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonDisconnected) + if s.Status.ExitCode != nil || h.exec.ran(killScript) != 1 { + t.Fatal("a client that stopped answering pings must have its command stopped") + } +} + +func TestClientDisconnectEndsAsDisconnected(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = (&backendRecorder{}).run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + c.send(t, opClose, closePayload(closeNormal)) + + requireReason(t, waitForEnd(t, h), computev1alpha.InstanceConsoleSessionReasonDisconnected) +} + +func dialGorilla(t *testing.T, addr, uid string, key ed25519.PrivateKey) *websocket.Conn { + t.Helper() + ts, sig := consolesession.Sign(key, uid, time.Now()) + header := http.Header{} + header.Set(consolesession.HeaderTimestamp, ts) + header.Set(consolesession.HeaderSignature, sig) + dialer := websocket.Dialer{Subprotocols: []string{consolesession.SubProtocol}} + conn, resp, err := dialer.Dial("ws://"+addr+consolesession.ExecPath(uid), header) + if err != nil { + t.Fatalf("dial: %v (%v)", err, resp) + } + t.Cleanup(func() { _ = conn.Close() }) + return conn +} + +// A gorilla/websocket client answers pings from inside its read loop with the +// library's default handler, as datumctl and the browser client do. +func TestAnsweringClientStaysConnected(t *testing.T) { + h := newHarness(t) + a := h.agent(fastPings) + addr := serve(t, h, a) + key := readySession(t, h, a) + backend := &backendRecorder{} + h.exec.onSession = backend.run + + conn := dialGorilla(t, addr, testUID, key) + go func() { + for { + if _, _, err := conn.ReadMessage(); err != nil { + return + } + } + }() + waitForConnected(t, h) + time.Sleep(4 * a.cfg.PongTimeout) + + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) + if n := backend.pongs(); n != 0 { + t.Fatalf("relayed %d pongs to the agent's own pings to the apiserver", n) + } +} + +// Any frame is a sign of life: a client whose output is backed up may answer +// with unsolicited empty pongs rather than replies to the agent's pings. +func TestUnsolicitedPongsKeepClientConnected(t *testing.T) { + h := newHarness(t) + a := h.agent(fastPings) + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = (&backendRecorder{}).run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + stop := time.After(4 * a.cfg.PongTimeout) + for done := false; !done; { + select { + case <-stop: + done = true + case <-time.After(a.cfg.PongTimeout / 3): + c.send(t, opPong, nil) + } + } + + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) +} + +// ttyCommand writes a prompt, echoes the first stdin message, and exits 0. +func ttyCommand(conn net.Conn, r *bufio.Reader) { + writeServerFrame(conn, channelStdout, []byte("prompt$ ")) + for { + f, err := readFrame(r) + if err != nil { + return + } + if f.opcode == opBinary && len(f.payload) > 1 && f.payload[0] == channelStdin { + writeServerFrame(conn, channelStdout, f.payload[1:]) + writeExitStatus(conn, 0) + return + } + } +} + +type signingTransport struct { + key ed25519.PrivateKey + uid string + rt http.RoundTripper +} + +func (s *signingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + ts, sig := consolesession.Sign(s.key, s.uid, time.Now()) + req = req.Clone(req.Context()) + req.Header.Set(consolesession.HeaderTimestamp, ts) + req.Header.Set(consolesession.HeaderSignature, sig) + return s.rt.RoundTrip(req) +} + +// client-go's WebSocket executor, as kubectl uses it, reads a terminal +// session's output, and answers the agent's pings. +func TestClientGoExecutor(t *testing.T) { + for _, tty := range []bool{true, false} { + t.Run(map[bool]string{true: "tty", false: "no tty"}[tty], func(t *testing.T) { + h := newHarness(t) + a := h.agent(fastPings) + addr := serve(t, h, a) + key := readySession(t, h, a, func(s *computev1alpha.InstanceConsoleSession) { + s.Spec.Terminal = tty + }) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + time.Sleep(4 * a.cfg.PongTimeout) + ttyCommand(conn, r) + } + + cfg := &rest.Config{ + Host: "http://" + addr, + WrapTransport: func(rt http.RoundTripper) http.RoundTripper { + return &signingTransport{key: key, uid: testUID, rt: rt} + }, + } + exec, err := remotecommand.NewWebSocketExecutor(cfg, http.MethodGet, + "http://"+addr+consolesession.ExecPath(testUID)) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var stdout, stderr bytes.Buffer + opts := remotecommand.StreamOptions{ + Stdin: io.MultiReader(strings.NewReader("hi\n"), blockUntil(ctx)), + Stdout: &stdout, + Tty: tty, + } + if !tty { + opts.Stderr = &stderr + } + if err := exec.StreamWithContext(ctx, opts); err != nil { + t.Fatalf("stream: %v", err) + } + if got := stdout.String(); got != "prompt$ hi\n" { + t.Fatalf("stdout = %q", got) + } + s := waitForEnd(t, h) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonCompleted) + if s.Status.ExitCode == nil || *s.Status.ExitCode != 0 { + t.Fatalf("exit code = %v", s.Status.ExitCode) + } + }) + } +} + +type blockingReader struct{ ctx context.Context } + +func (b blockingReader) Read([]byte) (int, error) { + <-b.ctx.Done() + return 0, io.EOF +} + +func blockUntil(ctx context.Context) io.Reader { return blockingReader{ctx: ctx} } + +func encodeFragment(opcode byte, payload []byte, fin bool) []byte { + f := encodeFrame(opcode, payload, true) + if !fin { + f[0] &^= 0x80 + } + return f +} + +// The apiserver reads only the first frame of a fragmented message, so the +// agent must deliver a fragmented stdin message as one frame. +func TestFragmentedClientMessageIsReassembled(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + backend := &backendRecorder{} + h.exec.onSession = backend.run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + raw := append(encodeFragment(opBinary, []byte{channelStdin, 'a', 'b'}, false), + encodeFrame(opPing, []byte("client"), true)...) + raw = append(raw, encodeFragment(opContinuation, []byte("cd"), false)...) + raw = append(raw, encodeFragment(opContinuation, []byte("ef"), true)...) + if _, err := c.conn.Write(raw); err != nil { + t.Fatal(err) + } + + deadline := time.Now().Add(5 * time.Second) + for { + backend.mu.Lock() + frames := append([]*frame(nil), backend.frames...) + backend.mu.Unlock() + if len(frames) >= 2 { + if frames[0].opcode != opPing { + t.Fatalf("first relayed frame opcode = %d, want the ping", frames[0].opcode) + } + f := frames[1] + if !f.fin || f.opcode != opBinary || string(f.payload) != "\x00abcdef" || f.raw[1]&0x80 == 0 { + t.Fatalf("relayed frame = fin %v, opcode %d, payload %q, masked %v", + f.fin, f.opcode, f.payload, f.raw[1]&0x80 != 0) + } + return + } + if time.Now().After(deadline) { + t.Fatalf("relayed %d frames, want 2", len(frames)) + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestStrayContinuationEndsSession(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = (&backendRecorder{}).run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + c.send(t, opContinuation, []byte("x")) + _, status, _ := c.readUntilClose(t) + + if status == nil || string(status.Reason) != computev1alpha.InstanceConsoleSessionReasonInvalid { + t.Fatalf("status = %+v", status) + } + requireReason(t, waitForEnd(t, h), computev1alpha.InstanceConsoleSessionReasonInvalid) +} diff --git a/internal/shellagent/sweep.go b/internal/shellagent/sweep.go new file mode 100644 index 00000000..193025f8 --- /dev/null +++ b/internal/shellagent/sweep.go @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "strings" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +type sessionStateKind int + +const ( + sessionActive sessionStateKind = iota + sessionEnded +) + +// sessionState reports whether a session, by project UID, is still open. A +// session whose cell copy is gone has ended. +func (a *Agent) sessionState(ctx context.Context, uid string) (sessionStateKind, error) { + cached, err := a.findSession(ctx, uid) + if err != nil { + return sessionActive, err + } + if cached == nil { + return sessionEnded, nil + } + var session computev1alpha.InstanceConsoleSession + err = a.cell.Get(ctx, client.ObjectKeyFromObject(cached), &session) + if apierrors.IsNotFound(err) { + return sessionEnded, nil + } + if err != nil { + return sessionActive, err + } + if sessionUID(&session) != uid || isTerminal(&session) { + return sessionEnded, nil + } + return sessionActive, nil +} + +// RunSweeper sweeps for orphaned processes at start and at every sweep +// interval until ctx ends. +func (a *Agent) RunSweeper(ctx context.Context) error { + ticker := time.NewTicker(a.cfg.SweepInterval) + defer ticker.Stop() + for { + if err := a.sweep(ctx); err != nil && ctx.Err() == nil { + log.FromContext(ctx).Error(err, "sweep for orphaned session processes") + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +type markerPlace struct { + pod types.NamespacedName + podUID types.UID + container string + dir string +} + +// sweep stops processes left behind by sessions that no longer exist or have +// ended, and frees their slots. Slots name the containers sessions ran in, and +// each of those containers is searched for marker files of other sessions. +func (a *Agent) sweep(ctx context.Context) error { + var leases coordinationv1.LeaseList + if err := a.cell.List(ctx, &leases, client.InNamespace(a.cfg.Namespace), + client.MatchingLabels{componentLabel: slotComponent}); err != nil { + return err + } + places := map[markerPlace]struct{}{} + for i := range leases.Items { + s := slotFromLease(&leases.Items[i]) + if s.markerDir != "" && s.pod.Name != "" { + places[markerPlace{pod: s.pod, podUID: s.podUID, container: s.container, dir: s.markerDir}] = struct{}{} + } + if a.isLive(s.sessionUID) { + continue + } + state, err := a.sessionState(ctx, s.sessionUID) + if err != nil || state != sessionEnded { + continue + } + reaped, err := a.reap(ctx, s) + if err != nil { + log.FromContext(ctx).Error(err, "reap orphaned session", "session", s.sessionUID) + continue + } + if reaped { + if err := a.markStopped(ctx, s.session, s.sessionUID); err != nil { + log.FromContext(ctx).Error(err, "record stopped session", "session", s.sessionUID) + } + } + } + for place := range places { + a.sweepContainer(ctx, place) + } + return nil +} + +func (a *Agent) sweepContainer(ctx context.Context, place markerPlace) { + logger := log.FromContext(ctx).WithValues("pod", place.pod.String(), "container", place.container) + var pod corev1.Pod + if err := a.cell.Get(ctx, place.pod, &pod); err != nil || pod.UID != place.podUID { + return + } + if status := containerStatus(&pod, place.container); status == nil || status.State.Running == nil { + return + } + out, code, err := a.run(ctx, place.pod, place.container, listMarkersCommand(place.dir)) + if err != nil || code != 0 { + logger.Info("could not list session markers", "exitCode", code, "error", err) + return + } + for _, uid := range strings.Fields(out) { + if a.isLive(uid) { + continue + } + state, err := a.sessionState(ctx, uid) + if err != nil || state != sessionEnded { + continue + } + grace := int(a.cfg.KillGrace.Seconds()) + if _, code, err := a.run(ctx, place.pod, place.container, killCommand(place.dir, uid, grace)); err != nil || code != 0 { + logger.Info("could not stop orphaned session processes", "session", uid, "exitCode", code, "error", err) + continue + } + logger.Info("stopped orphaned session processes", "session", uid) + } +}