From 15d994d201cc4af4ee8368a8c7beee981ad0fc3d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 18:08:59 -0500 Subject: [PATCH 01/20] feat: Serve shell sessions from a cell agent Each compute cell needs a process that turns a delivered InstanceConsoleSession into a running command in the instance and guarantees the command stops when the session ends, whatever ends it. Key changes: - Claim session copies with optimistic concurrency after checking the cell Instance, its pod, the container, a shell and the executable; publish the paired endpoint's ID, relays and target on the hub copy - Cap sessions per instance with three atomic Lease slots; a slot also records where the command runs and is held until its processes are confirmed stopped - Renew a liveness Lease; survivors end a lost agent's sessions with AgentLost and stop their processes - Serve connection protocol v1 through the shared consolesession package: signed requests, refusal codes, Connected recorded before any byte is relayed, and one closing status then close code 1000 - Wrap every command so its process group is stopped on exit, disconnect, expiry, revocation, takeover and shutdown - Drain on SIGTERM, sweep orphaned processes every minute, cap each agent at 200 open sessions, and create and rotate the endpoint key - Hold a finalizer on the hub and cell copies until processes stop, so a hub copy that disappears confirms the cell cleaned up --- cmd/shell-agent/main.go | 234 +++++++++ internal/shellagent/agent.go | 263 ++++++++++ internal/shellagent/checks.go | 214 ++++++++ internal/shellagent/drain.go | 65 +++ internal/shellagent/exec.go | 259 ++++++++++ internal/shellagent/exec_test.go | 70 +++ internal/shellagent/frames.go | 136 ++++++ internal/shellagent/harness_test.go | 405 ++++++++++++++++ internal/shellagent/identity.go | 259 ++++++++++ internal/shellagent/liveness.go | 114 +++++ internal/shellagent/reconciler.go | 400 +++++++++++++++ internal/shellagent/reconciler_test.go | 647 +++++++++++++++++++++++++ internal/shellagent/scripts.go | 63 +++ internal/shellagent/server.go | 219 +++++++++ internal/shellagent/server_test.go | 363 ++++++++++++++ internal/shellagent/slots.go | 240 +++++++++ internal/shellagent/status.go | 151 ++++++ internal/shellagent/stream.go | 196 ++++++++ internal/shellagent/sweep.go | 136 ++++++ 19 files changed, 4434 insertions(+) create mode 100644 cmd/shell-agent/main.go create mode 100644 internal/shellagent/agent.go create mode 100644 internal/shellagent/checks.go create mode 100644 internal/shellagent/drain.go create mode 100644 internal/shellagent/exec.go create mode 100644 internal/shellagent/exec_test.go create mode 100644 internal/shellagent/frames.go create mode 100644 internal/shellagent/harness_test.go create mode 100644 internal/shellagent/identity.go create mode 100644 internal/shellagent/liveness.go create mode 100644 internal/shellagent/reconciler.go create mode 100644 internal/shellagent/reconciler_test.go create mode 100644 internal/shellagent/scripts.go create mode 100644 internal/shellagent/server.go create mode 100644 internal/shellagent/server_test.go create mode 100644 internal/shellagent/slots.go create mode 100644 internal/shellagent/status.go create mode 100644 internal/shellagent/stream.go create mode 100644 internal/shellagent/sweep.go diff --git a/cmd/shell-agent/main.go b/cmd/shell-agent/main.go new file mode 100644 index 00000000..07365674 --- /dev/null +++ b/cmd/shell-agent/main.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Command shell-agent serves instance shell sessions in a compute cell. Each +// replica pairs by ordinal with a tunnel endpoint, which is the only way a +// client reaches it: the endpoint proxies one TCP target, this agent's exec +// port. The agent claims sessions delivered to the cell, runs their commands +// through the cell apiserver, and writes their status to the Karmada hub. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/features" + "go.datum.net/compute/internal/shellagent" +) + +var setupLog = ctrl.Log.WithName("setup") + +type options struct { + listen string + probeAddr string + federationKubeconfig string + federationContext string + target string + relayURLs string + managedBy string + endpointPodPrefix string + featureGates string + drainTimeout time.Duration + killGrace time.Duration +} + +func main() { + defaults := shellagent.DefaultConfig() + var opts options + flag.StringVar(&opts.listen, "listen", ":7777", "Address to serve session connections on.") + flag.StringVar(&opts.probeAddr, "health-probe-bind-address", ":8081", "Address the probe endpoint binds to.") + flag.StringVar(&opts.federationKubeconfig, "federation-kubeconfig", "", + "Path to the kubeconfig for the Karmada hub, where session status is written.") + flag.StringVar(&opts.federationContext, "federation-context", "", + "Context to use from the federation kubeconfig. When omitted, the current context is used.") + flag.StringVar(&opts.target, "target", "", + "host:port the paired tunnel endpoint proxies to this agent, published as the session's target.") + flag.StringVar(&opts.relayURLs, "relay-urls", os.Getenv("DATUM_CONNECT_RELAY_URLS"), + "Comma-separated relays the paired tunnel endpoint uses, nearest first.") + flag.StringVar(&opts.managedBy, "managed-by", strings.Join(defaults.ManagedBy, ","), + "Comma-separated managed-by label values of instance pods that may take sessions.") + flag.StringVar(&opts.endpointPodPrefix, "endpoint-pod-prefix", "exec-endpoint", + "Name of the tunnel endpoint StatefulSet; the paired pod is -.") + flag.StringVar(&opts.featureGates, "feature-gates", "", + "Feature gates, for example InstanceConsoleSessions=true.") + flag.DurationVar(&opts.drainTimeout, "drain-timeout", defaults.DrainTimeout, + "How long connected sessions may continue after the agent is asked to stop.") + flag.DurationVar(&opts.killGrace, "kill-grace", defaults.KillGrace, + "Time between SIGHUP and SIGKILL when a session's processes are stopped.") + zapOpts := zap.Options{} + zapOpts.BindFlags(flag.CommandLine) + flag.Parse() + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOpts))) + + if err := run(opts, defaults); err != nil { + setupLog.Error(err, "shell agent failed") + os.Exit(1) + } +} + +func run(opts options, cfg shellagent.Config) error { + if opts.featureGates != "" { + if err := features.MutableFeatureGate.Set(opts.featureGates); err != nil { + return fmt.Errorf("parse feature gates: %w", err) + } + } + signals := make(chan os.Signal, 2) + signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT) + + if !features.FeatureGate.Enabled(features.InstanceConsoleSessions) { + setupLog.Info("the InstanceConsoleSessions feature gate is off; not serving sessions") + <-signals + return nil + } + + podName, namespace := os.Getenv("POD_NAME"), os.Getenv("POD_NAMESPACE") + ordinal, err := ordinalOf(podName) + if err != nil || namespace == "" { + return fmt.Errorf("POD_NAME must be a StatefulSet pod name and POD_NAMESPACE must be set: %w", err) + } + if opts.federationKubeconfig == "" { + return errors.New("--federation-kubeconfig is required: session status is written to the Karmada hub") + } + cfg.Namespace = namespace + cfg.Ordinal = ordinal + cfg.EndpointPodName = fmt.Sprintf("%s-%d", opts.endpointPodPrefix, ordinal) + cfg.Target = opts.target + cfg.RelayURLs = splitList(opts.relayURLs) + cfg.ManagedBy = splitList(opts.managedBy) + cfg.DrainTimeout = opts.drainTimeout + cfg.KillGrace = opts.killGrace + + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(computev1alpha.AddToScheme(scheme)) + + cellConfig := ctrl.GetConfigOrDie() + hubConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: opts.federationKubeconfig}, + &clientcmd.ConfigOverrides{CurrentContext: opts.federationContext}, + ).ClientConfig() + if err != nil { + return fmt.Errorf("load federation kubeconfig: %w", err) + } + + mgr, err := ctrl.NewManager(cellConfig, ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: opts.probeAddr, + }) + if err != nil { + return fmt.Errorf("create manager: %w", err) + } + hubCluster, err := cluster.New(hubConfig, func(o *cluster.Options) { o.Scheme = scheme }) + if err != nil { + return fmt.Errorf("create hub cluster: %w", err) + } + if err := mgr.Add(hubCluster); err != nil { + return err + } + cellClient, err := client.New(cellConfig, client.Options{Scheme: scheme}) + if err != nil { + return err + } + hubClient, err := client.New(hubConfig, client.Options{Scheme: scheme}) + if err != nil { + return err + } + executor, err := shellagent.NewAPIServer(cellConfig) + if err != nil { + return fmt.Errorf("configure pod exec: %w", err) + } + + incarnation := podName + "/" + strconv.FormatInt(time.Now().UnixNano(), 36) + agent, err := shellagent.New(cfg, mgr.GetClient(), cellClient, hubClient, executor, incarnation) + if err != nil { + return err + } + ctx, cancel := context.WithCancel(ctrl.LoggerInto(context.Background(), ctrl.Log.WithName("shell-agent"))) + defer cancel() + if err := agent.EnsureIdentity(ctx); err != nil { + return err + } + if err := agent.SetupWithManager(mgr, hubCluster); err != nil { + return fmt.Errorf("set up controller: %w", err) + } + + server := &http.Server{Addr: opts.listen, Handler: agent.Handler(ctx), ReadHeaderTimeout: 10 * time.Second} + for _, r := range []manager.RunnableFunc{ + func(ctx context.Context) error { + go func() { + <-ctx.Done() + _ = server.Close() + }() + if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + }, + agent.RunLiveness, + agent.RunSweeper, + agent.RunKeyRotation, + } { + if err := mgr.Add(r); err != nil { + return err + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return err + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return err + } + + go func() { + <-signals + setupLog.Info("draining", "timeout", cfg.DrainTimeout) + agent.Drain(ctx) + setupLog.Info("drained") + cancel() + <-signals + os.Exit(1) + }() + + setupLog.Info("starting shell agent", "endpointID", agent.EndpointID(), "target", cfg.Target) + return mgr.Start(ctx) +} + +func ordinalOf(podName string) (int, error) { + i := strings.LastIndex(podName, "-") + if i < 0 { + return 0, fmt.Errorf("pod name %q has no ordinal", podName) + } + return strconv.Atoi(podName[i+1:]) +} + +func splitList(s string) []string { + var out []string + for _, v := range strings.Split(s, ",") { + if v = strings.TrimSpace(v); v != "" { + out = append(out, v) + } + } + return out +} diff --git a/internal/shellagent/agent.go b/internal/shellagent/agent.go new file mode 100644 index 00000000..d2bafa9a --- /dev/null +++ b/internal/shellagent/agent.go @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package shellagent is the cell side of instance shell sessions. An agent +// claims InstanceConsoleSessions delivered to its cell, serves each one's +// single connection through its paired tunnel endpoint, runs the session's +// command in the instance through the cell apiserver, and stops that command +// when the session ends for any reason. +// +// Every agent reads session copies from its cell and writes status to their +// hub copies, which the management plane copies back to the project. +package shellagent + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +const ( + // Finalizer is set on a session's hub and cell copies when an agent claims + // it, and removed once the session's processes are stopped. A hub copy + // that has disappeared therefore confirms the cell cleaned up. + Finalizer = "compute.datumapis.com/shell-agent" + + // SessionUIDLabel carries the project session's UID on its hub and cell + // copies. Clients connect with this UID and sign it. + SessionUIDLabel = computev1alpha.LabelNamespace + "/session-uid" + + // InstanceNameLabel carries the name of the Instance a session copy runs + // in. The cell Instance has this name in the copy's namespace. + InstanceNameLabel = computev1alpha.LabelNamespace + "/instance-name" + + managedByLabel = "managed-by" +) + +// Config is an agent's configuration. +type Config struct { + // Namespace holds the agent's Leases and its endpoint's key Secret. + Namespace string + // Ordinal pairs the agent with the tunnel endpoint of the same ordinal. + Ordinal int + // EndpointPodName is the paired tunnel endpoint's pod, restarted when its + // key rotates. + EndpointPodName string + // Target is the host:port the paired endpoint proxies to this agent. + Target string + // RelayURLs are the relays the paired endpoint is reachable through, + // nearest first. + RelayURLs []string + // ManagedBy lists the managed-by label values of pods that may take + // sessions. + ManagedBy []string + + ConnectTimeout time.Duration + SlotsPerInstance int + MaxOpenSessions int + DrainTimeout time.Duration + KillGrace time.Duration + KeyRotationPeriod time.Duration + UnclaimedTimeout time.Duration + SweepInterval time.Duration +} + +// DefaultConfig returns the contract's limits. +func DefaultConfig() Config { + return Config{ + ManagedBy: []string{"kata-provider"}, + ConnectTimeout: 60 * time.Second, + SlotsPerInstance: 3, + MaxOpenSessions: 200, + DrainTimeout: 30 * time.Second, + KillGrace: 3 * time.Second, + KeyRotationPeriod: 30 * 24 * time.Hour, + UnclaimedTimeout: 2 * time.Minute, + SweepInterval: time.Minute, + } +} + +// Agent serves shell sessions in one cell. +type Agent struct { + cfg Config + + sessions client.Reader + cell client.Client + hub client.Client + exec Executor + now func() time.Time + + incarnation string + identity atomic.Pointer[identity] + draining atomic.Bool + rotating atomic.Bool + + mu sync.Mutex + live map[string]*liveSession + open map[string]types.NamespacedName + + probes sync.Map +} + +// New returns an agent. sessions reads session copies and Instances from the +// cell, typically through a cache; cell reaches the cell apiserver directly; +// hub reaches the Karmada hub. +func New(cfg Config, sessions client.Reader, cell, hub client.Client, exec Executor, incarnation string) (*Agent, error) { + if cfg.Namespace == "" || cfg.Target == "" || len(cfg.RelayURLs) == 0 || len(cfg.ManagedBy) == 0 { + return nil, errors.New("namespace, target, relay URLs and managed-by values are required") + } + return &Agent{ + cfg: cfg, + sessions: sessions, + cell: cell, + hub: hub, + exec: exec, + now: time.Now, + incarnation: incarnation, + live: map[string]*liveSession{}, + open: map[string]types.NamespacedName{}, + }, nil +} + +// EndpointID is the paired tunnel endpoint's current iroh endpoint ID. +func (a *Agent) EndpointID() string { + if id := a.identity.Load(); id != nil { + return id.endpointID + } + return "" +} + +func (a *Agent) claiming() bool { + return !a.draining.Load() && !a.rotating.Load() && a.identity.Load() != nil +} + +type liveSession struct { + cancel context.CancelFunc + done chan struct{} + stream *clientStream + tty bool + reason string +} + +// register records a connection. It returns false when the session already +// has one. +func (a *Agent) register(uid string, cancel context.CancelFunc, tty bool) (*liveSession, bool) { + a.mu.Lock() + defer a.mu.Unlock() + if _, ok := a.live[uid]; ok { + return nil, false + } + l := &liveSession{cancel: cancel, done: make(chan struct{}), tty: tty} + a.live[uid] = l + return l, true +} + +func (a *Agent) attach(l *liveSession, s *clientStream) { + a.mu.Lock() + l.stream = s + a.mu.Unlock() +} + +func (a *Agent) unregister(uid string) { + a.mu.Lock() + delete(a.live, uid) + a.mu.Unlock() +} + +func (a *Agent) isLive(uid string) bool { + a.mu.Lock() + defer a.mu.Unlock() + _, ok := a.live[uid] + return ok +} + +func (a *Agent) stopReason(uid string) string { + a.mu.Lock() + defer a.mu.Unlock() + if l := a.live[uid]; l != nil { + return l.reason + } + return "" +} + +func (a *Agent) liveSessions() map[string]*liveSession { + a.mu.Lock() + defer a.mu.Unlock() + out := make(map[string]*liveSession, len(a.live)) + for uid, l := range a.live { + out[uid] = l + } + return out +} + +// stop ends a connected session for reason and waits until its processes +// have been stopped and its end recorded. +func (a *Agent) stop(uid, reason string) { + a.mu.Lock() + l := a.live[uid] + if l != nil && l.reason == "" { + l.reason = reason + } + a.mu.Unlock() + if l == nil { + return + } + l.cancel() + select { + case <-l.done: + case <-time.After(a.cfg.KillGrace + 30*time.Second): + } +} + +// track counts a session claimed by this agent toward its open sessions. +func (a *Agent) track(uid string, key types.NamespacedName) { + a.mu.Lock() + a.open[uid] = key + a.mu.Unlock() +} + +func (a *Agent) forget(uid string) { + a.mu.Lock() + delete(a.open, uid) + a.mu.Unlock() +} + +func (a *Agent) openSessions() map[string]types.NamespacedName { + a.mu.Lock() + defer a.mu.Unlock() + out := make(map[string]types.NamespacedName, len(a.open)) + for uid, key := range a.open { + out[uid] = key + } + return out +} + +func (a *Agent) openCount() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.open) +} + +func sessionUID(s *computev1alpha.InstanceConsoleSession) string { + return s.Labels[SessionUIDLabel] +} + +func endpointOf(s *computev1alpha.InstanceConsoleSession) string { + if s.Status.Connection == nil { + return "" + } + return s.Status.Connection.EndpointID +} + +func sessionTTL(s *computev1alpha.InstanceConsoleSession) time.Duration { + if s.Spec.TTL == nil || s.Spec.TTL.Duration <= 0 { + return 15 * time.Minute + } + return s.Spec.TTL.Duration +} diff --git a/internal/shellagent/checks.go b/internal/shellagent/checks.go new file mode 100644 index 00000000..aa8d0eb4 --- /dev/null +++ b/internal/shellagent/checks.go @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "sync" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +// errNotInCell means the cell does not run the session's workload +// deployment, so the session is another cell's to claim. +var errNotInCell = errors.New("the session's instance does not run in this cell") + +// rejection ends a session before it is claimed. +type rejection struct { + reason string + message string +} + +func reject(reason, format string, args ...any) *rejection { + return &rejection{reason: reason, message: fmt.Sprintf(format, args...)} +} + +// target is where a session's command runs. +type target struct { + instance *computev1alpha.Instance + pod *corev1.Pod + container string + containerID string + markerDir string +} + +// check runs the checks that come before a claim: the cell Instance, its pod, +// the container, and a shell and the requested executable in it. +func (a *Agent) check(ctx context.Context, s *computev1alpha.InstanceConsoleSession) (*target, *rejection, error) { + inst, rej, err := a.findInstance(ctx, s) + if rej != nil || err != nil { + return nil, rej, err + } + pod, containerID, rej, err := a.findPod(ctx, inst, s.Spec.ContainerName) + if rej != nil || err != nil { + return nil, rej, err + } + dir, rej, err := a.probe(ctx, pod, s.Spec.ContainerName, s.Spec.Command[0]) + if rej != nil || err != nil { + return nil, rej, err + } + return &target{ + instance: inst, + pod: pod, + container: s.Spec.ContainerName, + containerID: containerID, + markerDir: dir, + }, nil, nil +} + +// findInstance finds the cell Instance a session copy names. A same-name +// Instance from another workload deployment is a replacement, not the +// Instance the session was created for. +func (a *Agent) findInstance(ctx context.Context, s *computev1alpha.InstanceConsoleSession) (*computev1alpha.Instance, *rejection, error) { + name := s.Labels[InstanceNameLabel] + deploymentUID := s.Labels[computev1alpha.WorkloadDeploymentUIDLabel] + if name == "" || deploymentUID == "" || len(s.Spec.Command) == 0 { + return nil, reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + "The session was delivered without the instance it runs in."), nil + } + var inst computev1alpha.Instance + err := a.sessions.Get(ctx, client.ObjectKey{Namespace: s.Namespace, Name: name}, &inst) + if err != nil && !apierrors.IsNotFound(err) { + return nil, nil, err + } + if err == nil && inst.Labels[computev1alpha.WorkloadDeploymentUIDLabel] == deploymentUID { + if !inst.DeletionTimestamp.IsZero() { + return nil, reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + "The instance is being deleted."), nil + } + return &inst, nil, nil + } + var siblings computev1alpha.InstanceList + if err := a.sessions.List(ctx, &siblings, client.InNamespace(s.Namespace), + client.MatchingLabels{computev1alpha.WorkloadDeploymentUIDLabel: deploymentUID}); err != nil { + return nil, nil, err + } + if len(siblings.Items) == 0 { + return nil, nil, errNotInCell + } + return nil, reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotFound, + "The instance no longer exists or has been replaced."), nil +} + +// findPod finds the running pod that realizes a cell Instance, and the ID of +// the requested container in it. +func (a *Agent) findPod(ctx context.Context, inst *computev1alpha.Instance, container string) (*corev1.Pod, string, *rejection, error) { + notRunning := reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, "The instance is not running.") + var pod corev1.Pod + err := a.cell.Get(ctx, client.ObjectKeyFromObject(inst), &pod) + if apierrors.IsNotFound(err) { + return nil, "", notRunning, nil + } + if err != nil { + return nil, "", nil, err + } + if owner := metav1.GetControllerOf(&pod); owner == nil || owner.UID != inst.UID { + return nil, "", notRunning, nil + } + if !slices.Contains(a.cfg.ManagedBy, pod.Labels[managedByLabel]) { + return nil, "", reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + "Sessions are not available for this instance's runtime."), nil + } + if !slices.ContainsFunc(pod.Spec.Containers, func(c corev1.Container) bool { return c.Name == container }) { + return nil, "", reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + "The instance has no container named %q.", container), nil + } + if pod.Status.Phase != corev1.PodRunning || !pod.DeletionTimestamp.IsZero() { + return nil, "", notRunning, nil + } + status := containerStatus(&pod, container) + if status == nil || status.State.Running == nil { + return nil, "", reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + "The container %q is not running.", container), nil + } + return &pod, status.ContainerID, nil, nil +} + +func containerStatus(pod *corev1.Pod, name string) *corev1.ContainerStatus { + for i := range pod.Status.ContainerStatuses { + if pod.Status.ContainerStatuses[i].Name == name { + return &pod.Status.ContainerStatuses[i] + } + } + return nil +} + +type probeResult struct { + mu sync.Mutex + dir string + executables map[string]bool +} + +const maxProbeCacheEntries = 4096 + +// probe checks that a container has a shell, a writable directory for the +// marker file, and the requested executable. What it finds is cached per pod +// UID and container, so each check execs into an instance once. +func (a *Agent) probe(ctx context.Context, pod *corev1.Pod, container, executable string) (string, *rejection, error) { + key := string(pod.UID) + "/" + container + if v, ok := a.probes.Load(key); ok { + r := v.(*probeResult) + r.mu.Lock() + dir, known := r.dir, r.executables[executable] + r.mu.Unlock() + if known { + return dir, nil, nil + } + } + + out, code, err := run(ctx, a.exec, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, + container, probeCommand(executable)) + var failed *ExecFailedError + switch { + case errors.As(err, &failed): + return "", reject(computev1alpha.InstanceConsoleSessionReasonNoShell, + "The container has no shell (sh), which the platform needs to manage the command."), nil + case err != nil: + return "", nil, err + case code == probeExitCommandMissing: + return "", reject(computev1alpha.InstanceConsoleSessionReasonCommandUnavailable, + "The container has no executable named %q.", executable), nil + case code == probeExitNoWritableDir: + return "", reject(computev1alpha.InstanceConsoleSessionReasonNoShell, + "The container has no writable temporary directory, which the platform needs to manage the command."), nil + case code != 0: + return "", reject(computev1alpha.InstanceConsoleSessionReasonNoShell, + "The container's shell (sh) failed to run."), nil + } + + lines := strings.Fields(out) + if len(lines) == 0 { + return "", reject(computev1alpha.InstanceConsoleSessionReasonNoShell, + "The container's shell (sh) failed to run."), nil + } + dir := lines[len(lines)-1] + if a.probeCacheSize() >= maxProbeCacheEntries { + a.probes.Clear() + } + v, _ := a.probes.LoadOrStore(key, &probeResult{executables: map[string]bool{}}) + r := v.(*probeResult) + r.mu.Lock() + r.dir = dir + r.executables[executable] = true + r.mu.Unlock() + return dir, nil, nil +} + +func (a *Agent) probeCacheSize() int { + n := 0 + a.probes.Range(func(_, _ any) bool { + n++ + return true + }) + return n +} diff --git a/internal/shellagent/drain.go b/internal/shellagent/drain.go new file mode 100644 index 00000000..94c17720 --- /dev/null +++ b/internal/shellagent/drain.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "fmt" + "sync" + "time" + + "sigs.k8s.io/controller-runtime/pkg/log" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +// Drain stops the agent claiming and connecting sessions, ends the ones it +// claimed that no client connected to, warns connected sessions that have a +// terminal, and waits up to the drain timeout for connected sessions to end +// before ending the rest with AgentShutdown. +func (a *Agent) Drain(ctx context.Context) { + a.draining.Store(true) + logger := log.FromContext(ctx) + deadline := time.Now().Add(a.cfg.DrainTimeout) + + for uid, key := range a.openSessions() { + if a.isLive(uid) { + continue + } + var hub computev1alpha.InstanceConsoleSession + if err := a.hub.Get(ctx, key, &hub); err != nil { + continue + } + if err := a.endUnconnected(ctx, &hub, computev1alpha.InstanceConsoleSessionReasonAgentShutdown); err != nil { + logger.Error(err, "end unconnected session", "session", uid) + } + } + + notice := fmt.Sprintf("\r\n*** Maintenance: this session will close by %s. Start a new session to continue. ***\r\n", + deadline.UTC().Format("15:04:05 MST")) + for _, l := range a.liveSessions() { + a.mu.Lock() + stream, tty := l.stream, l.tty + a.mu.Unlock() + if tty && stream != nil { + _ = stream.Notify(notice) + } + } + + for time.Now().Before(deadline) && len(a.liveSessions()) > 0 { + select { + case <-ctx.Done(): + return + case <-time.After(250 * time.Millisecond): + } + } + + var wg sync.WaitGroup + for uid := range a.liveSessions() { + wg.Go(func() { + a.stop(uid, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + }) + } + wg.Wait() + a.releaseLease(ctx, a.EndpointID()) +} diff --git a/internal/shellagent/exec.go b/internal/shellagent/exec.go new file mode 100644 index 00000000..97602e96 --- /dev/null +++ b/internal/shellagent/exec.go @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "context" + "crypto/rand" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + + "go.datum.net/compute/internal/consolesession" +) + +// ExecOptions is one pods/exec request. +type ExecOptions struct { + Container string + Command []string + Stdin bool + TTY bool +} + +func (o ExecOptions) query() url.Values { + q := url.Values{} + for _, c := range o.Command { + q.Add("command", c) + } + q.Set("container", o.Container) + q.Set("stdout", "true") + if o.Stdin { + q.Set("stdin", "true") + } + if o.TTY { + q.Set("tty", "true") + } else { + q.Set("stderr", "true") + } + return q +} + +// Executor opens exec streams into pods. +type Executor interface { + // Open upgrades a pods/exec request to a v5.channel.k8s.io WebSocket and + // returns the apiserver's 101 response with the stream. handshake carries + // the client's WebSocket key when the stream is relayed to a client, so + // the apiserver's accept header matches it; a nil handshake gets a fresh + // key. + Open(ctx context.Context, pod types.NamespacedName, opts ExecOptions, handshake http.Header) (*http.Response, net.Conn, *bufio.Reader, error) +} + +// ExecRefusedError is an exec the apiserver answered with something other +// than an upgrade. +type ExecRefusedError struct { + StatusCode int + Message string +} + +func (e *ExecRefusedError) Error() string { + return fmt.Sprintf("exec refused with HTTP %d: %s", e.StatusCode, e.Message) +} + +// APIServer opens exec streams against the cell apiserver. +type APIServer struct { + base *url.URL + tlsConfig *tls.Config + transport http.RoundTripper +} + +// NewAPIServer returns an Executor that authenticates to the apiserver the +// way cfg does. +func NewAPIServer(cfg *rest.Config) (*APIServer, error) { + base, _, err := rest.DefaultServerUrlFor(cfg) + if err != nil { + return nil, err + } + tlsConfig, err := rest.TLSConfigFor(cfg) + if err != nil { + return nil, err + } + if tlsConfig == nil { + tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } + tlsConfig = tlsConfig.Clone() + tlsConfig.NextProtos = []string{"http/1.1"} + transport, err := rest.HTTPWrappersForConfig(cfg, &capture{}) + if err != nil { + return nil, err + } + return &APIServer{base: base, tlsConfig: tlsConfig, transport: transport}, nil +} + +// capture ends a round trip at the transport, after the client-go wrappers +// have added credentials, so the authorized request can be written onto a +// connection the agent owns. +type capture struct{} + +type capturedRequest struct { + req *http.Request +} + +func (e *capturedRequest) Error() string { return "request captured" } + +func (capture) RoundTrip(req *http.Request) (*http.Response, error) { + return nil, &capturedRequest{req: req} +} + +func (a *APIServer) authorize(req *http.Request) (*http.Request, error) { + _, err := a.transport.RoundTrip(req) + var captured *capturedRequest + if errors.As(err, &captured) { + return captured.req, nil + } + if err == nil { + err = errors.New("credentials transport returned a response") + } + return nil, err +} + +// Open implements Executor. +func (a *APIServer) Open(ctx context.Context, pod types.NamespacedName, opts ExecOptions, handshake http.Header) (*http.Response, net.Conn, *bufio.Reader, error) { + target := *a.base + target.Path = strings.TrimSuffix(target.Path, "/") + + fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/exec", pod.Namespace, pod.Name) + target.RawQuery = opts.query().Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil) + if err != nil { + return nil, nil, nil, err + } + setHandshake(req.Header, handshake) + req, err = a.authorize(req) + if err != nil { + return nil, nil, nil, fmt.Errorf("authorize exec request: %w", err) + } + + host := target.Host + if target.Port() == "" { + port := "443" + if target.Scheme == "http" { + port = "80" + } + host = net.JoinHostPort(target.Hostname(), port) + } + var conn net.Conn + if target.Scheme == "http" { + conn, err = (&net.Dialer{}).DialContext(ctx, "tcp", host) + } else { + conn, err = (&tls.Dialer{Config: a.tlsConfig}).DialContext(ctx, "tcp", host) + } + if err != nil { + return nil, nil, nil, fmt.Errorf("dial apiserver: %w", err) + } + return upgrade(conn, req) +} + +func setHandshake(h, from http.Header) { + key := from.Get("Sec-WebSocket-Key") + if key == "" { + raw := make([]byte, 16) + _, _ = rand.Read(raw) + key = base64.StdEncoding.EncodeToString(raw) + } + h.Set("Connection", "Upgrade") + h.Set("Upgrade", "websocket") + h.Set("Sec-WebSocket-Version", "13") + h.Set("Sec-WebSocket-Key", key) + h.Set("Sec-WebSocket-Protocol", consolesession.SubProtocol) +} + +func upgrade(conn net.Conn, req *http.Request) (*http.Response, net.Conn, *bufio.Reader, error) { + if err := req.Write(conn); err != nil { + _ = conn.Close() + return nil, nil, nil, fmt.Errorf("send exec request: %w", err) + } + reader := bufio.NewReader(conn) + resp, err := http.ReadResponse(reader, req) + if err != nil { + _ = conn.Close() + return nil, nil, nil, fmt.Errorf("read exec response: %w", err) + } + if resp.StatusCode != http.StatusSwitchingProtocols { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + _ = resp.Body.Close() + _ = conn.Close() + return nil, nil, nil, &ExecRefusedError{StatusCode: resp.StatusCode, Message: strings.TrimSpace(string(body))} + } + return resp, conn, reader, nil +} + +// ExecFailedError is a command the apiserver could not run, as opposed to a +// command that ran and exited. +type ExecFailedError struct { + Message string +} + +func (e *ExecFailedError) Error() string { + return "exec failed: " + e.Message +} + +// run runs a command to completion without stdin and returns its combined +// output and exit code. +func run(ctx context.Context, exec Executor, pod types.NamespacedName, container string, command []string) (string, int, error) { + _, conn, reader, err := exec.Open(ctx, pod, ExecOptions{Container: container, Command: command}, nil) + if err != nil { + return "", 0, err + } + defer func() { _ = conn.Close() }() + stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + defer stop() + + var out strings.Builder + var status []byte + var channel byte + for { + f, err := readFrame(reader) + if err != nil { + if ctx.Err() != nil { + return out.String(), 0, ctx.Err() + } + return out.String(), 0, fmt.Errorf("exec stream ended without a status: %w", err) + } + switch { + case f.opcode == opClose: + return out.String(), 0, errors.New("exec stream closed without a status") + case f.control(): + continue + case f.opcode != opContinuation: + if len(f.payload) == 0 { + continue + } + channel = f.payload[0] + f.payload = f.payload[1:] + } + switch channel { + case channelStdout, channelStderr: + out.Write(f.payload) + case channelStatus: + status = append(status, f.payload...) + } + if channel != channelStatus || !f.fin { + continue + } + code, message, ok := exitCodeFromStatus(status) + if !ok { + return out.String(), 0, &ExecFailedError{Message: message} + } + return out.String(), int(code), nil + } +} diff --git a/internal/shellagent/exec_test.go b/internal/shellagent/exec_test.go new file mode 100644 index 00000000..8a12c126 --- /dev/null +++ b/internal/shellagent/exec_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "encoding/pem" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + + "go.datum.net/compute/internal/consolesession" +) + +func TestAPIServerOpensAuthorizedExecStream(t *testing.T) { + var got *http.Request + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.Clone(r.Context()) + conn, _, err := http.NewResponseController(w).Hijack() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + _, _ = conn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + + "Sec-WebSocket-Protocol: " + consolesession.SubProtocol + "\r\n\r\n")) + writeServerFrame(conn, channelStdout, []byte("/tmp\n")) + writeExitStatus(conn, 0) + })) + defer srv.Close() + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) + exec, err := NewAPIServer(&rest.Config{ + Host: srv.URL, + BearerToken: "agent-token", + TLSClientConfig: rest.TLSClientConfig{CAData: caPEM}, + }) + if err != nil { + t.Fatal(err) + } + + pod := types.NamespacedName{Namespace: "ns-1", Name: "web-0"} + out, code, err := run(t.Context(), exec, pod, "app", probeCommand("bash")) + + if err != nil || code != 0 || out != "/tmp\n" { + t.Fatalf("run() = %q, %d, %v", out, code, err) + } + if got.URL.Path != "/api/v1/namespaces/ns-1/pods/web-0/exec" { + t.Fatalf("path = %s", got.URL.Path) + } + q := got.URL.Query() + if !reflect.DeepEqual(q["command"], probeCommand("bash")) || q.Get("container") != "app" || + q.Get("stdout") != "true" || q.Get("stderr") != "true" || q.Has("stdin") || q.Has("tty") { + t.Fatalf("query = %v", q) + } + if got.Header.Get("Authorization") != "Bearer agent-token" || + got.Header.Get("Sec-WebSocket-Protocol") != consolesession.SubProtocol || + got.Header.Get("Sec-WebSocket-Key") == "" { + t.Fatalf("headers = %v", got.Header) + } +} + +func TestWrappedCommandRecordsProcessID(t *testing.T) { + got := wrappedCommand("/tmp", "uid-1", []string{"ls", "-l"}) + want := []string{"sh", "-c", wrapperScript, "sh", "/tmp/.datum-exec-uid-1", "ls", "-l"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("wrappedCommand() = %q, want %q", got, want) + } +} diff --git a/internal/shellagent/frames.go b/internal/shellagent/frames.go new file mode 100644 index 00000000..3d8379a5 --- /dev/null +++ b/internal/shellagent/frames.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "crypto/rand" + "encoding/binary" + "fmt" + "io" +) + +// MaxFrameSize is the largest WebSocket frame the agent relays. A larger frame +// ends the session. +const MaxFrameSize = 16 << 20 + +const ( + opContinuation = 0x0 + opText = 0x1 + opBinary = 0x2 + opClose = 0x8 + opPing = 0x9 + opPong = 0xa + + closeNormal = 1000 + + channelStdin = 0 + channelStdout = 1 + channelStderr = 2 + channelStatus = 3 +) + +// frame is one WebSocket frame. raw holds the frame exactly as it arrived, so +// it can be relayed untouched; payload is its unmasked payload. +type frame struct { + fin bool + opcode byte + payload []byte + raw []byte +} + +func (f *frame) control() bool { + return f.opcode >= opClose +} + +// errFrameTooLarge reports a frame over MaxFrameSize. +type errFrameTooLarge struct { + size uint64 +} + +func (e errFrameTooLarge) Error() string { + return fmt.Sprintf("frame of %d bytes exceeds the %d byte limit", e.size, MaxFrameSize) +} + +// here be dragons +func readFrame(r *bufio.Reader) (*frame, error) { + head := make([]byte, 2, 14) + if _, err := io.ReadFull(r, head); err != nil { + return nil, err + } + length := uint64(head[1] & 0x7f) + switch length { + case 126: + ext := make([]byte, 2) + if _, err := io.ReadFull(r, ext); err != nil { + return nil, err + } + head = append(head, ext...) + length = uint64(binary.BigEndian.Uint16(ext)) + case 127: + ext := make([]byte, 8) + if _, err := io.ReadFull(r, ext); err != nil { + return nil, err + } + head = append(head, ext...) + length = binary.BigEndian.Uint64(ext) + } + var mask []byte + if head[1]&0x80 != 0 { + mask = make([]byte, 4) + if _, err := io.ReadFull(r, mask); err != nil { + return nil, err + } + head = append(head, mask...) + } + if length > MaxFrameSize { + return nil, errFrameTooLarge{size: length} + } + raw := make([]byte, len(head)+int(length)) + copy(raw, head) + if _, err := io.ReadFull(r, raw[len(head):]); err != nil { + return nil, err + } + payload := raw[len(head):] + if mask != nil { + payload = make([]byte, length) + for i := range payload { + payload[i] = raw[len(head)+i] ^ mask[i%4] + } + } + return &frame{fin: head[0]&0x80 != 0, opcode: head[0] & 0x0f, payload: payload, raw: raw}, nil +} + +// encodeFrame builds a single final frame. Frames a client sends must be +// masked; frames a server sends must not. +func encodeFrame(opcode byte, payload []byte, masked bool) []byte { + out := []byte{0x80 | opcode} + maskBit := byte(0) + if masked { + maskBit = 0x80 + } + switch n := len(payload); { + case n < 126: + out = append(out, maskBit|byte(n)) + case n <= 0xffff: + out = append(out, maskBit|126) + out = binary.BigEndian.AppendUint16(out, uint16(n)) + default: + out = append(out, maskBit|127) + out = binary.BigEndian.AppendUint64(out, uint64(n)) + } + if !masked { + return append(out, payload...) + } + mask := make([]byte, 4) + _, _ = rand.Read(mask) + out = append(out, mask...) + for i, b := range payload { + out = append(out, b^mask[i%4]) + } + return out +} + +func closePayload(code uint16) []byte { + return binary.BigEndian.AppendUint16(nil, code) +} diff --git a/internal/shellagent/harness_test.go b/internal/shellagent/harness_test.go new file mode 100644 index 00000000..b51f2417 --- /dev/null +++ b/internal/shellagent/harness_test.go @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/consolesession" +) + +const ( + testAgentNamespace = "compute-shell-system" + testNamespace = "ns-project-uid" + testInstance = "web-0" + testInstanceUID = types.UID("cell-instance-uid") + testDeploymentUID = "cell-wd-uid" + testContainer = "app" + testSession = "s1" + testUID = "uid-1" +) + +// clock is a settable time source shared by the agents in a test. +type clock struct { + mu sync.Mutex + t time.Time +} + +func (c *clock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *clock) advance(d time.Duration) { + c.mu.Lock() + c.t = c.t.Add(d) + c.mu.Unlock() +} + +type harness struct { + t *testing.T + ctx context.Context + cell client.Client + hub client.Client + exec *fakeAPIServer + clock *clock +} + +func newScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(computev1alpha.AddToScheme(scheme)) + return scheme +} + +func newHarness(t *testing.T) *harness { + t.Helper() + scheme := newScheme() + owner := metav1.OwnerReference{ + APIVersion: computev1alpha.GroupVersion.String(), + Kind: "Instance", + Name: testInstance, + UID: testInstanceUID, + Controller: ptr.To(true), + } + instance := &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{ + Name: testInstance, + Namespace: testNamespace, + UID: testInstanceUID, + Labels: map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID}, + }} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: testInstance, + Namespace: testNamespace, + UID: "pod-uid", + Labels: map[string]string{managedByLabel: "kata-provider"}, + OwnerReferences: []metav1.OwnerReference{owner}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: testContainer}}}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: testContainer, + ContainerID: "containerd://abc", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }}, + }, + } + session := &computev1alpha.InstanceConsoleSession{} + return &harness{ + t: t, + ctx: ctrl.LoggerInto(context.Background(), ctrl.Log), + cell: fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(session). + WithObjects(instance, pod).Build(), + hub: fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(session).Build(), + exec: newFakeAPIServer(), + clock: &clock{t: time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC)}, + } +} + +// agent returns an agent whose endpoint ID derives from a fresh key and whose +// liveness Lease is current. +func (h *harness) agent(mutate ...func(*Config)) *Agent { + h.t.Helper() + cfg := DefaultConfig() + cfg.Namespace = testAgentNamespace + cfg.Target = "exec-agent-0.exec-agent.compute-shell-system.svc.cluster.local:7777" + cfg.RelayURLs = []string{"https://relay.example.test"} + cfg.KillGrace = 0 + for _, m := range mutate { + m(&cfg) + } + a, err := New(cfg, h.cell, h.cell, h.hub, h.exec, "test") + if err != nil { + h.t.Fatal(err) + } + a.now = h.clock.now + seed := make([]byte, ed25519.SeedSize) + _, _ = rand.Read(seed) + a.identity.Store(&identity{endpointID: EndpointID(seed), created: h.clock.now()}) + if err := a.renewLease(h.ctx); err != nil { + h.t.Fatal(err) + } + return a +} + +// session creates a session's hub and cell copies and returns the client's +// key. +func (h *harness) session(name, uid string, mutate ...func(*computev1alpha.InstanceConsoleSession)) ed25519.PrivateKey { + h.t.Helper() + _, key, _ := ed25519.GenerateKey(rand.Reader) + for _, c := range []client.Client{h.hub, h.cell} { + s := &computev1alpha.InstanceConsoleSession{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + CreationTimestamp: metav1.NewTime(h.clock.now()), + Labels: map[string]string{ + SessionUIDLabel: uid, + InstanceNameLabel: testInstance, + computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID, + }, + }, + Spec: computev1alpha.InstanceConsoleSessionSpec{ + InstanceRef: computev1alpha.InstanceConsoleSessionInstanceRef{Name: testInstance, UID: "project-instance-uid"}, + ContainerName: testContainer, + Command: []string{"sh"}, + Stdin: true, + Terminal: true, + ClientPublicKey: consolesession.PublicKey(key), + TTL: &metav1.Duration{Duration: 15 * time.Minute}, + }, + } + for _, m := range mutate { + m(s) + } + if err := c.Create(h.ctx, s); err != nil { + h.t.Fatal(err) + } + } + return key +} + +func (h *harness) hubSession(name string) *computev1alpha.InstanceConsoleSession { + h.t.Helper() + var s computev1alpha.InstanceConsoleSession + if err := h.hub.Get(h.ctx, client.ObjectKey{Namespace: testNamespace, Name: name}, &s); err != nil { + h.t.Fatal(err) + } + return &s +} + +func (h *harness) reconcile(a *Agent, name string) ctrl.Result { + h.t.Helper() + res, err := a.Reconcile(h.ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: name}}) + if err != nil { + h.t.Fatalf("reconcile %s: %v", name, err) + } + return res +} + +func (h *harness) slots() []coordinationv1.Lease { + h.t.Helper() + var leases coordinationv1.LeaseList + if err := h.cell.List(h.ctx, &leases, client.MatchingLabels{componentLabel: slotComponent}); err != nil { + h.t.Fatal(err) + } + return leases.Items +} + +func requireReason(t *testing.T, s *computev1alpha.InstanceConsoleSession, want string) { + t.Helper() + if got := readyReason(s); got != want { + c := readyCondition(s) + t.Fatalf("Ready reason = %q, want %q (condition %+v)", got, want, c) + } +} + +// fakeAPIServer plays the apiserver's pods/exec endpoint. Commands the agent +// runs to completion answer from scripts; session streams are handed to +// onSession. +type fakeAPIServer struct { + mu sync.Mutex + commands [][]string + probeExit int + probeDir string + markers string + killExit int + onSession func(conn net.Conn, r *bufio.Reader) +} + +func newFakeAPIServer() *fakeAPIServer { + return &fakeAPIServer{probeDir: "/tmp"} +} + +func (f *fakeAPIServer) ran(script string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, c := range f.commands { + if len(c) > 2 && c[2] == script { + n++ + } + } + return n +} + +func (f *fakeAPIServer) Open(_ context.Context, _ types.NamespacedName, opts ExecOptions, handshake http.Header) (*http.Response, net.Conn, *bufio.Reader, error) { + f.mu.Lock() + f.commands = append(f.commands, opts.Command) + onSession := f.onSession + f.mu.Unlock() + + agentSide, serverSide := net.Pipe() + resp := &http.Response{ + StatusCode: http.StatusSwitchingProtocols, + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{ + "Upgrade": []string{"websocket"}, + "Connection": []string{"Upgrade"}, + "Sec-Websocket-Protocol": []string{consolesession.SubProtocol}, + }, + } + serverReader := bufio.NewReader(serverSide) + go func() { + defer func() { _ = serverSide.Close() }() + if handshake != nil && onSession != nil { + onSession(serverSide, serverReader) + return + } + f.answer(serverSide, opts.Command) + }() + return resp, agentSide, bufio.NewReader(agentSide), nil +} + +func (f *fakeAPIServer) answer(conn net.Conn, command []string) { + f.mu.Lock() + probeExit, probeDir, markers, killExit := f.probeExit, f.probeDir, f.markers, f.killExit + f.mu.Unlock() + switch command[2] { + case probeScript: + if probeExit == 0 { + writeServerFrame(conn, channelStdout, []byte(probeDir+"\n")) + } + writeExitStatus(conn, probeExit) + case listMarkersScript: + writeServerFrame(conn, channelStdout, []byte(markers)) + writeExitStatus(conn, 0) + case killScript: + writeExitStatus(conn, killExit) + default: + writeExitStatus(conn, 0) + } +} + +func writeServerFrame(conn net.Conn, channel byte, payload []byte) { + _, _ = conn.Write(encodeFrame(opBinary, append([]byte{channel}, payload...), false)) +} + +func writeExitStatus(conn net.Conn, code int) { + var status metav1.Status + if code == 0 { + status = metav1.Status{Status: metav1.StatusSuccess} + } else { + c := int32(code) + status = closingStatus(computev1alpha.InstanceConsoleSessionReasonCompleted, &c, "") + } + body, _ := json.Marshal(status) + writeServerFrame(conn, channelStatus, body) + _, _ = conn.Write(encodeFrame(opClose, closePayload(closeNormal), false)) +} + +// wsClient is a minimal client side of the connection protocol. +type wsClient struct { + conn net.Conn + reader *bufio.Reader + status int + body string +} + +func dialSession(t *testing.T, addr, uid string, key ed25519.PrivateKey, at time.Time) *wsClient { + t.Helper() + conn, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + req, _ := http.NewRequest(http.MethodGet, "http://"+addr+consolesession.ExecPath(uid), nil) + req.Header.Set("Connection", "Upgrade") + req.Header.Set("Upgrade", "websocket") + req.Header.Set("Sec-WebSocket-Version", "13") + req.Header.Set("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==") + req.Header.Set("Sec-WebSocket-Protocol", consolesession.SubProtocol) + if key != nil { + ts, sig := consolesession.Sign(key, uid, at) + req.Header.Set(consolesession.HeaderTimestamp, ts) + req.Header.Set(consolesession.HeaderSignature, sig) + } + if err := req.Write(conn); err != nil { + t.Fatal(err) + } + reader := bufio.NewReader(conn) + resp, err := http.ReadResponse(reader, req) + if err != nil { + t.Fatal(err) + } + c := &wsClient{conn: conn, reader: reader, status: resp.StatusCode} + if resp.StatusCode != http.StatusSwitchingProtocols { + var b strings.Builder + buf := make([]byte, 512) + n, _ := resp.Body.Read(buf) + b.Write(buf[:n]) + c.body = b.String() + } + return c +} + +// readUntilClose returns every data frame by channel, the status message and +// the close code. +func (c *wsClient) readUntilClose(t *testing.T) (map[byte]string, *metav1.Status, int) { + t.Helper() + _ = c.conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + out := map[byte]string{} + var status *metav1.Status + for { + f, err := readFrame(c.reader) + if err != nil { + t.Fatalf("read frame: %v", err) + } + if f.opcode == opClose { + code := 0 + if len(f.payload) >= 2 { + code = int(f.payload[0])<<8 | int(f.payload[1]) + } + return out, status, code + } + if f.control() || len(f.payload) == 0 { + continue + } + if f.payload[0] == channelStatus { + if status != nil { + t.Fatal("received more than one status message") + } + status = &metav1.Status{} + if err := json.Unmarshal(f.payload[1:], status); err != nil { + t.Fatal(err) + } + continue + } + out[f.payload[0]] += string(f.payload[1:]) + } +} + +func (c *wsClient) send(t *testing.T, opcode byte, payload []byte) { + t.Helper() + if _, err := c.conn.Write(encodeFrame(opcode, payload, true)); err != nil { + t.Fatal(err) + } +} diff --git a/internal/shellagent/identity.go b/internal/shellagent/identity.go new file mode 100644 index 00000000..fb9dc593 --- /dev/null +++ b/internal/shellagent/identity.go @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "fmt" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // KeySecretKey is the key in an endpoint's Secret that holds its raw + // 32-byte identity key. + KeySecretKey = "key" + + annotationKeyCreated = annotationPrefix + "key-created-at" + rotationLockName = "endpoint-key-rotation" + rotationLockDuration = 2 * time.Hour + endpointRestartWait = 5 * time.Minute +) + +var rotationPollInterval = 5 * time.Second + +type identity struct { + endpointID string + created time.Time +} + +// KeySecretName is the Secret holding the identity key of the tunnel endpoint +// with the given ordinal. +func KeySecretName(ordinal int) string { + return fmt.Sprintf("endpoint-key-%d", ordinal) +} + +// EndpointID returns the iroh endpoint ID of a raw ed25519 identity key. +func EndpointID(seed []byte) string { + return hex.EncodeToString(ed25519.NewKeyFromSeed(seed).Public().(ed25519.PublicKey)) +} + +func newSeed() ([]byte, error) { + seed := make([]byte, ed25519.SeedSize) + if _, err := rand.Read(seed); err != nil { + return nil, err + } + return seed, nil +} + +// EnsureIdentity creates the paired endpoint's key Secret if it is missing and +// loads the endpoint ID the agent publishes. +func (a *Agent) EnsureIdentity(ctx context.Context) error { + key := client.ObjectKey{Namespace: a.cfg.Namespace, Name: KeySecretName(a.cfg.Ordinal)} + var secret corev1.Secret + err := a.cell.Get(ctx, key, &secret) + if apierrors.IsNotFound(err) { + var seed []byte + if seed, err = newSeed(); err != nil { + return err + } + secret = corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + Labels: map[string]string{componentLabel: "shell-endpoint-key"}, + Annotations: map[string]string{annotationKeyCreated: a.now().UTC().Format(time.RFC3339)}, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{KeySecretKey: seed}, + } + err = a.cell.Create(ctx, &secret) + if apierrors.IsAlreadyExists(err) { + err = a.cell.Get(ctx, key, &secret) + } + } + if err != nil { + return fmt.Errorf("load endpoint key %s: %w", key.Name, err) + } + return a.loadIdentity(&secret) +} + +func (a *Agent) loadIdentity(secret *corev1.Secret) error { + seed := secret.Data[KeySecretKey] + if len(seed) != ed25519.SeedSize { + return fmt.Errorf("endpoint key %s holds %d bytes, want %d", secret.Name, len(seed), ed25519.SeedSize) + } + created := secret.CreationTimestamp.Time + if at, err := time.Parse(time.RFC3339, secret.Annotations[annotationKeyCreated]); err == nil { + created = at + } + a.identity.Store(&identity{endpointID: EndpointID(seed), created: created}) + return nil +} + +// RunKeyRotation replaces the endpoint's identity key once it is older than +// the rotation period, checking hourly until ctx ends. +func (a *Agent) RunKeyRotation(ctx context.Context) error { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + if id := a.identity.Load(); id != nil && a.now().Sub(id.created) >= a.cfg.KeyRotationPeriod { + if err := a.rotate(ctx); err != nil && ctx.Err() == nil { + log.FromContext(ctx).Error(err, "rotate endpoint key") + } + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +// rotate stops claiming, waits for open sessions to end or expire, replaces +// the key, restarts the endpoint so it serves the new key, and resumes. One +// agent in the cell rotates at a time, so the cell keeps taking sessions. +func (a *Agent) rotate(ctx context.Context) error { + locked, err := a.lockRotation(ctx) + if err != nil || !locked { + return err + } + defer a.unlockRotation(ctx) + a.rotating.Store(true) + defer a.rotating.Store(false) + logger := log.FromContext(ctx) + logger.Info("rotating endpoint key; waiting for open sessions to end") + + idle := 0 + for idle < 2 { + if a.draining.Load() { + return nil + } + if a.openCount() == 0 && len(a.liveSessions()) == 0 { + idle++ + } else { + idle = 0 + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(rotationPollInterval): + } + } + + previous := a.EndpointID() + key := client.ObjectKey{Namespace: a.cfg.Namespace, Name: KeySecretName(a.cfg.Ordinal)} + var secret corev1.Secret + if err := a.cell.Get(ctx, key, &secret); err != nil { + return err + } + seed, err := newSeed() + if err != nil { + return err + } + secret.Data = map[string][]byte{KeySecretKey: seed} + if secret.Annotations == nil { + secret.Annotations = map[string]string{} + } + secret.Annotations[annotationKeyCreated] = a.now().UTC().Format(time.RFC3339) + if err := a.cell.Update(ctx, &secret); err != nil { + return err + } + if err := a.loadIdentity(&secret); err != nil { + return err + } + a.releaseLease(ctx, previous) + logger.Info("replaced endpoint key", "endpointID", a.EndpointID()) + return a.restartEndpoint(ctx) +} + +// restartEndpoint deletes the paired endpoint's pod and waits for its +// replacement to be ready with the new key. +func (a *Agent) restartEndpoint(ctx context.Context) error { + if a.cfg.EndpointPodName == "" { + return nil + } + key := client.ObjectKey{Namespace: a.cfg.Namespace, Name: a.cfg.EndpointPodName} + var pod corev1.Pod + if err := a.cell.Get(ctx, key, &pod); client.IgnoreNotFound(err) != nil { + return err + } + previous := pod.UID + if previous != "" { + if err := a.cell.Delete(ctx, &pod, client.Preconditions{UID: &previous}); client.IgnoreNotFound(err) != nil { + return err + } + } + waitCtx, cancel := context.WithTimeout(ctx, endpointRestartWait) + defer cancel() + for { + var next corev1.Pod + err := a.cell.Get(waitCtx, key, &next) + if err == nil && next.UID != previous && podReady(&next) { + return nil + } + select { + case <-waitCtx.Done(): + return fmt.Errorf("endpoint %s did not become ready after its key rotated", key.Name) + case <-time.After(2 * time.Second): + } + } +} + +func podReady(pod *corev1.Pod) bool { + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +func (a *Agent) lockRotation(ctx context.Context) (bool, error) { + now := metav1.NewMicroTime(a.now()) + lock := &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: rotationLockName, Namespace: a.cfg.Namespace}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: ptr.To(a.incarnation), + LeaseDurationSeconds: ptr.To(int32(rotationLockDuration.Seconds())), + AcquireTime: &now, + RenewTime: &now, + }, + } + err := a.cell.Create(ctx, lock) + if err == nil { + return true, nil + } + if !apierrors.IsAlreadyExists(err) { + return false, err + } + var held coordinationv1.Lease + if err := a.cell.Get(ctx, client.ObjectKeyFromObject(lock), &held); err != nil { + return false, client.IgnoreNotFound(err) + } + if held.Spec.RenewTime != nil && a.now().Before(held.Spec.RenewTime.Add(rotationLockDuration)) { + return false, nil + } + _ = a.cell.Delete(ctx, &held, client.Preconditions{UID: &held.UID}) + return false, nil +} + +func (a *Agent) unlockRotation(ctx context.Context) { + err := a.cell.Delete(ctx, &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{ + Name: rotationLockName, + Namespace: a.cfg.Namespace, + }}) + if client.IgnoreNotFound(err) != nil { + log.FromContext(ctx).Error(err, "release key rotation lock") + } +} diff --git a/internal/shellagent/liveness.go b/internal/shellagent/liveness.go new file mode 100644 index 00000000..afd913b6 --- /dev/null +++ b/internal/shellagent/liveness.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // LeaseDuration is how long an agent's liveness Lease holds without a + // renewal before other agents take over its sessions. + LeaseDuration = 15 * time.Second + // LeaseRenewInterval is how often an agent renews its liveness Lease. + LeaseRenewInterval = 5 * time.Second + + livenessComponent = "shell-agent-liveness" +) + +func agentLeaseName(endpointID string) string { + if len(endpointID) > 16 { + endpointID = endpointID[:16] + } + return "agent-" + endpointID +} + +// RunLiveness renews the agent's liveness Lease until ctx ends. +func (a *Agent) RunLiveness(ctx context.Context) error { + ticker := time.NewTicker(LeaseRenewInterval) + defer ticker.Stop() + for { + if err := a.renewLease(ctx); err != nil && ctx.Err() == nil { + log.FromContext(ctx).Error(err, "renew liveness lease") + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +func (a *Agent) renewLease(ctx context.Context) error { + id := a.EndpointID() + if id == "" { + return nil + } + now := metav1.NewMicroTime(a.now()) + key := client.ObjectKey{Namespace: a.cfg.Namespace, Name: agentLeaseName(id)} + var lease coordinationv1.Lease + err := a.cell.Get(ctx, key, &lease) + if apierrors.IsNotFound(err) { + return a.cell.Create(ctx, &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + Labels: map[string]string{componentLabel: livenessComponent}, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: ptr.To(a.incarnation), + LeaseDurationSeconds: ptr.To(int32(LeaseDuration.Seconds())), + AcquireTime: &now, + RenewTime: &now, + }, + }) + } + if err != nil { + return err + } + lease.Spec.HolderIdentity = ptr.To(a.incarnation) + lease.Spec.LeaseDurationSeconds = ptr.To(int32(LeaseDuration.Seconds())) + lease.Spec.RenewTime = &now + return a.cell.Update(ctx, &lease) +} + +// releaseLease deletes a liveness Lease once its agent no longer serves +// sessions under that endpoint ID. +func (a *Agent) releaseLease(ctx context.Context, endpointID string) { + err := a.cell.Delete(ctx, &coordinationv1.Lease{ObjectMeta: metav1.ObjectMeta{ + Name: agentLeaseName(endpointID), + Namespace: a.cfg.Namespace, + }}) + if client.IgnoreNotFound(err) != nil { + log.FromContext(ctx).Error(err, "release liveness lease") + } +} + +// agentAlive reports whether the agent serving endpointID renewed its +// liveness Lease within the Lease's duration. +func (a *Agent) agentAlive(ctx context.Context, endpointID string) (bool, error) { + if endpointID == a.EndpointID() { + return true, nil + } + var lease coordinationv1.Lease + err := a.cell.Get(ctx, client.ObjectKey{Namespace: a.cfg.Namespace, Name: agentLeaseName(endpointID)}, &lease) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + if lease.Spec.RenewTime == nil || lease.Spec.LeaseDurationSeconds == nil { + return false, nil + } + expires := lease.Spec.RenewTime.Add(time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second) + return a.now().Before(expires), nil +} diff --git a/internal/shellagent/reconciler.go b/internal/shellagent/reconciler.go new file mode 100644 index 00000000..ab825f21 --- /dev/null +++ b/internal/shellagent/reconciler.go @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "errors" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/source" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +const ( + claimRetryInterval = 5 * time.Second + cleanupRetryInterval = 5 * time.Second +) + +// SetupWithManager reconciles session copies delivered to the cell, and their +// hub copies, which share a namespace and name with them. +func (a *Agent) SetupWithManager(mgr ctrl.Manager, hub cluster.Cluster) error { + return ctrl.NewControllerManagedBy(mgr). + Named("shell-agent"). + For(&computev1alpha.InstanceConsoleSession{}). + WatchesRawSource(source.Kind(hub.GetCache(), &computev1alpha.InstanceConsoleSession{}, + &handler.TypedEnqueueRequestForObject[*computev1alpha.InstanceConsoleSession]{})). + WithOptions(controller.Options{MaxConcurrentReconciles: 16}). + Complete(a) +} + +// Reconcile drives one session: claiming it, ending it at its deadlines, +// taking it over from a lost agent, and stopping its processes before its +// finalizers are released. +func (a *Agent) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var cell computev1alpha.InstanceConsoleSession + if err := a.sessions.Get(ctx, req.NamespacedName, &cell); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + uid := sessionUID(&cell) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("session", uid)) + + var hub computev1alpha.InstanceConsoleSession + err := a.hub.Get(ctx, req.NamespacedName, &hub) + if err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + if apierrors.IsNotFound(err) || sessionUID(&hub) != uid { + return a.cleanUp(ctx, &cell, nil) + } + if !hub.DeletionTimestamp.IsZero() || !cell.DeletionTimestamp.IsZero() { + return a.cleanUp(ctx, &cell, &hub) + } + if uid == "" { + return ctrl.Result{}, a.rejectUnclaimed(ctx, &hub, reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + "The session was delivered without its session UID.")) + } + if isTerminal(&hub) { + a.forget(uid) + return a.settle(ctx, &hub) + } + + owner := endpointOf(&hub) + switch { + case owner == "": + return a.claim(ctx, &cell, &hub) + case owner != a.EndpointID(): + return a.watchOwner(ctx, &hub, owner) + } + + a.track(uid, req.NamespacedName) + now := a.now() + switch readyReason(&hub) { + case computev1alpha.InstanceConsoleSessionReasonSessionReady: + if a.draining.Load() { + return ctrl.Result{}, a.endUnconnected(ctx, &hub, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + } + if hub.Status.ConnectBefore != nil && now.Before(hub.Status.ConnectBefore.Time) { + return ctrl.Result{RequeueAfter: hub.Status.ConnectBefore.Sub(now)}, nil + } + log.FromContext(ctx).Info("session was not connected in time") + return ctrl.Result{}, a.endUnconnected(ctx, &hub, computev1alpha.InstanceConsoleSessionReasonNotConnected) + case computev1alpha.InstanceConsoleSessionReasonConnected: + if !a.isLive(uid) { + log.FromContext(ctx).Info("ending a session this agent's previous run served") + return ctrl.Result{}, a.abandon(ctx, &hub, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) + } + if hub.Status.ExpiresAt != nil && now.Before(hub.Status.ExpiresAt.Time) { + return ctrl.Result{RequeueAfter: hub.Status.ExpiresAt.Sub(now)}, nil + } + log.FromContext(ctx).Info("session expired") + a.stop(uid, computev1alpha.InstanceConsoleSessionReasonExpired) + } + return ctrl.Result{}, nil +} + +// claim checks a session and, if it can run, takes a slot and publishes where +// to connect. Agents race to claim with optimistic concurrency. +func (a *Agent) claim(ctx context.Context, cell, hub *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + if !a.claiming() || a.openCount() >= a.cfg.MaxOpenSessions { + return ctrl.Result{RequeueAfter: claimRetryInterval}, nil + } + target, rej, err := a.check(ctx, cell) + if errors.Is(err, errNotInCell) { + age := a.now().Sub(hub.CreationTimestamp.Time) + if age < a.cfg.UnclaimedTimeout { + return ctrl.Result{RequeueAfter: a.cfg.UnclaimedTimeout - age}, nil + } + rej = reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotFound, + "The instance no longer exists or has been replaced.") + } else if err != nil { + return ctrl.Result{}, err + } + if rej != nil { + log.FromContext(ctx).Info("refusing session", "reason", rej.reason, "message", rej.message) + return ctrl.Result{}, a.rejectUnclaimed(ctx, hub, rej) + } + + uid := sessionUID(hub) + held, err := a.acquireSlot(ctx, target.instance.UID, &slot{ + sessionUID: uid, + session: client.ObjectKeyFromObject(cell), + pod: client.ObjectKeyFromObject(target.pod), + podUID: target.pod.UID, + container: target.container, + containerID: target.containerID, + markerDir: target.markerDir, + endpointID: a.EndpointID(), + }) + if errors.Is(err, errClaimInProgress) { + return ctrl.Result{RequeueAfter: claimRetryInterval}, nil + } + if err != nil { + return ctrl.Result{}, err + } + if held == nil { + return ctrl.Result{}, a.rejectUnclaimed(ctx, hub, reject( + computev1alpha.InstanceConsoleSessionReasonTooManySessions, + "The instance already has %d open sessions, the most it allows.", a.cfg.SlotsPerInstance)) + } + + if err := addFinalizer(ctx, a.hub, hub); err != nil { + return ctrl.Result{}, errors.Join(err, a.releaseSlot(ctx, held)) + } + if err := addFinalizer(ctx, a.cell, cell); err != nil { + return ctrl.Result{}, errors.Join(err, a.releaseSlot(ctx, held)) + } + + now := a.now() + connectBefore := metav1.NewTime(now.Add(a.cfg.ConnectTimeout)) + hub.Status.Connection = &computev1alpha.InstanceConsoleSessionConnection{ + EndpointID: a.EndpointID(), + RelayURLs: a.cfg.RelayURLs, + Target: a.cfg.Target, + } + hub.Status.ConnectBefore = &connectBefore + setReady(hub, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonSessionReady, + "Connect before the connection deadline.") + if err := a.hub.Status().Update(ctx, hub); err != nil { + if relErr := a.releaseSlot(ctx, held); relErr != nil { + return ctrl.Result{}, relErr + } + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, err + } + a.track(uid, client.ObjectKeyFromObject(cell)) + log.FromContext(ctx).Info("claimed session", "pod", target.pod.Name, "container", target.container) + return ctrl.Result{RequeueAfter: a.cfg.ConnectTimeout}, nil +} + +// watchOwner takes over a session in this cell whose agent stopped renewing +// its liveness Lease. A session claimed in another cell has no slot here and +// is left alone. +func (a *Agent) watchOwner(ctx context.Context, hub *computev1alpha.InstanceConsoleSession, owner string) (ctrl.Result, error) { + held, err := a.slotFor(ctx, sessionUID(hub)) + if err != nil || held == nil { + return ctrl.Result{}, err + } + alive, err := a.agentAlive(ctx, owner) + if err != nil { + return ctrl.Result{}, err + } + if alive { + return ctrl.Result{RequeueAfter: LeaseDuration}, nil + } + log.FromContext(ctx).Info("taking over a session from a lost agent", "agent", owner) + return ctrl.Result{}, a.abandon(ctx, hub, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) +} + +// abandon ends a session no agent is serving: it stops the session's process, +// which a dropped stream leaves running, and records why. The slot is kept if +// the process could not be confirmed stopped, for the sweep to retry. +func (a *Agent) abandon(ctx context.Context, hub *computev1alpha.InstanceConsoleSession, owner, reason string) error { + uid := sessionUID(hub) + held, err := a.slotFor(ctx, uid) + if err != nil { + return err + } + stopped := true + if held != nil { + if stopped, err = a.stopProcesses(ctx, held); err != nil { + return err + } + } + if _, err := a.end(ctx, client.ObjectKeyFromObject(hub), reason, endMessage(reason), nil, + func(s *computev1alpha.InstanceConsoleSession) bool { return endpointOf(s) == owner }); err != nil { + return err + } + a.forget(uid) + if stopped { + return a.releaseSlot(ctx, held) + } + return nil +} + +// endUnconnected ends a session this agent claimed but no client connected +// to. No process was started, so its slot is freed at once. +func (a *Agent) endUnconnected(ctx context.Context, hub *computev1alpha.InstanceConsoleSession, reason string) error { + uid := sessionUID(hub) + me := a.EndpointID() + ended, err := a.end(ctx, client.ObjectKeyFromObject(hub), reason, endMessage(reason), nil, + func(s *computev1alpha.InstanceConsoleSession) bool { + return endpointOf(s) == me && readyReason(s) == computev1alpha.InstanceConsoleSessionReasonSessionReady + }) + if err != nil || !ended { + return err + } + a.forget(uid) + held, err := a.slotFor(ctx, uid) + if err != nil { + return err + } + return a.releaseSlot(ctx, held) +} + +func (a *Agent) rejectUnclaimed(ctx context.Context, hub *computev1alpha.InstanceConsoleSession, rej *rejection) error { + _, err := a.end(ctx, client.ObjectKeyFromObject(hub), rej.reason, rej.message, nil, + func(s *computev1alpha.InstanceConsoleSession) bool { return s.Status.Connection == nil }) + return err +} + +// settle frees the slot of an ended session once its process is stopped, +// unless a live agent is still ending it. +func (a *Agent) settle(ctx context.Context, hub *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(hub) + if a.isLive(uid) { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, nil + } + held, err := a.slotFor(ctx, uid) + if err != nil || held == nil { + return ctrl.Result{}, err + } + if owner := endpointOf(hub); owner != "" && owner != a.EndpointID() { + if alive, err := a.agentAlive(ctx, owner); err != nil || alive { + return ctrl.Result{RequeueAfter: LeaseDuration}, err + } + } + reaped, err := a.reap(ctx, held) + if err != nil || !reaped { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, err + } + return ctrl.Result{}, nil +} + +// cleanUp handles a session whose hub or cell copy is being deleted, or whose +// hub copy is gone. The session is revoked if it is still open, and the +// finalizers are released only once its process is stopped. +func (a *Agent) cleanUp(ctx context.Context, cell, hub *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(cell) + me := a.EndpointID() + responsible := true + if hub == nil && a.isLive(uid) { + a.stop(uid, computev1alpha.InstanceConsoleSessionReasonRevoked) + } + if hub != nil && !isTerminal(hub) { + switch owner := endpointOf(hub); { + case owner == me && a.isLive(uid): + log.FromContext(ctx).Info("revoking session") + a.stop(uid, computev1alpha.InstanceConsoleSessionReasonRevoked) + case owner == me: + if err := a.endUnconnected(ctx, hub, computev1alpha.InstanceConsoleSessionReasonRevoked); err != nil { + return ctrl.Result{}, err + } + case owner == "": + if err := a.rejectUnclaimed(ctx, hub, reject(computev1alpha.InstanceConsoleSessionReasonRevoked, + "%s", endMessage(computev1alpha.InstanceConsoleSessionReasonRevoked))); err != nil { + return ctrl.Result{}, err + } + default: + held, err := a.slotFor(ctx, uid) + if err != nil { + return ctrl.Result{}, err + } + if held == nil { + responsible = false + break + } + alive, err := a.agentAlive(ctx, owner) + if err != nil { + return ctrl.Result{}, err + } + if alive { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, nil + } + if err := a.abandon(ctx, hub, owner, computev1alpha.InstanceConsoleSessionReasonRevoked); err != nil { + return ctrl.Result{}, err + } + } + } + + if a.isLive(uid) { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + held, err := a.slotFor(ctx, uid) + if err != nil { + return ctrl.Result{}, err + } + if held != nil { + reaped, err := a.reap(ctx, held) + if err != nil || !reaped { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, err + } + } + a.forget(uid) + + if hub != nil && responsible { + if err := removeFinalizer(ctx, a.hub, client.ObjectKeyFromObject(hub)); err != nil { + return ctrl.Result{}, err + } + } + if !cell.DeletionTimestamp.IsZero() || hub == nil { + return ctrl.Result{}, removeFinalizer(ctx, a.cell, client.ObjectKeyFromObject(cell)) + } + return ctrl.Result{}, nil +} + +// end records a terminal reason on the hub copy if the session has not ended +// yet and guard still holds for its latest state. It reports whether this +// call ended the session. +func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, message string, exitCode *int32, + guard func(*computev1alpha.InstanceConsoleSession) bool) (bool, error) { + ended := false + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var s computev1alpha.InstanceConsoleSession + if err := a.hub.Get(ctx, key, &s); err != nil { + return err + } + if isTerminal(&s) || guard != nil && !guard(&s) { + return nil + } + now := metav1.NewTime(a.now()) + s.Status.EndedAt = &now + s.Status.ExitCode = exitCode + setReady(&s, metav1.ConditionFalse, reason, message) + if err := a.hub.Status().Update(ctx, &s); err != nil { + return err + } + ended = true + return nil + }) + return ended, client.IgnoreNotFound(err) +} + +func addFinalizer(ctx context.Context, c client.Client, s *computev1alpha.InstanceConsoleSession) error { + if controllerutil.ContainsFinalizer(s, Finalizer) { + return nil + } + base := s.DeepCopy() + controllerutil.AddFinalizer(s, Finalizer) + return c.Patch(ctx, s, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) +} + +func removeFinalizer(ctx context.Context, c client.Client, key types.NamespacedName) error { + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var s computev1alpha.InstanceConsoleSession + if err := c.Get(ctx, key, &s); err != nil { + return err + } + if !controllerutil.ContainsFinalizer(&s, Finalizer) { + return nil + } + base := s.DeepCopy() + controllerutil.RemoveFinalizer(&s, Finalizer) + return c.Patch(ctx, &s, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) + }) + return client.IgnoreNotFound(err) +} diff --git a/internal/shellagent/reconciler_test.go b/internal/shellagent/reconciler_test.go new file mode 100644 index 00000000..2d715644 --- /dev/null +++ b/internal/shellagent/reconciler_test.go @@ -0,0 +1,647 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "fmt" + "net" + "sync" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +func TestClaimPublishesConnection(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + + res := h.reconcile(a, testSession) + + s := h.hubSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonSessionReady) + if s.Status.Connection == nil || s.Status.Connection.EndpointID != a.EndpointID() || + s.Status.Connection.Target != a.cfg.Target || len(s.Status.Connection.RelayURLs) != 1 { + t.Fatalf("connection = %+v", s.Status.Connection) + } + if want := h.clock.now().Add(time.Minute); !s.Status.ConnectBefore.Time.Equal(want) { + t.Fatalf("connectBefore = %v, want %v", s.Status.ConnectBefore, want) + } + if res.RequeueAfter != time.Minute { + t.Fatalf("requeue after %v, want the connect timeout", res.RequeueAfter) + } + if !controllerutil.ContainsFinalizer(s, Finalizer) { + t.Fatal("hub copy has no finalizer") + } + var cell computev1alpha.InstanceConsoleSession + _ = h.cell.Get(h.ctx, client.ObjectKeyFromObject(s), &cell) + if !controllerutil.ContainsFinalizer(&cell, Finalizer) { + t.Fatal("cell copy has no finalizer") + } + if n := len(h.slots()); n != 1 { + t.Fatalf("slots = %d, want 1", n) + } +} + +func TestClaimRaceHasOneWinner(t *testing.T) { + h := newHarness(t) + agents := []*Agent{h.agent(), h.agent()} + h.session(testSession, testUID) + + var wg sync.WaitGroup + for _, a := range agents { + wg.Go(func() { + for range 3 { + _, _ = a.Reconcile(h.ctx, reqFor(testSession)) + } + }) + } + wg.Wait() + + s := h.hubSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonSessionReady) + owners := 0 + for _, a := range agents { + if s.Status.Connection.EndpointID == a.EndpointID() { + owners++ + } + } + if owners != 1 { + t.Fatalf("session claimed by %d agents", owners) + } + if n := len(h.slots()); n != 1 { + t.Fatalf("slots = %d, want 1", n) + } +} + +func TestClaimWithStaleCopyLosesToWinner(t *testing.T) { + h := newHarness(t) + winner, loser := h.agent(), h.agent() + h.session(testSession, testUID) + stale := h.hubSession(testSession) + var cell computev1alpha.InstanceConsoleSession + _ = h.cell.Get(h.ctx, client.ObjectKeyFromObject(stale), &cell) + + h.reconcile(winner, testSession) + if _, err := loser.claim(h.ctx, &cell, stale); err != nil { + t.Logf("losing claim returned %v", err) + } + + s := h.hubSession(testSession) + if s.Status.Connection.EndpointID != winner.EndpointID() { + t.Fatal("a stale claim overwrote the winner's connection") + } + slots := h.slots() + if len(slots) != 1 || slots[0].Annotations[annotationEndpoint] != winner.EndpointID() { + t.Fatalf("slots = %+v, want only the winner's", slots) + } +} + +func TestSlotCapIsExactlyThree(t *testing.T) { + h := newHarness(t) + agents := []*Agent{h.agent(), h.agent()} + for i := range 6 { + h.session(fmt.Sprintf("s%d", i), fmt.Sprintf("uid-%d", i)) + } + + var wg sync.WaitGroup + for i := range 6 { + a := agents[i%2] + wg.Go(func() { + for range 3 { + _, _ = a.Reconcile(h.ctx, reqFor(fmt.Sprintf("s%d", i))) + } + }) + } + wg.Wait() + + counts := map[string]int{} + for i := range 6 { + counts[readyReason(h.hubSession(fmt.Sprintf("s%d", i)))]++ + } + if counts[computev1alpha.InstanceConsoleSessionReasonSessionReady] != 3 || + counts[computev1alpha.InstanceConsoleSessionReasonTooManySessions] != 3 { + t.Fatalf("outcomes = %v, want 3 ready and 3 refused", counts) + } + if n := len(h.slots()); n != 3 { + t.Fatalf("slots = %d, want 3", n) + } +} + +func TestSlotFreedByEndedSessionIsReused(t *testing.T) { + h := newHarness(t) + a := h.agent() + for i := range 4 { + h.session(fmt.Sprintf("s%d", i), fmt.Sprintf("uid-%d", i)) + } + for i := range 3 { + h.reconcile(a, fmt.Sprintf("s%d", i)) + } + s0 := h.hubSession("s0") + setReady(s0, metav1.ConditionFalse, computev1alpha.InstanceConsoleSessionReasonCompleted, "done") + if err := h.hub.Status().Update(h.ctx, s0); err != nil { + t.Fatal(err) + } + + h.reconcile(a, "s3") + + requireReason(t, h.hubSession("s3"), computev1alpha.InstanceConsoleSessionReasonSessionReady) +} + +func TestChecksBeforeClaim(t *testing.T) { + cases := []struct { + name string + mutate func(h *harness) + want string + }{ + {"no shell", func(h *harness) { h.exec.probeExit = 127 }, computev1alpha.InstanceConsoleSessionReasonNoShell}, + {"no writable directory", func(h *harness) { h.exec.probeExit = probeExitNoWritableDir }, computev1alpha.InstanceConsoleSessionReasonNoShell}, + {"missing executable", func(h *harness) { h.exec.probeExit = probeExitCommandMissing }, computev1alpha.InstanceConsoleSessionReasonCommandUnavailable}, + {"replaced instance", func(h *harness) { + patchInstance(h, func(i *computev1alpha.Instance) { + i.Labels[computev1alpha.WorkloadDeploymentUIDLabel] = "newer" + }) + if err := h.cell.Create(h.ctx, &computev1alpha.Instance{ObjectMeta: metav1.ObjectMeta{ + Name: "web-1", + Namespace: testNamespace, + Labels: map[string]string{computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID}, + }}); err != nil { + h.t.Fatal(err) + } + }, computev1alpha.InstanceConsoleSessionReasonInstanceNotFound}, + {"pod not running", func(h *harness) { + patchPod(h, func(p *corev1.Pod) { p.Status.Phase = "Pending" }) + }, computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning}, + {"pod from another instance", func(h *harness) { + patchPod(h, func(p *corev1.Pod) { p.OwnerReferences[0].UID = "other" }) + }, computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning}, + {"other runtime", func(h *harness) { + patchPod(h, func(p *corev1.Pod) { p.Labels[managedByLabel] = "unikraft-provider" }) + }, computev1alpha.InstanceConsoleSessionReasonInvalid}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + a := h.agent() + tc.mutate(h) + h.session(testSession, testUID) + + h.reconcile(a, testSession) + + s := h.hubSession(testSession) + requireReason(t, s, tc.want) + if s.Status.Connection != nil || len(h.slots()) != 0 { + t.Fatal("a refused session was claimed") + } + }) + } +} + +func TestSessionForAnotherCellIsIgnored(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID, func(s *computev1alpha.InstanceConsoleSession) { + s.Labels[computev1alpha.WorkloadDeploymentUIDLabel] = "elsewhere" + }) + + res := h.reconcile(a, testSession) + + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) + if res.RequeueAfter <= 0 { + t.Fatal("expected a retry until the unclaimed timeout") + } + h.clock.advance(a.cfg.UnclaimedTimeout) + h.reconcile(a, testSession) + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonInstanceNotFound) +} + +func TestProbeIsCachedPerPodAndContainer(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.session("s2", "uid-2") + + h.reconcile(a, testSession) + h.reconcile(a, "s2") + + if n := h.exec.ran(probeScript); n != 1 { + t.Fatalf("probe ran %d times, want 1", n) + } +} + +func TestNotConnectedDeadline(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + + h.clock.advance(time.Minute) + h.reconcile(a, testSession) + + s := h.hubSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonNotConnected) + if s.Status.EndedAt == nil || s.Status.ExitCode != nil { + t.Fatalf("status = %+v", s.Status) + } + if n := len(h.slots()); n != 0 { + t.Fatalf("slots = %d, want 0", n) + } + if a.openCount() != 0 { + t.Fatal("ended session still counts as open") + } +} + +func TestExpiryStopsConnectedSession(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Minute)) + stopped := fakeLive(a, testUID, false) + + if res := h.reconcile(a, testSession); res.RequeueAfter != time.Minute { + t.Fatalf("requeue after %v, want the time left", res.RequeueAfter) + } + h.clock.advance(time.Minute) + h.reconcile(a, testSession) + + if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonExpired { + t.Fatalf("stop reason = %q, want Expired", got) + } +} + +func TestLivenessTakeover(t *testing.T) { + h := newHarness(t) + lost, survivor := h.agent(), h.agent() + h.session(testSession, testUID) + h.reconcile(lost, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + + if res := h.reconcile(survivor, testSession); res.RequeueAfter != LeaseDuration { + t.Fatalf("requeue after %v while the owner is alive, want %v", res.RequeueAfter, LeaseDuration) + } + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) + + h.clock.advance(LeaseDuration + time.Second) + if err := survivor.renewLease(h.ctx); err != nil { + t.Fatal(err) + } + h.reconcile(survivor, testSession) + + s := h.hubSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonAgentLost) + if h.exec.ran(killScript) != 1 { + t.Fatal("the lost session's processes were not stopped") + } + if n := len(h.slots()); n != 0 { + t.Fatalf("slots = %d, want 0", n) + } +} + +func TestRestartedAgentEndsItsPreviousSessions(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + + h.reconcile(a, testSession) + + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonAgentLost) + if h.exec.ran(killScript) != 1 { + t.Fatal("the previous run's processes were not stopped") + } +} + +func TestSessionClaimedInAnotherCellIsLeftAlone(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + s := h.hubSession(testSession) + s.Status.Connection = &computev1alpha.InstanceConsoleSessionConnection{EndpointID: "another-cells-endpoint"} + setReady(s, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonSessionReady, "ready") + if err := h.hub.Status().Update(h.ctx, s); err != nil { + t.Fatal(err) + } + + h.reconcile(a, testSession) + + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) +} + +func TestDeletionReleasesFinalizersAfterProcessesStop(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + stopped := fakeLive(a, testUID, false) + + if err := h.hub.Delete(h.ctx, h.hubSession(testSession)); err != nil { + t.Fatal(err) + } + h.reconcile(a, testSession) + if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonRevoked { + t.Fatalf("stop reason = %q, want Revoked", got) + } + + var hub computev1alpha.InstanceConsoleSession + if err := h.hub.Get(h.ctx, reqFor(testSession).NamespacedName, &hub); err == nil { + t.Fatalf("hub copy still exists with finalizers %v", hub.Finalizers) + } + if h.exec.ran(killScript) == 0 || len(h.slots()) != 0 { + t.Fatal("processes were not stopped before the finalizer was released") + } + + var cell computev1alpha.InstanceConsoleSession + _ = h.cell.Get(h.ctx, reqFor(testSession).NamespacedName, &cell) + if err := h.cell.Delete(h.ctx, &cell); err != nil { + t.Fatal(err) + } + h.reconcile(a, testSession) + if err := h.cell.Get(h.ctx, reqFor(testSession).NamespacedName, &cell); err == nil { + t.Fatalf("cell copy still exists with finalizers %v", cell.Finalizers) + } +} + +func TestFinalizerHeldWhileProcessesCannotBeStopped(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + endAs(h, computev1alpha.InstanceConsoleSessionReasonAgentLost) + h.exec.killExit = 1 + + if err := h.hub.Delete(h.ctx, h.hubSession(testSession)); err != nil { + t.Fatal(err) + } + if res := h.reconcile(a, testSession); res.RequeueAfter == 0 { + t.Fatal("expected a retry while processes may be running") + } + if !controllerutil.ContainsFinalizer(h.hubSession(testSession), Finalizer) { + t.Fatal("finalizer released before processes were stopped") + } + + h.exec.killExit = 0 + h.reconcile(a, testSession) + var hub computev1alpha.InstanceConsoleSession + if err := h.hub.Get(h.ctx, reqFor(testSession).NamespacedName, &hub); err == nil { + t.Fatal("hub copy kept after processes stopped") + } +} + +func TestUnclaimedDeletionIsRevoked(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID, func(s *computev1alpha.InstanceConsoleSession) { + s.Finalizers = []string{"test/hold"} + }) + if err := h.hub.Delete(h.ctx, h.hubSession(testSession)); err != nil { + t.Fatal(err) + } + + h.reconcile(a, testSession) + + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonRevoked) +} + +func TestDrain(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.DrainTimeout = 100 * time.Millisecond }) + h.session("waiting", "uid-w") + h.session("tty", "uid-t") + h.session("pipe", "uid-p", func(s *computev1alpha.InstanceConsoleSession) { s.Spec.Terminal = false }) + for _, name := range []string{"waiting", "tty", "pipe"} { + h.reconcile(a, name) + } + markConnected(h, "tty", h.clock.now().Add(time.Hour)) + markConnected(h, "pipe", h.clock.now().Add(time.Hour)) + ttyClient, ttyStopped := liveWithStream(a, "uid-t", true) + pipeClient, pipeStopped := liveWithStream(a, "uid-p", false) + + a.Drain(h.ctx) + + requireReason(t, h.hubSession("waiting"), computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + for _, stopped := range []chan string{ttyStopped, pipeStopped} { + if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonAgentShutdown { + t.Fatalf("stop reason = %q, want AgentShutdown", got) + } + } + if got := <-ttyClient; got == "" { + t.Fatal("terminal session got no maintenance notice") + } + if got := <-pipeClient; got != "" { + t.Fatalf("session without a terminal got injected bytes %q", got) + } + if !a.draining.Load() || a.claiming() { + t.Fatal("a drained agent still claims") + } + h.session("late", "uid-l") + h.reconcile(a, "late") + requireReason(t, h.hubSession("late"), computev1alpha.InstanceConsoleSessionReasonPending) +} + +func TestConcurrencyCapDefersClaims(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.MaxOpenSessions = 1 }) + h.session(testSession, testUID) + h.session("s2", "uid-2") + + h.reconcile(a, testSession) + res := h.reconcile(a, "s2") + + requireReason(t, h.hubSession("s2"), computev1alpha.InstanceConsoleSessionReasonPending) + if res.RequeueAfter == 0 { + t.Fatal("a deferred claim must be retried") + } +} + +func TestSweepStopsOrphanedProcesses(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, "uid-live") + h.reconcile(a, testSession) + orphan := &slot{ + sessionUID: "uid-gone", + session: types.NamespacedName{Namespace: testNamespace, Name: "gone"}, + pod: types.NamespacedName{Namespace: testNamespace, Name: testInstance}, + podUID: "pod-uid", + container: testContainer, + markerDir: "/tmp", + } + if err := h.cell.Create(h.ctx, orphan.leaseFor(testAgentNamespace, "slot-orphan-0")); err != nil { + t.Fatal(err) + } + h.exec.markers = "uid-live\nuid-gone\nuid-older\n" + + if err := a.sweep(h.ctx); err != nil { + t.Fatal(err) + } + + killed := map[string]bool{} + for _, c := range h.exec.commands { + if len(c) > 4 && c[2] == killScript { + killed[c[4]] = true + } + } + for _, uid := range []string{"uid-gone", "uid-older"} { + if !killed[markerPath("/tmp", uid)] { + t.Fatalf("orphan %s was not stopped (killed %v)", uid, killed) + } + } + if killed[markerPath("/tmp", "uid-live")] { + t.Fatal("an open session's process was stopped") + } + slots := h.slots() + if len(slots) != 1 || ptr.Deref(slots[0].Spec.HolderIdentity, "") != "uid-live" { + t.Fatalf("slots = %v, want only the open session's", slots) + } +} + +func TestEnsureIdentityCreatesKey(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.Ordinal = 1 }) + a.identity.Store(nil) + + if err := a.EnsureIdentity(h.ctx); err != nil { + t.Fatal(err) + } + var secret corev1.Secret + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testAgentNamespace, Name: "endpoint-key-1"}, &secret); err != nil { + t.Fatal(err) + } + if len(secret.Data[KeySecretKey]) != 32 || a.EndpointID() != EndpointID(secret.Data[KeySecretKey]) { + t.Fatal("endpoint ID does not match the stored key") + } + first := a.EndpointID() + if err := a.EnsureIdentity(h.ctx); err != nil || a.EndpointID() != first { + t.Fatal("an existing key was replaced") + } +} + +func TestRotationWaitsForOpenSessions(t *testing.T) { + rotationPollInterval = 10 * time.Millisecond + h := newHarness(t) + a := h.agent() + a.identity.Store(nil) + if err := a.EnsureIdentity(h.ctx); err != nil { + t.Fatal(err) + } + before := a.EndpointID() + a.track("uid-open", types.NamespacedName{Name: "open"}) + + done := make(chan error, 1) + go func() { done <- a.rotate(h.ctx) }() + time.Sleep(50 * time.Millisecond) + if a.EndpointID() != before || !a.rotating.Load() || a.claiming() { + t.Fatal("rotation must stop claiming and wait for open sessions") + } + a.forget("uid-open") + if err := <-done; err != nil { + t.Fatal(err) + } + if a.EndpointID() == before || a.rotating.Load() { + t.Fatal("key was not rotated") + } + var lease coordinationv1.Lease + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testAgentNamespace, Name: agentLeaseName(before)}, &lease); err == nil { + t.Fatal("the old endpoint's liveness lease was kept") + } +} + +func reqFor(name string) ctrl.Request { + return ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: name}} +} + +func markConnected(h *harness, name string, expires time.Time) { + h.t.Helper() + s := h.hubSession(name) + started := metav1.NewTime(h.clock.now()) + s.Status.StartedAt = &started + s.Status.ExpiresAt = &metav1.Time{Time: expires} + setReady(s, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonConnected, "connected") + if err := h.hub.Status().Update(h.ctx, s); err != nil { + h.t.Fatal(err) + } +} + +func endAs(h *harness, reason string) { + h.t.Helper() + s := h.hubSession(testSession) + setReady(s, metav1.ConditionFalse, reason, "ended") + if err := h.hub.Status().Update(h.ctx, s); err != nil { + h.t.Fatal(err) + } +} + +// fakeLive registers a connection that reports the reason it was stopped for. +func fakeLive(a *Agent, uid string, tty bool) chan string { + stopped := make(chan string, 1) + var l *liveSession + l, _ = a.register(uid, func() { + go func() { + stopped <- a.stopReason(uid) + a.unregister(uid) + close(l.done) + }() + }, tty) + return stopped +} + +// liveWithStream registers a connection with a client stream, and reports +// what the client read before the stream was stopped. +func liveWithStream(a *Agent, uid string, tty bool) (chan string, chan string) { + clientSide, agentSide := net.Pipe() + read := make(chan string, 1) + go func() { + buf := make([]byte, 4096) + _ = clientSide.SetReadDeadline(time.Now().Add(time.Second)) + n, _ := clientSide.Read(buf) + read <- string(buf[:n]) + }() + stopped := fakeLive(a, uid, tty) + a.mu.Lock() + a.live[uid].stream = &clientStream{conn: agentSide} + a.mu.Unlock() + return read, stopped +} + +func patchInstance(h *harness, mutate func(*computev1alpha.Instance)) { + h.t.Helper() + var inst computev1alpha.Instance + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testNamespace, Name: testInstance}, &inst); err != nil { + h.t.Fatal(err) + } + mutate(&inst) + if err := h.cell.Update(h.ctx, &inst); err != nil { + h.t.Fatal(err) + } +} + +func patchPod(h *harness, mutate func(*corev1.Pod)) { + h.t.Helper() + var pod corev1.Pod + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testNamespace, Name: testInstance}, &pod); err != nil { + h.t.Fatal(err) + } + mutate(&pod) + status := pod.Status + if err := h.cell.Update(h.ctx, &pod); err != nil { + h.t.Fatal(err) + } + pod.Status = status + if err := h.cell.Status().Update(h.ctx, &pod); err != nil { + h.t.Fatal(err) + } +} diff --git a/internal/shellagent/scripts.go b/internal/shellagent/scripts.go new file mode 100644 index 00000000..2c5cc3ea --- /dev/null +++ b/internal/shellagent/scripts.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "strconv" + "strings" +) + +const markerPrefix = ".datum-exec-" + +// The probe exits 4 when the requested executable is missing and 3 when the +// container has no writable directory for the marker file. A container +// without sh cannot run it at all. +const probeScript = `command -v "$1" >/dev/null 2>&1 || exit 4 +for d in /tmp /dev/shm /run /var/tmp; do + if [ -d "$d" ] && [ -w "$d" ]; then echo "$d"; exit 0; fi +done +exit 3` + +const ( + probeExitCommandMissing = 4 + probeExitNoWritableDir = 3 +) + +// The wrapper records the command's process ID before it replaces itself with +// the command, so the agent can stop the process group on any ending. +const wrapperScript = `echo $$ > "$1"; shift; exec "$@"` + +const killScript = `f="$1"; grace="$2" +[ -f "$f" ] || exit 0 +p=$(cat "$f") +case "$p" in ''|*[!0-9]*) rm -f "$f"; exit 0 ;; esac +kill -HUP -"$p" 2>/dev/null; kill -HUP "$p" 2>/dev/null +i=0 +while [ "$i" -lt "$grace" ] && kill -0 "$p" 2>/dev/null; do sleep 1; i=$((i+1)); done +kill -KILL -"$p" 2>/dev/null; kill -KILL "$p" 2>/dev/null +rm -f "$f"` + +const listMarkersScript = `for f in "$1"/` + markerPrefix + `*; do + [ -f "$f" ] && echo "${f##*/` + markerPrefix + `}" +done +true` + +func markerPath(dir, sessionUID string) string { + return strings.TrimSuffix(dir, "/") + "/" + markerPrefix + sessionUID +} + +func probeCommand(executable string) []string { + return []string{"sh", "-c", probeScript, "sh", executable} +} + +func wrappedCommand(dir, sessionUID string, command []string) []string { + return append([]string{"sh", "-c", wrapperScript, "sh", markerPath(dir, sessionUID)}, command...) +} + +func killCommand(dir, sessionUID string, graceSeconds int) []string { + return []string{"sh", "-c", killScript, "sh", markerPath(dir, sessionUID), strconv.Itoa(graceSeconds)} +} + +func listMarkersCommand(dir string) []string { + return []string{"sh", "-c", listMarkersScript, "sh", dir} +} diff --git a/internal/shellagent/server.go b/internal/shellagent/server.go new file mode 100644 index 00000000..3d8ee177 --- /dev/null +++ b/internal/shellagent/server.go @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "net/http" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/consolesession" +) + +// Handler serves the connection protocol's exec endpoint. +func (a *Agent) Handler(ctx context.Context) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET "+consolesession.ExecPath("{uid}"), func(w http.ResponseWriter, r *http.Request) { + a.serveExec(log.IntoContext(r.Context(), log.FromContext(ctx)), w, r) + }) + return mux +} + +func wantsExecStream(r *http.Request) bool { + if !strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + return false + } + for _, value := range r.Header.Values("Sec-WebSocket-Protocol") { + for _, protocol := range strings.Split(value, ",") { + if strings.TrimSpace(protocol) == consolesession.SubProtocol { + return true + } + } + } + return false +} + +// findSession returns the cell copy of the session with a project UID. +func (a *Agent) findSession(ctx context.Context, uid string) (*computev1alpha.InstanceConsoleSession, error) { + var sessions computev1alpha.InstanceConsoleSessionList + if err := a.sessions.List(ctx, &sessions, client.MatchingLabels{SessionUIDLabel: uid}); err != nil { + return nil, err + } + if len(sessions.Items) == 0 { + return nil, nil + } + return &sessions.Items[0], nil +} + +//nolint:gocyclo // each refusal is one branch of the protocol, in its order +func (a *Agent) serveExec(ctx context.Context, w http.ResponseWriter, r *http.Request) { + uid := r.PathValue("uid") + logger := log.FromContext(ctx).WithValues("session", uid) + + if a.draining.Load() { + http.Error(w, "The platform is restarting this cell's shell service. Create a new session.", http.StatusServiceUnavailable) + return + } + if !wantsExecStream(r) { + http.Error(w, "Only "+consolesession.SubProtocol+" WebSocket upgrades are supported.", http.StatusBadRequest) + return + } + cell, err := a.findSession(ctx, uid) + if err != nil { + logger.Error(err, "look up session") + http.Error(w, "The session could not be looked up.", http.StatusInternalServerError) + return + } + if cell == nil { + http.Error(w, "Session not found.", http.StatusNotFound) + return + } + now := a.now() + if err := consolesession.Verify(cell.Spec.ClientPublicKey, uid, r.Header.Get(consolesession.HeaderTimestamp), + r.Header.Get(consolesession.HeaderSignature), now, consolesession.ClockSkew); err != nil { + logger.Info("refused connection", "reason", err.Error()) + http.Error(w, "Not authorized for this session.", http.StatusForbidden) + return + } + + var hub computev1alpha.InstanceConsoleSession + err = a.hub.Get(ctx, client.ObjectKeyFromObject(cell), &hub) + if err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "read session") + http.Error(w, "The session could not be read.", http.StatusInternalServerError) + return + } + if err != nil || sessionUID(&hub) != uid || endpointOf(&hub) != "" && endpointOf(&hub) != a.EndpointID() { + http.Error(w, "Session not found.", http.StatusNotFound) + return + } + if !hub.DeletionTimestamp.IsZero() || !cell.DeletionTimestamp.IsZero() { + http.Error(w, endMessage(computev1alpha.InstanceConsoleSessionReasonRevoked), http.StatusGone) + return + } + if readyReason(&hub) != computev1alpha.InstanceConsoleSessionReasonSessionReady || isTerminal(&hub) { + code, msg := refusal(&hub) + http.Error(w, msg, code) + return + } + if hub.Status.ConnectBefore != nil && !now.Before(hub.Status.ConnectBefore.Time) { + http.Error(w, endMessage(computev1alpha.InstanceConsoleSessionReasonNotConnected), http.StatusGone) + return + } + held, err := a.slotFor(ctx, uid) + if err != nil || held == nil { + http.Error(w, "The session is not ready for a connection.", http.StatusConflict) + return + } + + started := metav1.NewTime(now) + expires := metav1.NewTime(now.Add(sessionTTL(&hub))) + runCtx, cancel := context.WithDeadline(context.Background(), expires.Time) + defer cancel() + live, ok := a.register(uid, cancel, cell.Spec.Terminal) + if !ok { + http.Error(w, "The session already has a connection.", http.StatusConflict) + return + } + defer func() { + a.unregister(uid) + close(live.done) + }() + + hub.Status.StartedAt = &started + hub.Status.ExpiresAt = &expires + setReady(&hub, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonConnected, "A client is connected.") + if err := a.hub.Status().Update(ctx, &hub); err != nil { + if apierrors.IsConflict(err) { + http.Error(w, "The session already has a connection or has ended.", http.StatusConflict) + return + } + logger.Error(err, "record connection") + http.Error(w, "The session could not be started.", http.StatusInternalServerError) + return + } + logger.Info("session connected", "pod", held.pod.String(), "container", held.container) + + result := a.runSession(runCtx, w, r, &hub, held, live) + a.finishSession(uid, client.ObjectKeyFromObject(&hub), held, result) + logger.Info("session ended", "reason", result.reason, "duration", a.now().Sub(started.Time).Round(time.Millisecond)) +} + +// runSession runs the session's command and relays its stream until it ends, +// then sends the client the session's closing status. +func (a *Agent) runSession(ctx context.Context, w http.ResponseWriter, r *http.Request, + hub *computev1alpha.InstanceConsoleSession, held *slot, live *liveSession) outcome { + uid := sessionUID(hub) + opts := ExecOptions{ + Container: held.container, + Command: wrappedCommand(held.markerDir, uid, hub.Spec.Command), + Stdin: hub.Spec.Stdin, + TTY: hub.Spec.Terminal, + } + resp, backend, fromBackend, err := a.exec.Open(ctx, held.pod, opts, r.Header) + if err != nil { + log.FromContext(ctx).Error(err, "open exec stream", "session", uid) + http.Error(w, "The instance could not start the command.", http.StatusBadGateway) + return outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + message: "The instance could not start the command.", + } + } + defer func() { _ = backend.Close() }() + + conn, rw, err := http.NewResponseController(w).Hijack() + if err != nil { + return outcomeClientGone + } + if err := resp.Write(conn); err != nil { + _ = conn.Close() + return outcomeClientGone + } + stream := &clientStream{conn: conn} + a.attach(live, stream) + + result := relay(ctx, stream, rw.Reader, backend, fromBackend) + if reason := a.stopReason(uid); reason != "" { + result = &outcome{reason: reason, message: endMessage(reason)} + } else if result == nil { + reason := computev1alpha.InstanceConsoleSessionReasonExpired + result = &outcome{reason: reason, message: endMessage(reason)} + } + stream.End(closingStatus(result.reason, result.exitCode, result.message)) + return *result +} + +// finishSession stops the session's processes, records its end, and frees its +// slot once the processes are known to be stopped. +func (a *Agent) finishSession(uid string, key client.ObjectKey, held *slot, result outcome) { + ctx, cancel := context.WithTimeout(context.Background(), a.cfg.KillGrace+30*time.Second) + defer cancel() + logger := log.FromContext(ctx).WithValues("session", uid) + + stopped, err := a.stopProcesses(ctx, held) + if err != nil { + logger.Error(err, "stop session processes") + } + message := result.message + if message == "" { + message = "The command exited." + } + me := a.EndpointID() + if _, err := a.end(ctx, key, result.reason, message, result.exitCode, + func(s *computev1alpha.InstanceConsoleSession) bool { return endpointOf(s) == me }); err != nil { + logger.Error(err, "record session end") + } + a.forget(uid) + if stopped { + if err := a.releaseSlot(ctx, held); err != nil { + logger.Error(err, "release session slot") + } + } +} diff --git a/internal/shellagent/server_test.go b/internal/shellagent/server_test.go new file mode 100644 index 00000000..a0ef69c2 --- /dev/null +++ b/internal/shellagent/server_test.go @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "crypto/ed25519" + "crypto/rand" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/consolesession" +) + +// serve starts the agent's exec endpoint on real time, so the connection +// deadline and the signature window follow the wall clock. +func serve(t *testing.T, h *harness, a *Agent) string { + t.Helper() + a.now = time.Now + h.clock.t = time.Now() + srv := httptest.NewServer(a.Handler(h.ctx)) + t.Cleanup(srv.Close) + return strings.TrimPrefix(srv.URL, "http://") +} + +// readySession creates a session and has the agent claim it. +func readySession(t *testing.T, h *harness, a *Agent, mutate ...func(*computev1alpha.InstanceConsoleSession)) ed25519.PrivateKey { + t.Helper() + key := h.session(testSession, testUID, mutate...) + h.reconcile(a, testSession) + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) + return key +} + +func TestRefusals(t *testing.T) { + _, otherKey, _ := ed25519.GenerateKey(rand.Reader) + cases := []struct { + name string + setup func(t *testing.T, h *harness, a *Agent) (uid string, key ed25519.PrivateKey, at time.Time) + want int + }{ + {"unknown session", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + return "no-such-uid", otherKey, time.Now() + }, http.StatusNotFound}, + {"wrong key", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + readySession(t, h, a) + return testUID, otherKey, time.Now() + }, http.StatusForbidden}, + {"stale signature", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + return testUID, key, time.Now().Add(-consolesession.ClockSkew - 5*time.Second) + }, http.StatusForbidden}, + {"no signature", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + readySession(t, h, a) + return testUID, nil, time.Now() + }, http.StatusForbidden}, + {"claimed by another agent", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, h.agent()) + return testUID, key, time.Now() + }, http.StatusNotFound}, + {"not yet ready", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := h.session(testSession, testUID) + return testUID, key, time.Now() + }, http.StatusConflict}, + {"already connected", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + markConnected(h, testSession, time.Now().Add(time.Hour)) + return testUID, key, time.Now() + }, http.StatusConflict}, + {"completed", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + endAs(h, computev1alpha.InstanceConsoleSessionReasonCompleted) + return testUID, key, time.Now() + }, http.StatusConflict}, + {"revoked", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + endAs(h, computev1alpha.InstanceConsoleSessionReasonRevoked) + return testUID, key, time.Now() + }, http.StatusGone}, + {"being deleted", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + if err := h.hub.Delete(h.ctx, h.hubSession(testSession)); err != nil { + t.Fatal(err) + } + return testUID, key, time.Now() + }, http.StatusGone}, + {"expired", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + endAs(h, computev1alpha.InstanceConsoleSessionReasonExpired) + return testUID, key, time.Now() + }, http.StatusGone}, + {"past connectBefore", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + s := h.hubSession(testSession) + s.Status.ConnectBefore = &metav1.Time{Time: time.Now().Add(-time.Second)} + if err := h.hub.Status().Update(h.ctx, s); err != nil { + t.Fatal(err) + } + return testUID, key, time.Now() + }, http.StatusGone}, + {"draining", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + a.draining.Store(true) + return testUID, key, time.Now() + }, http.StatusServiceUnavailable}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + uid, key, at := tc.setup(t, h, a) + + c := dialSession(t, addr, uid, key, at) + + if c.status != tc.want { + t.Fatalf("status = %d (%s), want %d", c.status, c.body, tc.want) + } + }) + } +} + +func TestRefusesOtherSubprotocols(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + ts, sig := consolesession.Sign(key, testUID, time.Now()) + + req, _ := http.NewRequest(http.MethodGet, "http://"+addr+consolesession.ExecPath(testUID), nil) + req.Header.Set("Connection", "Upgrade") + req.Header.Set("Upgrade", "websocket") + req.Header.Set("Sec-WebSocket-Protocol", "v4.channel.k8s.io") + req.Header.Set(consolesession.HeaderTimestamp, ts) + req.Header.Set(consolesession.HeaderSignature, sig) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", resp.StatusCode) + } +} + +// scriptedExit has the fake apiserver echo stdin and exit with code. +func scriptedExit(code int) func(conn net.Conn, r *bufio.Reader) { + return func(conn net.Conn, r *bufio.Reader) { + writeServerFrame(conn, channelStdout, []byte("hello")) + f, err := readFrame(r) + if err != nil { + return + } + writeServerFrame(conn, channelStdout, f.payload[1:]) + writeExitStatus(conn, code) + } +} + +func TestClosingStatus(t *testing.T) { + cases := []struct { + name string + exit int + wantState string + wantCause string + }{ + {"normal exit", 0, metav1.StatusSuccess, ""}, + {"non-zero exit", 3, metav1.StatusFailure, "3"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = scriptedExit(tc.exit) + + c := dialSession(t, addr, testUID, key, time.Now()) + if c.status != http.StatusSwitchingProtocols { + t.Fatalf("status = %d (%s)", c.status, c.body) + } + c.send(t, opBinary, append([]byte{channelStdin}, " world"...)) + out, status, code := c.readUntilClose(t) + + if out[channelStdout] != "hello world" { + t.Fatalf("stdout = %q", out[channelStdout]) + } + if status == nil || status.Status != tc.wantState || code != closeNormal { + t.Fatalf("status = %+v, close code %d", status, code) + } + if tc.wantCause != "" { + if status.Reason != ReasonNonZeroExitCode || status.Details == nil || + status.Details.Causes[0].Type != "ExitCode" || status.Details.Causes[0].Message != tc.wantCause { + t.Fatalf("status = %+v, want NonZeroExitCode with cause %s", status, tc.wantCause) + } + } + s := waitForEnd(t, h) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonCompleted) + if s.Status.ExitCode == nil || int(*s.Status.ExitCode) != tc.exit || s.Status.StartedAt == nil || + s.Status.ExpiresAt == nil { + t.Fatalf("status = %+v", s.Status) + } + if h.exec.ran(killScript) != 1 || len(h.slots()) != 0 { + t.Fatal("process group was not stopped, or the slot was kept") + } + }) + } +} + +func TestPlatformEndings(t *testing.T) { + cases := []struct { + name string + ttl time.Duration + end func(a *Agent) + reason string + }{ + {"revoked", time.Hour, func(a *Agent) { + a.stop(testUID, computev1alpha.InstanceConsoleSessionReasonRevoked) + }, computev1alpha.InstanceConsoleSessionReasonRevoked}, + {"shutdown", time.Hour, func(a *Agent) { + a.stop(testUID, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + }, computev1alpha.InstanceConsoleSessionReasonAgentShutdown}, + {"expired", 500 * time.Millisecond, func(*Agent) {}, computev1alpha.InstanceConsoleSessionReasonExpired}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a, func(s *computev1alpha.InstanceConsoleSession) { + s.Spec.TTL = &metav1.Duration{Duration: tc.ttl} + }) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + writeServerFrame(conn, channelStdout, []byte("$ ")) + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForReason(t, h, testSession, computev1alpha.InstanceConsoleSessionReasonConnected) + go tc.end(a) + out, status, code := c.readUntilClose(t) + + if out[channelStdout] != "$ " { + t.Fatalf("stdout = %q", out[channelStdout]) + } + if status == nil || status.Status != metav1.StatusFailure || string(status.Reason) != tc.reason || + status.Message == "" || code != closeNormal { + t.Fatalf("status = %+v, close code %d", status, code) + } + s := waitForEnd(t, h) + requireReason(t, s, tc.reason) + if s.Status.ExitCode != nil || h.exec.ran(killScript) != 1 { + t.Fatalf("exit code %v, kills %d", s.Status.ExitCode, h.exec.ran(killScript)) + } + }) + } +} + +func TestClientDisconnectStopsCommand(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForReason(t, h, testSession, computev1alpha.InstanceConsoleSessionReasonConnected) + _ = c.conn.Close() + + s := waitForEnd(t, h) + if s.Status.ExitCode != nil || h.exec.ran(killScript) != 1 { + t.Fatal("a dropped connection must stop the command") + } +} + +func TestOversizedFrameEndsSession(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + + c := dialSession(t, addr, testUID, key, time.Now()) + head := []byte{0x80 | opBinary, 0x80 | 127, 0, 0, 0, 0, 0x01, 0, 0, 1, 0, 0, 0, 0} + if _, err := c.conn.Write(head); err != nil { + t.Fatal(err) + } + _, status, _ := c.readUntilClose(t) + + if status == nil || string(status.Reason) != computev1alpha.InstanceConsoleSessionReasonInvalid { + t.Fatalf("status = %+v", status) + } + requireReason(t, waitForEnd(t, h), computev1alpha.InstanceConsoleSessionReasonInvalid) +} + +func TestSecondConnectionIsRefused(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + + first := dialSession(t, addr, testUID, key, time.Now()) + second := dialSession(t, addr, testUID, key, time.Now()) + + if first.status != http.StatusSwitchingProtocols || second.status != http.StatusConflict { + t.Fatalf("statuses = %d, %d; want 101, 409", first.status, second.status) + } +} + +func waitForReason(t *testing.T, h *harness, name, reason string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if readyReason(h.hubSession(name)) == reason { + return + } + time.Sleep(10 * time.Millisecond) + } + requireReason(t, h.hubSession(name), reason) +} + +func waitForEnd(t *testing.T, h *harness) *computev1alpha.InstanceConsoleSession { + name := testSession + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if s := h.hubSession(name); isTerminal(s) && len(h.slots()) == 0 { + return s + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("session %s did not end: %+v", name, h.hubSession(name).Status) + return nil +} diff --git a/internal/shellagent/slots.go b/internal/shellagent/slots.go new file mode 100644 index 00000000..bdeea1b9 --- /dev/null +++ b/internal/shellagent/slots.go @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + componentLabel = "app.kubernetes.io/component" + slotComponent = "shell-session-slot" + annotationPrefix = "shell-agent.compute.datumapis.com/" + annotationSession = annotationPrefix + "session" + annotationPod = annotationPrefix + "pod" + annotationPodUID = annotationPrefix + "pod-uid" + annotationContainer = annotationPrefix + "container" + annotationContainerID = annotationPrefix + "container-id" + annotationMarkerDir = annotationPrefix + "marker-dir" + annotationEndpoint = annotationPrefix + "endpoint" +) + +// A slot is a Lease held by one session of one Instance. Creating a Lease is +// atomic, so agents claiming at once cannot take more slots than exist. A +// slot also records where the session's command runs, and outlives the +// session until that command is known to be stopped: a held slot is the +// cell's record that a process may still need stopping. +type slot struct { + lease *coordinationv1.Lease + sessionUID string + session types.NamespacedName + pod types.NamespacedName + podUID types.UID + container string + containerID string + markerDir string + endpointID string +} + +func slotName(instanceUID types.UID, n int) string { + sum := sha256.Sum256([]byte(instanceUID)) + return fmt.Sprintf("slot-%s-%d", hex.EncodeToString(sum[:8]), n) +} + +func slotFromLease(l *coordinationv1.Lease) *slot { + s := &slot{ + lease: l, + sessionUID: ptr.Deref(l.Spec.HolderIdentity, ""), + podUID: types.UID(l.Annotations[annotationPodUID]), + container: l.Annotations[annotationContainer], + containerID: l.Annotations[annotationContainerID], + markerDir: l.Annotations[annotationMarkerDir], + endpointID: l.Annotations[annotationEndpoint], + } + s.session = splitKey(l.Annotations[annotationSession]) + s.pod = splitKey(l.Annotations[annotationPod]) + return s +} + +func splitKey(key string) types.NamespacedName { + for i := 0; i < len(key); i++ { + if key[i] == '/' { + return types.NamespacedName{Namespace: key[:i], Name: key[i+1:]} + } + } + return types.NamespacedName{Name: key} +} + +func (s *slot) leaseFor(namespace, name string) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + componentLabel: slotComponent, + SessionUIDLabel: s.sessionUID, + }, + Annotations: map[string]string{ + annotationSession: s.session.String(), + annotationPod: s.pod.String(), + annotationPodUID: string(s.podUID), + annotationContainer: s.container, + annotationContainerID: s.containerID, + annotationMarkerDir: s.markerDir, + annotationEndpoint: s.endpointID, + }, + }, + Spec: coordinationv1.LeaseSpec{HolderIdentity: ptr.To(s.sessionUID)}, + } +} + +// errClaimInProgress means another agent in the cell is claiming the same +// session. +var errClaimInProgress = errors.New("another agent is claiming the session") + +// acquireSlot takes one of the Instance's slots for want. It returns nil when +// every slot is held by a session that may still be running. A slot already +// held by the same session is returned as is, so a retried claim is +// idempotent. +func (a *Agent) acquireSlot(ctx context.Context, instanceUID types.UID, want *slot) (*slot, error) { + for n := range a.cfg.SlotsPerInstance { + name := slotName(instanceUID, n) + for range 2 { + lease := want.leaseFor(a.cfg.Namespace, name) + err := a.cell.Create(ctx, lease) + if err == nil { + return slotFromLease(lease), nil + } + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + var held coordinationv1.Lease + err = a.cell.Get(ctx, client.ObjectKey{Namespace: a.cfg.Namespace, Name: name}, &held) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return nil, err + } + holder := slotFromLease(&held) + if holder.sessionUID == want.sessionUID { + if holder.endpointID == want.endpointID { + return holder, nil + } + alive, err := a.agentAlive(ctx, holder.endpointID) + if err != nil { + return nil, err + } + if alive { + return nil, errClaimInProgress + } + if err := a.releaseSlot(ctx, holder); err != nil { + return nil, err + } + continue + } + if a.isLive(holder.sessionUID) { + break + } + state, err := a.sessionState(ctx, holder.sessionUID) + if err != nil { + return nil, err + } + if state != sessionEnded { + break + } + if reaped, err := a.reap(ctx, holder); err != nil || !reaped { + break + } + } + } + return nil, nil +} + +// slotFor returns the slot a session holds in this cell, if any. +func (a *Agent) slotFor(ctx context.Context, sessionUID string) (*slot, error) { + var leases coordinationv1.LeaseList + if err := a.cell.List(ctx, &leases, client.InNamespace(a.cfg.Namespace), client.MatchingLabels{ + componentLabel: slotComponent, + SessionUIDLabel: sessionUID, + }); err != nil { + return nil, err + } + for i := range leases.Items { + if ptr.Deref(leases.Items[i].Spec.HolderIdentity, "") == sessionUID { + return slotFromLease(&leases.Items[i]), nil + } + } + return nil, nil +} + +// releaseSlot frees a slot whose session never started a process or whose +// process is known to be stopped. +func (a *Agent) releaseSlot(ctx context.Context, s *slot) error { + if s == nil { + return nil + } + var opts []client.DeleteOption + if s.lease.UID != "" { + opts = append(opts, client.Preconditions{UID: &s.lease.UID}) + } + err := a.cell.Delete(ctx, s.lease, opts...) + if apierrors.IsNotFound(err) || apierrors.IsConflict(err) { + return nil + } + return err +} + +// reap stops the process a slot records, then frees the slot. It reports +// false, keeping the slot, when the process could not be confirmed stopped. +func (a *Agent) reap(ctx context.Context, s *slot) (bool, error) { + stopped, err := a.stopProcesses(ctx, s) + if err != nil || !stopped { + return false, err + } + return true, a.releaseSlot(ctx, s) +} + +// stopProcesses stops a session's process group in the pod its slot records. +// A pod or container that stopped, or was replaced, took the process with it. +func (a *Agent) stopProcesses(ctx context.Context, s *slot) (bool, error) { + if s.markerDir == "" || s.pod.Name == "" { + return true, nil + } + var pod corev1.Pod + err := a.cell.Get(ctx, s.pod, &pod) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + if pod.UID != s.podUID { + return true, nil + } + if status := containerStatus(&pod, s.container); status == nil || status.State.Running == nil || + s.containerID != "" && status.ContainerID != s.containerID { + return true, nil + } + grace := int(a.cfg.KillGrace.Seconds()) + out, code, err := run(ctx, a.exec, s.pod, s.container, killCommand(s.markerDir, s.sessionUID, grace)) + if err != nil || code != 0 { + log.FromContext(ctx).Info("could not stop session processes", "session", s.sessionUID, + "pod", s.pod.String(), "exitCode", code, "output", out, "error", err) + return false, nil + } + return true, nil +} diff --git a/internal/shellagent/status.go b/internal/shellagent/status.go new file mode 100644 index 00000000..80136b00 --- /dev/null +++ b/internal/shellagent/status.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "encoding/json" + "fmt" + "strconv" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +// ReasonNonZeroExitCode is the reason Kubernetes reports on the exec status +// channel for a command that exited with a code other than zero. +const ReasonNonZeroExitCode = "NonZeroExitCode" + +const causeTypeExitCode = "ExitCode" + +var endMessages = map[string]string{ + computev1alpha.InstanceConsoleSessionReasonExpired: "The session reached its time limit and the command was stopped.", + computev1alpha.InstanceConsoleSessionReasonRevoked: "The session was deleted and the command was stopped.", + computev1alpha.InstanceConsoleSessionReasonNotConnected: "No client connected before the connection deadline.", + computev1alpha.InstanceConsoleSessionReasonAgentShutdown: "The platform stopped the session for maintenance. Start a new session to continue.", + computev1alpha.InstanceConsoleSessionReasonAgentLost: "The platform lost the session unexpectedly. Start a new session to continue.", + computev1alpha.InstanceConsoleSessionReasonTooManySessions: "The instance already has as many open sessions as it allows.", + computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning: "The instance is not running.", + computev1alpha.InstanceConsoleSessionReasonInstanceNotFound: "The instance no longer exists or has been replaced.", +} + +func endMessage(reason string) string { + if msg, ok := endMessages[reason]; ok { + return msg + } + return "The session ended." +} + +// terminalReason reports whether reason ends a session for good. +func terminalReason(reason string) bool { + switch reason { + case computev1alpha.InstanceConsoleSessionReasonCompleted, + computev1alpha.InstanceConsoleSessionReasonExpired, + computev1alpha.InstanceConsoleSessionReasonRevoked, + computev1alpha.InstanceConsoleSessionReasonNotConnected, + computev1alpha.InstanceConsoleSessionReasonAgentShutdown, + computev1alpha.InstanceConsoleSessionReasonAgentLost, + computev1alpha.InstanceConsoleSessionReasonTooManySessions, + computev1alpha.InstanceConsoleSessionReasonNoShell, + computev1alpha.InstanceConsoleSessionReasonCommandUnavailable, + computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + computev1alpha.InstanceConsoleSessionReasonInstanceNotFound, + computev1alpha.InstanceConsoleSessionReasonInvalid: + return true + } + return false +} + +func readyCondition(s *computev1alpha.InstanceConsoleSession) *metav1.Condition { + return meta.FindStatusCondition(s.Status.Conditions, computev1alpha.InstanceConsoleSessionReady) +} + +func readyReason(s *computev1alpha.InstanceConsoleSession) string { + if c := readyCondition(s); c != nil { + return c.Reason + } + return computev1alpha.InstanceConsoleSessionReasonPending +} + +func isTerminal(s *computev1alpha.InstanceConsoleSession) bool { + c := readyCondition(s) + return c != nil && c.Status == metav1.ConditionFalse && terminalReason(c.Reason) +} + +func setReady(s *computev1alpha.InstanceConsoleSession, status metav1.ConditionStatus, reason, message string) { + meta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{ + Type: computev1alpha.InstanceConsoleSessionReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: s.Generation, + }) +} + +// closingStatus is the one status message the agent sends on the status +// channel before it closes a session's stream. A command that exited reports +// what Kubernetes would; any other ending names its terminal reason. +func closingStatus(reason string, exitCode *int32, message string) metav1.Status { + status := metav1.Status{ + TypeMeta: metav1.TypeMeta{Kind: "Status", APIVersion: "v1"}, + Status: metav1.StatusFailure, + } + switch { + case reason == computev1alpha.InstanceConsoleSessionReasonCompleted && exitCode != nil && *exitCode == 0: + status.Status = metav1.StatusSuccess + case reason == computev1alpha.InstanceConsoleSessionReasonCompleted && exitCode != nil: + status.Reason = ReasonNonZeroExitCode + status.Message = fmt.Sprintf("command terminated with non-zero exit code: %d", *exitCode) + status.Details = &metav1.StatusDetails{Causes: []metav1.StatusCause{{ + Type: causeTypeExitCode, + Message: strconv.Itoa(int(*exitCode)), + }}} + default: + status.Reason = metav1.StatusReason(reason) + status.Message = message + } + return status +} + +// exitCodeFromStatus reads the command's exit code from the status message the +// apiserver sends when the command exits. ok is false when the status reports +// an error other than the command's exit. +func exitCodeFromStatus(raw []byte) (code int32, message string, ok bool) { + var status metav1.Status + if err := json.Unmarshal(raw, &status); err != nil { + return 0, "the platform could not read the command's result", false + } + if status.Status == metav1.StatusSuccess { + return 0, "", true + } + if status.Reason == ReasonNonZeroExitCode && status.Details != nil { + for _, cause := range status.Details.Causes { + if cause.Type != causeTypeExitCode { + continue + } + n, err := strconv.ParseInt(cause.Message, 10, 32) + if err == nil { + return int32(n), "", true + } + } + } + return 0, status.Message, false +} + +// refusal maps a session that cannot take a connection to the protocol's +// refusal code. +func refusal(s *computev1alpha.InstanceConsoleSession) (int, string) { + switch reason := readyReason(s); { + case reason == computev1alpha.InstanceConsoleSessionReasonRevoked, + reason == computev1alpha.InstanceConsoleSessionReasonExpired, + reason == computev1alpha.InstanceConsoleSessionReasonNotConnected: + return 410, endMessage(reason) + case isTerminal(s): + return 409, "the session has already ended" + case reason == computev1alpha.InstanceConsoleSessionReasonConnected: + return 409, "the session already has a connection" + default: + return 409, "the session is not ready for a connection" + } +} diff --git a/internal/shellagent/stream.go b/internal/shellagent/stream.go new file mode 100644 index 00000000..34cb639d --- /dev/null +++ b/internal/shellagent/stream.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "net" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +var errStreamEnded = errors.New("stream ended") + +// clientStream owns writes to a client's exec WebSocket. Frames from the +// apiserver are relayed whole, so the agent can write its own messages between +// them without splitting a fragmented message. +type clientStream struct { + conn net.Conn + + mu sync.Mutex + inFragment bool + ended bool + endOnce sync.Once +} + +func (c *clientStream) relayData(raw []byte, fin bool) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.ended { + return errStreamEnded + } + c.inFragment = !fin + _, err := c.conn.Write(raw) + return err +} + +func (c *clientStream) relayControl(raw []byte) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.ended { + return errStreamEnded + } + _, err := c.conn.Write(raw) + return err +} + +// writeBetweenMessages writes frames once no relayed message is half sent, +// waiting at most two seconds for one to finish. +func (c *clientStream) writeBetweenMessages(frames ...[]byte) error { + deadline := time.Now().Add(2 * time.Second) + for { + c.mu.Lock() + if c.ended { + c.mu.Unlock() + return errStreamEnded + } + if !c.inFragment || time.Now().After(deadline) { + var err error + for _, f := range frames { + if _, err = c.conn.Write(f); err != nil { + break + } + } + c.mu.Unlock() + return err + } + c.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } +} + +// Notify writes a notice to the session's standard output. Only sessions with +// a terminal get notices. +func (c *clientStream) Notify(text string) error { + return c.writeBetweenMessages(encodeFrame(opBinary, append([]byte{channelStdout}, text...), false)) +} + +// End sends the session's one status message and closes the stream with +// code 1000. +func (c *clientStream) End(status metav1.Status) { + c.endOnce.Do(func() { + body, _ := json.Marshal(status) + _ = c.writeBetweenMessages( + encodeFrame(opBinary, append([]byte{channelStatus}, body...), false), + encodeFrame(opClose, closePayload(closeNormal), false), + ) + c.mu.Lock() + c.ended = true + c.mu.Unlock() + _ = c.conn.Close() + }) +} + +// outcome is how a connected session ended. +type outcome struct { + reason string + exitCode *int32 + message string +} + +var ( + outcomeClientGone = outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonCompleted, + message: "The client disconnected and the command was stopped.", + } + outcomeStreamLost = outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + message: "The instance stopped the command's stream.", + } + outcomeFrameTooLarge = outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInvalid, + message: "The client sent a message larger than 16 MiB.", + } +) + +// relay copies frames both ways until the command exits, either side goes +// away, or ctx ends. It returns nil when ctx ended first. +func relay(ctx context.Context, c *clientStream, fromClient *bufio.Reader, backend net.Conn, fromBackend *bufio.Reader) *outcome { + results := make(chan outcome, 2) + go func() { results <- c.pump(fromBackend) }() + go func() { results <- forward(fromClient, backend) }() + select { + case o := <-results: + return &o + case <-ctx.Done(): + return nil + } +} + +// pump relays the apiserver's frames to the client, holding back the status +// message so the agent sends the stream's only one. +func (c *clientStream) pump(r *bufio.Reader) outcome { + var status []byte + intercept := false + for { + f, err := readFrame(r) + if err != nil || f.opcode == opClose { + return outcomeStreamLost + } + if f.control() { + if err := c.relayControl(f.raw); err != nil { + return outcomeClientGone + } + continue + } + if f.opcode != opContinuation { + intercept = len(f.payload) > 0 && f.payload[0] == channelStatus + if intercept { + status = append(status[:0], f.payload[1:]...) + } + } else if intercept { + status = append(status, f.payload...) + } + if !intercept { + if err := c.relayData(f.raw, f.fin); err != nil { + return outcomeClientGone + } + continue + } + if !f.fin { + continue + } + code, message, ok := exitCodeFromStatus(status) + if !ok { + return outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, + message: "The instance could not run the command: " + message, + } + } + return outcome{reason: computev1alpha.InstanceConsoleSessionReasonCompleted, exitCode: &code} + } +} + +// forward relays the client's frames to the apiserver unchanged. +func forward(r *bufio.Reader, backend net.Conn) outcome { + for { + f, err := readFrame(r) + var tooLarge errFrameTooLarge + switch { + case errors.As(err, &tooLarge): + return outcomeFrameTooLarge + case err != nil, f.opcode == opClose: + return outcomeClientGone + } + if _, err := backend.Write(f.raw); err != nil { + return outcomeStreamLost + } + } +} diff --git a/internal/shellagent/sweep.go b/internal/shellagent/sweep.go new file mode 100644 index 00000000..9a62d82c --- /dev/null +++ b/internal/shellagent/sweep.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "context" + "strings" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + computev1alpha "go.datum.net/compute/api/v1alpha" +) + +type sessionStateKind int + +const ( + sessionActive sessionStateKind = iota + sessionEnded +) + +// sessionState reports whether a session, by project UID, is still open. A +// session whose cell or hub copy is gone has ended. +func (a *Agent) sessionState(ctx context.Context, uid string) (sessionStateKind, error) { + cell, err := a.findSession(ctx, uid) + if err != nil { + return sessionActive, err + } + if cell == nil { + return sessionEnded, nil + } + var hub computev1alpha.InstanceConsoleSession + err = a.hub.Get(ctx, client.ObjectKeyFromObject(cell), &hub) + if apierrors.IsNotFound(err) { + return sessionEnded, nil + } + if err != nil { + return sessionActive, err + } + if sessionUID(&hub) != uid || isTerminal(&hub) { + return sessionEnded, nil + } + return sessionActive, nil +} + +// RunSweeper sweeps for orphaned processes at start and at every sweep +// interval until ctx ends. +func (a *Agent) RunSweeper(ctx context.Context) error { + ticker := time.NewTicker(a.cfg.SweepInterval) + defer ticker.Stop() + for { + if err := a.sweep(ctx); err != nil && ctx.Err() == nil { + log.FromContext(ctx).Error(err, "sweep for orphaned session processes") + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +type markerPlace struct { + pod types.NamespacedName + podUID types.UID + container string + dir string +} + +// sweep stops processes left behind by sessions that no longer exist or have +// ended, and frees their slots. Slots name the containers sessions ran in, and +// each of those containers is searched for marker files of other sessions. +func (a *Agent) sweep(ctx context.Context) error { + var leases coordinationv1.LeaseList + if err := a.cell.List(ctx, &leases, client.InNamespace(a.cfg.Namespace), + client.MatchingLabels{componentLabel: slotComponent}); err != nil { + return err + } + places := map[markerPlace]struct{}{} + for i := range leases.Items { + s := slotFromLease(&leases.Items[i]) + if s.markerDir != "" && s.pod.Name != "" { + places[markerPlace{pod: s.pod, podUID: s.podUID, container: s.container, dir: s.markerDir}] = struct{}{} + } + if a.isLive(s.sessionUID) { + continue + } + state, err := a.sessionState(ctx, s.sessionUID) + if err != nil || state != sessionEnded { + continue + } + if _, err := a.reap(ctx, s); err != nil { + log.FromContext(ctx).Error(err, "reap orphaned session", "session", s.sessionUID) + } + } + for place := range places { + a.sweepContainer(ctx, place) + } + return nil +} + +func (a *Agent) sweepContainer(ctx context.Context, place markerPlace) { + logger := log.FromContext(ctx).WithValues("pod", place.pod.String(), "container", place.container) + var pod corev1.Pod + if err := a.cell.Get(ctx, place.pod, &pod); err != nil || pod.UID != place.podUID { + return + } + if status := containerStatus(&pod, place.container); status == nil || status.State.Running == nil { + return + } + out, code, err := run(ctx, a.exec, place.pod, place.container, listMarkersCommand(place.dir)) + if err != nil || code != 0 { + logger.Info("could not list session markers", "exitCode", code, "error", err) + return + } + for _, uid := range strings.Fields(out) { + if a.isLive(uid) { + continue + } + state, err := a.sessionState(ctx, uid) + if err != nil || state != sessionEnded { + continue + } + grace := int(a.cfg.KillGrace.Seconds()) + if _, code, err := run(ctx, a.exec, place.pod, place.container, killCommand(place.dir, uid, grace)); err != nil || code != 0 { + logger.Info("could not stop orphaned session processes", "session", uid, "exitCode", code, "error", err) + continue + } + logger.Info("stopped orphaned session processes", "session", uid) + } +} From f0041be1a250f901e983d37b62804aa1d1c7a27f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 18:09:00 -0500 Subject: [PATCH 02/20] feat: Package the shell agent for compute cells A cell opts into shell sessions by deploying the agent beside Datum Connect tunnel endpoints, so the agent is reachable only through its paired endpoint and the endpoint holds no cluster credential. Key changes: - Two two-replica StatefulSets in compute-shell-system, paired by ordinal through a headless Service - Least-privilege RBAC for the agent; the endpoint mounts no service account token and reads its key read-only - Portable NetworkPolicies, and Cilium policies in their own component that also police traffic to the node's apiserver - An opt-in overlay that keeps the namespace intact, since the cell overlay renames every namespace it renders The datum-connect image tag is a placeholder until datum-cloud/app#192 is released. --- .../shell-agent-cilium/kustomization.yaml | 9 ++ .../components/shell-agent-cilium/policy.yaml | 93 +++++++++++++ config/components/shell-agent/agent.yaml | 120 +++++++++++++++++ config/components/shell-agent/endpoint.yaml | 105 +++++++++++++++ .../components/shell-agent/kustomization.yaml | 30 +++++ config/components/shell-agent/namespace.yaml | 6 + .../components/shell-agent/networkpolicy.yaml | 82 ++++++++++++ config/components/shell-agent/rbac.yaml | 123 ++++++++++++++++++ config/components/shell-agent/service.yaml | 18 +++ .../cell-shell-agent/kustomization.yaml | 10 ++ 10 files changed, 596 insertions(+) create mode 100644 config/components/shell-agent-cilium/kustomization.yaml create mode 100644 config/components/shell-agent-cilium/policy.yaml create mode 100644 config/components/shell-agent/agent.yaml create mode 100644 config/components/shell-agent/endpoint.yaml create mode 100644 config/components/shell-agent/kustomization.yaml create mode 100644 config/components/shell-agent/namespace.yaml create mode 100644 config/components/shell-agent/networkpolicy.yaml create mode 100644 config/components/shell-agent/rbac.yaml create mode 100644 config/components/shell-agent/service.yaml create mode 100644 config/overlays/cell-shell-agent/kustomization.yaml diff --git a/config/components/shell-agent-cilium/kustomization.yaml b/config/components/shell-agent-cilium/kustomization.yaml new file mode 100644 index 00000000..34d6641d --- /dev/null +++ b/config/components/shell-agent-cilium/kustomization.yaml @@ -0,0 +1,9 @@ +# Cilium policies for the shell agent and its tunnel endpoints. Include beside +# config/components/shell-agent on cells that run Cilium. Unlike the portable +# policies they police traffic to the node's own apiserver, kubelet and etcd, +# so the endpoint, which holds no credential, cannot reach them at all. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - policy.yaml diff --git a/config/components/shell-agent-cilium/policy.yaml b/config/components/shell-agent-cilium/policy.yaml new file mode 100644 index 00000000..7aae2d74 --- /dev/null +++ b/config/components/shell-agent-cilium/policy.yaml @@ -0,0 +1,93 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: exec-agent + namespace: compute-shell-system +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: exec-agent + enableDefaultDeny: + ingress: true + egress: true + ingress: + - fromEndpoints: + - matchLabels: + app.kubernetes.io/name: exec-endpoint + toPorts: + - ports: + - port: "7777" + protocol: TCP + - fromEntities: + - host + toPorts: + - ports: + - port: "8081" + protocol: TCP + egress: + - toEntities: + - kube-apiserver + toPorts: + - ports: + - port: "443" + protocol: TCP + - port: "6443" + protocol: TCP + # The Karmada hub is outside the cell. Narrow this to the hub's name with + # toFQDNs where the deployment knows it. + - toEntities: + - world + toPorts: + - ports: + - port: "443" + protocol: TCP + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: exec-endpoint + namespace: compute-shell-system +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + enableDefaultDeny: + ingress: true + egress: true + egress: + - toEndpoints: + - matchLabels: + app.kubernetes.io/name: exec-agent + toPorts: + - ports: + - port: "7777" + protocol: TCP + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP + # Relays only: world excludes the cluster's own nodes and apiserver. + - toEntities: + - world + toPorts: + - ports: + - port: "443" + protocol: TCP + - port: "7842" + protocol: UDP diff --git a/config/components/shell-agent/agent.yaml b/config/components/shell-agent/agent.yaml new file mode 100644 index 00000000..b84fc466 --- /dev/null +++ b/config/components/shell-agent/agent.yaml @@ -0,0 +1,120 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: exec-agent + namespace: compute-shell-system + labels: + app.kubernetes.io/name: exec-agent + app.kubernetes.io/part-of: compute +spec: + replicas: 2 + serviceName: exec-agent + podManagementPolicy: Parallel + selector: + matchLabels: + app.kubernetes.io/name: exec-agent + template: + metadata: + labels: + app.kubernetes.io/name: exec-agent + app.kubernetes.io/part-of: compute + spec: + serviceAccountName: exec-agent + # The drain window, then time to stop the processes of sessions still + # open at its end. + terminationGracePeriodSeconds: 75 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: exec-agent + containers: + - name: agent + # The manager's image, which carries this binary too. + image: ghcr.io/datum-cloud/compute:latest + imagePullPolicy: IfNotPresent + command: + - /shell-agent + args: + - --listen=:7777 + - --health-probe-bind-address=:8081 + - --target=$(POD_NAME).exec-agent.$(POD_NAMESPACE).svc.cluster.local:7777 + - --federation-kubeconfig=/etc/kubernetes/upstream/auth/kubeconfig + - --managed-by=kata-provider + - --endpoint-pod-prefix=exec-endpoint + - --drain-timeout=30s + - --feature-gates=$(FEATURE_GATES) + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: FEATURE_GATES + value: InstanceConsoleSessions=true + envFrom: + - configMapRef: + name: exec-relays + ports: + - name: exec + containerPort: 7777 + protocol: TCP + - name: probes + containerPort: 8081 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: probes + initialDelaySeconds: 10 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: probes + initialDelaySeconds: 5 + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 256Mi + volumeMounts: + - name: federation + mountPath: /etc/kubernetes/upstream/auth/ + readOnly: true + volumes: + - name: federation + projected: + sources: + - secret: + name: exec-agent-federation-client-cert + - configMap: + name: exec-agent-federation-kubeconfig diff --git a/config/components/shell-agent/endpoint.yaml b/config/components/shell-agent/endpoint.yaml new file mode 100644 index 00000000..33efac7e --- /dev/null +++ b/config/components/shell-agent/endpoint.yaml @@ -0,0 +1,105 @@ +# Datum Connect tunnel endpoints. Ordinal n serves the identity key in +# endpoint-key-, which its paired agent creates and rotates, and proxies to +# that agent only. Both keys are projected and the ordinal picks one, because a +# StatefulSet cannot name a different Secret per pod. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: exec-endpoint + namespace: compute-shell-system + labels: + app.kubernetes.io/name: exec-endpoint + app.kubernetes.io/part-of: compute +spec: + replicas: 2 + serviceName: exec-endpoint + podManagementPolicy: Parallel + selector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + template: + metadata: + labels: + app.kubernetes.io/name: exec-endpoint + app.kubernetes.io/part-of: compute + spec: + serviceAccountName: exec-endpoint + automountServiceAccountToken: false + enableServiceLinks: false + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + containers: + - name: endpoint + image: ghcr.io/datum-cloud/datum-connect + imagePullPolicy: IfNotPresent + args: + - serve + - --listen-key-file=/var/run/datum-connect/keys/$(POD_INDEX) + - --tcp-proxy=exec-agent-$(POD_INDEX).exec-agent.$(POD_NAMESPACE).svc.cluster.local:7777 + env: + - name: POD_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: RUST_LOG + value: info + envFrom: + - configMapRef: + name: exec-relays + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "1" + memory: 256Mi + volumeMounts: + - name: keys + mountPath: /var/run/datum-connect/keys + readOnly: true + - name: state + mountPath: /var/lib/datum-connect + volumes: + - name: keys + projected: + defaultMode: 0440 + sources: + - secret: + name: endpoint-key-0 + optional: true + items: + - key: key + path: "0" + - secret: + name: endpoint-key-1 + optional: true + items: + - key: key + path: "1" + - name: state + emptyDir: {} diff --git a/config/components/shell-agent/kustomization.yaml b/config/components/shell-agent/kustomization.yaml new file mode 100644 index 00000000..f9a918b8 --- /dev/null +++ b/config/components/shell-agent/kustomization.yaml @@ -0,0 +1,30 @@ +# Serves instance shell sessions in a cell: two shell agents, each paired by +# ordinal with a Datum Connect tunnel endpoint that is the agent's only way in. +# +# Everything lives in compute-shell-system, and the session target clients dial +# names that namespace, so include this component from a kustomization that does +# not rename namespaces (see config/overlays/cell-shell-agent). The deployment +# supplies three things this component only references: +# +# - ConfigMap exec-relays: DATUM_CONNECT_RELAY_URLS, the relays the endpoints +# register with and the agents publish, nearest first. +# - ConfigMap exec-agent-federation-kubeconfig (key kubeconfig) and Secret +# exec-agent-federation-client-cert: the Karmada hub credential, mounted +# together at /etc/kubernetes/upstream/auth/. On the hub it needs get, list, +# watch and patch on instanceconsolesessions and update on their status. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - namespace.yaml + - rbac.yaml + - service.yaml + - agent.yaml + - endpoint.yaml + - networkpolicy.yaml + +# datum-connect's CLI image is first published by datum-cloud/app#192. Until +# that release exists this tag is a placeholder; pin the released tag here. +images: + - name: ghcr.io/datum-cloud/datum-connect + newTag: v0.0.0-unreleased diff --git a/config/components/shell-agent/namespace.yaml b/config/components/shell-agent/namespace.yaml new file mode 100644 index 00000000..fe40cdb7 --- /dev/null +++ b/config/components/shell-agent/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: compute-shell-system + labels: + pod-security.kubernetes.io/enforce: restricted diff --git a/config/components/shell-agent/networkpolicy.yaml b/config/components/shell-agent/networkpolicy.yaml new file mode 100644 index 00000000..e71bc223 --- /dev/null +++ b/config/components/shell-agent/networkpolicy.yaml @@ -0,0 +1,82 @@ +# Portable policies. Most CNIs do not police a pod's traffic to its own node, +# so they cannot keep the endpoint off the node's apiserver; the Cilium +# policies in config/components/shell-agent-cilium do. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: exec-agent + namespace: compute-shell-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: exec-agent + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + ports: + - protocol: TCP + port: 7777 + egress: + # The cell apiserver and the Karmada hub. + - ports: + - protocol: TCP + port: 443 + - protocol: TCP + port: 6443 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: exec-endpoint + namespace: compute-shell-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: exec-endpoint + policyTypes: + - Ingress + - Egress + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: exec-agent + ports: + - protocol: TCP + port: 7777 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Relays, and QUIC address discovery. Traffic always goes through a relay; + # direct paths would need UDP egress to any address. + - ports: + - protocol: TCP + port: 443 + - protocol: UDP + port: 7842 diff --git a/config/components/shell-agent/rbac.yaml b/config/components/shell-agent/rbac.yaml new file mode 100644 index 00000000..45533da7 --- /dev/null +++ b/config/components/shell-agent/rbac.yaml @@ -0,0 +1,123 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exec-agent + namespace: compute-shell-system +--- +# The tunnel endpoint needs no Kubernetes API access at all. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: exec-endpoint + namespace: compute-shell-system +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: compute-shell-agent +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - get + - apiGroups: + - compute.datumapis.com + resources: + - instances + verbs: + - get + - list + - watch + # Session copies are read here and only their finalizer is written; status + # goes to the hub copy. + - apiGroups: + - compute.datumapis.com + resources: + - instanceconsolesessions + verbs: + - get + - list + - watch + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: compute-shell-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: compute-shell-agent +subjects: + - kind: ServiceAccount + name: exec-agent + namespace: compute-shell-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: exec-agent + namespace: compute-shell-system +rules: + # Liveness, per-instance slots and the key rotation lock. + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - create + - update + - delete + # Create cannot be limited by name; reading and rotating can. + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - endpoint-key-0 + - endpoint-key-1 + verbs: + - get + - update + # Restarting its paired endpoint after a key rotation. + - apiGroups: + - "" + resources: + - pods + resourceNames: + - exec-endpoint-0 + - exec-endpoint-1 + verbs: + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: exec-agent + namespace: compute-shell-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: exec-agent +subjects: + - kind: ServiceAccount + name: exec-agent + namespace: compute-shell-system diff --git a/config/components/shell-agent/service.yaml b/config/components/shell-agent/service.yaml new file mode 100644 index 00000000..6ee1e833 --- /dev/null +++ b/config/components/shell-agent/service.yaml @@ -0,0 +1,18 @@ +# Gives each agent the stable name its paired endpoint proxies to, +# exec-agent-.exec-agent.compute-shell-system.svc.cluster.local, which is +# also the target clients name. A draining agent keeps its name so clients get +# its refusal rather than a lookup failure. +apiVersion: v1 +kind: Service +metadata: + name: exec-agent + namespace: compute-shell-system +spec: + clusterIP: None + publishNotReadyAddresses: true + selector: + app.kubernetes.io/name: exec-agent + ports: + - name: exec + port: 7777 + targetPort: exec diff --git a/config/overlays/cell-shell-agent/kustomization.yaml b/config/overlays/cell-shell-agent/kustomization.yaml new file mode 100644 index 00000000..4a7274a5 --- /dev/null +++ b/config/overlays/cell-shell-agent/kustomization.yaml @@ -0,0 +1,10 @@ +# Instance shell sessions for a cell, opt-in and separate from the cell +# overlay: that overlay moves everything into compute-system, and the shell +# service must stay in compute-shell-system, whose name is part of every +# session's connection target. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../components/shell-agent + - ../../components/shell-agent-cilium From e7864d504e46dde3696bc3dd5ac4872d49f6111f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 18:09:00 -0500 Subject: [PATCH 03/20] chore: Ship the shell agent in the compute image The agent shares the session API and the connection protocol with the manager, so building it into the same image and stamping the same tag into its component keeps the two from skewing, and reuses the existing release plumbing instead of adding an image. --- .github/workflows/publish.yaml | 6 ++++-- Dockerfile | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 280d816d..ea667b8f 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -56,8 +56,9 @@ jobs: # can only stamp one image across N paths, which can't express "three # different images, four different paths" at all. # - # compute-mcp is stamped with the manager's image and the manager's tag: one - # image carries both binaries, so the two paths cannot name different builds. + # compute-mcp and the shell agent are stamped with the manager's image and + # the manager's tag: one image carries all three binaries, so the paths + # cannot name different builds. publish-kustomize-bundles: needs: [ @@ -78,4 +79,5 @@ jobs: - { path: config/components/consumer-ui-plugin, name: ghcr.io/datum-cloud/compute-consumer-ui-plugin, tag: "${{ needs.publish-consumer-ui-plugin-image.outputs.tag }}" } - { path: config/components/provider-ui-plugin, name: ghcr.io/datum-cloud/compute-provider-ui-plugin, tag: "${{ needs.publish-provider-ui-plugin-image.outputs.tag }}" } - { path: config/components/compute-mcp, name: ghcr.io/datum-cloud/compute, tag: "${{ needs.publish-container-image.outputs.tag }}" } + - { path: config/components/shell-agent, name: ghcr.io/datum-cloud/compute, tag: "${{ needs.publish-container-image.outputs.tag }}" } secrets: inherit diff --git a/Dockerfile b/Dockerfile index ea349bff..5ffd405e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,7 @@ RUN go mod download # package whose embed.FS carries the knowledge and runbooks compute-mcp serves. COPY cmd/main.go cmd/main.go COPY cmd/compute-mcp/ cmd/compute-mcp/ +COPY cmd/shell-agent/ cmd/shell-agent/ COPY api/ api/ COPY internal/ internal/ COPY pkg/ pkg/ @@ -61,6 +62,16 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \ -ldflags "-s -w" \ -o compute-mcp ./cmd/compute-mcp +# shell-agent runs in each cell beside its tunnel endpoint. It shares the +# session API and the connection protocol package with the manager, so it +# ships in the same image and cannot skew from the controllers that deliver +# its sessions. +RUN --mount=type=cache,target=/go/pkg/mod/ \ + --mount=type=cache,target="/root/.cache/go-build" \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ + -ldflags "-s -w" \ + -o shell-agent ./cmd/shell-agent + # Use distroless as minimal base image to package the binaries # Refer to https://github.com/GoogleContainerTools/distroless for more details # @@ -74,6 +85,7 @@ FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/manager . COPY --from=builder /workspace/compute-mcp . +COPY --from=builder /workspace/shell-agent . USER 65532:65532 # The manager keeps the entrypoint it has always had, so nothing that runs this From b4fc139626047630e889bf1f9a4f79ad9d5d1969 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 18:38:04 -0500 Subject: [PATCH 04/20] fix: Retry lost claim races without an error Two agents claiming at once is expected: the loser's finalizer or status write conflicts. Treat that as a retry rather than a reconcile error, and avoid the deprecated immediate requeue. --- internal/shellagent/reconciler.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/internal/shellagent/reconciler.go b/internal/shellagent/reconciler.go index ab825f21..0c00d86f 100644 --- a/internal/shellagent/reconciler.go +++ b/internal/shellagent/reconciler.go @@ -150,11 +150,19 @@ func (a *Agent) claim(ctx context.Context, cell, hub *computev1alpha.InstanceCon "The instance already has %d open sessions, the most it allows.", a.cfg.SlotsPerInstance)) } - if err := addFinalizer(ctx, a.hub, hub); err != nil { - return ctrl.Result{}, errors.Join(err, a.releaseSlot(ctx, held)) - } - if err := addFinalizer(ctx, a.cell, cell); err != nil { - return ctrl.Result{}, errors.Join(err, a.releaseSlot(ctx, held)) + for _, f := range []struct { + c client.Client + s *computev1alpha.InstanceConsoleSession + }{{a.hub, hub}, {a.cell, cell}} { + if err := addFinalizer(ctx, f.c, f.s); err != nil { + if relErr := a.releaseSlot(ctx, held); relErr != nil { + return ctrl.Result{}, relErr + } + if apierrors.IsConflict(err) { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return ctrl.Result{}, err + } } now := a.now() @@ -172,7 +180,7 @@ func (a *Agent) claim(ctx context.Context, cell, hub *computev1alpha.InstanceCon return ctrl.Result{}, relErr } if apierrors.IsConflict(err) { - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: time.Second}, nil } return ctrl.Result{}, err } From 4232c11bff1cb8284b593282ee9a8b90487f0881 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 19:23:30 -0500 Subject: [PATCH 05/20] fix: Leave unclaimed sessions to the control plane The control plane now ends sessions no cell claims within 30 seconds as Unavailable, deciding on the hub copy with optimistic concurrency. The agent's own two-minute InstanceNotFound fallback would race it, so cells without the session's instance now simply leave the session alone. A client that vanishes behind the tunnel is only noticed when the QUIC connection idles out, 25 to 30 seconds later, and it ended as Completed. The agent now pings clients and ends silent or departed clients as Disconnected, stopping the command's process group. Key changes: - Use the API's session labels and finalizers instead of local copies - Treat Unavailable and Disconnected as terminal, so an ended hub copy is never claimed; claims still lose to any concurrent write on the hub copy - Ping the client every 10s; a client silent for 20s is disconnected (--ping-interval, --pong-timeout); pongs to the agent's pings are not relayed to the apiserver - Test that gorilla/websocket and client-go's WebSocket executor answer the pings and read TTY and non-TTY output through the agent --- cmd/shell-agent/main.go | 8 + go.mod | 2 + go.sum | 6 + internal/shellagent/agent.go | 32 ++-- internal/shellagent/checks.go | 2 +- internal/shellagent/harness_test.go | 14 +- internal/shellagent/reconciler.go | 23 ++- internal/shellagent/reconciler_test.go | 66 ++++++- internal/shellagent/server.go | 4 +- internal/shellagent/server_test.go | 12 +- internal/shellagent/slots.go | 10 +- internal/shellagent/status.go | 5 +- internal/shellagent/stream.go | 46 ++++- internal/shellagent/stream_test.go | 235 +++++++++++++++++++++++++ 14 files changed, 403 insertions(+), 62 deletions(-) create mode 100644 internal/shellagent/stream_test.go diff --git a/cmd/shell-agent/main.go b/cmd/shell-agent/main.go index 07365674..5b1564c9 100644 --- a/cmd/shell-agent/main.go +++ b/cmd/shell-agent/main.go @@ -51,6 +51,8 @@ type options struct { featureGates string drainTimeout time.Duration killGrace time.Duration + pingInterval time.Duration + pongTimeout time.Duration } func main() { @@ -76,6 +78,10 @@ func main() { "How long connected sessions may continue after the agent is asked to stop.") flag.DurationVar(&opts.killGrace, "kill-grace", defaults.KillGrace, "Time between SIGHUP and SIGKILL when a session's processes are stopped.") + flag.DurationVar(&opts.pingInterval, "ping-interval", defaults.PingInterval, + "How often the agent pings a connected client.") + flag.DurationVar(&opts.pongTimeout, "pong-timeout", defaults.PongTimeout, + "How long a connected client may send nothing, not even a pong, before its session ends as Disconnected.") zapOpts := zap.Options{} zapOpts.BindFlags(flag.CommandLine) flag.Parse() @@ -118,6 +124,8 @@ func run(opts options, cfg shellagent.Config) error { cfg.ManagedBy = splitList(opts.managedBy) cfg.DrainTimeout = opts.drainTimeout cfg.KillGrace = opts.killGrace + cfg.PingInterval = opts.pingInterval + cfg.PongTimeout = opts.pongTimeout scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) diff --git a/go.mod b/go.mod index 7a20b8bd..e69d4e52 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( require ( github.com/distribution/reference v0.6.0 + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/modelcontextprotocol/go-sdk v1.7.0 ) @@ -70,6 +71,7 @@ require ( github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/signal v0.7.1 // indirect github.com/morikuni/aec v1.1.0 // indirect diff --git a/go.sum b/go.sum index 1214b6b6..f8a77325 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9 github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -180,6 +182,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -222,6 +226,8 @@ github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/policy-helpers v0.0.0-20260612073044-d5411a945cfc h1:dvhPFj1niuMP3CBCjhiZWJQr//+w1LOA8cHclFJnNe0= github.com/moby/policy-helpers v0.0.0-20260612073044-d5411a945cfc/go.mod h1:frGYJTxenVCGPa9doaqZSU9FqzT7bt+1dFeVAaCFoyQ= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= diff --git a/internal/shellagent/agent.go b/internal/shellagent/agent.go index d2bafa9a..ffafe20e 100644 --- a/internal/shellagent/agent.go +++ b/internal/shellagent/agent.go @@ -23,22 +23,7 @@ import ( computev1alpha "go.datum.net/compute/api/v1alpha" ) -const ( - // Finalizer is set on a session's hub and cell copies when an agent claims - // it, and removed once the session's processes are stopped. A hub copy - // that has disappeared therefore confirms the cell cleaned up. - Finalizer = "compute.datumapis.com/shell-agent" - - // SessionUIDLabel carries the project session's UID on its hub and cell - // copies. Clients connect with this UID and sign it. - SessionUIDLabel = computev1alpha.LabelNamespace + "/session-uid" - - // InstanceNameLabel carries the name of the Instance a session copy runs - // in. The cell Instance has this name in the copy's namespace. - InstanceNameLabel = computev1alpha.LabelNamespace + "/instance-name" - - managedByLabel = "managed-by" -) +const managedByLabel = "managed-by" // Config is an agent's configuration. type Config struct { @@ -64,8 +49,13 @@ type Config struct { DrainTimeout time.Duration KillGrace time.Duration KeyRotationPeriod time.Duration - UnclaimedTimeout time.Duration SweepInterval time.Duration + // PingInterval is how often the agent pings a connected client. + PingInterval time.Duration + // PongTimeout is how long a connected client may send nothing, not even a + // pong, before the agent ends the session as Disconnected. It must be + // longer than PingInterval. + PongTimeout time.Duration } // DefaultConfig returns the contract's limits. @@ -78,8 +68,9 @@ func DefaultConfig() Config { DrainTimeout: 30 * time.Second, KillGrace: 3 * time.Second, KeyRotationPeriod: 30 * 24 * time.Hour, - UnclaimedTimeout: 2 * time.Minute, SweepInterval: time.Minute, + PingInterval: 10 * time.Second, + PongTimeout: 20 * time.Second, } } @@ -112,6 +103,9 @@ func New(cfg Config, sessions client.Reader, cell, hub client.Client, exec Execu if cfg.Namespace == "" || cfg.Target == "" || len(cfg.RelayURLs) == 0 || len(cfg.ManagedBy) == 0 { return nil, errors.New("namespace, target, relay URLs and managed-by values are required") } + if cfg.PingInterval <= 0 || cfg.PongTimeout <= cfg.PingInterval { + return nil, errors.New("the ping interval must be positive and shorter than the pong timeout") + } return &Agent{ cfg: cfg, sessions: sessions, @@ -245,7 +239,7 @@ func (a *Agent) openCount() int { } func sessionUID(s *computev1alpha.InstanceConsoleSession) string { - return s.Labels[SessionUIDLabel] + return s.Labels[computev1alpha.InstanceConsoleSessionUIDLabel] } func endpointOf(s *computev1alpha.InstanceConsoleSession) string { diff --git a/internal/shellagent/checks.go b/internal/shellagent/checks.go index aa8d0eb4..a11a1e14 100644 --- a/internal/shellagent/checks.go +++ b/internal/shellagent/checks.go @@ -70,7 +70,7 @@ func (a *Agent) check(ctx context.Context, s *computev1alpha.InstanceConsoleSess // Instance from another workload deployment is a replacement, not the // Instance the session was created for. func (a *Agent) findInstance(ctx context.Context, s *computev1alpha.InstanceConsoleSession) (*computev1alpha.Instance, *rejection, error) { - name := s.Labels[InstanceNameLabel] + name := s.Labels[computev1alpha.InstanceConsoleSessionInstanceNameLabel] deploymentUID := s.Labels[computev1alpha.WorkloadDeploymentUIDLabel] if name == "" || deploymentUID == "" || len(s.Spec.Command) == 0 { return nil, reject(computev1alpha.InstanceConsoleSessionReasonInvalid, diff --git a/internal/shellagent/harness_test.go b/internal/shellagent/harness_test.go index b51f2417..914ede14 100644 --- a/internal/shellagent/harness_test.go +++ b/internal/shellagent/harness_test.go @@ -7,6 +7,8 @@ import ( "context" "crypto/ed25519" "crypto/rand" + "crypto/sha1" //nolint:gosec // RFC 6455 derives the accept key with SHA-1 + "encoding/base64" "encoding/json" "net" "net/http" @@ -161,9 +163,9 @@ func (h *harness) session(name, uid string, mutate ...func(*computev1alpha.Insta Namespace: testNamespace, CreationTimestamp: metav1.NewTime(h.clock.now()), Labels: map[string]string{ - SessionUIDLabel: uid, - InstanceNameLabel: testInstance, - computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID, + computev1alpha.InstanceConsoleSessionUIDLabel: uid, + computev1alpha.InstanceConsoleSessionInstanceNameLabel: testInstance, + computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID, }, }, Spec: computev1alpha.InstanceConsoleSessionSpec{ @@ -265,6 +267,7 @@ func (f *fakeAPIServer) Open(_ context.Context, _ types.NamespacedName, opts Exe "Upgrade": []string{"websocket"}, "Connection": []string{"Upgrade"}, "Sec-Websocket-Protocol": []string{consolesession.SubProtocol}, + "Sec-Websocket-Accept": []string{acceptKey(handshake.Get("Sec-WebSocket-Key"))}, }, } serverReader := bufio.NewReader(serverSide) @@ -279,6 +282,11 @@ func (f *fakeAPIServer) Open(_ context.Context, _ types.NamespacedName, opts Exe return resp, agentSide, bufio.NewReader(agentSide), nil } +func acceptKey(key string) string { + sum := sha1.Sum([]byte(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11")) + return base64.StdEncoding.EncodeToString(sum[:]) +} + func (f *fakeAPIServer) answer(conn net.Conn, command []string) { f.mu.Lock() probeExit, probeDir, markers, killExit := f.probeExit, f.probeDir, f.markers, f.killExit diff --git a/internal/shellagent/reconciler.go b/internal/shellagent/reconciler.go index 0c00d86f..bd98861b 100644 --- a/internal/shellagent/reconciler.go +++ b/internal/shellagent/reconciler.go @@ -106,20 +106,19 @@ func (a *Agent) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, e } // claim checks a session and, if it can run, takes a slot and publishes where -// to connect. Agents race to claim with optimistic concurrency. +// to connect. Agents and the control plane, which ends sessions no cell takes +// in time, race on the hub copy with optimistic concurrency, so a claim never +// lands on a session that has already ended. A cell that does not run the +// session's instance leaves it to the cell that does, or to the control plane. func (a *Agent) claim(ctx context.Context, cell, hub *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { if !a.claiming() || a.openCount() >= a.cfg.MaxOpenSessions { return ctrl.Result{RequeueAfter: claimRetryInterval}, nil } target, rej, err := a.check(ctx, cell) if errors.Is(err, errNotInCell) { - age := a.now().Sub(hub.CreationTimestamp.Time) - if age < a.cfg.UnclaimedTimeout { - return ctrl.Result{RequeueAfter: a.cfg.UnclaimedTimeout - age}, nil - } - rej = reject(computev1alpha.InstanceConsoleSessionReasonInstanceNotFound, - "The instance no longer exists or has been replaced.") - } else if err != nil { + return ctrl.Result{RequeueAfter: claimRetryInterval}, nil + } + if err != nil { return ctrl.Result{}, err } if rej != nil { @@ -383,11 +382,11 @@ func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, messa } func addFinalizer(ctx context.Context, c client.Client, s *computev1alpha.InstanceConsoleSession) error { - if controllerutil.ContainsFinalizer(s, Finalizer) { + if controllerutil.ContainsFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { return nil } base := s.DeepCopy() - controllerutil.AddFinalizer(s, Finalizer) + controllerutil.AddFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) return c.Patch(ctx, s, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) } @@ -397,11 +396,11 @@ func removeFinalizer(ctx context.Context, c client.Client, key types.NamespacedN if err := c.Get(ctx, key, &s); err != nil { return err } - if !controllerutil.ContainsFinalizer(&s, Finalizer) { + if !controllerutil.ContainsFinalizer(&s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { return nil } base := s.DeepCopy() - controllerutil.RemoveFinalizer(&s, Finalizer) + controllerutil.RemoveFinalizer(&s, computev1alpha.InstanceConsoleSessionAgentFinalizer) return c.Patch(ctx, &s, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) }) return client.IgnoreNotFound(err) diff --git a/internal/shellagent/reconciler_test.go b/internal/shellagent/reconciler_test.go index 2d715644..debbc3b8 100644 --- a/internal/shellagent/reconciler_test.go +++ b/internal/shellagent/reconciler_test.go @@ -40,12 +40,12 @@ func TestClaimPublishesConnection(t *testing.T) { if res.RequeueAfter != time.Minute { t.Fatalf("requeue after %v, want the connect timeout", res.RequeueAfter) } - if !controllerutil.ContainsFinalizer(s, Finalizer) { + if !controllerutil.ContainsFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { t.Fatal("hub copy has no finalizer") } var cell computev1alpha.InstanceConsoleSession _ = h.cell.Get(h.ctx, client.ObjectKeyFromObject(s), &cell) - if !controllerutil.ContainsFinalizer(&cell, Finalizer) { + if !controllerutil.ContainsFinalizer(&cell, computev1alpha.InstanceConsoleSessionAgentFinalizer) { t.Fatal("cell copy has no finalizer") } if n := len(h.slots()); n != 1 { @@ -214,15 +214,65 @@ func TestSessionForAnotherCellIsIgnored(t *testing.T) { s.Labels[computev1alpha.WorkloadDeploymentUIDLabel] = "elsewhere" }) - res := h.reconcile(a, testSession) + for range 3 { + if res := h.reconcile(a, testSession); res.RequeueAfter <= 0 { + t.Fatal("expected a retry while the session is unclaimed") + } + h.clock.advance(10 * time.Minute) + } requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) - if res.RequeueAfter <= 0 { - t.Fatal("expected a retry until the unclaimed timeout") + if len(h.slots()) != 0 { + t.Fatal("a cell without the instance took a slot") + } +} + +// endUnavailable ends a session the way the control plane does when no cell +// claims it in time. +func endUnavailable(t *testing.T, h *harness) { + t.Helper() + s := h.hubSession(testSession) + setReady(s, metav1.ConditionFalse, computev1alpha.InstanceConsoleSessionReasonUnavailable, "No cell took the session.") + if err := h.hub.Status().Update(h.ctx, s); err != nil { + t.Fatal(err) + } +} + +func requireUnclaimed(t *testing.T, h *harness) { + t.Helper() + s := h.hubSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonUnavailable) + if s.Status.Connection != nil || len(h.slots()) != 0 || + controllerutil.ContainsFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { + t.Fatalf("an ended session was claimed: %+v", s) } - h.clock.advance(a.cfg.UnclaimedTimeout) +} + +func TestEndedSessionIsNotClaimed(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + endUnavailable(t, h) + h.reconcile(a, testSession) - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonInstanceNotFound) + + requireUnclaimed(t, h) +} + +func TestClaimLosesToControlPlaneEnding(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + stale := h.hubSession(testSession) + var cell computev1alpha.InstanceConsoleSession + _ = h.cell.Get(h.ctx, client.ObjectKeyFromObject(stale), &cell) + endUnavailable(t, h) + + if _, err := a.claim(h.ctx, &cell, stale); err != nil { + t.Logf("losing claim returned %v", err) + } + + requireUnclaimed(t, h) } func TestProbeIsCachedPerPodAndContainer(t *testing.T) { @@ -389,7 +439,7 @@ func TestFinalizerHeldWhileProcessesCannotBeStopped(t *testing.T) { if res := h.reconcile(a, testSession); res.RequeueAfter == 0 { t.Fatal("expected a retry while processes may be running") } - if !controllerutil.ContainsFinalizer(h.hubSession(testSession), Finalizer) { + if !controllerutil.ContainsFinalizer(h.hubSession(testSession), computev1alpha.InstanceConsoleSessionAgentFinalizer) { t.Fatal("finalizer released before processes were stopped") } diff --git a/internal/shellagent/server.go b/internal/shellagent/server.go index 3d8ee177..3885bc61 100644 --- a/internal/shellagent/server.go +++ b/internal/shellagent/server.go @@ -43,7 +43,7 @@ func wantsExecStream(r *http.Request) bool { // findSession returns the cell copy of the session with a project UID. func (a *Agent) findSession(ctx context.Context, uid string) (*computev1alpha.InstanceConsoleSession, error) { var sessions computev1alpha.InstanceConsoleSessionList - if err := a.sessions.List(ctx, &sessions, client.MatchingLabels{SessionUIDLabel: uid}); err != nil { + if err := a.sessions.List(ctx, &sessions, client.MatchingLabels{computev1alpha.InstanceConsoleSessionUIDLabel: uid}); err != nil { return nil, err } if len(sessions.Items) == 0 { @@ -179,7 +179,7 @@ func (a *Agent) runSession(ctx context.Context, w http.ResponseWriter, r *http.R stream := &clientStream{conn: conn} a.attach(live, stream) - result := relay(ctx, stream, rw.Reader, backend, fromBackend) + result := relay(ctx, stream, rw.Reader, backend, fromBackend, a.cfg.PingInterval, a.cfg.PongTimeout) if reason := a.stopReason(uid); reason != "" { result = &outcome{reason: reason, message: endMessage(reason)} } else if result == nil { diff --git a/internal/shellagent/server_test.go b/internal/shellagent/server_test.go index a0ef69c2..9e59e214 100644 --- a/internal/shellagent/server_test.go +++ b/internal/shellagent/server_test.go @@ -246,7 +246,7 @@ func TestPlatformEndings(t *testing.T) { } c := dialSession(t, addr, testUID, key, time.Now()) - waitForReason(t, h, testSession, computev1alpha.InstanceConsoleSessionReasonConnected) + waitForConnected(t, h) go tc.end(a) out, status, code := c.readUntilClose(t) @@ -280,10 +280,11 @@ func TestClientDisconnectStopsCommand(t *testing.T) { } c := dialSession(t, addr, testUID, key, time.Now()) - waitForReason(t, h, testSession, computev1alpha.InstanceConsoleSessionReasonConnected) + waitForConnected(t, h) _ = c.conn.Close() s := waitForEnd(t, h) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonDisconnected) if s.Status.ExitCode != nil || h.exec.ran(killScript) != 1 { t.Fatal("a dropped connection must stop the command") } @@ -336,16 +337,17 @@ func TestSecondConnectionIsRefused(t *testing.T) { } } -func waitForReason(t *testing.T, h *harness, name, reason string) { +func waitForConnected(t *testing.T, h *harness) { t.Helper() + reason := computev1alpha.InstanceConsoleSessionReasonConnected deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { - if readyReason(h.hubSession(name)) == reason { + if readyReason(h.hubSession(testSession)) == reason { return } time.Sleep(10 * time.Millisecond) } - requireReason(t, h.hubSession(name), reason) + requireReason(t, h.hubSession(testSession), reason) } func waitForEnd(t *testing.T, h *harness) *computev1alpha.InstanceConsoleSession { diff --git a/internal/shellagent/slots.go b/internal/shellagent/slots.go index bdeea1b9..d6b2f390 100644 --- a/internal/shellagent/slots.go +++ b/internal/shellagent/slots.go @@ -17,6 +17,8 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + + computev1alpha "go.datum.net/compute/api/v1alpha" ) const ( @@ -84,8 +86,8 @@ func (s *slot) leaseFor(namespace, name string) *coordinationv1.Lease { Name: name, Namespace: namespace, Labels: map[string]string{ - componentLabel: slotComponent, - SessionUIDLabel: s.sessionUID, + componentLabel: slotComponent, + computev1alpha.InstanceConsoleSessionUIDLabel: s.sessionUID, }, Annotations: map[string]string{ annotationSession: s.session.String(), @@ -168,8 +170,8 @@ func (a *Agent) acquireSlot(ctx context.Context, instanceUID types.UID, want *sl func (a *Agent) slotFor(ctx context.Context, sessionUID string) (*slot, error) { var leases coordinationv1.LeaseList if err := a.cell.List(ctx, &leases, client.InNamespace(a.cfg.Namespace), client.MatchingLabels{ - componentLabel: slotComponent, - SessionUIDLabel: sessionUID, + componentLabel: slotComponent, + computev1alpha.InstanceConsoleSessionUIDLabel: sessionUID, }); err != nil { return nil, err } diff --git a/internal/shellagent/status.go b/internal/shellagent/status.go index 80136b00..291e88bd 100644 --- a/internal/shellagent/status.go +++ b/internal/shellagent/status.go @@ -28,6 +28,7 @@ var endMessages = map[string]string{ computev1alpha.InstanceConsoleSessionReasonTooManySessions: "The instance already has as many open sessions as it allows.", computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning: "The instance is not running.", computev1alpha.InstanceConsoleSessionReasonInstanceNotFound: "The instance no longer exists or has been replaced.", + computev1alpha.InstanceConsoleSessionReasonDisconnected: "The client disconnected and the command was stopped.", } func endMessage(reason string) string { @@ -51,7 +52,9 @@ func terminalReason(reason string) bool { computev1alpha.InstanceConsoleSessionReasonCommandUnavailable, computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, computev1alpha.InstanceConsoleSessionReasonInstanceNotFound, - computev1alpha.InstanceConsoleSessionReasonInvalid: + computev1alpha.InstanceConsoleSessionReasonInvalid, + computev1alpha.InstanceConsoleSessionReasonUnavailable, + computev1alpha.InstanceConsoleSessionReasonDisconnected: return true } return false diff --git a/internal/shellagent/stream.go b/internal/shellagent/stream.go index 34cb639d..6f7db99e 100644 --- a/internal/shellagent/stream.go +++ b/internal/shellagent/stream.go @@ -4,6 +4,7 @@ package shellagent import ( "bufio" + "bytes" "context" "encoding/json" "errors" @@ -107,8 +108,8 @@ type outcome struct { var ( outcomeClientGone = outcome{ - reason: computev1alpha.InstanceConsoleSessionReasonCompleted, - message: "The client disconnected and the command was stopped.", + reason: computev1alpha.InstanceConsoleSessionReasonDisconnected, + message: endMessage(computev1alpha.InstanceConsoleSessionReasonDisconnected), } outcomeStreamLost = outcome{ reason: computev1alpha.InstanceConsoleSessionReasonInstanceNotRunning, @@ -120,12 +121,22 @@ var ( } ) +// pingPayload marks the agent's own pings, so the client's pongs to them are +// not relayed to the apiserver. +var pingPayload = []byte("shell-agent") + // relay copies frames both ways until the command exits, either side goes -// away, or ctx ends. It returns nil when ctx ended first. -func relay(ctx context.Context, c *clientStream, fromClient *bufio.Reader, backend net.Conn, fromBackend *bufio.Reader) *outcome { +// away, or ctx ends. It returns nil when ctx ended first. The client is pinged +// every pingInterval and counts as gone once it has sent nothing, not even a +// pong, for pongTimeout. +func relay(ctx context.Context, c *clientStream, fromClient *bufio.Reader, backend net.Conn, fromBackend *bufio.Reader, + pingInterval, pongTimeout time.Duration) *outcome { results := make(chan outcome, 2) + done := make(chan struct{}) + defer close(done) go func() { results <- c.pump(fromBackend) }() - go func() { results <- forward(fromClient, backend) }() + go func() { results <- c.forward(fromClient, backend, pongTimeout) }() + go c.ping(pingInterval, done) select { case o := <-results: return &o @@ -134,6 +145,21 @@ func relay(ctx context.Context, c *clientStream, fromClient *bufio.Reader, backe } } +func (c *clientStream) ping(interval time.Duration, done <-chan struct{}) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + if err := c.relayControl(encodeFrame(opPing, pingPayload, false)); err != nil { + return + } + } + } +} + // pump relays the apiserver's frames to the client, holding back the status // message so the agent sends the stream's only one. func (c *clientStream) pump(r *bufio.Reader) outcome { @@ -178,9 +204,13 @@ func (c *clientStream) pump(r *bufio.Reader) outcome { } } -// forward relays the client's frames to the apiserver unchanged. -func forward(r *bufio.Reader, backend net.Conn) outcome { +// forward relays the client's frames to the apiserver unchanged, except pongs +// to the agent's own pings. +func (c *clientStream) forward(r *bufio.Reader, backend net.Conn, idle time.Duration) outcome { for { + if err := c.conn.SetReadDeadline(time.Now().Add(idle)); err != nil { + return outcomeClientGone + } f, err := readFrame(r) var tooLarge errFrameTooLarge switch { @@ -188,6 +218,8 @@ func forward(r *bufio.Reader, backend net.Conn) outcome { return outcomeFrameTooLarge case err != nil, f.opcode == opClose: return outcomeClientGone + case f.opcode == opPong && bytes.Equal(f.payload, pingPayload): + continue } if _, err := backend.Write(f.raw); err != nil { return outcomeStreamLost diff --git a/internal/shellagent/stream_test.go b/internal/shellagent/stream_test.go new file mode 100644 index 00000000..881e9cd3 --- /dev/null +++ b/internal/shellagent/stream_test.go @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "bufio" + "bytes" + "context" + "crypto/ed25519" + "io" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + computev1alpha "go.datum.net/compute/api/v1alpha" + "go.datum.net/compute/internal/consolesession" +) + +func fastPings(c *Config) { + c.PingInterval = 50 * time.Millisecond + c.PongTimeout = 300 * time.Millisecond +} + +// backendRecorder is a fake command that runs until the stream closes and +// records every frame the agent relays to it. +type backendRecorder struct { + mu sync.Mutex + frames []*frame +} + +func (b *backendRecorder) run(conn net.Conn, r *bufio.Reader) { + writeServerFrame(conn, channelStdout, []byte("$ ")) + for { + f, err := readFrame(r) + if err != nil { + return + } + b.mu.Lock() + b.frames = append(b.frames, f) + b.mu.Unlock() + } +} + +func (b *backendRecorder) pongs() int { + b.mu.Lock() + defer b.mu.Unlock() + n := 0 + for _, f := range b.frames { + if f.opcode == opPong { + n++ + } + } + return n +} + +func TestSilentClientIsDisconnected(t *testing.T) { + h := newHarness(t) + a := h.agent(fastPings) + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = (&backendRecorder{}).run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + pinged := false + _ = c.conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + for !pinged { + f, err := readFrame(c.reader) + if err != nil { + t.Fatalf("read frame: %v", err) + } + pinged = f.opcode == opPing + } + + s := waitForEnd(t, h) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonDisconnected) + if s.Status.ExitCode != nil || h.exec.ran(killScript) != 1 { + t.Fatal("a client that stopped answering pings must have its command stopped") + } +} + +func TestClientDisconnectEndsAsDisconnected(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = (&backendRecorder{}).run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + c.send(t, opClose, closePayload(closeNormal)) + + requireReason(t, waitForEnd(t, h), computev1alpha.InstanceConsoleSessionReasonDisconnected) +} + +func dialGorilla(t *testing.T, addr, uid string, key ed25519.PrivateKey) *websocket.Conn { + t.Helper() + ts, sig := consolesession.Sign(key, uid, time.Now()) + header := http.Header{} + header.Set(consolesession.HeaderTimestamp, ts) + header.Set(consolesession.HeaderSignature, sig) + dialer := websocket.Dialer{Subprotocols: []string{consolesession.SubProtocol}} + conn, resp, err := dialer.Dial("ws://"+addr+consolesession.ExecPath(uid), header) + if err != nil { + t.Fatalf("dial: %v (%v)", err, resp) + } + t.Cleanup(func() { _ = conn.Close() }) + return conn +} + +// A gorilla/websocket client answers pings from inside its read loop with the +// library's default handler, as datumctl and the browser client do. +func TestAnsweringClientStaysConnected(t *testing.T) { + h := newHarness(t) + a := h.agent(fastPings) + addr := serve(t, h, a) + key := readySession(t, h, a) + backend := &backendRecorder{} + h.exec.onSession = backend.run + + conn := dialGorilla(t, addr, testUID, key) + go func() { + for { + if _, _, err := conn.ReadMessage(); err != nil { + return + } + } + }() + waitForConnected(t, h) + time.Sleep(4 * a.cfg.PongTimeout) + + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) + if n := backend.pongs(); n != 0 { + t.Fatalf("relayed %d pongs to the agent's own pings to the apiserver", n) + } +} + +// ttyCommand writes a prompt, echoes the first stdin message, and exits 0. +func ttyCommand(conn net.Conn, r *bufio.Reader) { + writeServerFrame(conn, channelStdout, []byte("prompt$ ")) + for { + f, err := readFrame(r) + if err != nil { + return + } + if f.opcode == opBinary && len(f.payload) > 1 && f.payload[0] == channelStdin { + writeServerFrame(conn, channelStdout, f.payload[1:]) + writeExitStatus(conn, 0) + return + } + } +} + +type signingTransport struct { + key ed25519.PrivateKey + uid string + rt http.RoundTripper +} + +func (s *signingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + ts, sig := consolesession.Sign(s.key, s.uid, time.Now()) + req = req.Clone(req.Context()) + req.Header.Set(consolesession.HeaderTimestamp, ts) + req.Header.Set(consolesession.HeaderSignature, sig) + return s.rt.RoundTrip(req) +} + +// client-go's WebSocket executor, as kubectl uses it, reads a terminal +// session's output, and answers the agent's pings. +func TestClientGoExecutor(t *testing.T) { + for _, tty := range []bool{true, false} { + t.Run(map[bool]string{true: "tty", false: "no tty"}[tty], func(t *testing.T) { + h := newHarness(t) + a := h.agent(fastPings) + addr := serve(t, h, a) + key := readySession(t, h, a, func(s *computev1alpha.InstanceConsoleSession) { + s.Spec.Terminal = tty + }) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + time.Sleep(4 * a.cfg.PongTimeout) + ttyCommand(conn, r) + } + + cfg := &rest.Config{ + Host: "http://" + addr, + WrapTransport: func(rt http.RoundTripper) http.RoundTripper { + return &signingTransport{key: key, uid: testUID, rt: rt} + }, + } + exec, err := remotecommand.NewWebSocketExecutor(cfg, http.MethodGet, + "http://"+addr+consolesession.ExecPath(testUID)) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var stdout, stderr bytes.Buffer + opts := remotecommand.StreamOptions{ + Stdin: io.MultiReader(strings.NewReader("hi\n"), blockUntil(ctx)), + Stdout: &stdout, + Tty: tty, + } + if !tty { + opts.Stderr = &stderr + } + if err := exec.StreamWithContext(ctx, opts); err != nil { + t.Fatalf("stream: %v", err) + } + if got := stdout.String(); got != "prompt$ hi\n" { + t.Fatalf("stdout = %q", got) + } + s := waitForEnd(t, h) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonCompleted) + if s.Status.ExitCode == nil || *s.Status.ExitCode != 0 { + t.Fatalf("exit code = %v", s.Status.ExitCode) + } + }) + } +} + +type blockingReader struct{ ctx context.Context } + +func (b blockingReader) Read([]byte) (int, error) { + <-b.ctx.Done() + return 0, io.EOF +} + +func blockUntil(ctx context.Context) io.Reader { return blockingReader{ctx: ctx} } From 1cc34912afed04a1a64ed2d870b08d7bfb1c99e5 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 19:27:17 -0500 Subject: [PATCH 06/20] test: Keep clients alive on unsolicited pongs A client whose output is backed up may send unsolicited empty pongs instead of answering each ping. The agent already counts any inbound frame as a sign of life; pin that down so a later change cannot start matching pongs to pings and drop busy clients. --- internal/shellagent/stream.go | 3 ++- internal/shellagent/stream_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/shellagent/stream.go b/internal/shellagent/stream.go index 6f7db99e..3f4ee727 100644 --- a/internal/shellagent/stream.go +++ b/internal/shellagent/stream.go @@ -205,7 +205,8 @@ func (c *clientStream) pump(r *bufio.Reader) outcome { } // forward relays the client's frames to the apiserver unchanged, except pongs -// to the agent's own pings. +// to the agent's own pings. Any frame, including an unsolicited pong, shows +// the client is still there. func (c *clientStream) forward(r *bufio.Reader, backend net.Conn, idle time.Duration) outcome { for { if err := c.conn.SetReadDeadline(time.Now().Add(idle)); err != nil { diff --git a/internal/shellagent/stream_test.go b/internal/shellagent/stream_test.go index 881e9cd3..5f42247a 100644 --- a/internal/shellagent/stream_test.go +++ b/internal/shellagent/stream_test.go @@ -142,6 +142,30 @@ func TestAnsweringClientStaysConnected(t *testing.T) { } } +// Any frame is a sign of life: a client whose output is backed up may answer +// with unsolicited empty pongs rather than replies to the agent's pings. +func TestUnsolicitedPongsKeepClientConnected(t *testing.T) { + h := newHarness(t) + a := h.agent(fastPings) + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = (&backendRecorder{}).run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + stop := time.After(4 * a.cfg.PongTimeout) + for done := false; !done; { + select { + case <-stop: + done = true + case <-time.After(a.cfg.PongTimeout / 3): + c.send(t, opPong, nil) + } + } + + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) +} + // ttyCommand writes a prompt, echoes the first stdin message, and exits 0. func ttyCommand(conn net.Conn, r *bufio.Reader) { writeServerFrame(conn, channelStdout, []byte("prompt$ ")) From 9c33e834b3c0df82e522ed6470f17d1641f4ad9e Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 20:30:25 -0500 Subject: [PATCH 07/20] fix: Give commands without stdin an empty input Kata leaves an exec's standard input open when the request asks for none, so a one-off command that reads it, such as `datumctl compute exec -- cat`, waited for the whole session instead of ending. Runc gives such a command /dev/null. The wrapper now redirects standard input from /dev/null when the session asks for neither standard input nor a terminal, so the command sees an empty input on every runtime. --- internal/shellagent/exec_test.go | 22 ++++++++++++++++++---- internal/shellagent/scripts.go | 13 +++++++++++-- internal/shellagent/server.go | 2 +- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/internal/shellagent/exec_test.go b/internal/shellagent/exec_test.go index 8a12c126..7802afec 100644 --- a/internal/shellagent/exec_test.go +++ b/internal/shellagent/exec_test.go @@ -62,9 +62,23 @@ func TestAPIServerOpensAuthorizedExecStream(t *testing.T) { } func TestWrappedCommandRecordsProcessID(t *testing.T) { - got := wrappedCommand("/tmp", "uid-1", []string{"ls", "-l"}) - want := []string{"sh", "-c", wrapperScript, "sh", "/tmp/.datum-exec-uid-1", "ls", "-l"} - if !reflect.DeepEqual(got, want) { - t.Fatalf("wrappedCommand() = %q, want %q", got, want) + tests := []struct { + name string + stdin, tty bool + script string + }{ + {name: "stdin", stdin: true, script: wrapperScript}, + {name: "terminal", tty: true, script: wrapperScript}, + {name: "stdin and terminal", stdin: true, tty: true, script: wrapperScript}, + {name: "neither", script: wrapperScriptNoStdin}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := wrappedCommand("/tmp", "uid-1", []string{"ls", "-l"}, tt.stdin, tt.tty) + want := []string{"sh", "-c", tt.script, "sh", "/tmp/.datum-exec-uid-1", "ls", "-l"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("wrappedCommand() = %q, want %q", got, want) + } + }) } } diff --git a/internal/shellagent/scripts.go b/internal/shellagent/scripts.go index 2c5cc3ea..0f30fbb6 100644 --- a/internal/shellagent/scripts.go +++ b/internal/shellagent/scripts.go @@ -27,6 +27,11 @@ const ( // the command, so the agent can stop the process group on any ending. const wrapperScript = `echo $$ > "$1"; shift; exec "$@"` +// Kata leaves an exec's standard input open when the request asks for none, so +// a command that reads it would wait for the whole session. Without a +// terminal, the command gets /dev/null instead, as it does under runc. +const wrapperScriptNoStdin = `echo $$ > "$1"; shift; exec "$@" Date: Tue, 29 Sep 2026 20:54:03 -0500 Subject: [PATCH 08/20] fix: Claim sessions only while the endpoint is ready An agent whose paired tunnel endpoint was down, for example while its image could not be pulled, still claimed sessions and published the endpoint's ID. With two agents per cell, half of all sessions then failed at the client's dial with an unreachable-endpoint timeout and ended as NotConnected a minute later. The agent now leaves sessions to its peer, or to the control plane's Unavailable ending, until its endpoint's pod is ready. --- internal/shellagent/identity.go | 17 +++++++++++++++++ internal/shellagent/reconciler.go | 7 +++++++ internal/shellagent/reconciler_test.go | 26 ++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/internal/shellagent/identity.go b/internal/shellagent/identity.go index fb9dc593..21d92e0c 100644 --- a/internal/shellagent/identity.go +++ b/internal/shellagent/identity.go @@ -210,6 +210,23 @@ func (a *Agent) restartEndpoint(ctx context.Context) error { } } +// endpointReady reports whether the paired endpoint can carry a connection. A +// session claimed while it cannot fails at the client's dial. +func (a *Agent) endpointReady(ctx context.Context) (bool, error) { + if a.cfg.EndpointPodName == "" { + return true, nil + } + var pod corev1.Pod + err := a.cell.Get(ctx, client.ObjectKey{Namespace: a.cfg.Namespace, Name: a.cfg.EndpointPodName}, &pod) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + return podReady(&pod), nil +} + func podReady(pod *corev1.Pod) bool { for _, c := range pod.Status.Conditions { if c.Type == corev1.PodReady { diff --git a/internal/shellagent/reconciler.go b/internal/shellagent/reconciler.go index bd98861b..9ae91bec 100644 --- a/internal/shellagent/reconciler.go +++ b/internal/shellagent/reconciler.go @@ -114,6 +114,13 @@ func (a *Agent) claim(ctx context.Context, cell, hub *computev1alpha.InstanceCon if !a.claiming() || a.openCount() >= a.cfg.MaxOpenSessions { return ctrl.Result{RequeueAfter: claimRetryInterval}, nil } + ready, err := a.endpointReady(ctx) + if err != nil { + return ctrl.Result{}, err + } + if !ready { + return ctrl.Result{RequeueAfter: claimRetryInterval}, nil + } target, rej, err := a.check(ctx, cell) if errors.Is(err, errNotInCell) { return ctrl.Result{RequeueAfter: claimRetryInterval}, nil diff --git a/internal/shellagent/reconciler_test.go b/internal/shellagent/reconciler_test.go index debbc3b8..86fb0b4a 100644 --- a/internal/shellagent/reconciler_test.go +++ b/internal/shellagent/reconciler_test.go @@ -517,6 +517,32 @@ func TestConcurrencyCapDefersClaims(t *testing.T) { } } +func TestClaimWaitsForReadyEndpoint(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.EndpointPodName = "exec-endpoint-0" }) + h.session(testSession, testUID) + + res := h.reconcile(a, testSession) + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) + if res.RequeueAfter == 0 { + t.Fatal("a claim deferred for a missing endpoint must be retried") + } + + endpoint := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: testAgentNamespace, Name: "exec-endpoint-0"}} + if err := h.cell.Create(h.ctx, endpoint); err != nil { + t.Fatal(err) + } + h.reconcile(a, testSession) + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) + + endpoint.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if err := h.cell.Status().Update(h.ctx, endpoint); err != nil { + t.Fatal(err) + } + h.reconcile(a, testSession) + requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) +} + func TestSweepStopsOrphanedProcesses(t *testing.T) { h := newHarness(t) a := h.agent() From cc98709bd257efbc59c1199541e2174aa1e6a68f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 21:03:11 -0500 Subject: [PATCH 09/20] fix: Read hub sessions by name instead of watching Each cell's hub identity may now only get and patch sessions and write their status; it cannot list or watch them, so a leaked cell credential cannot enumerate other locations' sessions. The agent's hub informer would fail to start under that grant. The agent now watches only the session copies delivered to its cell and fetches each one's hub copy by the same namespace and name. Hub-side changes, such as a deletion or the control plane ending the session, are picked up by rereading the hub copy every 10 seconds while the session is open or still holds the agent's finalizer. Key changes: - Drop the hub cache and its watch from the controller and the binary - Poll the hub copy of open and finalizer-held sessions (HubPollInterval) - Run every test against a hub client that refuses list, watch, create, update and delete, as the hub now does --- cmd/shell-agent/main.go | 10 +--- .../components/shell-agent/kustomization.yaml | 6 ++- internal/shellagent/agent.go | 5 ++ internal/shellagent/harness_test.go | 34 +++++++++++- internal/shellagent/reconciler.go | 53 +++++++++++-------- internal/shellagent/reconciler_test.go | 48 +++++++++++++++-- 6 files changed, 117 insertions(+), 39 deletions(-) diff --git a/cmd/shell-agent/main.go b/cmd/shell-agent/main.go index 5b1564c9..4ea2a3ff 100644 --- a/cmd/shell-agent/main.go +++ b/cmd/shell-agent/main.go @@ -26,7 +26,6 @@ import ( "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -148,13 +147,6 @@ func run(opts options, cfg shellagent.Config) error { if err != nil { return fmt.Errorf("create manager: %w", err) } - hubCluster, err := cluster.New(hubConfig, func(o *cluster.Options) { o.Scheme = scheme }) - if err != nil { - return fmt.Errorf("create hub cluster: %w", err) - } - if err := mgr.Add(hubCluster); err != nil { - return err - } cellClient, err := client.New(cellConfig, client.Options{Scheme: scheme}) if err != nil { return err @@ -178,7 +170,7 @@ func run(opts options, cfg shellagent.Config) error { if err := agent.EnsureIdentity(ctx); err != nil { return err } - if err := agent.SetupWithManager(mgr, hubCluster); err != nil { + if err := agent.SetupWithManager(mgr); err != nil { return fmt.Errorf("set up controller: %w", err) } diff --git a/config/components/shell-agent/kustomization.yaml b/config/components/shell-agent/kustomization.yaml index f9a918b8..6ef47a35 100644 --- a/config/components/shell-agent/kustomization.yaml +++ b/config/components/shell-agent/kustomization.yaml @@ -10,8 +10,10 @@ # register with and the agents publish, nearest first. # - ConfigMap exec-agent-federation-kubeconfig (key kubeconfig) and Secret # exec-agent-federation-client-cert: the Karmada hub credential, mounted -# together at /etc/kubernetes/upstream/auth/. On the hub it needs get, list, -# watch and patch on instanceconsolesessions and update on their status. +# together at /etc/kubernetes/upstream/auth/. On the hub it needs only get +# and patch on instanceconsolesessions and get, update and patch on their +# status. The agent never lists or watches the hub: it reads each session's +# hub copy by the namespace and name of the copy delivered to its cell. apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component diff --git a/internal/shellagent/agent.go b/internal/shellagent/agent.go index ffafe20e..2a6711e3 100644 --- a/internal/shellagent/agent.go +++ b/internal/shellagent/agent.go @@ -56,6 +56,10 @@ type Config struct { // pong, before the agent ends the session as Disconnected. It must be // longer than PingInterval. PongTimeout time.Duration + // HubPollInterval is how often the agent rereads the hub copy of a session + // that is open or still holds its finalizer. The hub lets a cell read + // sessions only by name, so there is nothing to watch. + HubPollInterval time.Duration } // DefaultConfig returns the contract's limits. @@ -71,6 +75,7 @@ func DefaultConfig() Config { SweepInterval: time.Minute, PingInterval: 10 * time.Second, PongTimeout: 20 * time.Second, + HubPollInterval: 10 * time.Second, } } diff --git a/internal/shellagent/harness_test.go b/internal/shellagent/harness_test.go index 914ede14..90db5637 100644 --- a/internal/shellagent/harness_test.go +++ b/internal/shellagent/harness_test.go @@ -10,6 +10,7 @@ import ( "crypto/sha1" //nolint:gosec // RFC 6455 derives the accept key with SHA-1 "encoding/base64" "encoding/json" + "fmt" "net" "net/http" "strings" @@ -19,15 +20,18 @@ import ( coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" computev1alpha "go.datum.net/compute/api/v1alpha" "go.datum.net/compute/internal/consolesession" @@ -66,7 +70,7 @@ type harness struct { t *testing.T ctx context.Context cell client.Client - hub client.Client + hub client.WithWatch exec *fakeAPIServer clock *clock } @@ -125,6 +129,32 @@ func newHarness(t *testing.T) *harness { } } +// cellHubAccess limits a hub client to what the hub grants a cell's agent: +// get and patch on sessions and writes to their status, and nothing else. +func cellHubAccess(hub client.WithWatch) client.WithWatch { + forbidden := func(verb string) error { + return apierrors.NewForbidden(computev1alpha.GroupVersion.WithResource("instanceconsolesessions").GroupResource(), + "", fmt.Errorf("shell agents may not %s sessions on the hub", verb)) + } + return interceptor.NewClient(hub, interceptor.Funcs{ + List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { + return forbidden("list") + }, + Watch: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) (watch.Interface, error) { + return nil, forbidden("watch") + }, + Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + return forbidden("create") + }, + Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error { + return forbidden("update") + }, + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return forbidden("delete") + }, + }) +} + // agent returns an agent whose endpoint ID derives from a fresh key and whose // liveness Lease is current. func (h *harness) agent(mutate ...func(*Config)) *Agent { @@ -137,7 +167,7 @@ func (h *harness) agent(mutate ...func(*Config)) *Agent { for _, m := range mutate { m(&cfg) } - a, err := New(cfg, h.cell, h.cell, h.hub, h.exec, "test") + a, err := New(cfg, h.cell, h.cell, cellHubAccess(h.hub), h.exec, "test") if err != nil { h.t.Fatal(err) } diff --git a/internal/shellagent/reconciler.go b/internal/shellagent/reconciler.go index 9ae91bec..ef344b09 100644 --- a/internal/shellagent/reconciler.go +++ b/internal/shellagent/reconciler.go @@ -13,12 +13,9 @@ import ( "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/source" computev1alpha "go.datum.net/compute/api/v1alpha" ) @@ -28,14 +25,15 @@ const ( cleanupRetryInterval = 5 * time.Second ) -// SetupWithManager reconciles session copies delivered to the cell, and their -// hub copies, which share a namespace and name with them. -func (a *Agent) SetupWithManager(mgr ctrl.Manager, hub cluster.Cluster) error { +// SetupWithManager reconciles session copies delivered to the cell. The hub +// grants each cell only get, patch and status writes on sessions, so the agent +// cannot watch hub copies; it reads a session's hub copy by the cell copy's +// namespace and name, and polls it while the session is open or holds the +// agent's finalizer. +func (a *Agent) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). Named("shell-agent"). For(&computev1alpha.InstanceConsoleSession{}). - WatchesRawSource(source.Kind(hub.GetCache(), &computev1alpha.InstanceConsoleSession{}, - &handler.TypedEnqueueRequestForObject[*computev1alpha.InstanceConsoleSession]{})). WithOptions(controller.Options{MaxConcurrentReconciles: 16}). Complete(a) } @@ -48,53 +46,62 @@ func (a *Agent) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, e if err := a.sessions.Get(ctx, req.NamespacedName, &cell); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - uid := sessionUID(&cell) - ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("session", uid)) + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("session", sessionUID(&cell))) var hub computev1alpha.InstanceConsoleSession err := a.hub.Get(ctx, req.NamespacedName, &hub) if err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } - if apierrors.IsNotFound(err) || sessionUID(&hub) != uid { + if apierrors.IsNotFound(err) || sessionUID(&hub) != sessionUID(&cell) { return a.cleanUp(ctx, &cell, nil) } + res, err := a.reconcileSession(ctx, &cell, &hub) + poll := !isTerminal(&hub) || controllerutil.ContainsFinalizer(&hub, computev1alpha.InstanceConsoleSessionAgentFinalizer) + if err == nil && poll && (res.RequeueAfter <= 0 || res.RequeueAfter > a.cfg.HubPollInterval) { + res.RequeueAfter = a.cfg.HubPollInterval + } + return res, err +} + +func (a *Agent) reconcileSession(ctx context.Context, cell, hub *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(cell) if !hub.DeletionTimestamp.IsZero() || !cell.DeletionTimestamp.IsZero() { - return a.cleanUp(ctx, &cell, &hub) + return a.cleanUp(ctx, cell, hub) } if uid == "" { - return ctrl.Result{}, a.rejectUnclaimed(ctx, &hub, reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + return ctrl.Result{}, a.rejectUnclaimed(ctx, hub, reject(computev1alpha.InstanceConsoleSessionReasonInvalid, "The session was delivered without its session UID.")) } - if isTerminal(&hub) { + if isTerminal(hub) { a.forget(uid) - return a.settle(ctx, &hub) + return a.settle(ctx, hub) } - owner := endpointOf(&hub) + owner := endpointOf(hub) switch { case owner == "": - return a.claim(ctx, &cell, &hub) + return a.claim(ctx, cell, hub) case owner != a.EndpointID(): - return a.watchOwner(ctx, &hub, owner) + return a.watchOwner(ctx, hub, owner) } - a.track(uid, req.NamespacedName) + a.track(uid, client.ObjectKeyFromObject(cell)) now := a.now() - switch readyReason(&hub) { + switch readyReason(hub) { case computev1alpha.InstanceConsoleSessionReasonSessionReady: if a.draining.Load() { - return ctrl.Result{}, a.endUnconnected(ctx, &hub, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + return ctrl.Result{}, a.endUnconnected(ctx, hub, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) } if hub.Status.ConnectBefore != nil && now.Before(hub.Status.ConnectBefore.Time) { return ctrl.Result{RequeueAfter: hub.Status.ConnectBefore.Sub(now)}, nil } log.FromContext(ctx).Info("session was not connected in time") - return ctrl.Result{}, a.endUnconnected(ctx, &hub, computev1alpha.InstanceConsoleSessionReasonNotConnected) + return ctrl.Result{}, a.endUnconnected(ctx, hub, computev1alpha.InstanceConsoleSessionReasonNotConnected) case computev1alpha.InstanceConsoleSessionReasonConnected: if !a.isLive(uid) { log.FromContext(ctx).Info("ending a session this agent's previous run served") - return ctrl.Result{}, a.abandon(ctx, &hub, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) + return ctrl.Result{}, a.abandon(ctx, hub, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) } if hub.Status.ExpiresAt != nil && now.Before(hub.Status.ExpiresAt.Time) { return ctrl.Result{RequeueAfter: hub.Status.ExpiresAt.Sub(now)}, nil diff --git a/internal/shellagent/reconciler_test.go b/internal/shellagent/reconciler_test.go index 86fb0b4a..de401905 100644 --- a/internal/shellagent/reconciler_test.go +++ b/internal/shellagent/reconciler_test.go @@ -11,6 +11,7 @@ import ( coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" @@ -23,7 +24,7 @@ import ( func TestClaimPublishesConnection(t *testing.T) { h := newHarness(t) - a := h.agent() + a := h.agent(noPolling) h.session(testSession, testUID) res := h.reconcile(a, testSession) @@ -313,7 +314,7 @@ func TestNotConnectedDeadline(t *testing.T) { func TestExpiryStopsConnectedSession(t *testing.T) { h := newHarness(t) - a := h.agent() + a := h.agent(noPolling) h.session(testSession, testUID) h.reconcile(a, testSession) markConnected(h, testSession, h.clock.now().Add(time.Minute)) @@ -332,7 +333,7 @@ func TestExpiryStopsConnectedSession(t *testing.T) { func TestLivenessTakeover(t *testing.T) { h := newHarness(t) - lost, survivor := h.agent(), h.agent() + lost, survivor := h.agent(noPolling), h.agent(noPolling) h.session(testSession, testUID) h.reconcile(lost, testSession) markConnected(h, testSession, h.clock.now().Add(time.Hour)) @@ -721,3 +722,44 @@ func patchPod(h *harness, mutate func(*corev1.Pod)) { h.t.Fatal(err) } } + +// noPolling leaves a session's deadlines as the only reason to requeue it. +func noPolling(c *Config) { + c.HubPollInterval = 24 * time.Hour +} + +func TestOpenSessionsPollTheirHubCopy(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + + if res := h.reconcile(a, testSession); res.RequeueAfter != a.cfg.HubPollInterval { + t.Fatalf("requeue after %v for a claimed session, want the hub poll interval", res.RequeueAfter) + } + s := h.hubSession(testSession) + now := metav1.NewTime(h.clock.now()) + s.DeletionTimestamp = &now + if err := h.hub.Delete(h.ctx, s); err != nil { + t.Fatal(err) + } + + h.reconcile(a, testSession) + + if err := h.hub.Get(h.ctx, client.ObjectKeyFromObject(s), s); !apierrors.IsNotFound(err) { + t.Fatalf("hub copy still held after its deletion was polled: %v", err) + } + if len(h.slots()) != 0 { + t.Fatal("slot kept after the hub copy was deleted") + } +} + +func TestEndedSessionWithoutFinalizerIsNotPolled(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + endUnavailable(t, h) + + if res := h.reconcile(a, testSession); res.RequeueAfter != 0 { + t.Fatalf("requeue after %v for an ended session this agent never held", res.RequeueAfter) + } +} From 1c28ea2ea62838337d782bec5b046b0457eab94c Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 21:04:04 -0500 Subject: [PATCH 10/20] fix: Bound each exec the agent runs in an instance A pod exec that never answers, such as one into a wedged guest, held its caller forever. Stopping a session's processes runs from reconcile workers and from the cell-wide orphan sweep, so one hung instance could tie up workers and stall the sweep for every other session. Each probe, marker listing and process stop now gives up after the kill grace plus 30 seconds. A stop that times out counts as not confirmed, so the slot and marker are kept for the sweep to retry. --- internal/shellagent/agent.go | 7 +++++++ internal/shellagent/checks.go | 2 +- internal/shellagent/exec.go | 8 ++++++++ internal/shellagent/harness_test.go | 8 +++++++- internal/shellagent/reconciler_test.go | 24 ++++++++++++++++++++++++ internal/shellagent/slots.go | 2 +- internal/shellagent/sweep.go | 4 ++-- 7 files changed, 50 insertions(+), 5 deletions(-) diff --git a/internal/shellagent/agent.go b/internal/shellagent/agent.go index 2a6711e3..9b22ad15 100644 --- a/internal/shellagent/agent.go +++ b/internal/shellagent/agent.go @@ -60,6 +60,10 @@ type Config struct { // that is open or still holds its finalizer. The hub lets a cell read // sessions only by name, so there is nothing to watch. HubPollInterval time.Duration + // ExecTimeout bounds each command the agent runs in an instance to probe, + // list or stop session processes, so a hung exec cannot hold a reconcile + // worker or the sweep. Zero means KillGrace plus 30 seconds. + ExecTimeout time.Duration } // DefaultConfig returns the contract's limits. @@ -108,6 +112,9 @@ func New(cfg Config, sessions client.Reader, cell, hub client.Client, exec Execu if cfg.Namespace == "" || cfg.Target == "" || len(cfg.RelayURLs) == 0 || len(cfg.ManagedBy) == 0 { return nil, errors.New("namespace, target, relay URLs and managed-by values are required") } + if cfg.ExecTimeout <= 0 { + cfg.ExecTimeout = cfg.KillGrace + 30*time.Second + } if cfg.PingInterval <= 0 || cfg.PongTimeout <= cfg.PingInterval { return nil, errors.New("the ping interval must be positive and shorter than the pong timeout") } diff --git a/internal/shellagent/checks.go b/internal/shellagent/checks.go index a11a1e14..1e641a90 100644 --- a/internal/shellagent/checks.go +++ b/internal/shellagent/checks.go @@ -166,7 +166,7 @@ func (a *Agent) probe(ctx context.Context, pod *corev1.Pod, container, executabl } } - out, code, err := run(ctx, a.exec, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, + out, code, err := a.run(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, container, probeCommand(executable)) var failed *ExecFailedError switch { diff --git a/internal/shellagent/exec.go b/internal/shellagent/exec.go index 97602e96..8cbf99ac 100644 --- a/internal/shellagent/exec.go +++ b/internal/shellagent/exec.go @@ -207,6 +207,14 @@ func (e *ExecFailedError) Error() string { return "exec failed: " + e.Message } +// run runs a command in a pod to completion, giving up after the agent's exec +// timeout. +func (a *Agent) run(ctx context.Context, pod types.NamespacedName, container string, command []string) (string, int, error) { + ctx, cancel := context.WithTimeout(ctx, a.cfg.ExecTimeout) + defer cancel() + return run(ctx, a.exec, pod, container, command) +} + // run runs a command to completion without stdin and returns its combined // output and exit code. func run(ctx context.Context, exec Executor, pod types.NamespacedName, container string, command []string) (string, int, error) { diff --git a/internal/shellagent/harness_test.go b/internal/shellagent/harness_test.go index 90db5637..f8ed16e5 100644 --- a/internal/shellagent/harness_test.go +++ b/internal/shellagent/harness_test.go @@ -11,6 +11,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io" "net" "net/http" "strings" @@ -263,6 +264,7 @@ type fakeAPIServer struct { probeDir string markers string killExit int + killHangs bool onSession func(conn net.Conn, r *bufio.Reader) } @@ -319,7 +321,7 @@ func acceptKey(key string) string { func (f *fakeAPIServer) answer(conn net.Conn, command []string) { f.mu.Lock() - probeExit, probeDir, markers, killExit := f.probeExit, f.probeDir, f.markers, f.killExit + probeExit, probeDir, markers, killExit, killHangs := f.probeExit, f.probeDir, f.markers, f.killExit, f.killHangs f.mu.Unlock() switch command[2] { case probeScript: @@ -331,6 +333,10 @@ func (f *fakeAPIServer) answer(conn net.Conn, command []string) { writeServerFrame(conn, channelStdout, []byte(markers)) writeExitStatus(conn, 0) case killScript: + if killHangs { + _, _ = io.Copy(io.Discard, conn) + return + } writeExitStatus(conn, killExit) default: writeExitStatus(conn, 0) diff --git a/internal/shellagent/reconciler_test.go b/internal/shellagent/reconciler_test.go index de401905..c5f8041f 100644 --- a/internal/shellagent/reconciler_test.go +++ b/internal/shellagent/reconciler_test.go @@ -763,3 +763,27 @@ func TestEndedSessionWithoutFinalizerIsNotPolled(t *testing.T) { t.Fatalf("requeue after %v for an ended session this agent never held", res.RequeueAfter) } } + +func TestHungExecIsBounded(t *testing.T) { + h := newHarness(t) + a := h.agent(func(c *Config) { c.ExecTimeout = 200 * time.Millisecond }) + h.session(testSession, testUID) + h.reconcile(a, testSession) + held, err := a.slotFor(h.ctx, testUID) + if err != nil || held == nil { + t.Fatalf("slot = %v, %v", held, err) + } + h.exec.mu.Lock() + h.exec.killHangs = true + h.exec.mu.Unlock() + + start := time.Now() + stopped, err := a.stopProcesses(h.ctx, held) + + if err != nil || stopped { + t.Fatalf("stopProcesses() = %v, %v; want not stopped, no error", stopped, err) + } + if took := time.Since(start); took > 5*time.Second { + t.Fatalf("a hung exec held the caller for %v", took) + } +} diff --git a/internal/shellagent/slots.go b/internal/shellagent/slots.go index d6b2f390..c3fcaa69 100644 --- a/internal/shellagent/slots.go +++ b/internal/shellagent/slots.go @@ -232,7 +232,7 @@ func (a *Agent) stopProcesses(ctx context.Context, s *slot) (bool, error) { return true, nil } grace := int(a.cfg.KillGrace.Seconds()) - out, code, err := run(ctx, a.exec, s.pod, s.container, killCommand(s.markerDir, s.sessionUID, grace)) + out, code, err := a.run(ctx, s.pod, s.container, killCommand(s.markerDir, s.sessionUID, grace)) if err != nil || code != 0 { log.FromContext(ctx).Info("could not stop session processes", "session", s.sessionUID, "pod", s.pod.String(), "exitCode", code, "output", out, "error", err) diff --git a/internal/shellagent/sweep.go b/internal/shellagent/sweep.go index 9a62d82c..547a18ab 100644 --- a/internal/shellagent/sweep.go +++ b/internal/shellagent/sweep.go @@ -113,7 +113,7 @@ func (a *Agent) sweepContainer(ctx context.Context, place markerPlace) { if status := containerStatus(&pod, place.container); status == nil || status.State.Running == nil { return } - out, code, err := run(ctx, a.exec, place.pod, place.container, listMarkersCommand(place.dir)) + out, code, err := a.run(ctx, place.pod, place.container, listMarkersCommand(place.dir)) if err != nil || code != 0 { logger.Info("could not list session markers", "exitCode", code, "error", err) return @@ -127,7 +127,7 @@ func (a *Agent) sweepContainer(ctx context.Context, place markerPlace) { continue } grace := int(a.cfg.KillGrace.Seconds()) - if _, code, err := run(ctx, a.exec, place.pod, place.container, killCommand(place.dir, uid, grace)); err != nil || code != 0 { + if _, code, err := a.run(ctx, place.pod, place.container, killCommand(place.dir, uid, grace)); err != nil || code != 0 { logger.Info("could not stop orphaned session processes", "session", uid, "exitCode", code, "error", err) continue } From 0aa741722b9b25357679270a286090d6f3065ec0 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 21:04:44 -0500 Subject: [PATCH 11/20] fix: Name liveness Leases by the full endpoint hash An agent's liveness Lease was named by the first 16 hex digits of its endpoint ID. Two endpoint keys sharing that prefix would share a Lease, so one agent's renewals could keep a crashed peer looking alive and delay the takeover of its sessions. The Lease is now named by the full SHA-256 of the endpoint ID. Lease names are DNS subdomains, which fit all 64 hex digits, and hashing keeps the name valid whatever the hub copy's status claims the ID is. --- internal/shellagent/harness_test.go | 2 +- internal/shellagent/liveness.go | 11 +++++++---- internal/shellagent/reconciler_test.go | 13 +++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/internal/shellagent/harness_test.go b/internal/shellagent/harness_test.go index f8ed16e5..58535450 100644 --- a/internal/shellagent/harness_test.go +++ b/internal/shellagent/harness_test.go @@ -25,8 +25,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/watch" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/watch" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" diff --git a/internal/shellagent/liveness.go b/internal/shellagent/liveness.go index afd913b6..fd88d5ed 100644 --- a/internal/shellagent/liveness.go +++ b/internal/shellagent/liveness.go @@ -4,6 +4,8 @@ package shellagent import ( "context" + "crypto/sha256" + "encoding/hex" "time" coordinationv1 "k8s.io/api/coordination/v1" @@ -24,11 +26,12 @@ const ( livenessComponent = "shell-agent-liveness" ) +// agentLeaseName names an agent's liveness Lease by the full SHA-256 of its +// endpoint ID. Lease names are DNS subdomains, which fit all 64 hex digits, so +// two agents never share a Lease. func agentLeaseName(endpointID string) string { - if len(endpointID) > 16 { - endpointID = endpointID[:16] - } - return "agent-" + endpointID + sum := sha256.Sum256([]byte(endpointID)) + return "agent-" + hex.EncodeToString(sum[:]) } // RunLiveness renews the agent's liveness Lease until ctx ends. diff --git a/internal/shellagent/reconciler_test.go b/internal/shellagent/reconciler_test.go index c5f8041f..9ded5dc7 100644 --- a/internal/shellagent/reconciler_test.go +++ b/internal/shellagent/reconciler_test.go @@ -5,6 +5,7 @@ package shellagent import ( "fmt" "net" + "strings" "sync" "testing" "time" @@ -14,6 +15,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -787,3 +789,14 @@ func TestHungExecIsBounded(t *testing.T) { t.Fatalf("a hung exec held the caller for %v", took) } } + +func TestAgentLeaseNamesAreDistinctAndValid(t *testing.T) { + a := agentLeaseName(strings.Repeat("a", 16) + strings.Repeat("0", 48)) + b := agentLeaseName(strings.Repeat("a", 16) + strings.Repeat("1", 48)) + if a == b { + t.Fatal("endpoint IDs sharing a prefix share a liveness Lease") + } + if errs := validation.IsDNS1123Subdomain(a); len(errs) != 0 { + t.Fatalf("lease name %q is invalid: %v", a, errs) + } +} From 64910e9208dc3968ddc3466e85739c5142da4b45 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 21:07:34 -0500 Subject: [PATCH 12/20] fix: Reassemble fragmented client messages The agent relayed a client's frames to the apiserver as they arrived. The apiserver reads only the first frame of a fragmented message and drops the rest, so input a client sent in several frames, such as a large paste, silently lost everything after the first frame. The agent now joins a fragmented text or binary message and sends it to the apiserver as one masked frame. Control frames in the middle of a message are still relayed at once. A continuation with no message to continue, or a new message inside an unfinished one, ends the session as Invalid, and a reassembled message over 16 MiB ends it as a single oversized frame does. --- internal/shellagent/stream.go | 38 ++++++++++++++-- internal/shellagent/stream_test.go | 69 ++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/internal/shellagent/stream.go b/internal/shellagent/stream.go index 3f4ee727..0a48d9a6 100644 --- a/internal/shellagent/stream.go +++ b/internal/shellagent/stream.go @@ -119,6 +119,10 @@ var ( reason: computev1alpha.InstanceConsoleSessionReasonInvalid, message: "The client sent a message larger than 16 MiB.", } + outcomeMalformed = outcome{ + reason: computev1alpha.InstanceConsoleSessionReasonInvalid, + message: "The client sent a malformed WebSocket message.", + } ) // pingPayload marks the agent's own pings, so the client's pongs to them are @@ -204,10 +208,14 @@ func (c *clientStream) pump(r *bufio.Reader) outcome { } } -// forward relays the client's frames to the apiserver unchanged, except pongs -// to the agent's own pings. Any frame, including an unsolicited pong, shows -// the client is still there. +// forward relays the client's frames to the apiserver, except pongs to the +// agent's own pings. The apiserver reads only the first frame of a fragmented +// message, so fragmented messages are reassembled and sent as one frame. Any +// frame, including an unsolicited pong, shows the client is still there. func (c *clientStream) forward(r *bufio.Reader, backend net.Conn, idle time.Duration) outcome { + var message []byte + var messageOp byte + fragmented := false for { if err := c.conn.SetReadDeadline(time.Now().Add(idle)); err != nil { return outcomeClientGone @@ -222,7 +230,29 @@ func (c *clientStream) forward(r *bufio.Reader, backend net.Conn, idle time.Dura case f.opcode == opPong && bytes.Equal(f.payload, pingPayload): continue } - if _, err := backend.Write(f.raw); err != nil { + out := f.raw + switch { + case f.control(): + case f.opcode == opContinuation: + if !fragmented { + return outcomeMalformed + } + if len(message)+len(f.payload) > MaxFrameSize { + return outcomeFrameTooLarge + } + message = append(message, f.payload...) + if !f.fin { + continue + } + out = encodeFrame(messageOp, message, true) + message, fragmented = nil, false + case fragmented: + return outcomeMalformed + case !f.fin: + message, messageOp, fragmented = append([]byte(nil), f.payload...), f.opcode, true + continue + } + if _, err := backend.Write(out); err != nil { return outcomeStreamLost } } diff --git a/internal/shellagent/stream_test.go b/internal/shellagent/stream_test.go index 5f42247a..e0e904dc 100644 --- a/internal/shellagent/stream_test.go +++ b/internal/shellagent/stream_test.go @@ -257,3 +257,72 @@ func (b blockingReader) Read([]byte) (int, error) { } func blockUntil(ctx context.Context) io.Reader { return blockingReader{ctx: ctx} } + +func encodeFragment(opcode byte, payload []byte, fin bool) []byte { + f := encodeFrame(opcode, payload, true) + if !fin { + f[0] &^= 0x80 + } + return f +} + +// The apiserver reads only the first frame of a fragmented message, so the +// agent must deliver a fragmented stdin message as one frame. +func TestFragmentedClientMessageIsReassembled(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + backend := &backendRecorder{} + h.exec.onSession = backend.run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + raw := append(encodeFragment(opBinary, []byte{channelStdin, 'a', 'b'}, false), + encodeFrame(opPing, []byte("client"), true)...) + raw = append(raw, encodeFragment(opContinuation, []byte("cd"), false)...) + raw = append(raw, encodeFragment(opContinuation, []byte("ef"), true)...) + if _, err := c.conn.Write(raw); err != nil { + t.Fatal(err) + } + + deadline := time.Now().Add(5 * time.Second) + for { + backend.mu.Lock() + frames := append([]*frame(nil), backend.frames...) + backend.mu.Unlock() + if len(frames) >= 2 { + if frames[0].opcode != opPing { + t.Fatalf("first relayed frame opcode = %d, want the ping", frames[0].opcode) + } + f := frames[1] + if !f.fin || f.opcode != opBinary || string(f.payload) != "\x00abcdef" || f.raw[1]&0x80 == 0 { + t.Fatalf("relayed frame = fin %v, opcode %d, payload %q, masked %v", + f.fin, f.opcode, f.payload, f.raw[1]&0x80 != 0) + } + return + } + if time.Now().After(deadline) { + t.Fatalf("relayed %d frames, want 2", len(frames)) + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestStrayContinuationEndsSession(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = (&backendRecorder{}).run + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + c.send(t, opContinuation, []byte("x")) + _, status, _ := c.readUntilClose(t) + + if status == nil || string(status.Reason) != computev1alpha.InstanceConsoleSessionReasonInvalid { + t.Fatalf("status = %+v", status) + } + requireReason(t, waitForEnd(t, h), computev1alpha.InstanceConsoleSessionReasonInvalid) +} From 0bedcd96fa2f584d004223e5f28231f2d2f4ebcd Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 21:19:06 -0500 Subject: [PATCH 13/20] fix: Stop background jobs when a session ends Background jobs outlived the sessions that started them. The agent stopped a session by signalling the process group of the command it started, but an interactive shell's job control moves each background job to a process group of its own. On Kata, `sleep 60 &` and `nohup sleep 60 &` in busybox sh, and `nohup ... & disown` in bash, kept running after a disconnect or a revocation, and the orphan sweep could not find them because the kill had already removed their marker. Job control never moves a job out of its session, and every exec leads a session of its own under Kata and runc. The wrapper now records the session it leads, starting one with setsid where it does not, and the kill script signals every process in that session. Key changes: - Record the wrapper's PID, start time and whether it leads a session or only a process group - Signal each live member with SIGHUP and SIGCONT, then SIGKILL after the grace period, and skip zombies and processes older than the wrapper - Skip the kill when the recorded PID now belongs to a younger process, which means the session is long gone - Keep the marker, and report failure, until no member is left, so the sweep retries stragglers - Test the kill script against a fake /proc, and the wrapper against the real one on Linux A process that calls setsid itself leaves the session and is not stopped. Verified on Kata with busybox sh and bash, and on runc. --- internal/shellagent/exec_test.go | 14 +- internal/shellagent/scripts.go | 86 ++++++++-- internal/shellagent/scripts_test.go | 234 ++++++++++++++++++++++++++++ 3 files changed, 310 insertions(+), 24 deletions(-) create mode 100644 internal/shellagent/scripts_test.go diff --git a/internal/shellagent/exec_test.go b/internal/shellagent/exec_test.go index 7802afec..4643bafa 100644 --- a/internal/shellagent/exec_test.go +++ b/internal/shellagent/exec_test.go @@ -61,21 +61,21 @@ func TestAPIServerOpensAuthorizedExecStream(t *testing.T) { } } -func TestWrappedCommandRecordsProcessID(t *testing.T) { +func TestWrappedCommandArguments(t *testing.T) { tests := []struct { name string stdin, tty bool - script string + mode string }{ - {name: "stdin", stdin: true, script: wrapperScript}, - {name: "terminal", tty: true, script: wrapperScript}, - {name: "stdin and terminal", stdin: true, tty: true, script: wrapperScript}, - {name: "neither", script: wrapperScriptNoStdin}, + {name: "stdin", stdin: true, mode: "-"}, + {name: "terminal", tty: true, mode: "-"}, + {name: "stdin and terminal", stdin: true, tty: true, mode: "-"}, + {name: "neither", mode: "null"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := wrappedCommand("/tmp", "uid-1", []string{"ls", "-l"}, tt.stdin, tt.tty) - want := []string{"sh", "-c", tt.script, "sh", "/tmp/.datum-exec-uid-1", "ls", "-l"} + want := []string{"sh", "-c", wrapperScript, "sh", wrapperScript, "/tmp/.datum-exec-uid-1", tt.mode, "0", "ls", "-l"} if !reflect.DeepEqual(got, want) { t.Fatalf("wrappedCommand() = %q, want %q", got, want) } diff --git a/internal/shellagent/scripts.go b/internal/shellagent/scripts.go index 0f30fbb6..66a4ec4f 100644 --- a/internal/shellagent/scripts.go +++ b/internal/shellagent/scripts.go @@ -23,24 +23,76 @@ const ( probeExitNoWritableDir = 3 ) -// The wrapper records the command's process ID before it replaces itself with -// the command, so the agent can stop the process group on any ending. -const wrapperScript = `echo $$ > "$1"; shift; exec "$@"` +// procStatFunction reads /proc//stat into Z (state), G (process group), +// S (session) and T (start time). The command name may hold spaces and +// parentheses, so the fields are taken after its last ")". +const procStatFunction = `st() { Z=; G=; S=; T=; read -r l < "$P/$1/stat" 2>/dev/null || return 1; set -- ${l##*)}; Z=$1 G=$3 S=$4 T=${20}; } +` -// Kata leaves an exec's standard input open when the request asks for none, so -// a command that reads it would wait for the whole session. Without a -// terminal, the command gets /dev/null instead, as it does under runc. -const wrapperScriptNoStdin = `echo $$ > "$1"; shift; exec "$@" /dev/null 2>&1; then + exec setsid sh -c "$w" sh "$w" "$m" "$mode" 1 "$@" +fi +if ! st $$; then echo "$$" > "$m" +elif [ "$S" = "$$" ]; then echo "$$ $T session" > "$m" +else echo "$$ $T group" > "$m"; fi +[ "$mode" = null ] && exec "$@" /dev/null; kill -HUP "$p" 2>/dev/null +case "$t" in *[!0-9]*) t= ;; esac +` + procStatFunction + `if [ -n "$t" ] && st "$p" && [ "$T" != "$t" ]; then rm -f "$f"; exit 0; fi +members() { + M= + if [ -z "$t" ]; then + kill -0 -"$p" 2>/dev/null && M="-$p" + kill -0 "$p" 2>/dev/null && M="$M $p" + return 0 + fi + for d in "$P"/[0-9]*; do + q=${d##*/} + [ "$q" = "$$" ] && continue + st "$q" || continue + [ "$Z" = Z ] && continue + [ "$T" -ge "$t" ] 2>/dev/null || continue + if [ "$S" = "$p" ] && [ "$kind" = session ] || [ "$G" = "$p" ]; then M="$M $q"; fi + done +} +members +[ -z "$M" ] && { rm -f "$f"; exit 0; } +kill -HUP $M 2>/dev/null; kill -CONT $M 2>/dev/null +i=0 +while [ "$i" -lt "$grace" ]; do + sleep 1; i=$((i+1)) + members + [ -z "$M" ] && break +done i=0 -while [ "$i" -lt "$grace" ] && kill -0 "$p" 2>/dev/null; do sleep 1; i=$((i+1)); done -kill -KILL -"$p" 2>/dev/null; kill -KILL "$p" 2>/dev/null -rm -f "$f"` +while :; do + members + [ -z "$M" ] && { rm -f "$f"; exit 0; } + [ "$i" -ge 5 ] && exit 5 + kill -KILL $M 2>/dev/null + sleep 1; i=$((i+1)) +done` const listMarkersScript = `for f in "$1"/` + markerPrefix + `*; do [ -f "$f" ] && echo "${f##*/` + markerPrefix + `}" @@ -56,15 +108,15 @@ func probeCommand(executable string) []string { } func wrappedCommand(dir, sessionUID string, command []string, stdin, tty bool) []string { - script := wrapperScript + mode := "-" if !stdin && !tty { - script = wrapperScriptNoStdin + mode = "null" } - return append([]string{"sh", "-c", script, "sh", markerPath(dir, sessionUID)}, command...) + return append([]string{"sh", "-c", wrapperScript, "sh", wrapperScript, markerPath(dir, sessionUID), mode, "0"}, command...) } func killCommand(dir, sessionUID string, graceSeconds int) []string { - return []string{"sh", "-c", killScript, "sh", markerPath(dir, sessionUID), strconv.Itoa(graceSeconds)} + return []string{"sh", "-c", killScript, "sh", markerPath(dir, sessionUID), strconv.Itoa(graceSeconds), "/proc"} } func listMarkersCommand(dir string) []string { diff --git a/internal/shellagent/scripts_test.go b/internal/shellagent/scripts_test.go new file mode 100644 index 00000000..741b8926 --- /dev/null +++ b/internal/shellagent/scripts_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package shellagent + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "testing" +) + +type fakeProcess struct { + pid, pgrp, sid, start int + comm, state string + ignoresHUP bool + unkillable bool +} + +// fakeKill replaces kill and sleep in the kill script: a signalled process +// dies unless it ignores SIGHUP or cannot be killed, and every call is logged. +const fakeKill = `kill() { + sig=$1; shift + echo "$sig $*" >> "$LOG" + for q in "$@"; do + case "$sig" in + -HUP) grep -qx "$q" "$NOHUP" || rm -rf "$P/$q" ;; + -KILL) grep -qx "$q" "$UNKILLABLE" || rm -rf "$P/$q" ;; + esac + done +} +sleep() { :; } +` + +type killRun struct { + exit int + signalled map[string][]int + survivors []int + markerLeft bool +} + +func runKillScript(t *testing.T, marker string, procs []fakeProcess) killRun { + t.Helper() + dir := t.TempDir() + proc := filepath.Join(dir, "proc") + var nohup, unkillable []string + for _, p := range procs { + comm, state := p.comm, p.state + if comm == "" { + comm = "sh" + } + if state == "" { + state = "S" + } + stat := fmt.Sprintf("%d (%s) %s 1 %d %d 34816 %d 0 0 0 0 0 0 0 0 0 20 0 1 0 %d 4210688 489\n", + p.pid, comm, state, p.pgrp, p.sid, p.pgrp, p.start) + if err := os.MkdirAll(filepath.Join(proc, strconv.Itoa(p.pid)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proc, strconv.Itoa(p.pid), "stat"), []byte(stat), 0o644); err != nil { + t.Fatal(err) + } + if p.ignoresHUP { + nohup = append(nohup, strconv.Itoa(p.pid)) + } + if p.unkillable { + unkillable = append(unkillable, strconv.Itoa(p.pid)) + } + } + markerFile := filepath.Join(dir, ".datum-exec-uid") + if err := os.WriteFile(markerFile, []byte(marker+"\n"), 0o644); err != nil { + t.Fatal(err) + } + logFile := filepath.Join(dir, "log") + write := func(name string, lines []string) string { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + return path + } + cmd := exec.Command("sh", "-c", fakeKill+killScript, "sh", markerFile, "3", proc) + cmd.Env = append(os.Environ(), "LOG="+logFile, "NOHUP="+write("nohup", nohup), + "UNKILLABLE="+write("unkillable", unkillable)) + out, err := cmd.CombinedOutput() + res := killRun{signalled: map[string][]int{}} + if exitErr, ok := err.(*exec.ExitError); ok { + res.exit = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("kill script: %v: %s", err, out) + } + log, _ := os.ReadFile(logFile) + for _, line := range strings.Split(strings.TrimSpace(string(log)), "\n") { + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + for _, f := range fields[1:] { + n, _ := strconv.Atoi(f) + res.signalled[fields[0]] = append(res.signalled[fields[0]], n) + } + } + for sig := range res.signalled { + sort.Ints(res.signalled[sig]) + res.signalled[sig] = uniq(res.signalled[sig]) + } + entries, _ := os.ReadDir(proc) + for _, e := range entries { + n, _ := strconv.Atoi(e.Name()) + res.survivors = append(res.survivors, n) + } + sort.Ints(res.survivors) + _, statErr := os.Stat(markerFile) + res.markerLeft = statErr == nil + return res +} + +func uniq(in []int) []int { + var out []int + for i, v := range in { + if i == 0 || v != in[i-1] { + out = append(out, v) + } + } + return out +} + +func TestKillScriptStopsTheWholeSession(t *testing.T) { + res := runKillScript(t, "40 1000 session", []fakeProcess{ + {pid: 1, pgrp: 1, sid: 1, start: 10, comm: "init"}, + {pid: 40, pgrp: 40, sid: 40, start: 1000}, + {pid: 41, pgrp: 41, sid: 40, start: 1100, comm: "sleep"}, + {pid: 42, pgrp: 42, sid: 40, start: 1200, comm: "nohup sleep", ignoresHUP: true}, + {pid: 43, pgrp: 43, sid: 40, start: 1300, comm: "odd) name (x", state: "Z"}, + {pid: 44, pgrp: 44, sid: 44, start: 1400, comm: "other session"}, + {pid: 45, pgrp: 45, sid: 40, start: 900, comm: "older"}, + }) + + if res.exit != 0 || res.markerLeft { + t.Fatalf("exit %d, marker kept %v", res.exit, res.markerLeft) + } + if got := fmt.Sprint(res.signalled["-HUP"]); got != "[40 41 42]" { + t.Fatalf("SIGHUP sent to %s", got) + } + if got := fmt.Sprint(res.signalled["-KILL"]); got != "[42]" { + t.Fatalf("SIGKILL sent to %s, want only the process that ignored SIGHUP", got) + } + if got := fmt.Sprint(res.survivors); got != "[1 43 44 45]" { + t.Fatalf("survivors = %s", got) + } +} + +func TestKillScriptKeepsMarkerWhileProcessesSurvive(t *testing.T) { + res := runKillScript(t, "40 1000 session", []fakeProcess{ + {pid: 40, pgrp: 40, sid: 40, start: 1000}, + {pid: 41, pgrp: 41, sid: 40, start: 1100, ignoresHUP: true, unkillable: true}, + }) + + if res.exit != 5 || !res.markerLeft { + t.Fatalf("exit %d, marker kept %v; want 5 and the marker kept for the sweep", res.exit, res.markerLeft) + } +} + +func TestKillScriptLeavesAReusedProcessIDAlone(t *testing.T) { + res := runKillScript(t, "40 1000 session", []fakeProcess{ + {pid: 40, pgrp: 40, sid: 40, start: 5000}, + {pid: 41, pgrp: 41, sid: 40, start: 5100}, + }) + + if res.exit != 0 || res.markerLeft || len(res.signalled) != 0 { + t.Fatalf("exit %d, marker kept %v, signalled %v", res.exit, res.markerLeft, res.signalled) + } +} + +func TestKillScriptStopsOnlyTheGroupWithoutASession(t *testing.T) { + res := runKillScript(t, "40 1000 group", []fakeProcess{ + {pid: 30, pgrp: 30, sid: 30, start: 900}, + {pid: 40, pgrp: 40, sid: 30, start: 1000}, + {pid: 41, pgrp: 40, sid: 30, start: 1100}, + {pid: 42, pgrp: 42, sid: 30, start: 1200}, + }) + + if res.exit != 0 || res.markerLeft { + t.Fatalf("exit %d, marker kept %v", res.exit, res.markerLeft) + } + if got := fmt.Sprint(res.survivors); got != "[30 42]" { + t.Fatalf("survivors = %s, want the rest of the shared session", got) + } +} + +func TestKillScriptWithoutRecordDoesNothing(t *testing.T) { + res := runKillScript(t, "not-a-pid", []fakeProcess{{pid: 40, pgrp: 40, sid: 40, start: 1000}}) + + if res.exit != 0 || res.markerLeft || len(res.signalled) != 0 { + t.Fatalf("exit %d, marker kept %v, signalled %v", res.exit, res.markerLeft, res.signalled) + } +} + +// The wrapper runs for real where there is a /proc: it records a session it +// leads, or starts one with setsid, and gives a command without stdin or a +// terminal an empty input. +func TestWrapperRecordsItsSession(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("needs /proc") + } + dir := t.TempDir() + marker := filepath.Join(dir, ".datum-exec-uid") + args := wrappedCommand(dir, "uid", []string{"sh", "-c", `cat; read -r l < /proc/$$/stat; set -- ${l##*)}; echo "$4"`}, false, false) + cmd := exec.Command(args[0], args[1:]...) + cmd.Stdin = strings.NewReader("must not be read\n") + out, err := cmd.Output() + if err != nil { + t.Fatalf("wrapper: %v", err) + } + record, err := os.ReadFile(marker) + if err != nil { + t.Fatal(err) + } + fields := strings.Fields(string(record)) + if len(fields) != 3 { + t.Fatalf("marker = %q", record) + } + if strings.Contains(string(out), "must not be read") { + t.Fatal("the command read stdin the session did not ask for") + } + if _, err := exec.LookPath("setsid"); err == nil && + (fields[2] != "session" || strings.TrimSpace(string(out)) != fields[0]) { + t.Fatalf("marker = %q, command session %q; want a session the wrapper leads", record, out) + } +} From b9926a15941997acace8cd8931fee7e825252120 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 21:20:24 -0500 Subject: [PATCH 14/20] feat: Limit the shell agent's exec to tenant pods The agent's ClusterRole grants pod exec in every namespace, because tenant namespaces are created per project and cannot be listed in RBAC. A compromised agent could therefore exec into any pod in the cell, including system workloads and the agent's own namespace. The shell-agent component now ships a ValidatingAdmissionPolicy on CONNECT that lets the agent's service account exec only into pods in tenant namespaces (the `ns-` prefix), and refuses it attach and port forwarding. The agent's own checks before a claim are unchanged. Verified on a Kubernetes 1.34 kind cluster: the agent's identity can exec into a pod in a tenant namespace and is refused in `default` and in `compute-shell-system`; other users are unaffected. --- .../shell-agent/exec-admission-policy.yaml | 42 +++++++++++++++++++ .../components/shell-agent/kustomization.yaml | 1 + config/components/shell-agent/rbac.yaml | 3 ++ 3 files changed, 46 insertions(+) create mode 100644 config/components/shell-agent/exec-admission-policy.yaml diff --git a/config/components/shell-agent/exec-admission-policy.yaml b/config/components/shell-agent/exec-admission-policy.yaml new file mode 100644 index 00000000..2e73526c --- /dev/null +++ b/config/components/shell-agent/exec-admission-policy.yaml @@ -0,0 +1,42 @@ +# The agent's ClusterRole grants pods/exec in every namespace, because tenant +# namespaces are created per project. This policy narrows what the apiserver +# lets the agent exec into to tenant namespaces, so a compromised agent cannot +# exec into system workloads, including this component's own. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: compute-shell-agent-exec +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CONNECT + resources: + - pods/exec + - pods/attach + - pods/portforward + matchConditions: + - name: shell-agent + expression: request.userInfo.username == 'system:serviceaccount:compute-shell-system:exec-agent' + validations: + - expression: request.resource.resource == 'pods' && request.subResource == 'exec' + message: The shell agent may only exec into pods. + reason: Forbidden + - expression: request.namespace.startsWith('ns-') + messageExpression: >- + 'the shell agent may only exec into tenant namespaces, not ' + request.namespace + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: compute-shell-agent-exec +spec: + policyName: compute-shell-agent-exec + validationActions: + - Deny diff --git a/config/components/shell-agent/kustomization.yaml b/config/components/shell-agent/kustomization.yaml index 6ef47a35..54f00965 100644 --- a/config/components/shell-agent/kustomization.yaml +++ b/config/components/shell-agent/kustomization.yaml @@ -20,6 +20,7 @@ kind: Component resources: - namespace.yaml - rbac.yaml + - exec-admission-policy.yaml - service.yaml - agent.yaml - endpoint.yaml diff --git a/config/components/shell-agent/rbac.yaml b/config/components/shell-agent/rbac.yaml index 45533da7..cdaf5b23 100644 --- a/config/components/shell-agent/rbac.yaml +++ b/config/components/shell-agent/rbac.yaml @@ -23,6 +23,9 @@ rules: - pods verbs: - get + # Tenant namespaces are created per project, so exec is granted cluster-wide + # and narrowed to tenant namespaces by the compute-shell-agent-exec + # ValidatingAdmissionPolicy. - apiGroups: - "" resources: From 0b3c8dd717fb8d5bd89de41f5e66c99e325cf560 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 29 Sep 2026 21:20:49 -0500 Subject: [PATCH 15/20] fix: Allow the shell agent out only to its hub The base Cilium policy let the agent reach any address on the internet over HTTPS and left narrowing it to each deployment. A deployment that forgot the patch shipped an agent holding cell and hub credentials with open egress. The base policy now allows the agent out of the cell only to one hub name, through Cilium's DNS proxy. The name is a placeholder that resolves nowhere, so an unset deployment reaches no hub instead of the whole internet, and deployments replace it with their hub's name. --- .../shell-agent-cilium/kustomization.yaml | 15 +++++++++++++++ config/components/shell-agent-cilium/policy.yaml | 14 ++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/config/components/shell-agent-cilium/kustomization.yaml b/config/components/shell-agent-cilium/kustomization.yaml index 34d6641d..d2f1fdf9 100644 --- a/config/components/shell-agent-cilium/kustomization.yaml +++ b/config/components/shell-agent-cilium/kustomization.yaml @@ -2,6 +2,21 @@ # config/components/shell-agent on cells that run Cilium. Unlike the portable # policies they police traffic to the node's own apiserver, kubelet and etcd, # so the endpoint, which holds no credential, cannot reach them at all. +# +# The agent may leave the cell only for the Karmada hub. Set the hub's name in +# the deployment, replacing the placeholder karmada-hub.invalid: +# +# - target: +# group: cilium.io +# kind: CiliumNetworkPolicy +# name: exec-agent +# patch: |- +# - op: test +# path: /spec/egress/1/toFQDNs/0/matchName +# value: karmada-hub.invalid +# - op: replace +# path: /spec/egress/1/toFQDNs/0/matchName +# value: karmada.example.net apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component diff --git a/config/components/shell-agent-cilium/policy.yaml b/config/components/shell-agent-cilium/policy.yaml index 7aae2d74..522cd2a8 100644 --- a/config/components/shell-agent-cilium/policy.yaml +++ b/config/components/shell-agent-cilium/policy.yaml @@ -33,14 +33,17 @@ spec: protocol: TCP - port: "6443" protocol: TCP - # The Karmada hub is outside the cell. Narrow this to the hub's name with - # toFQDNs where the deployment knows it. - - toEntities: - - world + # The Karmada hub, and nothing else outside the cell. The name is a + # placeholder that resolves nowhere, so a deployment that does not set its + # hub's name reaches no hub rather than the whole internet. + - toFQDNs: + - matchName: karmada-hub.invalid toPorts: - ports: - port: "443" protocol: TCP + # Cilium learns the hub's addresses from the agent's DNS answers, so DNS + # goes through its proxy. - toEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: kube-system @@ -51,6 +54,9 @@ spec: protocol: UDP - port: "53" protocol: TCP + rules: + dns: + - matchPattern: "*" --- apiVersion: cilium.io/v2 kind: CiliumNetworkPolicy From c2b3577f0194f0a4b1122ad4b0e3137f6c80ced9 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 30 Sep 2026 06:32:04 -0500 Subject: [PATCH 16/20] fix: Keep the shell agent off the Karmada hub No cell component may write to the Karmada hub any longer. The agent now claims, connects and ends sessions only on the cell copy, and Karmada's status reflection carries that status back to the hub copy. Karmada deletes a cell copy without waiting for it, so a deleted hub copy cannot report the session's end. The control plane therefore revokes an open session with an annotation first, and the agent records Revoked only after it has stopped the session's processes, which is what the control plane takes as confirmed cleanup. Key changes: - drop the hub client, its kubeconfig flags and the hub poll interval - claim with optimistic concurrency on the cell copy, and keep the slot of a claim that already stands when a lagging cache retries it - hold only the cell copy with the shell-agent finalizer - end sessions on the revoke annotation, and refuse connections to them - read the cell copy for the orphan sweep and for draining --- cmd/shell-agent/main.go | 48 ++-- internal/shellagent/agent.go | 17 +- internal/shellagent/drain.go | 6 +- internal/shellagent/harness_test.go | 93 +++----- internal/shellagent/reconciler.go | 294 ++++++++++++------------- internal/shellagent/reconciler_test.go | 258 ++++++++++++---------- internal/shellagent/server.go | 38 ++-- internal/shellagent/server_test.go | 21 +- internal/shellagent/stream_test.go | 4 +- internal/shellagent/sweep.go | 12 +- 10 files changed, 379 insertions(+), 412 deletions(-) diff --git a/cmd/shell-agent/main.go b/cmd/shell-agent/main.go index 4ea2a3ff..c2e6a1a3 100644 --- a/cmd/shell-agent/main.go +++ b/cmd/shell-agent/main.go @@ -4,7 +4,8 @@ // replica pairs by ordinal with a tunnel endpoint, which is the only way a // client reaches it: the endpoint proxies one TCP target, this agent's exec // port. The agent claims sessions delivered to the cell, runs their commands -// through the cell apiserver, and writes their status to the Karmada hub. +// through the cell apiserver, and writes their status on the cell's copies, +// which Karmada reflects back to the hub. It needs no hub credential. package main import ( @@ -23,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/clientcmd" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -39,19 +39,17 @@ import ( var setupLog = ctrl.Log.WithName("setup") type options struct { - listen string - probeAddr string - federationKubeconfig string - federationContext string - target string - relayURLs string - managedBy string - endpointPodPrefix string - featureGates string - drainTimeout time.Duration - killGrace time.Duration - pingInterval time.Duration - pongTimeout time.Duration + listen string + probeAddr string + target string + relayURLs string + managedBy string + endpointPodPrefix string + featureGates string + drainTimeout time.Duration + killGrace time.Duration + pingInterval time.Duration + pongTimeout time.Duration } func main() { @@ -59,10 +57,6 @@ func main() { var opts options flag.StringVar(&opts.listen, "listen", ":7777", "Address to serve session connections on.") flag.StringVar(&opts.probeAddr, "health-probe-bind-address", ":8081", "Address the probe endpoint binds to.") - flag.StringVar(&opts.federationKubeconfig, "federation-kubeconfig", "", - "Path to the kubeconfig for the Karmada hub, where session status is written.") - flag.StringVar(&opts.federationContext, "federation-context", "", - "Context to use from the federation kubeconfig. When omitted, the current context is used.") flag.StringVar(&opts.target, "target", "", "host:port the paired tunnel endpoint proxies to this agent, published as the session's target.") flag.StringVar(&opts.relayURLs, "relay-urls", os.Getenv("DATUM_CONNECT_RELAY_URLS"), @@ -112,9 +106,6 @@ func run(opts options, cfg shellagent.Config) error { if err != nil || namespace == "" { return fmt.Errorf("POD_NAME must be a StatefulSet pod name and POD_NAMESPACE must be set: %w", err) } - if opts.federationKubeconfig == "" { - return errors.New("--federation-kubeconfig is required: session status is written to the Karmada hub") - } cfg.Namespace = namespace cfg.Ordinal = ordinal cfg.EndpointPodName = fmt.Sprintf("%s-%d", opts.endpointPodPrefix, ordinal) @@ -131,13 +122,6 @@ func run(opts options, cfg shellagent.Config) error { utilruntime.Must(computev1alpha.AddToScheme(scheme)) cellConfig := ctrl.GetConfigOrDie() - hubConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: opts.federationKubeconfig}, - &clientcmd.ConfigOverrides{CurrentContext: opts.federationContext}, - ).ClientConfig() - if err != nil { - return fmt.Errorf("load federation kubeconfig: %w", err) - } mgr, err := ctrl.NewManager(cellConfig, ctrl.Options{ Scheme: scheme, @@ -151,17 +135,13 @@ func run(opts options, cfg shellagent.Config) error { if err != nil { return err } - hubClient, err := client.New(hubConfig, client.Options{Scheme: scheme}) - if err != nil { - return err - } executor, err := shellagent.NewAPIServer(cellConfig) if err != nil { return fmt.Errorf("configure pod exec: %w", err) } incarnation := podName + "/" + strconv.FormatInt(time.Now().UnixNano(), 36) - agent, err := shellagent.New(cfg, mgr.GetClient(), cellClient, hubClient, executor, incarnation) + agent, err := shellagent.New(cfg, mgr.GetClient(), cellClient, executor, incarnation) if err != nil { return err } diff --git a/internal/shellagent/agent.go b/internal/shellagent/agent.go index 9b22ad15..91f7e6c3 100644 --- a/internal/shellagent/agent.go +++ b/internal/shellagent/agent.go @@ -6,8 +6,9 @@ // command in the instance through the cell apiserver, and stops that command // when the session ends for any reason. // -// Every agent reads session copies from its cell and writes status to their -// hub copies, which the management plane copies back to the project. +// Every agent reads and writes only the session copies in its cell. Karmada +// reflects their status back to the hub, and the management plane copies it +// on to the project. The agent holds no hub credential. package shellagent import ( @@ -56,10 +57,6 @@ type Config struct { // pong, before the agent ends the session as Disconnected. It must be // longer than PingInterval. PongTimeout time.Duration - // HubPollInterval is how often the agent rereads the hub copy of a session - // that is open or still holds its finalizer. The hub lets a cell read - // sessions only by name, so there is nothing to watch. - HubPollInterval time.Duration // ExecTimeout bounds each command the agent runs in an instance to probe, // list or stop session processes, so a hung exec cannot hold a reconcile // worker or the sweep. Zero means KillGrace plus 30 seconds. @@ -79,7 +76,6 @@ func DefaultConfig() Config { SweepInterval: time.Minute, PingInterval: 10 * time.Second, PongTimeout: 20 * time.Second, - HubPollInterval: 10 * time.Second, } } @@ -89,7 +85,6 @@ type Agent struct { sessions client.Reader cell client.Client - hub client.Client exec Executor now func() time.Time @@ -106,9 +101,8 @@ type Agent struct { } // New returns an agent. sessions reads session copies and Instances from the -// cell, typically through a cache; cell reaches the cell apiserver directly; -// hub reaches the Karmada hub. -func New(cfg Config, sessions client.Reader, cell, hub client.Client, exec Executor, incarnation string) (*Agent, error) { +// cell, typically through a cache; cell reaches the cell apiserver directly. +func New(cfg Config, sessions client.Reader, cell client.Client, exec Executor, incarnation string) (*Agent, error) { if cfg.Namespace == "" || cfg.Target == "" || len(cfg.RelayURLs) == 0 || len(cfg.ManagedBy) == 0 { return nil, errors.New("namespace, target, relay URLs and managed-by values are required") } @@ -122,7 +116,6 @@ func New(cfg Config, sessions client.Reader, cell, hub client.Client, exec Execu cfg: cfg, sessions: sessions, cell: cell, - hub: hub, exec: exec, now: time.Now, incarnation: incarnation, diff --git a/internal/shellagent/drain.go b/internal/shellagent/drain.go index 94c17720..b8ce72ed 100644 --- a/internal/shellagent/drain.go +++ b/internal/shellagent/drain.go @@ -26,11 +26,11 @@ func (a *Agent) Drain(ctx context.Context) { if a.isLive(uid) { continue } - var hub computev1alpha.InstanceConsoleSession - if err := a.hub.Get(ctx, key, &hub); err != nil { + var session computev1alpha.InstanceConsoleSession + if err := a.cell.Get(ctx, key, &session); err != nil { continue } - if err := a.endUnconnected(ctx, &hub, computev1alpha.InstanceConsoleSessionReasonAgentShutdown); err != nil { + if err := a.endUnconnected(ctx, &session, computev1alpha.InstanceConsoleSessionReasonAgentShutdown); err != nil { logger.Error(err, "end unconnected session", "session", uid) } } diff --git a/internal/shellagent/harness_test.go b/internal/shellagent/harness_test.go index 58535450..9d87c7a4 100644 --- a/internal/shellagent/harness_test.go +++ b/internal/shellagent/harness_test.go @@ -10,7 +10,6 @@ import ( "crypto/sha1" //nolint:gosec // RFC 6455 derives the accept key with SHA-1 "encoding/base64" "encoding/json" - "fmt" "io" "net" "net/http" @@ -21,18 +20,15 @@ import ( coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/watch" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" computev1alpha "go.datum.net/compute/api/v1alpha" "go.datum.net/compute/internal/consolesession" @@ -71,7 +67,6 @@ type harness struct { t *testing.T ctx context.Context cell client.Client - hub client.WithWatch exec *fakeAPIServer clock *clock } @@ -124,38 +119,11 @@ func newHarness(t *testing.T) *harness { cell: fake.NewClientBuilder().WithScheme(scheme). WithStatusSubresource(session). WithObjects(instance, pod).Build(), - hub: fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(session).Build(), exec: newFakeAPIServer(), clock: &clock{t: time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC)}, } } -// cellHubAccess limits a hub client to what the hub grants a cell's agent: -// get and patch on sessions and writes to their status, and nothing else. -func cellHubAccess(hub client.WithWatch) client.WithWatch { - forbidden := func(verb string) error { - return apierrors.NewForbidden(computev1alpha.GroupVersion.WithResource("instanceconsolesessions").GroupResource(), - "", fmt.Errorf("shell agents may not %s sessions on the hub", verb)) - } - return interceptor.NewClient(hub, interceptor.Funcs{ - List: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) error { - return forbidden("list") - }, - Watch: func(context.Context, client.WithWatch, client.ObjectList, ...client.ListOption) (watch.Interface, error) { - return nil, forbidden("watch") - }, - Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { - return forbidden("create") - }, - Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error { - return forbidden("update") - }, - Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { - return forbidden("delete") - }, - }) -} - // agent returns an agent whose endpoint ID derives from a fresh key and whose // liveness Lease is current. func (h *harness) agent(mutate ...func(*Config)) *Agent { @@ -168,7 +136,7 @@ func (h *harness) agent(mutate ...func(*Config)) *Agent { for _, m := range mutate { m(&cfg) } - a, err := New(cfg, h.cell, h.cell, cellHubAccess(h.hub), h.exec, "test") + a, err := New(cfg, h.cell, h.cell, h.exec, "test") if err != nil { h.t.Fatal(err) } @@ -182,47 +150,44 @@ func (h *harness) agent(mutate ...func(*Config)) *Agent { return a } -// session creates a session's hub and cell copies and returns the client's -// key. +// session creates a session's cell copy and returns the client's key. func (h *harness) session(name, uid string, mutate ...func(*computev1alpha.InstanceConsoleSession)) ed25519.PrivateKey { h.t.Helper() _, key, _ := ed25519.GenerateKey(rand.Reader) - for _, c := range []client.Client{h.hub, h.cell} { - s := &computev1alpha.InstanceConsoleSession{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: testNamespace, - CreationTimestamp: metav1.NewTime(h.clock.now()), - Labels: map[string]string{ - computev1alpha.InstanceConsoleSessionUIDLabel: uid, - computev1alpha.InstanceConsoleSessionInstanceNameLabel: testInstance, - computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID, - }, - }, - Spec: computev1alpha.InstanceConsoleSessionSpec{ - InstanceRef: computev1alpha.InstanceConsoleSessionInstanceRef{Name: testInstance, UID: "project-instance-uid"}, - ContainerName: testContainer, - Command: []string{"sh"}, - Stdin: true, - Terminal: true, - ClientPublicKey: consolesession.PublicKey(key), - TTL: &metav1.Duration{Duration: 15 * time.Minute}, + s := &computev1alpha.InstanceConsoleSession{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + CreationTimestamp: metav1.NewTime(h.clock.now()), + Labels: map[string]string{ + computev1alpha.InstanceConsoleSessionUIDLabel: uid, + computev1alpha.InstanceConsoleSessionInstanceNameLabel: testInstance, + computev1alpha.WorkloadDeploymentUIDLabel: testDeploymentUID, }, - } - for _, m := range mutate { - m(s) - } - if err := c.Create(h.ctx, s); err != nil { - h.t.Fatal(err) - } + }, + Spec: computev1alpha.InstanceConsoleSessionSpec{ + InstanceRef: computev1alpha.InstanceConsoleSessionInstanceRef{Name: testInstance, UID: "project-instance-uid"}, + ContainerName: testContainer, + Command: []string{"sh"}, + Stdin: true, + Terminal: true, + ClientPublicKey: consolesession.PublicKey(key), + TTL: &metav1.Duration{Duration: 15 * time.Minute}, + }, + } + for _, m := range mutate { + m(s) + } + if err := h.cell.Create(h.ctx, s); err != nil { + h.t.Fatal(err) } return key } -func (h *harness) hubSession(name string) *computev1alpha.InstanceConsoleSession { +func (h *harness) cellSession(name string) *computev1alpha.InstanceConsoleSession { h.t.Helper() var s computev1alpha.InstanceConsoleSession - if err := h.hub.Get(h.ctx, client.ObjectKey{Namespace: testNamespace, Name: name}, &s); err != nil { + if err := h.cell.Get(h.ctx, client.ObjectKey{Namespace: testNamespace, Name: name}, &s); err != nil { h.t.Fatal(err) } return &s diff --git a/internal/shellagent/reconciler.go b/internal/shellagent/reconciler.go index ef344b09..2b516b6a 100644 --- a/internal/shellagent/reconciler.go +++ b/internal/shellagent/reconciler.go @@ -25,11 +25,8 @@ const ( cleanupRetryInterval = 5 * time.Second ) -// SetupWithManager reconciles session copies delivered to the cell. The hub -// grants each cell only get, patch and status writes on sessions, so the agent -// cannot watch hub copies; it reads a session's hub copy by the cell copy's -// namespace and name, and polls it while the session is open or holds the -// agent's finalizer. +// SetupWithManager reconciles session copies delivered to the cell. The agent +// writes status only on these copies; Karmada carries it back to the hub. func (a *Agent) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). Named("shell-agent"). @@ -38,73 +35,62 @@ func (a *Agent) SetupWithManager(mgr ctrl.Manager) error { Complete(a) } -// Reconcile drives one session: claiming it, ending it at its deadlines, -// taking it over from a lost agent, and stopping its processes before its -// finalizers are released. +// Reconcile drives one session: claiming it, ending it at its deadlines or +// when it is revoked, taking it over from a lost agent, and stopping its +// processes before its finalizer is released. func (a *Agent) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - var cell computev1alpha.InstanceConsoleSession - if err := a.sessions.Get(ctx, req.NamespacedName, &cell); err != nil { + var session computev1alpha.InstanceConsoleSession + if err := a.sessions.Get(ctx, req.NamespacedName, &session); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("session", sessionUID(&cell))) - - var hub computev1alpha.InstanceConsoleSession - err := a.hub.Get(ctx, req.NamespacedName, &hub) - if err != nil && !apierrors.IsNotFound(err) { - return ctrl.Result{}, err - } - if apierrors.IsNotFound(err) || sessionUID(&hub) != sessionUID(&cell) { - return a.cleanUp(ctx, &cell, nil) - } - res, err := a.reconcileSession(ctx, &cell, &hub) - poll := !isTerminal(&hub) || controllerutil.ContainsFinalizer(&hub, computev1alpha.InstanceConsoleSessionAgentFinalizer) - if err == nil && poll && (res.RequeueAfter <= 0 || res.RequeueAfter > a.cfg.HubPollInterval) { - res.RequeueAfter = a.cfg.HubPollInterval - } - return res, err + ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("session", sessionUID(&session))) + return a.reconcileSession(ctx, &session) } -func (a *Agent) reconcileSession(ctx context.Context, cell, hub *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { - uid := sessionUID(cell) - if !hub.DeletionTimestamp.IsZero() || !cell.DeletionTimestamp.IsZero() { - return a.cleanUp(ctx, cell, hub) +func (a *Agent) reconcileSession(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(session) + if !session.DeletionTimestamp.IsZero() { + return a.cleanUp(ctx, session) } if uid == "" { - return ctrl.Result{}, a.rejectUnclaimed(ctx, hub, reject(computev1alpha.InstanceConsoleSessionReasonInvalid, + return ctrl.Result{}, a.rejectUnclaimed(ctx, session, reject(computev1alpha.InstanceConsoleSessionReasonInvalid, "The session was delivered without its session UID.")) } - if isTerminal(hub) { + if revoked(session) && !isTerminal(session) { + return a.revoke(ctx, session) + } + if isTerminal(session) { a.forget(uid) - return a.settle(ctx, hub) + return a.settle(ctx, session) } - owner := endpointOf(hub) + owner := endpointOf(session) switch { case owner == "": - return a.claim(ctx, cell, hub) + return a.claim(ctx, session) case owner != a.EndpointID(): - return a.watchOwner(ctx, hub, owner) + return a.watchOwner(ctx, session, owner) } - a.track(uid, client.ObjectKeyFromObject(cell)) + a.track(uid, client.ObjectKeyFromObject(session)) now := a.now() - switch readyReason(hub) { + switch readyReason(session) { case computev1alpha.InstanceConsoleSessionReasonSessionReady: if a.draining.Load() { - return ctrl.Result{}, a.endUnconnected(ctx, hub, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + return ctrl.Result{}, a.endUnconnected(ctx, session, computev1alpha.InstanceConsoleSessionReasonAgentShutdown) } - if hub.Status.ConnectBefore != nil && now.Before(hub.Status.ConnectBefore.Time) { - return ctrl.Result{RequeueAfter: hub.Status.ConnectBefore.Sub(now)}, nil + if session.Status.ConnectBefore != nil && now.Before(session.Status.ConnectBefore.Time) { + return ctrl.Result{RequeueAfter: session.Status.ConnectBefore.Sub(now)}, nil } log.FromContext(ctx).Info("session was not connected in time") - return ctrl.Result{}, a.endUnconnected(ctx, hub, computev1alpha.InstanceConsoleSessionReasonNotConnected) + return ctrl.Result{}, a.endUnconnected(ctx, session, computev1alpha.InstanceConsoleSessionReasonNotConnected) case computev1alpha.InstanceConsoleSessionReasonConnected: if !a.isLive(uid) { log.FromContext(ctx).Info("ending a session this agent's previous run served") - return ctrl.Result{}, a.abandon(ctx, hub, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) + return ctrl.Result{}, a.abandon(ctx, session, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) } - if hub.Status.ExpiresAt != nil && now.Before(hub.Status.ExpiresAt.Time) { - return ctrl.Result{RequeueAfter: hub.Status.ExpiresAt.Sub(now)}, nil + if session.Status.ExpiresAt != nil && now.Before(session.Status.ExpiresAt.Time) { + return ctrl.Result{RequeueAfter: session.Status.ExpiresAt.Sub(now)}, nil } log.FromContext(ctx).Info("session expired") a.stop(uid, computev1alpha.InstanceConsoleSessionReasonExpired) @@ -113,11 +99,10 @@ func (a *Agent) reconcileSession(ctx context.Context, cell, hub *computev1alpha. } // claim checks a session and, if it can run, takes a slot and publishes where -// to connect. Agents and the control plane, which ends sessions no cell takes -// in time, race on the hub copy with optimistic concurrency, so a claim never -// lands on a session that has already ended. A cell that does not run the +// to connect. The cell's agents race on the cell copy with optimistic +// concurrency, so exactly one claim lands. A cell that does not run the // session's instance leaves it to the cell that does, or to the control plane. -func (a *Agent) claim(ctx context.Context, cell, hub *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { +func (a *Agent) claim(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { if !a.claiming() || a.openCount() >= a.cfg.MaxOpenSessions { return ctrl.Result{RequeueAfter: claimRetryInterval}, nil } @@ -128,7 +113,7 @@ func (a *Agent) claim(ctx context.Context, cell, hub *computev1alpha.InstanceCon if !ready { return ctrl.Result{RequeueAfter: claimRetryInterval}, nil } - target, rej, err := a.check(ctx, cell) + target, rej, err := a.check(ctx, session) if errors.Is(err, errNotInCell) { return ctrl.Result{RequeueAfter: claimRetryInterval}, nil } @@ -137,13 +122,13 @@ func (a *Agent) claim(ctx context.Context, cell, hub *computev1alpha.InstanceCon } if rej != nil { log.FromContext(ctx).Info("refusing session", "reason", rej.reason, "message", rej.message) - return ctrl.Result{}, a.rejectUnclaimed(ctx, hub, rej) + return ctrl.Result{}, a.rejectUnclaimed(ctx, session, rej) } - uid := sessionUID(hub) + uid := sessionUID(session) held, err := a.acquireSlot(ctx, target.instance.UID, &slot{ sessionUID: uid, - session: client.ObjectKeyFromObject(cell), + session: client.ObjectKeyFromObject(session), pod: client.ObjectKeyFromObject(target.pod), podUID: target.pod.UID, container: target.container, @@ -158,55 +143,60 @@ func (a *Agent) claim(ctx context.Context, cell, hub *computev1alpha.InstanceCon return ctrl.Result{}, err } if held == nil { - return ctrl.Result{}, a.rejectUnclaimed(ctx, hub, reject( + return ctrl.Result{}, a.rejectUnclaimed(ctx, session, reject( computev1alpha.InstanceConsoleSessionReasonTooManySessions, "The instance already has %d open sessions, the most it allows.", a.cfg.SlotsPerInstance)) } - for _, f := range []struct { - c client.Client - s *computev1alpha.InstanceConsoleSession - }{{a.hub, hub}, {a.cell, cell}} { - if err := addFinalizer(ctx, f.c, f.s); err != nil { - if relErr := a.releaseSlot(ctx, held); relErr != nil { - return ctrl.Result{}, relErr - } - if apierrors.IsConflict(err) { - return ctrl.Result{RequeueAfter: time.Second}, nil - } - return ctrl.Result{}, err - } + claimed := session.DeepCopy() + if err := addFinalizer(ctx, a.cell, claimed); err != nil { + return a.abortClaim(ctx, client.ObjectKeyFromObject(session), held, err) } - now := a.now() connectBefore := metav1.NewTime(now.Add(a.cfg.ConnectTimeout)) - hub.Status.Connection = &computev1alpha.InstanceConsoleSessionConnection{ + claimed.Status.Connection = &computev1alpha.InstanceConsoleSessionConnection{ EndpointID: a.EndpointID(), RelayURLs: a.cfg.RelayURLs, Target: a.cfg.Target, } - hub.Status.ConnectBefore = &connectBefore - setReady(hub, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonSessionReady, + claimed.Status.ConnectBefore = &connectBefore + setReady(claimed, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonSessionReady, "Connect before the connection deadline.") - if err := a.hub.Status().Update(ctx, hub); err != nil { - if relErr := a.releaseSlot(ctx, held); relErr != nil { - return ctrl.Result{}, relErr + if err := a.cell.Status().Update(ctx, claimed); err != nil { + return a.abortClaim(ctx, client.ObjectKeyFromObject(session), held, err) + } + a.track(uid, client.ObjectKeyFromObject(session)) + log.FromContext(ctx).Info("claimed session", "pod", target.pod.Name, "container", target.container) + return ctrl.Result{RequeueAfter: a.cfg.ConnectTimeout}, nil +} + +// abortClaim frees the slot of a claim that did not land. A conflict means +// the session changed, most likely because another agent claimed it, and the +// retry sees the change. The cache can also lag this agent's own claim, and +// the slot of a claim that already stands is kept. +func (a *Agent) abortClaim(ctx context.Context, key types.NamespacedName, held *slot, err error) (ctrl.Result, error) { + if apierrors.IsConflict(err) { + var current computev1alpha.InstanceConsoleSession + if getErr := a.cell.Get(ctx, key, ¤t); getErr != nil { + return ctrl.Result{}, client.IgnoreNotFound(getErr) } - if apierrors.IsConflict(err) { + if endpointOf(¤t) == a.EndpointID() && !isTerminal(¤t) { return ctrl.Result{RequeueAfter: time.Second}, nil } - return ctrl.Result{}, err } - a.track(uid, client.ObjectKeyFromObject(cell)) - log.FromContext(ctx).Info("claimed session", "pod", target.pod.Name, "container", target.container) - return ctrl.Result{RequeueAfter: a.cfg.ConnectTimeout}, nil + if relErr := a.releaseSlot(ctx, held); relErr != nil { + return ctrl.Result{}, relErr + } + if apierrors.IsConflict(err) { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return ctrl.Result{}, err } -// watchOwner takes over a session in this cell whose agent stopped renewing -// its liveness Lease. A session claimed in another cell has no slot here and -// is left alone. -func (a *Agent) watchOwner(ctx context.Context, hub *computev1alpha.InstanceConsoleSession, owner string) (ctrl.Result, error) { - held, err := a.slotFor(ctx, sessionUID(hub)) +// watchOwner takes over a session whose agent stopped renewing its liveness +// Lease. A session with no slot has nothing left to take over. +func (a *Agent) watchOwner(ctx context.Context, session *computev1alpha.InstanceConsoleSession, owner string) (ctrl.Result, error) { + held, err := a.slotFor(ctx, sessionUID(session)) if err != nil || held == nil { return ctrl.Result{}, err } @@ -218,14 +208,14 @@ func (a *Agent) watchOwner(ctx context.Context, hub *computev1alpha.InstanceCons return ctrl.Result{RequeueAfter: LeaseDuration}, nil } log.FromContext(ctx).Info("taking over a session from a lost agent", "agent", owner) - return ctrl.Result{}, a.abandon(ctx, hub, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) + return ctrl.Result{}, a.abandon(ctx, session, owner, computev1alpha.InstanceConsoleSessionReasonAgentLost) } // abandon ends a session no agent is serving: it stops the session's process, // which a dropped stream leaves running, and records why. The slot is kept if // the process could not be confirmed stopped, for the sweep to retry. -func (a *Agent) abandon(ctx context.Context, hub *computev1alpha.InstanceConsoleSession, owner, reason string) error { - uid := sessionUID(hub) +func (a *Agent) abandon(ctx context.Context, session *computev1alpha.InstanceConsoleSession, owner, reason string) error { + uid := sessionUID(session) held, err := a.slotFor(ctx, uid) if err != nil { return err @@ -236,7 +226,7 @@ func (a *Agent) abandon(ctx context.Context, hub *computev1alpha.InstanceConsole return err } } - if _, err := a.end(ctx, client.ObjectKeyFromObject(hub), reason, endMessage(reason), nil, + if _, err := a.end(ctx, client.ObjectKeyFromObject(session), reason, endMessage(reason), nil, func(s *computev1alpha.InstanceConsoleSession) bool { return endpointOf(s) == owner }); err != nil { return err } @@ -249,10 +239,10 @@ func (a *Agent) abandon(ctx context.Context, hub *computev1alpha.InstanceConsole // endUnconnected ends a session this agent claimed but no client connected // to. No process was started, so its slot is freed at once. -func (a *Agent) endUnconnected(ctx context.Context, hub *computev1alpha.InstanceConsoleSession, reason string) error { - uid := sessionUID(hub) +func (a *Agent) endUnconnected(ctx context.Context, session *computev1alpha.InstanceConsoleSession, reason string) error { + uid := sessionUID(session) me := a.EndpointID() - ended, err := a.end(ctx, client.ObjectKeyFromObject(hub), reason, endMessage(reason), nil, + ended, err := a.end(ctx, client.ObjectKeyFromObject(session), reason, endMessage(reason), nil, func(s *computev1alpha.InstanceConsoleSession) bool { return endpointOf(s) == me && readyReason(s) == computev1alpha.InstanceConsoleSessionReasonSessionReady }) @@ -267,16 +257,16 @@ func (a *Agent) endUnconnected(ctx context.Context, hub *computev1alpha.Instance return a.releaseSlot(ctx, held) } -func (a *Agent) rejectUnclaimed(ctx context.Context, hub *computev1alpha.InstanceConsoleSession, rej *rejection) error { - _, err := a.end(ctx, client.ObjectKeyFromObject(hub), rej.reason, rej.message, nil, +func (a *Agent) rejectUnclaimed(ctx context.Context, session *computev1alpha.InstanceConsoleSession, rej *rejection) error { + _, err := a.end(ctx, client.ObjectKeyFromObject(session), rej.reason, rej.message, nil, func(s *computev1alpha.InstanceConsoleSession) bool { return s.Status.Connection == nil }) return err } // settle frees the slot of an ended session once its process is stopped, // unless a live agent is still ending it. -func (a *Agent) settle(ctx context.Context, hub *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { - uid := sessionUID(hub) +func (a *Agent) settle(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(session) if a.isLive(uid) { return ctrl.Result{RequeueAfter: cleanupRetryInterval}, nil } @@ -284,7 +274,7 @@ func (a *Agent) settle(ctx context.Context, hub *computev1alpha.InstanceConsoleS if err != nil || held == nil { return ctrl.Result{}, err } - if owner := endpointOf(hub); owner != "" && owner != a.EndpointID() { + if owner := endpointOf(session); owner != "" && owner != a.EndpointID() { if alive, err := a.agentAlive(ctx, owner); err != nil || alive { return ctrl.Result{RequeueAfter: LeaseDuration}, err } @@ -296,52 +286,54 @@ func (a *Agent) settle(ctx context.Context, hub *computev1alpha.InstanceConsoleS return ctrl.Result{}, nil } -// cleanUp handles a session whose hub or cell copy is being deleted, or whose -// hub copy is gone. The session is revoked if it is still open, and the -// finalizers are released only once its process is stopped. -func (a *Agent) cleanUp(ctx context.Context, cell, hub *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { - uid := sessionUID(cell) - me := a.EndpointID() - responsible := true - if hub == nil && a.isLive(uid) { - a.stop(uid, computev1alpha.InstanceConsoleSessionReasonRevoked) - } - if hub != nil && !isTerminal(hub) { - switch owner := endpointOf(hub); { - case owner == me && a.isLive(uid): - log.FromContext(ctx).Info("revoking session") - a.stop(uid, computev1alpha.InstanceConsoleSessionReasonRevoked) - case owner == me: - if err := a.endUnconnected(ctx, hub, computev1alpha.InstanceConsoleSessionReasonRevoked); err != nil { - return ctrl.Result{}, err - } - case owner == "": - if err := a.rejectUnclaimed(ctx, hub, reject(computev1alpha.InstanceConsoleSessionReasonRevoked, - "%s", endMessage(computev1alpha.InstanceConsoleSessionReasonRevoked))); err != nil { - return ctrl.Result{}, err - } - default: - held, err := a.slotFor(ctx, uid) - if err != nil { - return ctrl.Result{}, err - } - if held == nil { - responsible = false - break - } - alive, err := a.agentAlive(ctx, owner) - if err != nil { - return ctrl.Result{}, err - } - if alive { - return ctrl.Result{RequeueAfter: cleanupRetryInterval}, nil - } - if err := a.abandon(ctx, hub, owner, computev1alpha.InstanceConsoleSessionReasonRevoked); err != nil { - return ctrl.Result{}, err - } +// revoke ends a session that is still open because the control plane revoked +// it or its copy is being deleted. The end is recorded only once the +// session's processes have been stopped, so the control plane can take it as +// confirmation that the command is gone. +func (a *Agent) revoke(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(session) + reason := computev1alpha.InstanceConsoleSessionReasonRevoked + switch owner := endpointOf(session); { + case owner == a.EndpointID() && a.isLive(uid): + log.FromContext(ctx).Info("revoking session") + a.stop(uid, reason) + case owner == a.EndpointID(): + if err := a.endUnconnected(ctx, session, reason); err != nil { + return ctrl.Result{}, err + } + case owner == "": + if err := a.rejectUnclaimed(ctx, session, reject(reason, "%s", endMessage(reason))); err != nil { + return ctrl.Result{}, err + } + default: + held, err := a.slotFor(ctx, uid) + if err != nil || held == nil { + return ctrl.Result{}, err + } + alive, err := a.agentAlive(ctx, owner) + if err != nil { + return ctrl.Result{}, err + } + if alive { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, nil + } + if err := a.abandon(ctx, session, owner, reason); err != nil { + return ctrl.Result{}, err } } + return ctrl.Result{}, nil +} +// cleanUp handles a session whose cell copy is being deleted. The session is +// revoked if it is still open, and the finalizer is released only once its +// process is stopped. +func (a *Agent) cleanUp(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { + uid := sessionUID(session) + if !isTerminal(session) { + if res, err := a.revoke(ctx, session); err != nil || res.RequeueAfter > 0 { + return res, err + } + } if a.isLive(uid) { return ctrl.Result{RequeueAfter: time.Second}, nil } @@ -356,19 +348,10 @@ func (a *Agent) cleanUp(ctx context.Context, cell, hub *computev1alpha.InstanceC } } a.forget(uid) - - if hub != nil && responsible { - if err := removeFinalizer(ctx, a.hub, client.ObjectKeyFromObject(hub)); err != nil { - return ctrl.Result{}, err - } - } - if !cell.DeletionTimestamp.IsZero() || hub == nil { - return ctrl.Result{}, removeFinalizer(ctx, a.cell, client.ObjectKeyFromObject(cell)) - } - return ctrl.Result{}, nil + return ctrl.Result{}, removeFinalizer(ctx, a.cell, client.ObjectKeyFromObject(session)) } -// end records a terminal reason on the hub copy if the session has not ended +// end records a terminal reason on the cell copy if the session has not ended // yet and guard still holds for its latest state. It reports whether this // call ended the session. func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, message string, exitCode *int32, @@ -376,7 +359,7 @@ func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, messa ended := false err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { var s computev1alpha.InstanceConsoleSession - if err := a.hub.Get(ctx, key, &s); err != nil { + if err := a.cell.Get(ctx, key, &s); err != nil { return err } if isTerminal(&s) || guard != nil && !guard(&s) { @@ -386,7 +369,7 @@ func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, messa s.Status.EndedAt = &now s.Status.ExitCode = exitCode setReady(&s, metav1.ConditionFalse, reason, message) - if err := a.hub.Status().Update(ctx, &s); err != nil { + if err := a.cell.Status().Update(ctx, &s); err != nil { return err } ended = true @@ -395,6 +378,13 @@ func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, messa return ended, client.IgnoreNotFound(err) } +// revoked reports whether the control plane asked the cell to end the +// session. +func revoked(s *computev1alpha.InstanceConsoleSession) bool { + _, ok := s.Annotations[computev1alpha.InstanceConsoleSessionRevokeAnnotation] + return ok +} + func addFinalizer(ctx context.Context, c client.Client, s *computev1alpha.InstanceConsoleSession) error { if controllerutil.ContainsFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { return nil diff --git a/internal/shellagent/reconciler_test.go b/internal/shellagent/reconciler_test.go index 9ded5dc7..1d0c97ea 100644 --- a/internal/shellagent/reconciler_test.go +++ b/internal/shellagent/reconciler_test.go @@ -26,12 +26,12 @@ import ( func TestClaimPublishesConnection(t *testing.T) { h := newHarness(t) - a := h.agent(noPolling) + a := h.agent() h.session(testSession, testUID) res := h.reconcile(a, testSession) - s := h.hubSession(testSession) + s := h.cellSession(testSession) requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonSessionReady) if s.Status.Connection == nil || s.Status.Connection.EndpointID != a.EndpointID() || s.Status.Connection.Target != a.cfg.Target || len(s.Status.Connection.RelayURLs) != 1 { @@ -44,11 +44,6 @@ func TestClaimPublishesConnection(t *testing.T) { t.Fatalf("requeue after %v, want the connect timeout", res.RequeueAfter) } if !controllerutil.ContainsFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { - t.Fatal("hub copy has no finalizer") - } - var cell computev1alpha.InstanceConsoleSession - _ = h.cell.Get(h.ctx, client.ObjectKeyFromObject(s), &cell) - if !controllerutil.ContainsFinalizer(&cell, computev1alpha.InstanceConsoleSessionAgentFinalizer) { t.Fatal("cell copy has no finalizer") } if n := len(h.slots()); n != 1 { @@ -71,7 +66,7 @@ func TestClaimRaceHasOneWinner(t *testing.T) { } wg.Wait() - s := h.hubSession(testSession) + s := h.cellSession(testSession) requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonSessionReady) owners := 0 for _, a := range agents { @@ -91,16 +86,14 @@ func TestClaimWithStaleCopyLosesToWinner(t *testing.T) { h := newHarness(t) winner, loser := h.agent(), h.agent() h.session(testSession, testUID) - stale := h.hubSession(testSession) - var cell computev1alpha.InstanceConsoleSession - _ = h.cell.Get(h.ctx, client.ObjectKeyFromObject(stale), &cell) + stale := h.cellSession(testSession) h.reconcile(winner, testSession) - if _, err := loser.claim(h.ctx, &cell, stale); err != nil { + if _, err := loser.claim(h.ctx, stale); err != nil { t.Logf("losing claim returned %v", err) } - s := h.hubSession(testSession) + s := h.cellSession(testSession) if s.Status.Connection.EndpointID != winner.EndpointID() { t.Fatal("a stale claim overwrote the winner's connection") } @@ -110,6 +103,26 @@ func TestClaimWithStaleCopyLosesToWinner(t *testing.T) { } } +func TestStaleRetryKeepsTheAgentsOwnClaim(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + stale := h.cellSession(testSession) + h.reconcile(a, testSession) + + if _, err := a.claim(h.ctx, stale); err != nil { + t.Fatalf("retried claim returned %v", err) + } + + s := h.cellSession(testSession) + if s.Status.Connection == nil || s.Status.Connection.EndpointID != a.EndpointID() { + t.Fatalf("connection = %+v", s.Status.Connection) + } + if n := len(h.slots()); n != 1 { + t.Fatalf("slots = %d, want the claim's slot kept", n) + } +} + func TestSlotCapIsExactlyThree(t *testing.T) { h := newHarness(t) agents := []*Agent{h.agent(), h.agent()} @@ -130,7 +143,7 @@ func TestSlotCapIsExactlyThree(t *testing.T) { counts := map[string]int{} for i := range 6 { - counts[readyReason(h.hubSession(fmt.Sprintf("s%d", i)))]++ + counts[readyReason(h.cellSession(fmt.Sprintf("s%d", i)))]++ } if counts[computev1alpha.InstanceConsoleSessionReasonSessionReady] != 3 || counts[computev1alpha.InstanceConsoleSessionReasonTooManySessions] != 3 { @@ -150,15 +163,15 @@ func TestSlotFreedByEndedSessionIsReused(t *testing.T) { for i := range 3 { h.reconcile(a, fmt.Sprintf("s%d", i)) } - s0 := h.hubSession("s0") + s0 := h.cellSession("s0") setReady(s0, metav1.ConditionFalse, computev1alpha.InstanceConsoleSessionReasonCompleted, "done") - if err := h.hub.Status().Update(h.ctx, s0); err != nil { + if err := h.cell.Status().Update(h.ctx, s0); err != nil { t.Fatal(err) } h.reconcile(a, "s3") - requireReason(t, h.hubSession("s3"), computev1alpha.InstanceConsoleSessionReasonSessionReady) + requireReason(t, h.cellSession("s3"), computev1alpha.InstanceConsoleSessionReasonSessionReady) } func TestChecksBeforeClaim(t *testing.T) { @@ -201,7 +214,7 @@ func TestChecksBeforeClaim(t *testing.T) { h.reconcile(a, testSession) - s := h.hubSession(testSession) + s := h.cellSession(testSession) requireReason(t, s, tc.want) if s.Status.Connection != nil || len(h.slots()) != 0 { t.Fatal("a refused session was claimed") @@ -224,58 +237,64 @@ func TestSessionForAnotherCellIsIgnored(t *testing.T) { h.clock.advance(10 * time.Minute) } - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) if len(h.slots()) != 0 { t.Fatal("a cell without the instance took a slot") } } -// endUnavailable ends a session the way the control plane does when no cell -// claims it in time. -func endUnavailable(t *testing.T, h *harness) { +// revokeSession annotates the cell copy the way Karmada does once the control +// plane revokes the session's hub copy. +func revokeSession(t *testing.T, h *harness) { t.Helper() - s := h.hubSession(testSession) - setReady(s, metav1.ConditionFalse, computev1alpha.InstanceConsoleSessionReasonUnavailable, "No cell took the session.") - if err := h.hub.Status().Update(h.ctx, s); err != nil { + s := h.cellSession(testSession) + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + s.Annotations[computev1alpha.InstanceConsoleSessionRevokeAnnotation] = h.clock.now().Format(time.RFC3339) + if err := h.cell.Update(h.ctx, s); err != nil { t.Fatal(err) } } -func requireUnclaimed(t *testing.T, h *harness) { +func requireRevokedUnclaimed(t *testing.T, h *harness) { t.Helper() - s := h.hubSession(testSession) - requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonUnavailable) + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonRevoked) if s.Status.Connection != nil || len(h.slots()) != 0 || controllerutil.ContainsFinalizer(s, computev1alpha.InstanceConsoleSessionAgentFinalizer) { - t.Fatalf("an ended session was claimed: %+v", s) + t.Fatalf("a revoked session was claimed: %+v", s) } } -func TestEndedSessionIsNotClaimed(t *testing.T) { +func TestRevokedSessionIsNotClaimed(t *testing.T) { h := newHarness(t) a := h.agent() h.session(testSession, testUID) - endUnavailable(t, h) + revokeSession(t, h) h.reconcile(a, testSession) - requireUnclaimed(t, h) + requireRevokedUnclaimed(t, h) } -func TestClaimLosesToControlPlaneEnding(t *testing.T) { +func TestClaimLosesToRevoke(t *testing.T) { h := newHarness(t) a := h.agent() h.session(testSession, testUID) - stale := h.hubSession(testSession) - var cell computev1alpha.InstanceConsoleSession - _ = h.cell.Get(h.ctx, client.ObjectKeyFromObject(stale), &cell) - endUnavailable(t, h) + stale := h.cellSession(testSession) + revokeSession(t, h) - if _, err := a.claim(h.ctx, &cell, stale); err != nil { + if _, err := a.claim(h.ctx, stale); err != nil { t.Logf("losing claim returned %v", err) } + s := h.cellSession(testSession) + if s.Status.Connection != nil || len(h.slots()) != 0 { + t.Fatalf("a claim read before the revoke landed: %+v", s) + } - requireUnclaimed(t, h) + h.reconcile(a, testSession) + requireRevokedUnclaimed(t, h) } func TestProbeIsCachedPerPodAndContainer(t *testing.T) { @@ -301,7 +320,7 @@ func TestNotConnectedDeadline(t *testing.T) { h.clock.advance(time.Minute) h.reconcile(a, testSession) - s := h.hubSession(testSession) + s := h.cellSession(testSession) requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonNotConnected) if s.Status.EndedAt == nil || s.Status.ExitCode != nil { t.Fatalf("status = %+v", s.Status) @@ -316,7 +335,7 @@ func TestNotConnectedDeadline(t *testing.T) { func TestExpiryStopsConnectedSession(t *testing.T) { h := newHarness(t) - a := h.agent(noPolling) + a := h.agent() h.session(testSession, testUID) h.reconcile(a, testSession) markConnected(h, testSession, h.clock.now().Add(time.Minute)) @@ -335,7 +354,7 @@ func TestExpiryStopsConnectedSession(t *testing.T) { func TestLivenessTakeover(t *testing.T) { h := newHarness(t) - lost, survivor := h.agent(noPolling), h.agent(noPolling) + lost, survivor := h.agent(), h.agent() h.session(testSession, testUID) h.reconcile(lost, testSession) markConnected(h, testSession, h.clock.now().Add(time.Hour)) @@ -343,7 +362,7 @@ func TestLivenessTakeover(t *testing.T) { if res := h.reconcile(survivor, testSession); res.RequeueAfter != LeaseDuration { t.Fatalf("requeue after %v while the owner is alive, want %v", res.RequeueAfter, LeaseDuration) } - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) h.clock.advance(LeaseDuration + time.Second) if err := survivor.renewLease(h.ctx); err != nil { @@ -351,7 +370,7 @@ func TestLivenessTakeover(t *testing.T) { } h.reconcile(survivor, testSession) - s := h.hubSession(testSession) + s := h.cellSession(testSession) requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonAgentLost) if h.exec.ran(killScript) != 1 { t.Fatal("the lost session's processes were not stopped") @@ -370,60 +389,105 @@ func TestRestartedAgentEndsItsPreviousSessions(t *testing.T) { h.reconcile(a, testSession) - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonAgentLost) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonAgentLost) if h.exec.ran(killScript) != 1 { t.Fatal("the previous run's processes were not stopped") } } -func TestSessionClaimedInAnotherCellIsLeftAlone(t *testing.T) { +func TestSessionOwnedByAnotherAgentWithoutSlotIsLeftAlone(t *testing.T) { h := newHarness(t) a := h.agent() h.session(testSession, testUID) - s := h.hubSession(testSession) - s.Status.Connection = &computev1alpha.InstanceConsoleSessionConnection{EndpointID: "another-cells-endpoint"} + s := h.cellSession(testSession) + s.Status.Connection = &computev1alpha.InstanceConsoleSessionConnection{EndpointID: "another-agents-endpoint"} setReady(s, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonSessionReady, "ready") - if err := h.hub.Status().Update(h.ctx, s); err != nil { + if err := h.cell.Status().Update(h.ctx, s); err != nil { t.Fatal(err) } h.reconcile(a, testSession) - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) } -func TestDeletionReleasesFinalizersAfterProcessesStop(t *testing.T) { +func TestRevokeStopsConnectedSession(t *testing.T) { h := newHarness(t) a := h.agent() h.session(testSession, testUID) h.reconcile(a, testSession) markConnected(h, testSession, h.clock.now().Add(time.Hour)) stopped := fakeLive(a, testUID, false) + revokeSession(t, h) - if err := h.hub.Delete(h.ctx, h.hubSession(testSession)); err != nil { - t.Fatal(err) - } h.reconcile(a, testSession) + if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonRevoked { t.Fatalf("stop reason = %q, want Revoked", got) } + if !controllerutil.ContainsFinalizer(h.cellSession(testSession), computev1alpha.InstanceConsoleSessionAgentFinalizer) { + t.Fatal("the finalizer holds the copy until the control plane deletes it") + } +} + +func TestRevokeEndsUnconnectedSessionAndFreesItsSlot(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + revokeSession(t, h) + + h.reconcile(a, testSession) - var hub computev1alpha.InstanceConsoleSession - if err := h.hub.Get(h.ctx, reqFor(testSession).NamespacedName, &hub); err == nil { - t.Fatalf("hub copy still exists with finalizers %v", hub.Finalizers) + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonRevoked) + if s.Status.EndedAt == nil { + t.Fatal("the end has no time") + } + if len(h.slots()) != 0 { + t.Fatal("slot kept after the revoke") } +} + +func TestRevokeOfLostAgentsSessionStopsProcessesFirst(t *testing.T) { + h := newHarness(t) + lost, survivor := h.agent(), h.agent() + h.session(testSession, testUID) + h.reconcile(lost, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + lost.releaseLease(h.ctx, lost.EndpointID()) + revokeSession(t, h) + + h.reconcile(survivor, testSession) + + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonRevoked) if h.exec.ran(killScript) == 0 || len(h.slots()) != 0 { - t.Fatal("processes were not stopped before the finalizer was released") + t.Fatal("the end was reported before the lost agent's processes were stopped") } +} - var cell computev1alpha.InstanceConsoleSession - _ = h.cell.Get(h.ctx, reqFor(testSession).NamespacedName, &cell) - if err := h.cell.Delete(h.ctx, &cell); err != nil { +func TestDeletionReleasesFinalizerAfterProcessesStop(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + stopped := fakeLive(a, testUID, false) + + if err := h.cell.Delete(h.ctx, h.cellSession(testSession)); err != nil { t.Fatal(err) } h.reconcile(a, testSession) - if err := h.cell.Get(h.ctx, reqFor(testSession).NamespacedName, &cell); err == nil { - t.Fatalf("cell copy still exists with finalizers %v", cell.Finalizers) + if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonRevoked { + t.Fatalf("stop reason = %q, want Revoked", got) + } + + var cell computev1alpha.InstanceConsoleSession + if err := h.cell.Get(h.ctx, reqFor(testSession).NamespacedName, &cell); !apierrors.IsNotFound(err) { + t.Fatalf("cell copy still exists with finalizers %v: %v", cell.Finalizers, err) + } + if h.exec.ran(killScript) == 0 || len(h.slots()) != 0 { + t.Fatal("processes were not stopped before the finalizer was released") } } @@ -436,21 +500,21 @@ func TestFinalizerHeldWhileProcessesCannotBeStopped(t *testing.T) { endAs(h, computev1alpha.InstanceConsoleSessionReasonAgentLost) h.exec.killExit = 1 - if err := h.hub.Delete(h.ctx, h.hubSession(testSession)); err != nil { + if err := h.cell.Delete(h.ctx, h.cellSession(testSession)); err != nil { t.Fatal(err) } if res := h.reconcile(a, testSession); res.RequeueAfter == 0 { t.Fatal("expected a retry while processes may be running") } - if !controllerutil.ContainsFinalizer(h.hubSession(testSession), computev1alpha.InstanceConsoleSessionAgentFinalizer) { + if !controllerutil.ContainsFinalizer(h.cellSession(testSession), computev1alpha.InstanceConsoleSessionAgentFinalizer) { t.Fatal("finalizer released before processes were stopped") } h.exec.killExit = 0 h.reconcile(a, testSession) - var hub computev1alpha.InstanceConsoleSession - if err := h.hub.Get(h.ctx, reqFor(testSession).NamespacedName, &hub); err == nil { - t.Fatal("hub copy kept after processes stopped") + var cell computev1alpha.InstanceConsoleSession + if err := h.cell.Get(h.ctx, reqFor(testSession).NamespacedName, &cell); !apierrors.IsNotFound(err) { + t.Fatalf("cell copy kept after processes stopped: %v", err) } } @@ -460,13 +524,13 @@ func TestUnclaimedDeletionIsRevoked(t *testing.T) { h.session(testSession, testUID, func(s *computev1alpha.InstanceConsoleSession) { s.Finalizers = []string{"test/hold"} }) - if err := h.hub.Delete(h.ctx, h.hubSession(testSession)); err != nil { + if err := h.cell.Delete(h.ctx, h.cellSession(testSession)); err != nil { t.Fatal(err) } h.reconcile(a, testSession) - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonRevoked) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonRevoked) } func TestDrain(t *testing.T) { @@ -485,7 +549,7 @@ func TestDrain(t *testing.T) { a.Drain(h.ctx) - requireReason(t, h.hubSession("waiting"), computev1alpha.InstanceConsoleSessionReasonAgentShutdown) + requireReason(t, h.cellSession("waiting"), computev1alpha.InstanceConsoleSessionReasonAgentShutdown) for _, stopped := range []chan string{ttyStopped, pipeStopped} { if got := <-stopped; got != computev1alpha.InstanceConsoleSessionReasonAgentShutdown { t.Fatalf("stop reason = %q, want AgentShutdown", got) @@ -502,7 +566,7 @@ func TestDrain(t *testing.T) { } h.session("late", "uid-l") h.reconcile(a, "late") - requireReason(t, h.hubSession("late"), computev1alpha.InstanceConsoleSessionReasonPending) + requireReason(t, h.cellSession("late"), computev1alpha.InstanceConsoleSessionReasonPending) } func TestConcurrencyCapDefersClaims(t *testing.T) { @@ -514,7 +578,7 @@ func TestConcurrencyCapDefersClaims(t *testing.T) { h.reconcile(a, testSession) res := h.reconcile(a, "s2") - requireReason(t, h.hubSession("s2"), computev1alpha.InstanceConsoleSessionReasonPending) + requireReason(t, h.cellSession("s2"), computev1alpha.InstanceConsoleSessionReasonPending) if res.RequeueAfter == 0 { t.Fatal("a deferred claim must be retried") } @@ -526,7 +590,7 @@ func TestClaimWaitsForReadyEndpoint(t *testing.T) { h.session(testSession, testUID) res := h.reconcile(a, testSession) - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) if res.RequeueAfter == 0 { t.Fatal("a claim deferred for a missing endpoint must be retried") } @@ -536,14 +600,14 @@ func TestClaimWaitsForReadyEndpoint(t *testing.T) { t.Fatal(err) } h.reconcile(a, testSession) - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonPending) endpoint.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} if err := h.cell.Status().Update(h.ctx, endpoint); err != nil { t.Fatal(err) } h.reconcile(a, testSession) - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) } func TestSweepStopsOrphanedProcesses(t *testing.T) { @@ -645,21 +709,21 @@ func reqFor(name string) ctrl.Request { func markConnected(h *harness, name string, expires time.Time) { h.t.Helper() - s := h.hubSession(name) + s := h.cellSession(name) started := metav1.NewTime(h.clock.now()) s.Status.StartedAt = &started s.Status.ExpiresAt = &metav1.Time{Time: expires} setReady(s, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonConnected, "connected") - if err := h.hub.Status().Update(h.ctx, s); err != nil { + if err := h.cell.Status().Update(h.ctx, s); err != nil { h.t.Fatal(err) } } func endAs(h *harness, reason string) { h.t.Helper() - s := h.hubSession(testSession) + s := h.cellSession(testSession) setReady(s, metav1.ConditionFalse, reason, "ended") - if err := h.hub.Status().Update(h.ctx, s); err != nil { + if err := h.cell.Status().Update(h.ctx, s); err != nil { h.t.Fatal(err) } } @@ -725,41 +789,11 @@ func patchPod(h *harness, mutate func(*corev1.Pod)) { } } -// noPolling leaves a session's deadlines as the only reason to requeue it. -func noPolling(c *Config) { - c.HubPollInterval = 24 * time.Hour -} - -func TestOpenSessionsPollTheirHubCopy(t *testing.T) { - h := newHarness(t) - a := h.agent() - h.session(testSession, testUID) - - if res := h.reconcile(a, testSession); res.RequeueAfter != a.cfg.HubPollInterval { - t.Fatalf("requeue after %v for a claimed session, want the hub poll interval", res.RequeueAfter) - } - s := h.hubSession(testSession) - now := metav1.NewTime(h.clock.now()) - s.DeletionTimestamp = &now - if err := h.hub.Delete(h.ctx, s); err != nil { - t.Fatal(err) - } - - h.reconcile(a, testSession) - - if err := h.hub.Get(h.ctx, client.ObjectKeyFromObject(s), s); !apierrors.IsNotFound(err) { - t.Fatalf("hub copy still held after its deletion was polled: %v", err) - } - if len(h.slots()) != 0 { - t.Fatal("slot kept after the hub copy was deleted") - } -} - -func TestEndedSessionWithoutFinalizerIsNotPolled(t *testing.T) { +func TestTerminalSessionIsNotRequeued(t *testing.T) { h := newHarness(t) a := h.agent() h.session(testSession, testUID) - endUnavailable(t, h) + endAs(h, computev1alpha.InstanceConsoleSessionReasonNoShell) if res := h.reconcile(a, testSession); res.RequeueAfter != 0 { t.Fatalf("requeue after %v for an ended session this agent never held", res.RequeueAfter) diff --git a/internal/shellagent/server.go b/internal/shellagent/server.go index 56f4f9c1..324b5aee 100644 --- a/internal/shellagent/server.go +++ b/internal/shellagent/server.go @@ -83,27 +83,27 @@ func (a *Agent) serveExec(ctx context.Context, w http.ResponseWriter, r *http.Re return } - var hub computev1alpha.InstanceConsoleSession - err = a.hub.Get(ctx, client.ObjectKeyFromObject(cell), &hub) + var current computev1alpha.InstanceConsoleSession + err = a.cell.Get(ctx, client.ObjectKeyFromObject(cell), ¤t) if err != nil && !apierrors.IsNotFound(err) { logger.Error(err, "read session") http.Error(w, "The session could not be read.", http.StatusInternalServerError) return } - if err != nil || sessionUID(&hub) != uid || endpointOf(&hub) != "" && endpointOf(&hub) != a.EndpointID() { + if err != nil || sessionUID(¤t) != uid || endpointOf(¤t) != "" && endpointOf(¤t) != a.EndpointID() { http.Error(w, "Session not found.", http.StatusNotFound) return } - if !hub.DeletionTimestamp.IsZero() || !cell.DeletionTimestamp.IsZero() { + if !current.DeletionTimestamp.IsZero() || revoked(¤t) { http.Error(w, endMessage(computev1alpha.InstanceConsoleSessionReasonRevoked), http.StatusGone) return } - if readyReason(&hub) != computev1alpha.InstanceConsoleSessionReasonSessionReady || isTerminal(&hub) { - code, msg := refusal(&hub) + if readyReason(¤t) != computev1alpha.InstanceConsoleSessionReasonSessionReady || isTerminal(¤t) { + code, msg := refusal(¤t) http.Error(w, msg, code) return } - if hub.Status.ConnectBefore != nil && !now.Before(hub.Status.ConnectBefore.Time) { + if current.Status.ConnectBefore != nil && !now.Before(current.Status.ConnectBefore.Time) { http.Error(w, endMessage(computev1alpha.InstanceConsoleSessionReasonNotConnected), http.StatusGone) return } @@ -114,7 +114,7 @@ func (a *Agent) serveExec(ctx context.Context, w http.ResponseWriter, r *http.Re } started := metav1.NewTime(now) - expires := metav1.NewTime(now.Add(sessionTTL(&hub))) + expires := metav1.NewTime(now.Add(sessionTTL(¤t))) runCtx, cancel := context.WithDeadline(context.Background(), expires.Time) defer cancel() live, ok := a.register(uid, cancel, cell.Spec.Terminal) @@ -127,10 +127,10 @@ func (a *Agent) serveExec(ctx context.Context, w http.ResponseWriter, r *http.Re close(live.done) }() - hub.Status.StartedAt = &started - hub.Status.ExpiresAt = &expires - setReady(&hub, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonConnected, "A client is connected.") - if err := a.hub.Status().Update(ctx, &hub); err != nil { + current.Status.StartedAt = &started + current.Status.ExpiresAt = &expires + setReady(¤t, metav1.ConditionTrue, computev1alpha.InstanceConsoleSessionReasonConnected, "A client is connected.") + if err := a.cell.Status().Update(ctx, ¤t); err != nil { if apierrors.IsConflict(err) { http.Error(w, "The session already has a connection or has ended.", http.StatusConflict) return @@ -141,21 +141,21 @@ func (a *Agent) serveExec(ctx context.Context, w http.ResponseWriter, r *http.Re } logger.Info("session connected", "pod", held.pod.String(), "container", held.container) - result := a.runSession(runCtx, w, r, &hub, held, live) - a.finishSession(uid, client.ObjectKeyFromObject(&hub), held, result) + result := a.runSession(runCtx, w, r, ¤t, held, live) + a.finishSession(uid, client.ObjectKeyFromObject(¤t), held, result) logger.Info("session ended", "reason", result.reason, "duration", a.now().Sub(started.Time).Round(time.Millisecond)) } // runSession runs the session's command and relays its stream until it ends, // then sends the client the session's closing status. func (a *Agent) runSession(ctx context.Context, w http.ResponseWriter, r *http.Request, - hub *computev1alpha.InstanceConsoleSession, held *slot, live *liveSession) outcome { - uid := sessionUID(hub) + session *computev1alpha.InstanceConsoleSession, held *slot, live *liveSession) outcome { + uid := sessionUID(session) opts := ExecOptions{ Container: held.container, - Command: wrappedCommand(held.markerDir, uid, hub.Spec.Command, hub.Spec.Stdin, hub.Spec.Terminal), - Stdin: hub.Spec.Stdin, - TTY: hub.Spec.Terminal, + Command: wrappedCommand(held.markerDir, uid, session.Spec.Command, session.Spec.Stdin, session.Spec.Terminal), + Stdin: session.Spec.Stdin, + TTY: session.Spec.Terminal, } resp, backend, fromBackend, err := a.exec.Open(ctx, held.pod, opts, r.Header) if err != nil { diff --git a/internal/shellagent/server_test.go b/internal/shellagent/server_test.go index 9e59e214..583f1f15 100644 --- a/internal/shellagent/server_test.go +++ b/internal/shellagent/server_test.go @@ -35,7 +35,7 @@ func readySession(t *testing.T, h *harness, a *Agent, mutate ...func(*computev1a t.Helper() key := h.session(testSession, testUID, mutate...) h.reconcile(a, testSession) - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonSessionReady) return key } @@ -84,9 +84,14 @@ func TestRefusals(t *testing.T) { endAs(h, computev1alpha.InstanceConsoleSessionReasonRevoked) return testUID, key, time.Now() }, http.StatusGone}, + {"revoke requested", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { + key := readySession(t, h, a) + revokeSession(t, h) + return testUID, key, time.Now() + }, http.StatusGone}, {"being deleted", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { key := readySession(t, h, a) - if err := h.hub.Delete(h.ctx, h.hubSession(testSession)); err != nil { + if err := h.cell.Delete(h.ctx, h.cellSession(testSession)); err != nil { t.Fatal(err) } return testUID, key, time.Now() @@ -98,9 +103,9 @@ func TestRefusals(t *testing.T) { }, http.StatusGone}, {"past connectBefore", func(t *testing.T, h *harness, a *Agent) (string, ed25519.PrivateKey, time.Time) { key := readySession(t, h, a) - s := h.hubSession(testSession) + s := h.cellSession(testSession) s.Status.ConnectBefore = &metav1.Time{Time: time.Now().Add(-time.Second)} - if err := h.hub.Status().Update(h.ctx, s); err != nil { + if err := h.cell.Status().Update(h.ctx, s); err != nil { t.Fatal(err) } return testUID, key, time.Now() @@ -342,12 +347,12 @@ func waitForConnected(t *testing.T, h *harness) { reason := computev1alpha.InstanceConsoleSessionReasonConnected deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { - if readyReason(h.hubSession(testSession)) == reason { + if readyReason(h.cellSession(testSession)) == reason { return } time.Sleep(10 * time.Millisecond) } - requireReason(t, h.hubSession(testSession), reason) + requireReason(t, h.cellSession(testSession), reason) } func waitForEnd(t *testing.T, h *harness) *computev1alpha.InstanceConsoleSession { @@ -355,11 +360,11 @@ func waitForEnd(t *testing.T, h *harness) *computev1alpha.InstanceConsoleSession t.Helper() deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { - if s := h.hubSession(name); isTerminal(s) && len(h.slots()) == 0 { + if s := h.cellSession(name); isTerminal(s) && len(h.slots()) == 0 { return s } time.Sleep(10 * time.Millisecond) } - t.Fatalf("session %s did not end: %+v", name, h.hubSession(name).Status) + t.Fatalf("session %s did not end: %+v", name, h.cellSession(name).Status) return nil } diff --git a/internal/shellagent/stream_test.go b/internal/shellagent/stream_test.go index e0e904dc..77959646 100644 --- a/internal/shellagent/stream_test.go +++ b/internal/shellagent/stream_test.go @@ -136,7 +136,7 @@ func TestAnsweringClientStaysConnected(t *testing.T) { waitForConnected(t, h) time.Sleep(4 * a.cfg.PongTimeout) - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) if n := backend.pongs(); n != 0 { t.Fatalf("relayed %d pongs to the agent's own pings to the apiserver", n) } @@ -163,7 +163,7 @@ func TestUnsolicitedPongsKeepClientConnected(t *testing.T) { } } - requireReason(t, h.hubSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) + requireReason(t, h.cellSession(testSession), computev1alpha.InstanceConsoleSessionReasonConnected) } // ttyCommand writes a prompt, echoes the first stdin message, and exits 0. diff --git a/internal/shellagent/sweep.go b/internal/shellagent/sweep.go index 547a18ab..869ac54c 100644 --- a/internal/shellagent/sweep.go +++ b/internal/shellagent/sweep.go @@ -25,24 +25,24 @@ const ( ) // sessionState reports whether a session, by project UID, is still open. A -// session whose cell or hub copy is gone has ended. +// session whose cell copy is gone has ended. func (a *Agent) sessionState(ctx context.Context, uid string) (sessionStateKind, error) { - cell, err := a.findSession(ctx, uid) + cached, err := a.findSession(ctx, uid) if err != nil { return sessionActive, err } - if cell == nil { + if cached == nil { return sessionEnded, nil } - var hub computev1alpha.InstanceConsoleSession - err = a.hub.Get(ctx, client.ObjectKeyFromObject(cell), &hub) + var session computev1alpha.InstanceConsoleSession + err = a.cell.Get(ctx, client.ObjectKeyFromObject(cached), &session) if apierrors.IsNotFound(err) { return sessionEnded, nil } if err != nil { return sessionActive, err } - if sessionUID(&hub) != uid || isTerminal(&hub) { + if sessionUID(&session) != uid || isTerminal(&session) { return sessionEnded, nil } return sessionActive, nil From 6313c76429710ae51cb98c908fb4d35863b972a1 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 30 Sep 2026 06:32:04 -0500 Subject: [PATCH 17/20] fix: Drop the shell agent's hub credential and egress The agent no longer talks to the Karmada hub, so the cell packaging needs neither the hub kubeconfig and client certificate nor a way out of the cell. The agent may now reach only the cell apiserver and DNS. Key changes: - remove the federation kubeconfig flag and its projected volume - let the agent write session status in the cell - remove the hub FQDN egress rule and its DNS proxy from the Cilium policy --- .../shell-agent-cilium/kustomization.yaml | 18 ++---------------- .../components/shell-agent-cilium/policy.yaml | 14 -------------- config/components/shell-agent/agent.yaml | 13 ------------- .../components/shell-agent/kustomization.yaml | 14 +++++--------- .../components/shell-agent/networkpolicy.yaml | 2 +- config/components/shell-agent/rbac.yaml | 13 +++++++++++-- 6 files changed, 19 insertions(+), 55 deletions(-) diff --git a/config/components/shell-agent-cilium/kustomization.yaml b/config/components/shell-agent-cilium/kustomization.yaml index d2f1fdf9..9b7bc752 100644 --- a/config/components/shell-agent-cilium/kustomization.yaml +++ b/config/components/shell-agent-cilium/kustomization.yaml @@ -1,22 +1,8 @@ # Cilium policies for the shell agent and its tunnel endpoints. Include beside # config/components/shell-agent on cells that run Cilium. Unlike the portable # policies they police traffic to the node's own apiserver, kubelet and etcd, -# so the endpoint, which holds no credential, cannot reach them at all. -# -# The agent may leave the cell only for the Karmada hub. Set the hub's name in -# the deployment, replacing the placeholder karmada-hub.invalid: -# -# - target: -# group: cilium.io -# kind: CiliumNetworkPolicy -# name: exec-agent -# patch: |- -# - op: test -# path: /spec/egress/1/toFQDNs/0/matchName -# value: karmada-hub.invalid -# - op: replace -# path: /spec/egress/1/toFQDNs/0/matchName -# value: karmada.example.net +# so the endpoint, which holds no credential, cannot reach them at all. The +# agent never leaves the cell: it talks only to the cell apiserver. apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component diff --git a/config/components/shell-agent-cilium/policy.yaml b/config/components/shell-agent-cilium/policy.yaml index 522cd2a8..75d4f35e 100644 --- a/config/components/shell-agent-cilium/policy.yaml +++ b/config/components/shell-agent-cilium/policy.yaml @@ -33,17 +33,6 @@ spec: protocol: TCP - port: "6443" protocol: TCP - # The Karmada hub, and nothing else outside the cell. The name is a - # placeholder that resolves nowhere, so a deployment that does not set its - # hub's name reaches no hub rather than the whole internet. - - toFQDNs: - - matchName: karmada-hub.invalid - toPorts: - - ports: - - port: "443" - protocol: TCP - # Cilium learns the hub's addresses from the agent's DNS answers, so DNS - # goes through its proxy. - toEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: kube-system @@ -54,9 +43,6 @@ spec: protocol: UDP - port: "53" protocol: TCP - rules: - dns: - - matchPattern: "*" --- apiVersion: cilium.io/v2 kind: CiliumNetworkPolicy diff --git a/config/components/shell-agent/agent.yaml b/config/components/shell-agent/agent.yaml index b84fc466..883af530 100644 --- a/config/components/shell-agent/agent.yaml +++ b/config/components/shell-agent/agent.yaml @@ -55,7 +55,6 @@ spec: - --listen=:7777 - --health-probe-bind-address=:8081 - --target=$(POD_NAME).exec-agent.$(POD_NAMESPACE).svc.cluster.local:7777 - - --federation-kubeconfig=/etc/kubernetes/upstream/auth/kubeconfig - --managed-by=kata-provider - --endpoint-pod-prefix=exec-endpoint - --drain-timeout=30s @@ -106,15 +105,3 @@ spec: limits: cpu: "1" memory: 256Mi - volumeMounts: - - name: federation - mountPath: /etc/kubernetes/upstream/auth/ - readOnly: true - volumes: - - name: federation - projected: - sources: - - secret: - name: exec-agent-federation-client-cert - - configMap: - name: exec-agent-federation-kubeconfig diff --git a/config/components/shell-agent/kustomization.yaml b/config/components/shell-agent/kustomization.yaml index 54f00965..e216d89a 100644 --- a/config/components/shell-agent/kustomization.yaml +++ b/config/components/shell-agent/kustomization.yaml @@ -4,16 +4,12 @@ # Everything lives in compute-shell-system, and the session target clients dial # names that namespace, so include this component from a kustomization that does # not rename namespaces (see config/overlays/cell-shell-agent). The deployment -# supplies three things this component only references: +# supplies ConfigMap exec-relays: DATUM_CONNECT_RELAY_URLS, the relays the +# endpoints register with and the agents publish, nearest first. # -# - ConfigMap exec-relays: DATUM_CONNECT_RELAY_URLS, the relays the endpoints -# register with and the agents publish, nearest first. -# - ConfigMap exec-agent-federation-kubeconfig (key kubeconfig) and Secret -# exec-agent-federation-client-cert: the Karmada hub credential, mounted -# together at /etc/kubernetes/upstream/auth/. On the hub it needs only get -# and patch on instanceconsolesessions and get, update and patch on their -# status. The agent never lists or watches the hub: it reads each session's -# hub copy by the namespace and name of the copy delivered to its cell. +# The agent needs no Karmada hub credential. It writes status only on the +# session copies in its cell, and Karmada reflects it back to the hub with the +# InstanceConsoleSession interpreter in config/components/federation. apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component diff --git a/config/components/shell-agent/networkpolicy.yaml b/config/components/shell-agent/networkpolicy.yaml index e71bc223..792cc84e 100644 --- a/config/components/shell-agent/networkpolicy.yaml +++ b/config/components/shell-agent/networkpolicy.yaml @@ -22,7 +22,7 @@ spec: - protocol: TCP port: 7777 egress: - # The cell apiserver and the Karmada hub. + # The cell apiserver. - ports: - protocol: TCP port: 443 diff --git a/config/components/shell-agent/rbac.yaml b/config/components/shell-agent/rbac.yaml index cdaf5b23..0db0ba13 100644 --- a/config/components/shell-agent/rbac.yaml +++ b/config/components/shell-agent/rbac.yaml @@ -41,8 +41,9 @@ rules: - get - list - watch - # Session copies are read here and only their finalizer is written; status - # goes to the hub copy. + # The agent claims and ends sessions on their cell copies, whose status + # Karmada reflects back to the hub, and holds each copy with its finalizer + # until the session's processes are stopped. - apiGroups: - compute.datumapis.com resources: @@ -52,6 +53,14 @@ rules: - list - watch - patch + - apiGroups: + - compute.datumapis.com + resources: + - instanceconsolesessions/status + verbs: + - get + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From 81681dee81e5cd9974efa044ba25873b5be292ee Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 30 Sep 2026 07:13:06 -0500 Subject: [PATCH 18/20] test: Return after nil-check fatals in build tests Lint on this branch fails with staticcheck SA5011 on three nil checks in the build analysis tests, a file this branch never touches. It is a golangci-lint cache bug, not a code defect (golangci/golangci-lint#5979, same root cause as #3228): with a warm analysis cache, staticcheck loses the fact that t.Fatal never returns, so every nil check followed by a dereference reads as a possible nil pointer. Key evidence: - A clean golangci-lint v2.12.2 run of this head reports 0 issues, on macOS and in a Linux container. - The go.mod change gives this branch its own lint cache key. The cold runs that built that cache passed in 130-165s; every warm run since has failed in 14-22s with the same six findings. An explicit return after each fatal makes the nil path terminate without relying on that fact, so lint passes whatever state the cache is in. --- internal/cmd/compute/build/analysis_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/cmd/compute/build/analysis_test.go b/internal/cmd/compute/build/analysis_test.go index 9a4be064..dbe51cf6 100644 --- a/internal/cmd/compute/build/analysis_test.go +++ b/internal/cmd/compute/build/analysis_test.go @@ -39,6 +39,7 @@ ENTRYPOINT ["/usr/local/bin/app"] node, sourceStage := df.findEntrypointProducer("/usr/local/bin/app", toolchain{Kind: toolchainGo}) if node == nil { t.Fatal("expected producer") + return } if node.StartLine != 3 { t.Fatalf("expected line 3, got %d", node.StartLine) @@ -62,6 +63,7 @@ ENTRYPOINT ["/server"] node := df.findEntrypointCopy("/server") if node == nil { t.Fatal("expected entrypoint copy") + return } if node.StartLine != 4 { t.Fatalf("expected line 4, got %d", node.StartLine) @@ -399,6 +401,7 @@ func TestCheckMissingLibrariesReportsDynamicLoader(t *testing.T) { _, elfFile := detectToolchain(data) if elfFile == nil { t.Fatal("expected ELF") + return } if interp := elfInterp(elfFile); interp == "" { t.Skip("test toolchain produced no PT_INTERP") From 8cd6012997caf43a68dfa36e3facd376584fab8b Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 30 Sep 2026 07:22:31 -0500 Subject: [PATCH 19/20] fix: Confirm a session's end only once its processes stop The agent recorded a session's end with endedAt even when it could not stop the session's processes, and the control plane took that end as confirmed cleanup while the processes might still run. The agent still records the end at once, so the client and the project learn of it, but now sets endedAt only when nothing the session started is left running. When the stop fails it keeps the slot, retries from the reconcile and the orphan sweep, and sets endedAt once a retry succeeds. The control plane waits for endedAt, and records CleanupUnconfirmed if it does not arrive within five minutes. No API field changes: endedAt already means when the session ended, and a session whose processes still run has not. Key changes: - record endedAt with the end only when the stop succeeded - set endedAt from the settle, cleanup and sweep paths once a later stop succeeds, only for the session that owns the copy - test failed stops on takeover and on a dropped client, and the sweep --- internal/shellagent/reconciler.go | 60 ++++++++++++++++++++------ internal/shellagent/reconciler_test.go | 58 +++++++++++++++++++++++++ internal/shellagent/server.go | 2 +- internal/shellagent/server_test.go | 34 +++++++++++++++ internal/shellagent/sweep.go | 9 +++- 5 files changed, 147 insertions(+), 16 deletions(-) diff --git a/internal/shellagent/reconciler.go b/internal/shellagent/reconciler.go index 2b516b6a..5f71bb2a 100644 --- a/internal/shellagent/reconciler.go +++ b/internal/shellagent/reconciler.go @@ -226,7 +226,7 @@ func (a *Agent) abandon(ctx context.Context, session *computev1alpha.InstanceCon return err } } - if _, err := a.end(ctx, client.ObjectKeyFromObject(session), reason, endMessage(reason), nil, + if _, err := a.end(ctx, client.ObjectKeyFromObject(session), reason, endMessage(reason), nil, stopped, func(s *computev1alpha.InstanceConsoleSession) bool { return endpointOf(s) == owner }); err != nil { return err } @@ -242,7 +242,7 @@ func (a *Agent) abandon(ctx context.Context, session *computev1alpha.InstanceCon func (a *Agent) endUnconnected(ctx context.Context, session *computev1alpha.InstanceConsoleSession, reason string) error { uid := sessionUID(session) me := a.EndpointID() - ended, err := a.end(ctx, client.ObjectKeyFromObject(session), reason, endMessage(reason), nil, + ended, err := a.end(ctx, client.ObjectKeyFromObject(session), reason, endMessage(reason), nil, true, func(s *computev1alpha.InstanceConsoleSession) bool { return endpointOf(s) == me && readyReason(s) == computev1alpha.InstanceConsoleSessionReasonSessionReady }) @@ -258,32 +258,37 @@ func (a *Agent) endUnconnected(ctx context.Context, session *computev1alpha.Inst } func (a *Agent) rejectUnclaimed(ctx context.Context, session *computev1alpha.InstanceConsoleSession, rej *rejection) error { - _, err := a.end(ctx, client.ObjectKeyFromObject(session), rej.reason, rej.message, nil, + _, err := a.end(ctx, client.ObjectKeyFromObject(session), rej.reason, rej.message, nil, true, func(s *computev1alpha.InstanceConsoleSession) bool { return s.Status.Connection == nil }) return err } -// settle frees the slot of an ended session once its process is stopped, -// unless a live agent is still ending it. +// settle frees the slot of an ended session once its process is stopped, and +// then records when it was, unless a live agent is still ending it. func (a *Agent) settle(ctx context.Context, session *computev1alpha.InstanceConsoleSession) (ctrl.Result, error) { uid := sessionUID(session) if a.isLive(uid) { return ctrl.Result{RequeueAfter: cleanupRetryInterval}, nil } held, err := a.slotFor(ctx, uid) - if err != nil || held == nil { + if err != nil { return ctrl.Result{}, err } + if held == nil && session.Status.EndedAt != nil { + return ctrl.Result{}, nil + } if owner := endpointOf(session); owner != "" && owner != a.EndpointID() { if alive, err := a.agentAlive(ctx, owner); err != nil || alive { return ctrl.Result{RequeueAfter: LeaseDuration}, err } } - reaped, err := a.reap(ctx, held) - if err != nil || !reaped { - return ctrl.Result{RequeueAfter: cleanupRetryInterval}, err + if held != nil { + reaped, err := a.reap(ctx, held) + if err != nil || !reaped { + return ctrl.Result{RequeueAfter: cleanupRetryInterval}, err + } } - return ctrl.Result{}, nil + return ctrl.Result{}, a.markStopped(ctx, client.ObjectKeyFromObject(session), uid) } // revoke ends a session that is still open because the control plane revoked @@ -348,14 +353,20 @@ func (a *Agent) cleanUp(ctx context.Context, session *computev1alpha.InstanceCon } } a.forget(uid) + if err := a.markStopped(ctx, client.ObjectKeyFromObject(session), uid); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{}, removeFinalizer(ctx, a.cell, client.ObjectKeyFromObject(session)) } // end records a terminal reason on the cell copy if the session has not ended // yet and guard still holds for its latest state. It reports whether this -// call ended the session. +// call ended the session. endedAt is set only when stopped says nothing the +// session started is still running, because the control plane takes it as +// confirmation of cleanup; otherwise markStopped sets it once the processes +// are stopped. func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, message string, exitCode *int32, - guard func(*computev1alpha.InstanceConsoleSession) bool) (bool, error) { + stopped bool, guard func(*computev1alpha.InstanceConsoleSession) bool) (bool, error) { ended := false err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { var s computev1alpha.InstanceConsoleSession @@ -365,8 +376,10 @@ func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, messa if isTerminal(&s) || guard != nil && !guard(&s) { return nil } - now := metav1.NewTime(a.now()) - s.Status.EndedAt = &now + if stopped { + now := metav1.NewTime(a.now()) + s.Status.EndedAt = &now + } s.Status.ExitCode = exitCode setReady(&s, metav1.ConditionFalse, reason, message) if err := a.cell.Status().Update(ctx, &s); err != nil { @@ -378,6 +391,25 @@ func (a *Agent) end(ctx context.Context, key types.NamespacedName, reason, messa return ended, client.IgnoreNotFound(err) } +// markStopped records endedAt on an ended session whose processes are now +// known to be stopped. A copy with the same name that belongs to another +// session is left alone. +func (a *Agent) markStopped(ctx context.Context, key types.NamespacedName, uid string) error { + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var s computev1alpha.InstanceConsoleSession + if err := a.cell.Get(ctx, key, &s); err != nil { + return err + } + if sessionUID(&s) != uid || !isTerminal(&s) || s.Status.EndedAt != nil { + return nil + } + now := metav1.NewTime(a.now()) + s.Status.EndedAt = &now + return a.cell.Status().Update(ctx, &s) + }) + return client.IgnoreNotFound(err) +} + // revoked reports whether the control plane asked the cell to end the // session. func revoked(s *computev1alpha.InstanceConsoleSession) bool { diff --git a/internal/shellagent/reconciler_test.go b/internal/shellagent/reconciler_test.go index 1d0c97ea..6275becf 100644 --- a/internal/shellagent/reconciler_test.go +++ b/internal/shellagent/reconciler_test.go @@ -834,3 +834,61 @@ func TestAgentLeaseNamesAreDistinctAndValid(t *testing.T) { t.Fatalf("lease name %q is invalid: %v", a, errs) } } + +func TestEndIsConfirmedOnlyOnceProcessesStop(t *testing.T) { + h := newHarness(t) + lost, survivor := h.agent(), h.agent() + h.session(testSession, testUID) + h.reconcile(lost, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + lost.releaseLease(h.ctx, lost.EndpointID()) + h.exec.killExit = 1 + + h.reconcile(survivor, testSession) + + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonAgentLost) + if s.Status.EndedAt != nil { + t.Fatal("endedAt set while the session's processes may still run") + } + if len(h.slots()) != 1 { + t.Fatal("slot released before the processes were stopped") + } + + if res := h.reconcile(survivor, testSession); res.RequeueAfter == 0 { + t.Fatal("expected a retry while processes may be running") + } + if h.cellSession(testSession).Status.EndedAt != nil { + t.Fatal("endedAt set after another failed stop") + } + + h.exec.killExit = 0 + h.reconcile(survivor, testSession) + s = h.cellSession(testSession) + if s.Status.EndedAt == nil { + t.Fatal("endedAt not set once the processes stopped") + } + if len(h.slots()) != 0 { + t.Fatal("slot kept after the processes stopped") + } +} + +func TestSweepConfirmsEndOnceProcessesStop(t *testing.T) { + h := newHarness(t) + a := h.agent() + h.session(testSession, testUID) + h.reconcile(a, testSession) + markConnected(h, testSession, h.clock.now().Add(time.Hour)) + endAs(h, computev1alpha.InstanceConsoleSessionReasonDisconnected) + + if err := a.sweep(h.ctx); err != nil { + t.Fatal(err) + } + + if h.cellSession(testSession).Status.EndedAt == nil { + t.Fatal("the sweep stopped the processes but did not record it") + } + if len(h.slots()) != 0 { + t.Fatal("slot kept after the sweep stopped the processes") + } +} diff --git a/internal/shellagent/server.go b/internal/shellagent/server.go index 324b5aee..2cdb1bc8 100644 --- a/internal/shellagent/server.go +++ b/internal/shellagent/server.go @@ -206,7 +206,7 @@ func (a *Agent) finishSession(uid string, key client.ObjectKey, held *slot, resu message = "The command exited." } me := a.EndpointID() - if _, err := a.end(ctx, key, result.reason, message, result.exitCode, + if _, err := a.end(ctx, key, result.reason, message, result.exitCode, stopped, func(s *computev1alpha.InstanceConsoleSession) bool { return endpointOf(s) == me }); err != nil { logger.Error(err, "record session end") } diff --git a/internal/shellagent/server_test.go b/internal/shellagent/server_test.go index 583f1f15..4f1a7cea 100644 --- a/internal/shellagent/server_test.go +++ b/internal/shellagent/server_test.go @@ -295,6 +295,40 @@ func TestClientDisconnectStopsCommand(t *testing.T) { } } +func TestFailedStopLeavesEndUnconfirmed(t *testing.T) { + h := newHarness(t) + a := h.agent() + addr := serve(t, h, a) + key := readySession(t, h, a) + h.exec.onSession = func(conn net.Conn, r *bufio.Reader) { + for { + if _, err := readFrame(r); err != nil { + return + } + } + } + h.exec.mu.Lock() + h.exec.killExit = 1 + h.exec.mu.Unlock() + + c := dialSession(t, addr, testUID, key, time.Now()) + waitForConnected(t, h) + _ = c.conn.Close() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && !isTerminal(h.cellSession(testSession)) { + time.Sleep(10 * time.Millisecond) + } + s := h.cellSession(testSession) + requireReason(t, s, computev1alpha.InstanceConsoleSessionReasonDisconnected) + if s.Status.EndedAt != nil { + t.Fatal("endedAt set although the command could not be confirmed stopped") + } + if len(h.slots()) != 1 { + t.Fatal("slot released although the command could not be confirmed stopped") + } +} + func TestOversizedFrameEndsSession(t *testing.T) { h := newHarness(t) a := h.agent() diff --git a/internal/shellagent/sweep.go b/internal/shellagent/sweep.go index 869ac54c..193025f8 100644 --- a/internal/shellagent/sweep.go +++ b/internal/shellagent/sweep.go @@ -94,8 +94,15 @@ func (a *Agent) sweep(ctx context.Context) error { if err != nil || state != sessionEnded { continue } - if _, err := a.reap(ctx, s); err != nil { + reaped, err := a.reap(ctx, s) + if err != nil { log.FromContext(ctx).Error(err, "reap orphaned session", "session", s.sessionUID) + continue + } + if reaped { + if err := a.markStopped(ctx, s.session, s.sessionUID); err != nil { + log.FromContext(ctx).Error(err, "record stopped session", "session", s.sessionUID) + } } } for place := range places { From 34f3026d1db252758bbcdcfc700e1871de3424d7 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 30 Sep 2026 07:48:38 -0500 Subject: [PATCH 20/20] test: Read the prompt before ending a platform-ended session TestPlatformEndings ended the session as soon as it reported Connected, which could race the relay of the command's first output, so the client sometimes read no prompt and the test failed on CI. The client now reads the prompt before the session is ended. --- internal/shellagent/server_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/shellagent/server_test.go b/internal/shellagent/server_test.go index 4f1a7cea..18bf52fa 100644 --- a/internal/shellagent/server_test.go +++ b/internal/shellagent/server_test.go @@ -252,11 +252,16 @@ func TestPlatformEndings(t *testing.T) { c := dialSession(t, addr, testUID, key, time.Now()) waitForConnected(t, h) + _ = c.conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + first, err := readFrame(c.reader) + if err != nil || len(first.payload) == 0 || first.payload[0] != channelStdout || string(first.payload[1:]) != "$ " { + t.Fatalf("first frame = %+v, %v", first, err) + } go tc.end(a) out, status, code := c.readUntilClose(t) - if out[channelStdout] != "$ " { - t.Fatalf("stdout = %q", out[channelStdout]) + if out[channelStdout] != "" { + t.Fatalf("stdout after the end = %q", out[channelStdout]) } if status == nil || status.Status != metav1.StatusFailure || string(status.Reason) != tc.reason || status.Message == "" || code != closeNormal {