From 6893f69ec207251f5d0599a9911b27fe307f1a59 Mon Sep 17 00:00:00 2001 From: Ron Kahn Date: Thu, 23 Jul 2026 16:07:13 +0300 Subject: [PATCH 1/2] docs(snapshot): document PVC storage access options Signed-off-by: Ron Kahn --- .../STORAGE-ACCESS-RESEARCH.md | 1152 +++++++++++++++++ 1 file changed, 1152 insertions(+) create mode 100644 plans/snapshot-migration/STORAGE-ACCESS-RESEARCH.md diff --git a/plans/snapshot-migration/STORAGE-ACCESS-RESEARCH.md b/plans/snapshot-migration/STORAGE-ACCESS-RESEARCH.md new file mode 100644 index 000000000000..8dad77b0acfc --- /dev/null +++ b/plans/snapshot-migration/STORAGE-ACCESS-RESEARCH.md @@ -0,0 +1,1152 @@ + + +# PodSnapshot PVC storage access research + +**Status:** research and option analysis; no architecture is selected by this document. + +**Question:** how should Dynamo represent PodSnapshot storage when the snapshot agent reaches a +PVC either through the workload Pod or through storage mounted directly on the node agent? + +**Current backend boundary:** PVC only. S3 and OCI are intentionally outside the implementation +scope of this analysis, but the API options are evaluated for whether they leave room for another +backend later. + +## 1. Executive summary + +Dynamo already implements two PVC data-access paths, but the choice is deployment-global and is not +represented in `PodSnapshot` or `PodSnapshotContent`: + +- `agentMount`: the snapshot-agent DaemonSet mounts the checkpoint PVC and reads or writes through + its own filesystem namespace. +- `podMount`: the workload Pod mounts the checkpoint PVC and the privileged node agent reaches that + mount through `/host/proc//root`. + +The distinction describes how the agent reaches the files. It does **not** mean that only one Pod +mounts the PVC. Restore still needs the target container to see the checkpoint directory because +`nsrestore` receives both a host-visible checkpoint path and the path visible inside the target +container. + +The current implementation has four separate sources of storage truth: + +1. operator configuration can define and optionally create a namespace-local PVC; +2. the operator injects the PVC and storage annotations into capture and restore Pods; +3. the snapshot-agent ConfigMap defines `type`, `basePath`, and `accessMode`; +4. the agent combines Pod annotations with its own configuration at execution time. + +`PodSnapshotContent` is an immutable node work order, but it currently carries no resolved storage +input. Consequently, a work order does not say which PVC contains the artifact or which access path +the selected node agent must use. + +A reusable, administrator-owned storage object is a reasonable direction, but PVC storage makes its +shape less straightforward than GKE's object-storage configuration: + +- PVCs are namespaced. +- A cluster-scoped object cannot directly mount a PVC. +- A running DaemonSet Pod cannot acquire a new PVC mount without recreation. +- A `ReadWriteOnce` PVC can be shared by multiple Pods only while they are on the same node. +- A restore Pod must be able to mount the artifact PVC on its destination node. + +The main unresolved architectural choice is therefore not the CRD name. It is the storage data +plane: + +- keep accessing a volume already mounted into the workload Pod; +- pre-mount one or more stores into every relevant node agent; +- access kubelet's host-side Pod volume mount; +- or create a privileged, same-node operation Pod that mounts the PVC for each save or restore. + +## 2. Terminology + +The following concepts must remain separate in the API and documentation: + +| Term | Meaning | +|---|---| +| Backend | The durable storage technology. This document considers only `pvc`. | +| Access strategy | How snapshot code reaches the mounted filesystem: through the workload Pod, a node-agent mount, a host kubelet path, or an operation Pod. | +| PVC access mode | Kubernetes `ReadWriteOnce`, `ReadWriteMany`, or `ReadWriteOncePod`. This is not the snapshot access strategy. | +| Storage configuration | Administrator policy that selects a backend and an access strategy. | +| Resolved storage binding | Immutable, operation-specific identity of the claim, mount contract, and artifact key. | +| Snapshot handle | Agent-produced identifier for an artifact that can later be resolved for restore. | +| Base path | Mount point at which a PVC is visible to a Pod or agent, such as `/checkpoints`. | +| Artifact path | System-owned path below the storage root for one captured artifact. | + +Names such as `accessMode` are ambiguous because Kubernetes already uses that term for PVC access +modes. A future CRD should prefer `accessStrategy`, `accessor`, or a structural union such as +`pod: { ... }` versus `nodeAgent: { ... }`. + +## 3. Current Dynamo implementation + +### 3.1 API objects + +`PodSnapshot` is namespaced and contains only: + +```yaml +spec: + source: + podRef: + name: string + uid: string +status: + boundSnapshotContentName: string + conditions: [] +``` + +`PodSnapshotContent` is cluster-scoped and contains only the back-reference, source Pod identity, +and source node: + +```yaml +spec: + snapshotRef: + namespace: string + name: string + uid: string + source: + podRef: + name: string + uid: string + nodeName: string +status: + conditions: [] +``` + +The definitions are in: + +- `deploy/operator/api/v1alpha1/podsnapshot_types.go` +- `deploy/operator/api/v1alpha1/podsnapshotcontent_types.go` + +There is no storage configuration reference, resolved claim reference, access strategy, artifact +path, or snapshot handle in either object. + +### 3.2 Operator storage configuration + +`CheckpointConfiguration.Storage` in +`deploy/operator/api/config/v1alpha1/types.go` currently supports PVC settings: + +- `pvcName` +- `basePath` +- `create` +- `size` +- `storageClassName` +- Kubernetes PVC `accessMode` + +`deploy/operator/internal/checkpoint/storage.go`: + +- normalizes and validates the PVC configuration; +- optionally creates a claim with `ReadWriteMany` as the default; +- resolves the checkpoint location as + `//versions/`; +- falls back to discovering a snapshot-agent DaemonSet when operator storage is omitted. + +The operator injects the claim and base-path mount into capture and restore Pod specs. It also writes +the storage type and base path as Pod annotations. The claim name is represented by the Pod volume, +not by a storage annotation. + +### 3.3 Agent configuration + +`deploy/snapshot/internal/types/config.go` defines: + +```yaml +storage: + type: pvc + basePath: /checkpoints + accessMode: agentMount | podMount +``` + +The default is `agentMount`. Only PVC is accepted. + +The Helm chart in `deploy/helm/charts/snapshot` renders the agent ConfigMap and: + +- mounts the PVC into the DaemonSet only for `agentMount`; +- does not mount it into the DaemonSet for `podMount`; +- always gives the privileged agent host PID access, host `/proc`, the runtime socket, overlay + storage, cgroups, and the kubelet Pod directory. + +Because the access strategy is agent configuration, all snapshots handled by one agent deployment +use the same strategy. + +### 3.4 Capture path + +The operator creates a `PodSnapshotContent` pinned to the source Pod UID and node. The node agent: + +1. watches the content; +2. retrieves and validates the source Pod; +3. reads checkpoint identity, target container, storage type, and base path from the Pod and its + own configuration; +4. resolves the target container's host PID; +5. derives host-visible and container-visible storage paths; +6. runs CRIU/CUDA/rootfs capture; +7. atomically promotes the temporary directory to the final artifact directory; +8. marks `PodSnapshotContent` ready. + +The relevant code is: + +- `deploy/operator/internal/controller/podsnapshot_reconciler.go` +- `deploy/snapshot/internal/controller/podsnapshotcontent.go` +- `deploy/snapshot/internal/controller/controller.go` +- `deploy/snapshot/internal/executor/checkpoint.go` + +For `agentMount`: + +```text +HostPath = /checkpoints//versions/ +ContainerPath = /checkpoints//versions/ +``` + +For `podMount`: + +```text +ContainerPath = /checkpoints//versions/ +HostPath = /host/proc//root/checkpoints//versions/ +``` + +The `podMount` path is recomputed after resolving the live PID, and the agent validates that the PID +still represents the same container before accessing it. + +### 3.5 Restore path + +The operator or mutation webhook: + +- chooses a ready checkpoint; +- injects the checkpoint PVC and mount into the target container; +- places the target process in restore standby; +- adds control-volume and readiness plumbing; +- stamps restore metadata on the Pod. + +The node agent: + +1. resolves the placeholder container and its host PID; +2. computes the checkpoint host path according to its configured access strategy; +3. verifies the artifact; +4. passes both the host-visible path and container-visible path to restore; +5. runs the external restore; +6. writes completion state. + +This is implemented in: + +- `deploy/operator/internal/checkpoint/podspec.go` +- `deploy/operator/internal/webhook/mutation/pod_checkpoint_restore_handler.go` +- `deploy/snapshot/protocol/restore.go` +- `deploy/snapshot/internal/controller/controller.go` +- `deploy/snapshot/internal/executor/restore.go` + +Even in `agentMount`, the restore target needs a mount at the container checkpoint path. Direct agent +access therefore does not eliminate workload-Pod storage mutation for the current restore mechanism. + +### 3.6 Current storage discovery + +When explicit operator storage is absent, restore discovers a snapshot-agent DaemonSet in the +workload namespace and looks for: + +- container name `agent`; +- volume name `checkpoints`; +- a PVC-backed DaemonSet volume; +- the corresponding agent mount path. + +This legacy discovery assumes a namespace-local agent. It cannot discover a central DaemonSet in an +infrastructure namespace for a workload in another namespace. The operator's explicit storage +configuration is the current bridge for central `podMount` deployments. + +### 3.7 Current correctness and security gaps + +- A `PodSnapshotContent` cannot independently explain where its artifact will be written. +- Capture success records only conditions; no resolved artifact handle is persisted. +- Restore recomposes the path from checkpoint identity, annotations, and live configuration. +- A configuration change between capture and restore can redirect resolution. +- Storage base path can come from a Pod annotation. Although the implementation requires an + absolute clean path and validates the live PID, the annotation still influences a privileged + node-agent filesystem access. Future storage placement should come from an administrator-owned + object or a controller-resolved immutable binding. +- `agentMount` has no explicit containment check proving the final path remains under a designated + mounted storage root. +- `podMount` proves that the path exists through the container root, but it does not currently bind + that path in the CRD to a particular Pod volume and PVC UID. +- Artifact identity remains `/versions/`, so delete/recreate and concurrent + work orders require separate Lease and in-process guards. + +## 4. Kubernetes and CSI constraints + +### 4.1 PVC namespace + +A Pod can reference only a PVC in the same namespace. Kubernetes resolves +`persistentVolumeClaim.claimName` in the Pod's namespace and mounts the backing PV into the Pod. + +Implications: + +- a cluster-scoped storage object should not treat an unqualified claim name as one unique cluster + resource; +- a Pod-based strategy can resolve the real claim from the source Pod and snapshot namespace; +- an agent-mounted claim belongs to the agent Pod's namespace, which may differ from the + `PodSnapshot` namespace; +- cross-namespace restore requires an explicit copy/replication mechanism or a different backend; + a claim reference alone cannot cross the namespace boundary. + +Source: [Kubernetes Persistent Volumes: Claims as +Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#claims-as-volumes). + +### 4.2 PVC access modes + +- `ReadWriteOnce` means a volume may be mounted read-write by one node. Multiple Pods on that same + node may use it. +- `ReadWriteMany` permits read-write mounts from many nodes when the storage backend supports it. +- `ReadWriteOncePod` permits one Pod cluster-wide and is incompatible with a design that requires + the workload plus a helper or agent Pod to mount the claim simultaneously. + +Implications: + +- `podMount` works naturally with RWO because only the workload Pod must mount the claim. +- a same-node operation Pod can share an RWO claim with the workload, but not an RWOP claim; +- one DaemonSet replica per node mounting the same claim normally requires RWX; +- restore to another node requires the storage backend to detach/reattach an RWO volume or provide + multi-node access. + +Sources: + +- [Kubernetes Persistent Volume access + modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) +- [Kubernetes ReadWriteOncePod + guidance](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-access-mode-readwriteoncepod/) + +### 4.3 Running Pod immutability + +Kubernetes does not permit adding volumes or volume mounts to a running Pod. A Pod template change +causes a controller to create replacement Pods. + +Implication: a storage CRD cannot dynamically add an arbitrary PVC to an existing snapshot-agent +DaemonSet Pod. A direct claim reference for node-agent access must be paired with: + +- a pre-existing mount; +- a controlled DaemonSet rollout; +- or a separate operation Pod. + +Source: [Kubernetes Pod update and +replacement](https://kubernetes.io/docs/concepts/workloads/pods/#pod-update-and-replacement). + +### 4.4 CSI node publication + +Kubelet directly calls CSI `NodeStageVolume` and `NodePublishVolume` over the registered CSI node +plugin socket. Normal Kubernetes components request a mount by creating/scheduling a Pod that +references a PVC; they do not invoke these CSI node methods themselves. + +Implications: + +- a Dynamo agent directly issuing CSI node calls would duplicate kubelet ownership, target-path + lifecycle, credentials, idempotency, and unpublish behavior; +- the Kubernetes-native way to obtain a dynamic mount is to schedule a Pod that references the + claim; +- an operation Pod should use scheduler-visible node affinity rather than bypassing scheduling + constraints blindly, so PVC topology and attachment predicates are evaluated. + +Sources: + +- [Kubernetes CSI developer + documentation](https://kubernetes-csi.github.io/docs/) +- [Assigning Pods to + Nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/) + +### 4.5 Host-side Pod volume paths + +A privileged DaemonSet can mount the kubelet Pod directory and access volumes already published for +Pods on that node. Velero File System Backup uses this design. It also documents important +limitations: + +- the kubelet root path varies across platforms; +- access may require root and privileged mode; +- only volumes already mounted by a Pod are available; +- some environments, including some virtual-cluster layouts, do not expose the expected + `/pods/` layout. + +Sources: + +- [Velero File System + Backup](https://velero.io/docs/main/file-system-backup/) +- [Velero installation and kubelet root + customization](https://velero.io/docs/main/customize-installation/#customize-the-kubelet-root-path-of-the-node-agent) + +## 5. Prior API and controller patterns + +### 5.1 GKE Pod snapshots + +GKE separates: + +- cluster-scoped storage configuration; +- namespaced snapshot policy and snapshot resources; +- node-agent execution; +- controller lifecycle and cleanup; +- agent-produced state stored in Cloud Storage. + +The workload selects no raw filesystem path. `PodSnapshotStorageConfig` contains the administrative +storage destination, while the policy references it by name. + +This is strong precedent for a reusable administrator-owned storage configuration, but the backend +is Cloud Storage. It does not solve PVC namespacing or dynamic node attachment. + +Sources: + +- [GKE Pod snapshot + architecture](https://docs.cloud.google.com/kubernetes-engine/docs/concepts/pod-snapshots) +- [GKE Pod snapshot CRD + reference](https://docs.cloud.google.com/kubernetes-engine/docs/reference/crds/podsnapshot) + +### 5.2 CSI VolumeSnapshot + +CSI separates: + +- `VolumeSnapshot`: namespaced user request; +- `VolumeSnapshotClass`: cluster-scoped administrator policy; +- `VolumeSnapshotContent`: cluster-scoped binding and backend identity; +- `snapshotHandle`: driver-produced backend identifier. + +This supports the following Dynamo principles: + +- a user object references an administrator-selected class/configuration; +- the content object records the resolved binding; +- the actor that creates the physical artifact publishes the handle; +- user-provided paths should not become privileged filesystem destinations. + +Sources: + +- [Kubernetes Volume + Snapshots](https://kubernetes.io/docs/concepts/storage/volume-snapshots/) +- [Kubernetes VolumeSnapshotClass](https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes/) + +### 5.3 Velero + +Velero separates reusable storage locations from individual backup requests. Its node agent +demonstrates two relevant data-plane patterns: + +- privileged host traversal of volumes already mounted for Pods; +- short-lived data-mover Pods for some backup and restore operations. + +Velero also makes the kubelet root configurable and documents environment-specific host-path +limitations. This is evidence that host traversal is workable, but not a portable abstraction that +should be hidden. + +Sources: + +- [Velero storage + locations](https://velero.io/docs/main/locations/) +- [Velero File System + Backup](https://velero.io/docs/main/file-system-backup/) +- [Velero node-agent + configuration](https://velero.io/docs/main/supported-configmaps/node-agent-configmap/) + +## 6. PVC access architecture options + +### Option A: workload-Pod mount through `/proc//root` + +This is the current `podMount` implementation. + +Capture: + +```text +PodSnapshotContent + -> node agent resolves source container PID + -> agent validates PID and Pod UID + -> /host/proc//root/ + -> checkpoint files on the Pod's PVC +``` + +Restore: + +```text +restore Pod mounts the artifact PVC + -> node agent resolves placeholder PID + -> /host/proc//root/ + -> nsrestore receives host path and container path +``` + +Advantages: + +- supports namespace-local PVCs without mounting them in the central agent namespace; +- works with RWO because the workload itself owns the node attachment; +- supports different claims in different namespaces; +- no additional privileged operation Pod; +- already implemented and tested; +- no dependency on kubelet's on-disk volume directory layout. + +Disadvantages and risks: + +- depends on a stable live container PID and mount namespace; +- the agent must remain privileged with host PID and `/proc` access; +- a path alone is insufficient provenance; it should be resolved to a specific Pod volume and PVC; +- a compromised or incorrectly prepared workload could present an unexpected filesystem at the + configured path; +- restore cannot begin until the placeholder Pod and its PVC mount exist; +- RWOP remains possible only when no other helper Pod also mounts the claim. + +Required hardening: + +- storage configuration identifies a Pod `volumeName` or expected mount contract, not a raw + user-controlled destination; +- controller resolves and pins claim namespace, name, and UID into the content work order; +- agent verifies that the target container mount uses that volume and expected mount path; +- reject `subPath`, mount propagation, read-only capture destinations, and non-PVC volume sources + unless each is explicitly supported; +- derive the artifact path from system-owned identity and assert containment below the mounted root. + +### Option B: PVC mounted directly into the node-agent DaemonSet + +This is the current `agentMount` implementation. + +Capture: + +```text +PodSnapshotContent + -> selected node agent + -> agent's own /checkpoints mount + -> shared PVC +``` + +Restore still requires: + +```text +agent reads its own PVC mount + + restore target sees the same artifact through its container mount +``` + +Advantages: + +- simplest and most direct filesystem access; +- independent of source container PID for storage access; +- easy atomic rename and cleanup in the agent namespace; +- explicit mount path controlled by the agent deployment; +- no kubelet internal path discovery. + +Disadvantages and risks: + +- all DaemonSet replicas mounting one claim generally requires RWX; +- the PVC must exist in the DaemonSet namespace; +- a running agent cannot dynamically mount another claim; +- per-tenant or per-namespace claims require separate agent deployments or many static mounts; +- an RWO claim prevents simultaneous mounts from agent replicas on different nodes; +- restore target still needs a compatible view of the same storage; +- a global base-path annotation must not be allowed to redirect writes outside the mounted store. + +Best fit: + +- one trusted, shared RWX checkpoint store; +- a namespace-local agent installation with one claim; +- environments where predictable simplicity is more important than tenant isolation. + +### Option C: named, pre-mounted node-agent bindings + +The DaemonSet mounts one or more stores at installation time and gives each a logical name: + +```yaml +storageBindings: + primary: + backend: pvc + mountPath: /stores/primary + archive: + backend: pvc + mountPath: /stores/archive +``` + +A cluster-scoped storage configuration selects `bindingName: primary`. The agent validates that the +binding exists locally. + +Advantages: + +- keeps PVC namespace and mount details under administrator control; +- a snapshot work order can state which configured store it requires; +- supports multiple pre-mounted stores without raw paths in user objects; +- agent validation is local and deterministic; +- gives a future object-store backend a similar logical-selection mechanism. + +Disadvantages and risks: + +- still static: adding or changing a binding rolls the DaemonSet; +- every applicable node must have the same logical binding; +- many bindings increase DaemonSet spec size and PVC attachment pressure; +- shared block PVCs still face RWO/RWX constraints; +- a config reference can become invalid if a binding is removed after capture. + +Required lifecycle rule: + +- content must pin the resolved binding and final handle at capture time; +- changing or deleting the reusable storage configuration must not change how an existing snapshot + resolves; +- agent deployment validation should report binding readiness before accepting work. + +### Option D: direct PVC reference with automated DaemonSet rollout + +A storage configuration names a claim in the agent namespace. A controller patches the DaemonSet +template to mount it and waits for rollout. + +Advantages: + +- self-describing Kubernetes API; +- administrators can add a store by creating a CR instead of editing Helm values; +- normal kubelet/CSI attachment remains responsible for mounting. + +Disadvantages and risks: + +- storage policy now mutates infrastructure deployment state; +- a new storage config can restart every node agent; +- concurrent configuration changes require a deterministic merge and ownership model; +- removing a config may strand existing snapshot handles; +- RWO/RWX constraints remain; +- cross-controller ownership of a Helm-managed DaemonSet creates drift; +- rollout during active checkpoint/restore can interrupt operations. + +Conclusion: + +This option is feasible only as a larger storage-binding controller. A plain CRD field without that +controller is misleading and must not be presented as dynamic agent access. + +### Option E: privileged, same-node operation Pod + +The resident controller or agent creates a short-lived Pod for one checkpoint or restore. The Pod: + +- mounts the selected namespace-local PVC normally; +- is constrained to the workload's node through scheduler-visible affinity; +- receives the runtime socket, host PID/proc, cgroup, and overlay access needed for CRIU; +- writes operation status back to `PodSnapshotContent`. + +Advantages: + +- Kubernetes and kubelet dynamically mount the PVC; +- supports different namespace-local claims without changing the DaemonSet; +- RWO can work when workload and operation Pod run on the same node; +- storage access is explicit in the operation Pod spec; +- avoids accessing checkpoint storage through the workload container root; +- operation resources, priority, timeout, and retries can be controlled per job. + +Disadvantages and risks: + +- the operation Pod is itself highly privileged; +- RWOP cannot be shared with the workload Pod; +- scheduling and PVC topology must converge on the already-selected workload node; +- startup latency is added to every save and restore; +- operation Pod creation, cancellation, garbage collection, retry, and status recovery become new + controller state machines; +- runtime/container identity must be handed off safely; +- restore coordination becomes more complex because the placeholder Pod and operation Pod coexist; +- a node failure may leave operation CRs and temporary directories requiring cleanup. + +Required design work: + +- define an operation resource or durable status transaction marker; +- use node affinity that allows the scheduler to evaluate PVC constraints; +- establish owner references/finalizers and bounded retries; +- define whether the operation Pod or resident agent owns the Lease; +- prevent two worker Pods from writing the same artifact; +- prove that CRIU and CUDA operations work from the worker's namespace and host mounts. + +### Option F: host traversal of kubelet-managed Pod volumes + +The agent mounts the kubelet root, finds the source Pod UID and volume name, and accesses the +host-side published volume. Velero uses this general approach. + +Advantages: + +- the workload remains the Kubernetes consumer of the PVC; +- no `/proc//root` dependency; +- no extra operation Pod; +- RWO works because the published volume is already on the workload node; +- volume identity can be based on Pod UID and volume name. + +Disadvantages and risks: + +- kubelet root differs by platform and must be configurable; +- CSI/in-tree volume directory shapes and mount propagation vary; +- vCluster and some managed environments may not expose expected paths; +- requires privileged hostPath access to all Pod volumes on the node; +- couples Dynamo to kubelet storage internals; +- only works after a Pod has caused the volume to be published; +- restore ordering must keep the placeholder and published mount alive. + +This is credible precedent, not automatically preferable to current `/proc//root` access. +A cluster compatibility matrix is required before selecting it. + +### Option G: injected workload sidecar/helper + +The operator injects a privileged or RPC-enabled helper into the workload Pod. The helper mounts the +checkpoint PVC and cooperates with the node agent. + +Advantages: + +- the PVC is naturally in the workload namespace; +- no kubelet path discovery; +- helper lifecycle follows the workload; +- communication can use a narrow protocol instead of arbitrary host filesystem traversal. + +Disadvantages and risks: + +- mutates every workload Pod; +- the helper may need privileges approaching those of the node agent; +- GPU/CRIU operations still require host runtime access or an agent RPC; +- sidecar resource overhead persists for the workload lifetime; +- restore bootstrap and ordering become more complex; +- users may object to an injected container and image supply-chain dependency. + +This option has little advantage over `podMount` unless it can substantially reduce node-agent +privilege, which must be demonstrated. + +### Option H: node agent calls CSI directly + +The node agent calls `NodeStageVolume`/`NodePublishVolume` on CSI driver sockets itself. + +Potential advantage: + +- dynamic mount without an operation Pod. + +Reasons to reject: + +- duplicates kubelet's ownership of CSI publication; +- requires CSI-driver-specific volume IDs and secrets, not only PVC names; +- must reproduce target-path, staging, SELinux, fsGroup, mount-option, retry, and unpublish logic; +- can race kubelet and leave leaked mounts; +- creates a new compatibility surface with every CSI driver. + +This should remain rejected unless Dynamo intentionally becomes a kubelet-integrated storage +component. + +## 7. API representation options + +### API option 1: cluster-scoped `PodSnapshotStorageConfig` + +`PodSnapshot` references an administrator-owned configuration: + +```yaml +spec: + storageConfigName: shared-pod-pvc +``` + +The configuration selects exactly one backend and one access strategy. Separate objects represent +pod access and node-agent access. + +Advantages: + +- separates administrator infrastructure policy from namespaced snapshot requests; +- mirrors GKE, CSI class, and Velero storage-location patterns; +- avoids repeating storage policy in every snapshot; +- prevents ordinary snapshot users from supplying privileged host paths; +- permits status conditions for readiness and agent compatibility. + +Challenges: + +- a cluster-scoped object cannot directly own a namespaced PVC; +- pod access should describe a volume contract and resolve the real claim from the Pod; +- node-agent access must describe a pre-mounted binding or worker strategy, not promise an + impossible live mount; +- RBAC must control who can create/update storage configs and which namespaces may reference them. + +### API option 2: inline storage in `PodSnapshot.spec` + +Example: + +```yaml +spec: + storage: + pvc: + pod: + volumeName: checkpoint-storage + mountPath: /checkpoints +``` + +Advantages: + +- smallest object count; +- clear per-snapshot selection; +- natural access to the snapshot namespace. + +Disadvantages: + +- user-facing object exposes infrastructure and path details; +- storage configuration is repeated; +- node-agent mounts cannot be safely user-selected; +- future credentials/backends expand the namespaced API; +- immutable snapshots preserve stale infrastructure configuration; +- weaker separation between request and privileged placement. + +### API option 3: resolved storage only in `PodSnapshotContent.spec` + +The operator continues to take policy from Helm/operator configuration, but copies a resolved, +immutable binding into the content work order. + +Advantages: + +- smallest behavioral migration; +- agent no longer reads storage placement from Pod annotations; +- content becomes self-describing; +- no new CRD or user-facing choice; +- suitable if there will remain one storage policy per operator/agent installation. + +Disadvantages: + +- no declarative Kubernetes object for storage readiness; +- per-namespace/per-tenant routing remains configuration-driven; +- operator and agent configuration must still agree out of band; +- changing deployment configuration is the only way to add a storage target. + +### API option 4: handle only in `PodSnapshotContent.status` + +Keep all input configuration outside snapshot CRDs and add only an agent-produced +`status.snapshotHandle`. + +Advantages: + +- records the result needed by restore; +- follows CSI's driver-produced handle pattern; +- avoids exposing storage policy. + +Disadvantages: + +- the work order still does not tell the agent which store or access strategy to use; +- retries before the handle is written remain dependent on mutable external configuration; +- cannot validate an agent/config mismatch before starting capture. + +This is useful but insufficient by itself. + +### API option 5: no CRD storage representation + +Keep the existing operator config, Pod annotations, and agent ConfigMap. + +Advantages: + +- no API migration; +- current behavior remains operational. + +Disadvantages: + +- preserves multiple mutable sources of truth; +- snapshot objects remain unable to identify their artifacts; +- storage changes can break restore; +- privileged placement remains influenced by Pod metadata; +- per-tenant routing and future backends remain difficult. + +## 8. Non-final schema sketches + +These sketches exist to make tradeoffs concrete. They are not an approved API. + +### 8.1 Reusable storage configuration + +Pod access: + +```yaml +apiVersion: nvidia.com/v1alpha1 +kind: PodSnapshotStorageConfig +metadata: + name: workload-pvc +spec: + pvc: + pod: + volumeName: checkpoint-storage + mountPath: /checkpoints +status: + conditions: [] +``` + +Semantics: + +- the object is cluster-scoped and administrator-owned; +- `volumeName` identifies a volume in the source and restore Pod specs; +- the operator resolves that volume to a PVC in the PodSnapshot namespace; +- `mountPath` is the expected path in the target container; +- the controller pins the actual claim identity into `PodSnapshotContent`; +- the storage config does not create the claim. + +Named node-agent binding: + +```yaml +apiVersion: nvidia.com/v1alpha1 +kind: PodSnapshotStorageConfig +metadata: + name: shared-agent-store +spec: + pvc: + nodeAgent: + bindingName: primary +status: + conditions: [] +``` + +Semantics: + +- `bindingName` is a logical binding installed in the agent DaemonSet configuration; +- each applicable agent must report or validate the same binding; +- the CR alone does not add a volume to the DaemonSet; +- existing contents pin the resolved binding and handle so later config changes do not redirect + them. + +Operation-Pod variant: + +```yaml +apiVersion: nvidia.com/v1alpha1 +kind: PodSnapshotStorageConfig +metadata: + name: dynamic-operation-pvc +spec: + pvc: + operationPod: + volumeName: checkpoint-storage + mountPath: /checkpoints +status: + conditions: [] +``` + +This can use the same source-Pod volume-resolution contract while changing the execution strategy. + +The CRD should enforce exactly one of `pod`, `nodeAgent`, or `operationPod` with structural fields +and CEL validation. It should not combine an enum with unrelated optional fields that permit invalid +shapes. + +### 8.2 PodSnapshot reference + +```yaml +spec: + source: + podRef: + name: worker-0 + uid: ... + storageConfigName: workload-pvc +``` + +Open questions: + +- whether a default storage config is allowed; +- whether namespace policy restricts which configs a namespace may reference; +- whether the reference pins config UID/generation at creation; +- whether changing `storageConfigName` is forbidden by the existing immutable spec rule. + +### 8.3 Resolved content binding + +For pod access: + +```yaml +spec: + storage: + configRef: + name: workload-pvc + uid: ... + pvc: + claimRef: + namespace: inference + name: snapshot-pvc + uid: ... + pod: + volumeName: checkpoint-storage + mountPath: /checkpoints + artifactKey: podsnapshotcontent-/versions/1 +``` + +For a pre-mounted agent binding: + +```yaml +spec: + storage: + configRef: + name: shared-agent-store + uid: ... + pvc: + claimRef: + namespace: snapshot-system + name: snapshot-pvc + uid: ... + nodeAgent: + bindingName: primary + mountPath: /stores/primary + artifactKey: podsnapshotcontent-/versions/1 +``` + +Properties: + +- `PodSnapshotContent.spec` remains immutable; +- paths are operator-derived and agent-validated; +- `artifactKey` is relative and system-owned; +- claim UID prevents a same-named replacement from silently becoming the restore source; +- content carries enough information to reject an incompatible agent before capture; +- the agent, not the user, writes the final handle. + +### 8.4 Result handle + +```yaml +status: + snapshotHandle: pvc://inference/snapshot-pvc/podsnapshotcontent-/versions/1 + conditions: + - type: Ready + status: "True" +``` + +The final format must: + +- include claim namespace and name or another stable claim identity; +- include a system-owned artifact key; +- avoid embedding a host `/proc`, kubelet, or agent-container path; +- be parseable without consulting Pod annotations; +- be paired with claim UID or equivalent provenance in structured status/spec; +- remain resolvable after storage configuration changes; +- define escaping and canonicalization rather than relying on string concatenation. + +Whether the handle is opaque or formally versioned remains open. + +## 9. Decision matrix + +| Architecture | Dynamic per namespace | RWO | RWOP | RWX | Extra privileged Pod | Depends on live workload mount | Depends on kubelet layout | Agent rollout for new store | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Current `podMount` via `/proc` | Yes | Yes | Yes | Yes | No | Yes | No | No | +| Static `agentMount` | Limited | Poor across nodes | No | Yes | No | Restore target only | No | Yes | +| Named agent binding | Limited | Poor across nodes | No | Yes | No | Restore target only | No | Yes | +| Direct PVC + DS rollout | Yes, with controller | Poor across nodes | No | Yes | No | Restore target only | No | Yes, automated | +| Operation Pod | Yes | Yes, same node | No | Yes | Yes | Source/restore coordination | No | No | +| Kubelet host traversal | Yes | Yes | Yes | Yes | No | Yes | Yes | No | +| Injected sidecar | Yes | Yes | Usually no | Yes | Persistent helper | Yes | No | No | +| Direct CSI calls | Theoretical | Driver-specific | Driver-specific | Driver-specific | No | No | No | No | + +Notes: + +- “RWO: Yes” assumes all simultaneous consumers are scheduled on the same node. +- `ReadWriteOncePod` excludes any design with a second Pod mounting the claim. +- Static agent designs may use per-node claims, but an artifact then needs replication or + node-pinned restore; that is a distinct backend architecture. + +## 10. Failure-mode checklist + +Any selected design must define behavior for: + +- source Pod deleted before capture; +- source Pod recreated with the same name and different UID; +- container PID changes between resolution and filesystem access; +- referenced PVC missing, pending, terminating, or recreated; +- volume not mounted at the expected path; +- mount uses `subPath`, is read-only, or refers to a non-PVC volume; +- selected node lacks the required agent binding; +- RWO volume is attached to another node; +- operation Pod cannot schedule on the source/restore node; +- node fails during temporary-directory creation or atomic rename; +- checkpoint succeeds but status update fails; +- storage config changes or is deleted after capture; +- content is deleted while an operation is active; +- artifact directory exists from an earlier content lifetime; +- restore occurs on a different node or Kubernetes namespace; +- cleanup runs after the claim or storage binding disappears; +- agent version does not understand the stored handle version; +- two snapshots resolve to the same artifact key; +- a malicious or malformed path attempts traversal outside the storage root. + +## 11. Security requirements + +The implementation should preserve these invariants regardless of API choice: + +1. Ordinary snapshot users never provide a host path. +2. Artifact placement is derived from system-owned UID or equivalent collision-resistant identity. +3. Every joined path is canonicalized and checked to remain below the resolved storage root. +4. Pod access is tied to a verified Pod UID, container, volume, mount, PVC name, and PVC UID. +5. Node-agent access selects only administrator-installed bindings. +6. A namespaced `PodSnapshot` cannot use a storage config to access an arbitrary infrastructure PVC. +7. `PodSnapshotContent.spec` is immutable after resolution. +8. The agent writes only status and the artifact location assigned by the resolved work order. +9. Restore uses the content's resolved handle, not mutable Pod annotations or current defaults. +10. RBAC for storage configuration and agent bindings is separate from permission to request a + snapshot. +11. Cleanup verifies containment before recursive deletion. +12. Logs and status do not expose credentials or kubelet-internal paths. + +## 12. Migration implications + +A storage CRD cannot be introduced as an isolated field addition. A safe migration must address: + +- dual-read precedence between CRD fields, Pod annotations, operator config, and agent config; +- whether old snapshots without handles remain restorable; +- cutover from checkpoint-ID paths to content-UID paths; +- how restore resolves old annotation-composed artifacts; +- whether storage configs are created automatically from current Helm values; +- how namespace-local legacy agent installations map to new configs; +- how central `podMount` installations map to new configs; +- status-handle population for captures completed before the upgrade; +- cleanup ownership for both path schemes; +- CRD conversion is unaffected because PodSnapshot types currently exist only in `v1alpha1`, but + generated deepcopy, CRD bases, Helm CRD copies, RBAC, and API tests still require updates. + +A likely compatibility strategy is: + +1. introduce configuration and resolved-binding fields; +2. write the resolved content handle for new captures; +3. restore new captures from the handle; +4. retain a bounded legacy read path for old snapshots; +5. remove Pod storage annotations only after no supported snapshot depends on them. + +This sequence is not approved; it is listed to expose the migration dependency. + +## 13. Experiments required before a decision + +### Experiment 1: harden and benchmark current `podMount` + +- Verify capture and restore on containerd and CRI-O. +- Exercise PID replacement between resolution and access. +- Test a PVC mounted through `subPath`. +- Test RWO and RWOP claims. +- Validate behavior under SELinux/OpenShift. +- Measure save/restore overhead versus direct agent mount. +- Prove a controller can resolve and pin the actual PVC from `volumeName`. + +### Experiment 2: kubelet host-volume traversal portability + +- Locate the same PVC mount on GKE, EKS, AKS, OpenShift, and vCluster. +- Test CSI file and block-backed filesystem drivers. +- Compare paths with configurable kubelet roots. +- Verify mount propagation and SELinux access. +- Compare reliability with `/proc//root`. + +### Experiment 3: operation-Pod prototype + +- Create a privileged Pod constrained to the source node with the same RWO PVC. +- Let the scheduler process both node affinity and PVC topology. +- Prove CRIU, CUDA checkpoint, overlay capture, and restore work from the worker. +- Measure startup overhead. +- Kill the worker at each phase and verify status/temporary-directory recovery. +- Test RWOP rejection explicitly. + +### Experiment 4: static binding validation + +- Configure two logical agent bindings. +- Verify every applicable agent reports the same binding. +- Remove or roll a binding during an active capture. +- Confirm an existing handle remains resolvable after config changes. +- Test one RWX store and per-node RWO stores. + +### Experiment 5: handle and cleanup model + +- Define canonical URI parsing and escaping. +- Prove claim recreation is detected by UID. +- Prove content recreation cannot overwrite an old artifact. +- Test orphan discovery and safe containment checks. +- Exercise capture-success/status-failure recovery. + +## 14. Open decisions + +The following decisions remain deliberately unresolved: + +1. Which data-access architecture is the default? +2. Is host `/proc` traversal acceptable as the long-term pod strategy? +3. Is kubelet path traversal sufficiently portable for supported distributions? +4. Is an operation Pod's complexity justified by dynamic PVC access? +5. Does node-agent access need multiple named bindings or only one default store? +6. Should storage configuration be cluster-scoped, or should PVC policy be namespaced? +7. How are namespaces authorized to reference a cluster-scoped storage config? +8. Does the storage config merely describe an existing Pod volume, or can it provision claims? +9. Must restore remain in the capture namespace? +10. Are RWO and RWOP explicit support requirements? +11. Is `PodSnapshotContent.status.snapshotHandle` opaque or versioned and parseable? +12. Does the artifact path change from checkpoint ID to content UID? +13. Which controller owns deletion, retention, and orphan cleanup? +14. How long must legacy annotation-based snapshots remain restorable? +15. How does an agent advertise binding readiness and supported access strategies? + +## 15. Evaluation guidance + +The options should be evaluated in this order: + +1. Define required storage topologies: namespace-local RWO, shared RWX, RWOP, multi-tenant, and + cross-node restore. +2. Run the experiments for `podMount`, kubelet traversal, and operation Pods. +3. Decide the data plane before finalizing the CRD. +4. Select whether storage policy is reusable or deployment-global. +5. Define the immutable resolved binding and handle. +6. Design migration and cleanup together with restore. + +Choosing a CRD schema before the data plane is proven risks encoding an access promise that the +DaemonSet or CSI layer cannot fulfill. From 4603f6598919f95f2097f9c99446060014210a8d Mon Sep 17 00:00:00 2001 From: Ron Kahn Date: Thu, 23 Jul 2026 18:40:42 +0300 Subject: [PATCH 2/2] feat(snapshot): add agentInject storage access mode Adds an opt-in storage access mode where workload pods never mount the checkpoint PVC. The node agent mounts it (like agentMount) and, for restore, grafts the checkpoint dir into the target container's mount namespace via open_tree(OPEN_TREE_CLONE)+move_mount inside nsrestore, before CRIU reads it. A plain /proc/self/fd bind cannot cross mount namespaces (EINVAL); open_tree clones a detached mount that move_mount attaches in the container ns. Operator skips PVC injection into capture/restore pods (PVCName empty) and stamps the resolved access mode; the agent treats the stamped value as authoritative over its own config. Signed-off-by: Ron Kahn --- .../operator/templates/operator-config.yaml | 11 ++++- .../charts/snapshot/templates/configmap.yaml | 4 +- .../charts/snapshot/templates/daemonset.yaml | 7 +-- .../helm/charts/snapshot/templates/pvc.yaml | 2 +- deploy/helm/charts/snapshot/values.yaml | 6 +++ deploy/operator/api/config/v1alpha1/types.go | 6 +++ .../internal/checkpoint/checkpoint_test.go | 26 +++++++++++ .../operator/internal/checkpoint/podspec.go | 1 + .../operator/internal/checkpoint/storage.go | 31 ++++++++++--- .../internal/controller/checkpoint_job.go | 9 +++- deploy/snapshot/cmd/nsrestore/main.go | 2 + .../internal/controller/controller.go | 14 ++++++ .../snapshot/internal/executor/nsrestore.go | 17 ++++++++ deploy/snapshot/internal/executor/restore.go | 21 +++++++++ deploy/snapshot/internal/runtime/nsmount.go | 43 +++++++++++++++++++ deploy/snapshot/internal/types/config.go | 9 +++- deploy/snapshot/internal/types/config_test.go | 12 ++++++ deploy/snapshot/protocol/common.go | 26 +++++++++-- deploy/snapshot/protocol/restore_test.go | 34 +++++++++++++++ 19 files changed, 259 insertions(+), 22 deletions(-) create mode 100644 deploy/snapshot/internal/runtime/nsmount.go diff --git a/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml b/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml index bc99863e608c..ec6c046487d6 100644 --- a/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml @@ -180,9 +180,18 @@ data: {{- $pvc := .pvc | default dict }} storage: type: {{ .type | quote }} + {{- if .accessMode }} + accessMode: {{ .accessMode | quote }} + {{- end }} {{- if eq .type "pvc" }} pvc: - pvcName: {{ required "checkpoint.storage.pvc.pvcName is required when checkpoint.storage.type=pvc" $pvc.pvcName | quote }} + {{- if eq (.accessMode | default "") "agentInject" }} + {{- if $pvc.pvcName }} + pvcName: {{ $pvc.pvcName | quote }} + {{- end }} + {{- else }} + pvcName: {{ required "checkpoint.storage.pvc.pvcName is required when checkpoint.storage.type=pvc and accessMode is not agentInject" $pvc.pvcName | quote }} + {{- end }} basePath: {{ required "checkpoint.storage.pvc.basePath is required when checkpoint.storage.type=pvc" $pvc.basePath | quote }} {{- if $pvc.create }} create: true diff --git a/deploy/helm/charts/snapshot/templates/configmap.yaml b/deploy/helm/charts/snapshot/templates/configmap.yaml index 9c57102c1a9c..062ecd1c1c27 100644 --- a/deploy/helm/charts/snapshot/templates/configmap.yaml +++ b/deploy/helm/charts/snapshot/templates/configmap.yaml @@ -6,8 +6,8 @@ kind: ConfigMap {{- if ne .Values.storage.type "pvc" }} {{- fail (printf "snapshot.storage.type=%q is not supported yet; only pvc is currently implemented" .Values.storage.type) }} {{- end }} -{{- if not (has .Values.storage.accessMode (list "agentMount" "podMount")) }} -{{- fail (printf "snapshot.storage.accessMode=%q is not supported; expected agentMount or podMount" .Values.storage.accessMode) }} +{{- if not (has .Values.storage.accessMode (list "agentMount" "podMount" "agentInject")) }} +{{- fail (printf "snapshot.storage.accessMode=%q is not supported; expected agentMount, podMount, or agentInject" .Values.storage.accessMode) }} {{- end }} metadata: name: {{ include "snapshot.fullname" . }}-config diff --git a/deploy/helm/charts/snapshot/templates/daemonset.yaml b/deploy/helm/charts/snapshot/templates/daemonset.yaml index b780784be487..00e5206de39e 100644 --- a/deploy/helm/charts/snapshot/templates/daemonset.yaml +++ b/deploy/helm/charts/snapshot/templates/daemonset.yaml @@ -125,8 +125,9 @@ spec: - name: config mountPath: /etc/snapshot readOnly: true - {{- if and (eq .Values.storage.type "pvc") (eq .Values.storage.accessMode "agentMount") }} - # Mount the checkpoint PVC directly in namespace-local agent mode + {{- if and (eq .Values.storage.type "pvc") (has .Values.storage.accessMode (list "agentMount" "agentInject")) }} + # Mount the checkpoint PVC directly. agentMount serves it to the pod; + # agentInject clones it into the target container via move_mount. - name: checkpoints mountPath: {{ .Values.storage.pvc.basePath }} {{- end }} @@ -182,7 +183,7 @@ spec: path: /var/lib/kubelet/seccomp type: DirectoryOrCreate {{- end }} - {{- if and (eq .Values.storage.type "pvc") (eq .Values.storage.accessMode "agentMount") }} + {{- if and (eq .Values.storage.type "pvc") (has .Values.storage.accessMode (list "agentMount" "agentInject")) }} - name: checkpoints persistentVolumeClaim: claimName: {{ .Values.storage.pvc.name }} diff --git a/deploy/helm/charts/snapshot/templates/pvc.yaml b/deploy/helm/charts/snapshot/templates/pvc.yaml index 1ce7cd24e9a7..d2331fb3aaf2 100644 --- a/deploy/helm/charts/snapshot/templates/pvc.yaml +++ b/deploy/helm/charts/snapshot/templates/pvc.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if and (eq .Values.storage.type "pvc") (eq .Values.storage.accessMode "agentMount") .Values.storage.pvc.create }} +{{- if and (eq .Values.storage.type "pvc") (has .Values.storage.accessMode (list "agentMount" "agentInject")) .Values.storage.pvc.create }} apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/deploy/helm/charts/snapshot/values.yaml b/deploy/helm/charts/snapshot/values.yaml index cff299b1db2e..0fd656b3378a 100644 --- a/deploy/helm/charts/snapshot/values.yaml +++ b/deploy/helm/charts/snapshot/values.yaml @@ -46,6 +46,12 @@ storage: # DaemonSet in an infrastructure namespace. Because the DaemonSet no longer # mounts the PVC from every GPU node, suitable ReadWriteOnce storage # classes can be used for sequential checkpoint/restore workflows. + # - agentInject: workload pods NEVER mount the checkpoint PVC. The agent mounts + # it (like agentMount) and, for restore, grafts the checkpoint dir into the + # target container's mount namespace via open_tree(OPEN_TREE_CLONE)+move_mount + # before nsrestore reads it. Set the operator's checkpoint.storage.accessMode + # to agentInject to match (the operator then skips injecting the PVC into + # workload pods). accessMode: agentMount # PVC configuration (when type=pvc) diff --git a/deploy/operator/api/config/v1alpha1/types.go b/deploy/operator/api/config/v1alpha1/types.go index e462300be109..1224b5d86554 100644 --- a/deploy/operator/api/config/v1alpha1/types.go +++ b/deploy/operator/api/config/v1alpha1/types.go @@ -390,6 +390,12 @@ func (c *CheckpointConfiguration) EffectiveSeccompProfile() string { type CheckpointStorageConfiguration struct { // Type is the storage backend type. Only pvc is implemented today. Type string `json:"type"` + // AccessMode selects how the snapshot-agent reaches storage and must match the + // snapshot chart's storage.accessMode. "agentInject" tells the operator to + // stamp checkpoint metadata but NOT mount the PVC into workload pods (the agent + // grafts the checkpoint into the container instead); pvcName is then not + // required. Empty/"agentMount"/"podMount" preserve the existing behavior. + AccessMode string `json:"accessMode,omitempty"` // PVC configuration for pvc-based settings. PVC CheckpointPVCConfig `json:"pvc"` // Deprecated: S3 is retained for compatibility and ignored. diff --git a/deploy/operator/internal/checkpoint/checkpoint_test.go b/deploy/operator/internal/checkpoint/checkpoint_test.go index e138c34de84f..87a50279efc6 100644 --- a/deploy/operator/internal/checkpoint/checkpoint_test.go +++ b/deploy/operator/internal/checkpoint/checkpoint_test.go @@ -165,6 +165,32 @@ func TestStorageFromConfig(t *testing.T) { assert.Equal(t, "/snapshots", storage.BasePath) }) + t.Run("agentInject resolves storage without a pvcName", func(t *testing.T) { + storage, ok, err := StorageFromConfig(configv1alpha1.CheckpointStorageConfiguration{ + Type: snapshotprotocol.StorageTypePVC, + AccessMode: snapshotprotocol.StorageAccessModeAgentInject, + PVC: configv1alpha1.CheckpointPVCConfig{ + BasePath: "/checkpoints", + }, + }) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, snapshotprotocol.StorageTypePVC, storage.Type) + assert.Equal(t, "", storage.PVCName, "agentInject must not carry a workload PVC name") + assert.Equal(t, "/checkpoints", storage.BasePath) + assert.Equal(t, snapshotprotocol.StorageAccessModeAgentInject, storage.AccessMode) + }) + + t.Run("non-agentInject still requires a pvcName", func(t *testing.T) { + _, _, err := StorageFromConfig(configv1alpha1.CheckpointStorageConfiguration{ + Type: snapshotprotocol.StorageTypePVC, + PVC: configv1alpha1.CheckpointPVCConfig{ + BasePath: "/checkpoints", + }, + }) + require.Error(t, err) + }) + t.Run("pvc config normalizes clean base path", func(t *testing.T) { storage, ok, err := StorageFromConfig(configv1alpha1.CheckpointStorageConfiguration{ Type: snapshotprotocol.StorageTypePVC, diff --git a/deploy/operator/internal/checkpoint/podspec.go b/deploy/operator/internal/checkpoint/podspec.go index ecaf016bb86f..59b6f351c643 100644 --- a/deploy/operator/internal/checkpoint/podspec.go +++ b/deploy/operator/internal/checkpoint/podspec.go @@ -73,6 +73,7 @@ func ApplyRestorePodMetadataWithStorageConfig( delete(annotations, snapshotprotocol.TargetContainersAnnotation) delete(annotations, snapshotprotocol.CheckpointStorageTypeAnnotation) delete(annotations, snapshotprotocol.CheckpointStorageBasePathAnnotation) + delete(annotations, snapshotprotocol.CheckpointStorageAccessModeAnnotation) delete(annotations, commonconsts.CheckpointRestoreCandidateAnnotation) delete(annotations, commonconsts.CheckpointNameAnnotation) delete(annotations, commonconsts.CheckpointStartupPolicyAnnotation) diff --git a/deploy/operator/internal/checkpoint/storage.go b/deploy/operator/internal/checkpoint/storage.go index c6416392bb68..1bb0ec9465be 100644 --- a/deploy/operator/internal/checkpoint/storage.go +++ b/deploy/operator/internal/checkpoint/storage.go @@ -23,12 +23,14 @@ import ( func StorageFromConfig(config configv1alpha1.CheckpointStorageConfiguration) (snapshotprotocol.Storage, bool, error) { storageType := strings.TrimSpace(config.Type) + storageAccessMode := strings.TrimSpace(config.AccessMode) + agentInject := storageAccessMode == snapshotprotocol.StorageAccessModeAgentInject pvcName := strings.TrimSpace(config.PVC.PVCName) basePath := strings.TrimSpace(config.PVC.BasePath) size := strings.TrimSpace(config.PVC.Size) storageClassName := strings.TrimSpace(config.PVC.StorageClassName) accessMode := strings.TrimSpace(config.PVC.AccessMode) - hasPVCConfig := pvcName != "" || basePath != "" || config.PVC.Create || size != "" || storageClassName != "" || accessMode != "" + hasPVCConfig := pvcName != "" || basePath != "" || config.PVC.Create || size != "" || storageClassName != "" || accessMode != "" || storageAccessMode != "" if storageType == "" && !hasPVCConfig { return snapshotprotocol.Storage{}, false, nil } @@ -46,22 +48,32 @@ func StorageFromConfig(config configv1alpha1.CheckpointStorageConfiguration) (sn if storageType != snapshotprotocol.StorageTypePVC { return snapshotprotocol.Storage{}, false, fmt.Errorf("checkpoint storage type %q is not supported; only pvc is implemented today", storageType) } - if pvcName == "" || basePath == "" { - return snapshotprotocol.Storage{}, false, fmt.Errorf("checkpoint.storage.pvc.pvcName and checkpoint.storage.pvc.basePath are required when checkpoint storage is configured") + // agentInject: the workload pod never mounts the checkpoint PVC, so pvcName is + // not required (and is deliberately left empty so no PVC is injected into the + // pod). Only basePath is needed to locate the artifact. + if basePath == "" || (!agentInject && pvcName == "") { + return snapshotprotocol.Storage{}, false, fmt.Errorf("checkpoint.storage.pvc.basePath (and pvcName unless accessMode=agentInject) are required when checkpoint storage is configured") } basePath, err := normalizeStorageBasePath(basePath) if err != nil { return snapshotprotocol.Storage{}, false, err } - if config.PVC.Create { + // agentInject does not create a workload-namespace PVC; skip PVC access-mode + // validation, which only governs operator-created claims. + if config.PVC.Create && !agentInject { if _, err := storagePVCAccessMode(accessMode); err != nil { return snapshotprotocol.Storage{}, false, err } } + resolvedPVCName := pvcName + if agentInject { + resolvedPVCName = "" + } return snapshotprotocol.Storage{ - Type: snapshotprotocol.StorageTypePVC, - PVCName: pvcName, - BasePath: basePath, + Type: snapshotprotocol.StorageTypePVC, + PVCName: resolvedPVCName, + BasePath: basePath, + AccessMode: storageAccessMode, }, true, nil } @@ -78,6 +90,11 @@ func EnsureStoragePVC( if !ok { return nil } + // agentInject leaves PVCName empty: the workload pod never mounts a PVC, so + // there is no namespace-local claim for the operator to ensure or create. + if storage.PVCName == "" { + return nil + } if kubeClient == nil { return fmt.Errorf("checkpoint storage client is required") } diff --git a/deploy/operator/internal/controller/checkpoint_job.go b/deploy/operator/internal/controller/checkpoint_job.go index ade7dd358d0b..5e1ff266e6b5 100644 --- a/deploy/operator/internal/controller/checkpoint_job.go +++ b/deploy/operator/internal/controller/checkpoint_job.go @@ -78,8 +78,13 @@ func buildCheckpointJob( if storage, ok, err := checkpoint.StorageFromConfig(config.Checkpoint.Storage); err != nil { return nil, err } else if ok { - snapshotprotocol.InjectCheckpointVolume(&podTemplate.Spec, storage.PVCName) - snapshotprotocol.InjectCheckpointVolumeMount(targetContainer, storage.BasePath) + // agentInject leaves PVCName empty: skip injecting the PVC into the + // checkpoint Job pod. The agent writes the dump through its own mount, so + // only the storage metadata (type/basePath/accessMode) needs stamping. + if storage.PVCName != "" { + snapshotprotocol.InjectCheckpointVolume(&podTemplate.Spec, storage.PVCName) + snapshotprotocol.InjectCheckpointVolumeMount(targetContainer, storage.BasePath) + } if podTemplate.Annotations == nil { podTemplate.Annotations = map[string]string{} } diff --git a/deploy/snapshot/cmd/nsrestore/main.go b/deploy/snapshot/cmd/nsrestore/main.go index f19841328b55..368f69b8a33b 100644 --- a/deploy/snapshot/cmd/nsrestore/main.go +++ b/deploy/snapshot/cmd/nsrestore/main.go @@ -20,6 +20,7 @@ func main() { cudaDeviceMap := flag.String("cuda-device-map", "", "CUDA device map for cuda-checkpoint-helper restore") cgroupRoot := flag.String("cgroup-root", "", "CRIU cgroup root remap path") targetPodIP := flag.String("target-pod-ip", "", "Restore pod IP for CRIU TCP socket remapping") + checkpointFD := flag.Int("checkpoint-fd", -1, "Inherited fd of a detached checkpoint mount (agentInject); grafted at --checkpoint-path before restore") flag.Parse() if *checkpointPath == "" { @@ -31,6 +32,7 @@ func main() { CUDADeviceMap: *cudaDeviceMap, CgroupRoot: *cgroupRoot, TargetPodIP: *targetPodIP, + CheckpointFD: *checkpointFD, } result, err := executor.RestoreInNamespace(context.Background(), opts, log) diff --git a/deploy/snapshot/internal/controller/controller.go b/deploy/snapshot/internal/controller/controller.go index 88481c0e0ce9..0dcd1687f21a 100644 --- a/deploy/snapshot/internal/controller/controller.go +++ b/deploy/snapshot/internal/controller/controller.go @@ -583,6 +583,7 @@ func (w *NodeController) runRestore(ctx context.Context, pod *corev1.Pod, contai TargetPodIP: pod.Status.PodIP, ContainerName: containerName, Clientset: w.clientset, + AccessMode: w.effectiveAccessMode(pod), } placeholderHostPID, err := executor.Restore(restoreCtx, w.runtime, log, req) if err != nil { @@ -696,6 +697,19 @@ func chooseActiveContent(objs []interface{}) string { return chosen.Name } +// effectiveAccessMode returns the operator-stamped storage access mode from the +// pod (the single source of truth), falling back to the agent's own configured +// mode when the annotation is absent (legacy pods). Making the stamped value +// authoritative prevents the operator and agent configs from silently disagreeing. +func (w *NodeController) effectiveAccessMode(pod *corev1.Pod) string { + if pod != nil { + if m := strings.TrimSpace(pod.Annotations[snapshotprotocol.CheckpointStorageAccessModeAnnotation]); m != "" { + return m + } + } + return strings.TrimSpace(w.config.Storage.AccessMode) +} + func (w *NodeController) checkpointLocationsFromPod(pod *corev1.Pod, checkpointID string, hostPID int) (checkpointLocations, error) { rawBasePath, hasBasePathAnnotation := pod.Annotations[snapshotprotocol.CheckpointStorageBasePathAnnotation] basePath := strings.TrimSpace(rawBasePath) diff --git a/deploy/snapshot/internal/executor/nsrestore.go b/deploy/snapshot/internal/executor/nsrestore.go index 7f4ed4ec687a..49043b3d9ddc 100644 --- a/deploy/snapshot/internal/executor/nsrestore.go +++ b/deploy/snapshot/internal/executor/nsrestore.go @@ -21,6 +21,11 @@ type RestoreOptions struct { CUDADeviceMap string CgroupRoot string TargetPodIP string + // CheckpointFD, when >= 0, is an inherited fd of a detached checkpoint mount + // (agentInject mode). nsrestore grafts it at CheckpointPath inside this + // container's mount namespace before reading the checkpoint. -1 means the + // checkpoint is already visible (agentMount/podMount). + CheckpointFD int } type RestoreInNamespaceResult struct { @@ -38,8 +43,20 @@ func RestoreInNamespace(ctx context.Context, opts RestoreOptions, log logr.Logge "has_cuda_map", opts.CUDADeviceMap != "", "cgroup_root", opts.CgroupRoot, "target_pod_ip_present", opts.TargetPodIP != "", + "checkpoint_fd", opts.CheckpointFD, ) + // agentInject: graft the agent-supplied detached checkpoint mount into this + // container's mount namespace at CheckpointPath before anything reads it. This + // must precede ReadManifest/ApplyRootfsDiff, which both read the checkpoint dir. + if opts.CheckpointFD >= 0 { + detach, err := snapshotruntime.AttachCheckpointTree(opts.CheckpointFD, opts.CheckpointPath) + if err != nil { + return nil, fmt.Errorf("failed to graft checkpoint mount: %w", err) + } + defer detach() + } + manifestReadStart := time.Now() m, err := types.ReadManifest(opts.CheckpointPath) if err != nil { diff --git a/deploy/snapshot/internal/executor/restore.go b/deploy/snapshot/internal/executor/restore.go index df97f63ec7af..a74bef4432e6 100644 --- a/deploy/snapshot/internal/executor/restore.go +++ b/deploy/snapshot/internal/executor/restore.go @@ -35,6 +35,10 @@ type RestoreRequest struct { TargetPodIP string ContainerName string Clientset kubernetes.Interface + // AccessMode is the operator-stamped storage access mode. For agentInject the + // workload pod has no checkpoint PVC mount, so the agent clones its own + // checkpoint dir and grafts it into the container ns via nsrestore. + AccessMode string } // Restore performs external restore for the given request. @@ -218,7 +222,24 @@ func execNSRestore(ctx context.Context, log logr.Logger, req RestoreRequest, sna args = append(args, "--target-pod-ip", req.TargetPodIP) } + // agentInject: the workload pod does not mount the checkpoint PVC. Clone the + // agent's own checkpoint dir into a detached mount and hand it to nsrestore, + // which grafts it into the container's mount namespace at --checkpoint-path. + var extraFiles []*os.File + if req.AccessMode == types.StorageAccessModeAgentInject { + tree, err := snapshotruntime.OpenCheckpointTree(req.CheckpointLocation) + if err != nil { + return nil, fmt.Errorf("agentInject checkpoint clone: %w", err) + } + defer tree.Close() + extraFiles = append(extraFiles, tree) + // First ExtraFiles entry lands at fd 3 in nsenter and is inherited by + // nsrestore across the execve chain (nsenter does not close it). + args = append(args, "--checkpoint-fd", "3") + } + cmd := exec.CommandContext(ctx, "nsenter", args...) + cmd.ExtraFiles = extraFiles // Inherit the agent environment so nsrestore uses the same logger settings. cmd.Env = os.Environ() log.V(1).Info("Executing nsenter + nsrestore", "cmd", cmd.String()) diff --git a/deploy/snapshot/internal/runtime/nsmount.go b/deploy/snapshot/internal/runtime/nsmount.go new file mode 100644 index 000000000000..90cc8ec515ac --- /dev/null +++ b/deploy/snapshot/internal/runtime/nsmount.go @@ -0,0 +1,43 @@ +package runtime + +import ( + "fmt" + "os" + + "golang.org/x/sys/unix" +) + +// OpenCheckpointTree creates a detached clone of the mount subtree rooted at +// path (the agent's own checkpoint dir) and returns it as an *os.File so it can +// be handed to a child process via exec.Cmd.ExtraFiles. +// +// The clone is an anonymous mount (attached to no mount namespace), which is the +// only thing that can be grafted into a different mount namespace. A plain bind +// of /proc/self/fd/N fails with EINVAL from inside another namespace because the +// source mount belongs to this (the agent's) namespace; open_tree(OPEN_TREE_CLONE) +// plus move_mount is the kernel API built for exactly this transfer (Linux 5.2+). +func OpenCheckpointTree(path string) (*os.File, error) { + fd, err := unix.OpenTree(unix.AT_FDCWD, path, + uint(unix.OPEN_TREE_CLONE|unix.AT_RECURSIVE|unix.OPEN_TREE_CLOEXEC)) + if err != nil { + return nil, fmt.Errorf("open_tree(%s): %w", path, err) + } + return os.NewFile(uintptr(fd), "checkpoint-tree:"+path), nil +} + +// AttachCheckpointTree grafts the detached mount referred to by treeFD onto +// target inside the CURRENT mount namespace, creating target if absent. It +// returns a cleanup func that lazily detaches the mount. It is meant to run from +// inside the target container's mount namespace (i.e. from nsrestore), so the +// checkpoint dir becomes visible to CRIU without the workload pod ever mounting +// the PVC. +func AttachCheckpointTree(treeFD int, target string) (func(), error) { + if err := os.MkdirAll(target, 0o755); err != nil { + return nil, fmt.Errorf("create checkpoint mount target %s: %w", target, err) + } + if err := unix.MoveMount(treeFD, "", unix.AT_FDCWD, target, + unix.MOVE_MOUNT_F_EMPTY_PATH); err != nil { + return nil, fmt.Errorf("move_mount -> %s: %w", target, err) + } + return func() { _ = unix.Unmount(target, unix.MNT_DETACH) }, nil +} diff --git a/deploy/snapshot/internal/types/config.go b/deploy/snapshot/internal/types/config.go index 5454a3706740..b9d31fc17a04 100644 --- a/deploy/snapshot/internal/types/config.go +++ b/deploy/snapshot/internal/types/config.go @@ -26,6 +26,11 @@ const ( // StorageAccessModePodMount means workload pods mount the checkpoint PVC, // and snapshot-agent reaches it through /host/proc//root. StorageAccessModePodMount = "podMount" + // StorageAccessModeAgentInject means workload pods do NOT mount the checkpoint + // PVC. The snapshot-agent mounts it (like agentMount) and, for restore, grafts + // the checkpoint dir into the target container's mount namespace via + // open_tree(OPEN_TREE_CLONE)+move_mount before nsrestore reads it. + StorageAccessModeAgentInject = "agentInject" ) func (c *AgentConfig) LoadEnvOverrides() { @@ -58,11 +63,11 @@ func (c *AgentConfig) Validate() error { accessMode = StorageAccessModeAgentMount } switch accessMode { - case StorageAccessModeAgentMount, StorageAccessModePodMount: + case StorageAccessModeAgentMount, StorageAccessModePodMount, StorageAccessModeAgentInject: default: return &ConfigError{ Field: "storage.accessMode", - Message: fmt.Sprintf("unsupported access mode %q; expected %q or %q", c.Storage.AccessMode, StorageAccessModeAgentMount, StorageAccessModePodMount), + Message: fmt.Sprintf("unsupported access mode %q; expected %q, %q, or %q", c.Storage.AccessMode, StorageAccessModeAgentMount, StorageAccessModePodMount, StorageAccessModeAgentInject), } } c.Storage.AccessMode = accessMode diff --git a/deploy/snapshot/internal/types/config_test.go b/deploy/snapshot/internal/types/config_test.go index 2c8be135d864..83cbfec1b8ba 100644 --- a/deploy/snapshot/internal/types/config_test.go +++ b/deploy/snapshot/internal/types/config_test.go @@ -51,3 +51,15 @@ func TestAgentConfigValidateDefaultsStorageAccessMode(t *testing.T) { t.Fatalf("Storage.AccessMode = %q, want %q", cfg.Storage.AccessMode, StorageAccessModeAgentMount) } } + +func TestAgentConfigValidateAcceptsAgentInject(t *testing.T) { + cfg := validAgentConfig() + cfg.Storage.AccessMode = StorageAccessModeAgentInject + + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error = %v", err) + } + if cfg.Storage.AccessMode != StorageAccessModeAgentInject { + t.Fatalf("Storage.AccessMode = %q, want %q", cfg.Storage.AccessMode, StorageAccessModeAgentInject) + } +} diff --git a/deploy/snapshot/protocol/common.go b/deploy/snapshot/protocol/common.go index 1cccb3d9df7f..00a28883cde1 100644 --- a/deploy/snapshot/protocol/common.go +++ b/deploy/snapshot/protocol/common.go @@ -46,11 +46,20 @@ const ( CheckpointStorageTypeAnnotation = "nvidia.com/snapshot-storage-type" CheckpointStorageBasePathAnnotation = "nvidia.com/snapshot-storage-base-path" - CheckpointVolumeName = "checkpoint-storage" + // CheckpointStorageAccessModeAnnotation carries the operator-resolved storage + // access mode to the agent. The operator is the sole writer; the agent treats + // this stamped value as authoritative over its own config so the two cannot + // silently disagree. + CheckpointStorageAccessModeAnnotation = "nvidia.com/snapshot-storage-access-mode" + CheckpointVolumeName = "checkpoint-storage" DefaultCheckpointArtifactVersion = "1" DefaultCheckpointJobTTLSeconds = int32(300) DefaultSeccompLocalhostProfile = "profiles/block-iouring.json" StorageTypePVC = "pvc" + // StorageAccessModeAgentInject mirrors types.StorageAccessModeAgentInject so + // the operator (which imports protocol, not internal/types) can recognize the + // mode. In this mode the workload pod does not mount the checkpoint PVC. + StorageAccessModeAgentInject = "agentInject" CheckpointStatusCompleted = "completed" CheckpointStatusFailed = "failed" @@ -64,6 +73,10 @@ type Storage struct { Location string PVCName string BasePath string + // AccessMode mirrors the agent storage access mode (agentMount, podMount, + // agentInject). For agentInject the workload pod never mounts the PVC, so + // PVCName is empty and the agent grafts the checkpoint into the container. + AccessMode string } type RestoreStatusAnnotationKeys struct { @@ -218,10 +231,14 @@ func ApplyCheckpointStorageMetadata(annotations map[string]string, storage Stora } delete(annotations, CheckpointStorageTypeAnnotation) delete(annotations, CheckpointStorageBasePathAnnotation) + delete(annotations, CheckpointStorageAccessModeAnnotation) storageType := strings.TrimSpace(storage.Type) if storageType != "" { annotations[CheckpointStorageTypeAnnotation] = storageType } + if accessMode := strings.TrimSpace(storage.AccessMode); accessMode != "" { + annotations[CheckpointStorageAccessModeAnnotation] = accessMode + } basePath := strings.TrimSpace(storage.BasePath) if basePath != "" { basePath = strings.TrimRight(basePath, "/") @@ -264,8 +281,9 @@ func resolveStorageConfig(storage Storage) (Storage, error) { basePath = "/" } return Storage{ - Type: storageType, - PVCName: strings.TrimSpace(storage.PVCName), - BasePath: basePath, + Type: storageType, + PVCName: strings.TrimSpace(storage.PVCName), + BasePath: basePath, + AccessMode: strings.TrimSpace(storage.AccessMode), }, nil } diff --git a/deploy/snapshot/protocol/restore_test.go b/deploy/snapshot/protocol/restore_test.go index d11ba1ffe6da..a1b765b1b61f 100644 --- a/deploy/snapshot/protocol/restore_test.go +++ b/deploy/snapshot/protocol/restore_test.go @@ -255,6 +255,40 @@ func TestNewRestorePodRejectsUnknownContainer(t *testing.T) { } } +func TestPrepareRestorePodSpecAgentInjectSkipsPVC(t *testing.T) { + podSpec := corev1.PodSpec{ + Containers: []corev1.Container{{Name: "main"}}, + } + // agentInject resolves with an empty PVCName: the workload pod must NOT mount + // the checkpoint PVC; the agent grafts the checkpoint in at restore time. + storage := Storage{ + Type: StorageTypePVC, + PVCName: "", + BasePath: "/checkpoints", + AccessMode: StorageAccessModeAgentInject, + } + annotations := map[string]string{TargetContainersAnnotation: "main"} + if err := PrepareRestorePodSpec(&podSpec, annotations, storage, DefaultSeccompLocalhostProfile, true); err != nil { + t.Fatalf("PrepareRestorePodSpec error: %v", err) + } + + for _, v := range podSpec.Volumes { + if v.Name == CheckpointVolumeName { + t.Fatalf("agentInject must not inject the %s PVC volume: %#v", CheckpointVolumeName, podSpec.Volumes) + } + } + for _, m := range podSpec.Containers[0].VolumeMounts { + if m.Name == CheckpointVolumeName { + t.Fatalf("agentInject must not inject the %s volume mount: %#v", CheckpointVolumeName, podSpec.Containers[0].VolumeMounts) + } + } + // The control volume/probe are still required and present, so validation passes + // even though the PVC is absent. + if err := ValidateRestorePodSpec(&podSpec, annotations, storage, DefaultSeccompLocalhostProfile); err != nil { + t.Fatalf("ValidateRestorePodSpec rejected an agentInject pod: %v", err) + } +} + func TestPrepareRestorePodSpec(t *testing.T) { podSpec := corev1.PodSpec{} readinessProbe := &corev1.Probe{PeriodSeconds: 13, SuccessThreshold: 1}