Support durable directories and combined restore for gVisor

cmd/ateapi/internal/controlapi/actor_template_test.go

@@ -436,6 +436,11 @@ func TestValidateActorTemplate(t *testing.T) {
 		mutate: func(tmpl *ateapipb.ActorTemplate) {
 			tmpl.SnapshotsConfig.OnResume = &ateapipb.OnResumeConfig{FromData: ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT}
 		},
+	}, {
+		name: "valid on_resume with golden",
+		mutate: func(tmpl *ateapipb.ActorTemplate) {
+			tmpl.SnapshotsConfig.OnResume = &ateapipb.OnResumeConfig{FromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN}
+		},
 	}, {
 		name: "negative on_resume from_data",
 		mutate: func(tmpl *ateapipb.ActorTemplate) {

cmd/atelet/main.go

@@ -919,21 +919,16 @@ func (s *AteomHerder) uploadLocalCheckpointDir(ctx context.Context, req *ateletp
 // full checkpoint is monolithic until split checkpoints land.
 func narrowFullCaptureToData(rec *sandboxAssetsRecord) error {
 	switch atev1alpha1.SandboxClass(rec.SandboxClass) {
-	case atev1alpha1.SandboxClassMicroVM:
+	case atev1alpha1.SandboxClassMicroVM, atev1alpha1.SandboxClassGvisor:
 		if !slices.Contains(rec.SnapshotFiles, ateompath.DurableDirTarFile) {
 			// No durable-dir volumes were attached at pause: this snapshot
 			// holds no data, and never will — not retryable.
-			return status.Errorf(codes.FailedPrecondition, "full micro-VM capture has no %s; the actor has no durable data to upload as %s", ateompath.DurableDirTarFile, ateattr.SnapshotScopeData)
+			return status.Errorf(codes.FailedPrecondition, "full %s capture has no %s; the actor has no durable data to upload as %s", rec.SandboxClass, ateompath.DurableDirTarFile, ateattr.SnapshotScopeData)
 		}
 		rec.SnapshotFiles = []string{ateompath.DurableDirTarFile}
 		rec.Scope = ateattr.SnapshotScopeData
 		return nil
 
-	case atev1alpha1.SandboxClassGvisor:
-		// TODO(#790): split-checkpoint runsc will let a full gVisor checkpoint
-		// yield its durable data; implement this branch when it lands.
-		return status.Errorf(codes.Unimplemented, "gVisor cannot extract durable data from a full checkpoint yet (see #790)")
-
 	default:
 		// The manifest's class is unvalidated input from disk/object storage.
 		return status.Errorf(codes.FailedPrecondition, "unknown sandbox class %q in snapshot manifest", rec.SandboxClass)

cmd/atelet/main_test.go

@@ -1713,7 +1713,7 @@ func TestUploadLocalCheckpointDir(t *testing.T) {
 		}
 	})
 
-	t.Run("gvisor full capture cannot become data yet", func(t *testing.T) {
+	t.Run("gvisor full capture without durable tar has no data", func(t *testing.T) {
 		s := &AteomHerder{gcsClient: &recordingObjectStorage{}}
 		dir := filepath.Join(t.TempDir(), "pause-snap-1")
 		writeLocalSnapshot(t, dir, sandboxAssetsRecord{
@@ -1726,8 +1726,8 @@ func TestUploadLocalCheckpointDir(t *testing.T) {
 		req := validUploadPausedCheckpointRequest()
 		req.DesiredScope = ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA
 		_, err := s.uploadLocalCheckpointDir(ctx, req, dir, uri)
-		if got := status.Code(err); got != codes.Unimplemented {
-			t.Fatalf("status.Code = %v (err %v), want Unimplemented", got, err)
+		if got := status.Code(err); got != codes.FailedPrecondition {
+			t.Fatalf("status.Code = %v (err %v), want FailedPrecondition", got, err)
 		}
 	})
 

cmd/ateom-gvisor/durable.go

@@ -0,0 +1,80 @@
+//go:build linux
+
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/agent-substrate/substrate/internal/ateompath"
+	"github.com/agent-substrate/substrate/internal/proto/ateompb"
+	"github.com/agent-substrate/substrate/internal/tarutil"
+)
+
+// durableTarFile is the snapshot file holding the tar of the actor's durable-dir
+// volumes. Its entries are <volumeName>/... relative to
+// ateompath.DurableDirVolumeMountsDir, so extraction restores the same layout.
+// The name is shared with atelet, which uses it to carve durable data out of a
+// FULL snapshot's file set when uploading a paused checkpoint as DATA.
+const durableTarFile = ateompath.DurableDirTarFile
+
+// hasDurableVolumes reports whether any container mounts a durable-dir volume.
+func hasDurableVolumes(containers []*ateompb.Container) bool {
+	for _, c := range containers {
+		if len(c.GetDurableDirVolumeMounts()) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// tarDurableVolumes archives the actor's durable-dir volumes (dir) into the
+// checkpoint directory. The caller must have paused the guest first.
+//
+// Sockets the workload left behind and gVisor internal files (.gvisor.*) are
+// skipped rather than archived.
+func tarDurableVolumes(ctx context.Context, dir, checkpointDir string) error {
+	skip := func(rel string) bool {
+		base := filepath.Base(rel)
+		return strings.HasPrefix(base, ".gvisor.")
+	}
+	if err := tarutil.CreateFiltered(ctx, filepath.Join(checkpointDir, durableTarFile), dir, skip); err != nil {
+		return fmt.Errorf("while archiving durable-dir volumes from %q: %w", dir, err)
+	}
+	return nil
+}
+
+// untarDurableVolumes restores the durable-dir volumes from a snapshot into the
+// actor's host directory (dir, which atelet has already created, empty).
+func untarDurableVolumes(dir, snapshotDir string) error {
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("while creating durable-dir volumes dir %q: %w", dir, err)
+	}
+	if err := tarutil.Extract(filepath.Join(snapshotDir, durableTarFile), dir); err != nil {
+		return fmt.Errorf("while restoring durable-dir volumes into %q: %w", dir, err)
+	}
+	_ = filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
+		if err == nil && !info.IsDir() && strings.HasPrefix(info.Name(), ".gvisor.") {
+			_ = os.Remove(p)
+		}
+		return nil
+	})
+	return nil
+}

cmd/ateom-gvisor/main.go

@@ -95,6 +95,9 @@ const actorHTTPUpstream = "http://" + ateomnet.ActorVethIP + ":80"
 // termination grace period for the ateom.
 const workloadGracePeriod = 1 * time.Minute
 
+// resumeTimeout is the conservative ceiling for unpausing a paused sandbox.
+const resumeTimeout = 30 * time.Second
+
 func main() {
 	pflag.Parse()
 	if *showVersion {
@@ -750,23 +753,35 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
 	// TODO(dberkov): this is a temporary workaround until gVisor supports taking durable-dir snapshots in a single request with the process snapshot.
 	switch req.GetScope() {
 	case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA:
-		var ddv []string
-		for _, ctr := range req.GetSpec().GetContainers() {
-			for _, m := range ctr.GetDurableDirVolumeMounts() {
-				ddv = append(ddv, m.GetMountPath())
-			}
-		}
-		if len(ddv) == 0 {
+		if !hasDurableVolumes(req.GetSpec().GetContainers()) {
 			return nil, fmt.Errorf("no durable-dir volumes found for DATA snapshot")
 		}
-		if err := rcmd.cmdFsCheckpoint(ctx, "pause", checkpointPath, ddv); err != nil {
-			return nil, fmt.Errorf("while fscheckpointing durable-dir %q: %w", ddv[0], err)
+		if err := rcmd.cmdPause(ctx, "pause"); err != nil {
+			return nil, fmt.Errorf("while pausing pause container: %w", err)
+		}
+		tarErr := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(req.GetActorUid()), checkpointPath)
+		// Undoing our own pause must not depend on the caller's context:
+		// tarutil does not check ctx, so a deadline expiring mid-tar would
+		// fail the resume instantly and leave the sandbox paused forever.
+		resumeCtx, cancelResume := context.WithTimeout(context.WithoutCancel(ctx), resumeTimeout)
+		defer cancelResume()
+		if err := rcmd.cmdResume(resumeCtx, "pause"); err != nil {
+			return nil, fmt.Errorf("while resuming pause container: %w", err)
+		}
+		if tarErr != nil {
+			return nil, fmt.Errorf("while archiving durable-dir volumes: %w", tarErr)
 		}
 	case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL:
 		// Checkpoint pause container (root of the sandbox)
+		// TODO: Consider pause -> tar -> resume -> checkpoint order for better failure handling.
 		if err := rcmd.cmdCheckpoint(ctx, "pause", checkpointPath); err != nil {
 			return nil, fmt.Errorf("while checkpointing pause: %w", err)
 		}
+		if hasDurableVolumes(req.GetSpec().GetContainers()) {
+			if err := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(req.GetActorUid()), checkpointPath); err != nil {
+				return nil, fmt.Errorf("while archiving durable-dir volumes: %w", err)
+			}
+		}
 	default:
 		return nil, fmt.Errorf("unsupported snapshot scope: %v", req.GetScope())
 	}
@@ -927,6 +942,11 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
 	}()
 	checkpointDir := ateompath.RestoreStateDir(req.GetActorUid())
 
+	if hasDurableVolumes(req.GetSpec().GetContainers()) {
+		if err := untarDurableVolumes(ateompath.DurableDirVolumeMountsDir(req.GetActorUid()), checkpointDir); err != nil {
+			return nil, fmt.Errorf("while restoring durable-dir volumes: %w", err)
+		}
+	}
 	// Compose the pause rootfs before create (see RunWorkload). runsc restore
 	// only needs the rootfs to hold the correct content; whether it came from
 	// an untar or an overlay of cached layers is transparent to it.
@@ -936,22 +956,22 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
 
 	switch req.GetScope() {
 	case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA:
-		// Create and restore pause container
+		// Create and start pause container (cold boot with durable-dir volumes restored)
 		containersToDelete = append(containersToDelete, "pause")
-		if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", []string{"--fs-restore-image-path", checkpointDir}); err != nil {
+		if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil {
 			return nil, fmt.Errorf("while creating pause container: %w", err)
 		}
 		if err := rcmd.cmdStart(ctx, os.Stdout, "pause"); err != nil {
 			return nil, fmt.Errorf("while starting pause container: %w", err)
 		}
-	case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL:
+	case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN:
 		// Create and restore pause container
 		containersToDelete = append(containersToDelete, "pause")
 		if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil {
 			return nil, fmt.Errorf("while creating pause container: %w", err)
 		}
 		if err := rcmd.cmdRestore(ctx, os.Stdout, "pause", checkpointDir); err != nil {
-			return nil, fmt.Errorf("while starting pause container: %w", err)
+			return nil, fmt.Errorf("while restoring pause container: %w", err)
 		}
 	default:
 		return nil, fmt.Errorf("unexpected snapshot scope: %v", req.GetScope())
@@ -977,13 +997,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore
 			if err := rcmd.cmdStart(ctx, pw, ac.GetName()); err != nil {
 				return nil, fmt.Errorf("while starting %q application container: %w", ac.GetName(), err)
 			}
-		case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL:
+		case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN:
 			containersToDelete = append(containersToDelete, ac.GetName())
 			if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil {
 				return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err)
 			}
 			if err := rcmd.cmdRestore(ctx, pw, ac.GetName(), checkpointDir); err != nil {
-				return nil, fmt.Errorf("while starting %q application container: %w", ac.GetName(), err)
+				return nil, fmt.Errorf("while restoring %q application container: %w", ac.GetName(), err)
 			}
 		default:
 			return nil, fmt.Errorf("unexpected snapshot scope: %v", req.GetScope())

cmd/ateom-gvisor/runsc.go

@@ -170,6 +170,7 @@ func (r *runsc) cmdCheckpoint(ctx context.Context, containerName, checkpointPath
 	return nil
 }
 
+//nolint:unused
 func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPath string, durableDirMounts []string) error {
 	slog.InfoContext(ctx, "About to run runsc fscheckpoint", slog.String("container", containerName))
 
@@ -206,6 +207,56 @@ func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPa
 	return nil
 }
 
+// pauseArgs builds the argv for `runsc pause <container>`. Factored out so the
+// argument construction can be unit-tested without executing runsc.
+func (r *runsc) pauseArgs(containerName string) []string {
+	return []string{
+		"-log-format", "json",
+		"--alsologtostderr",
+		"-root", ateompath.RunSCStateDir(r.actorUID),
+		"pause",
+		containerName,
+	}
+}
+
+// cmdPause pauses all processes in the container (or sandbox, if pause).
+func (r *runsc) cmdPause(ctx context.Context, containerName string) error {
+	slog.InfoContext(ctx, "About to run runsc pause", slog.String("container", containerName))
+
+	cmd := exec.CommandContext(ctx, r.path, r.pauseArgs(containerName)...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := reaper.RunCommand(cmd); err != nil {
+		return fmt.Errorf("while running `runsc pause`: %w", err)
+	}
+	return nil
+}
+
+// resumeArgs builds the argv for `runsc resume <container>`. Factored out so the
+// argument construction can be unit-tested without executing runsc.
+func (r *runsc) resumeArgs(containerName string) []string {
+	return []string{
+		"-log-format", "json",
+		"--alsologtostderr",
+		"-root", ateompath.RunSCStateDir(r.actorUID),
+		"resume",
+		containerName,
+	}
+}
+
+// cmdResume unpauses a paused container (or sandbox, if pause).
+func (r *runsc) cmdResume(ctx context.Context, containerName string) error {
+	slog.InfoContext(ctx, "About to run runsc resume", slog.String("container", containerName))
+
+	cmd := exec.CommandContext(ctx, r.path, r.resumeArgs(containerName)...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := reaper.RunCommand(cmd); err != nil {
+		return fmt.Errorf("while running `runsc resume`: %w", err)
+	}
+	return nil
+}
+
 // We take a checkpoint only of the root container of the sandbox, but we need
 // to call restore on each container, using the same checkpoint.
 func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, checkpointPath string) error {

cmd/ateom-gvisor/runsc_test.go

@@ -63,3 +63,43 @@ func TestWaitArgs(t *testing.T) {
 		t.Errorf("waitArgs() = %v, want %v", got, want)
 	}
 }
+
+func TestPauseArgs(t *testing.T) {
+	r := &runsc{
+		path:     "/usr/bin/runsc",
+		actorUID: "test-actor-123",
+	}
+
+	got := r.pauseArgs("pause")
+	want := []string{
+		"-log-format", "json",
+		"--alsologtostderr",
+		"-root", ateompath.RunSCStateDir("test-actor-123"),
+		"pause",
+		"pause",
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("pauseArgs() = %v, want %v", got, want)
+	}
+}
+
+func TestResumeArgs(t *testing.T) {
+	r := &runsc{
+		path:     "/usr/bin/runsc",
+		actorUID: "test-actor-123",
+	}
+
+	got := r.resumeArgs("pause")
+	want := []string{
+		"-log-format", "json",
+		"--alsologtostderr",
+		"-root", ateompath.RunSCStateDir("test-actor-123"),
+		"resume",
+		"pause",
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("resumeArgs() = %v, want %v", got, want)
+	}
+}

cmd/ateom-microvm/durable.go

@@ -42,10 +42,10 @@ import (
 	"path/filepath"
 
 	"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata"
-	"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/tarutil"
 	"github.com/agent-substrate/substrate/internal/ateompath"
 	"github.com/agent-substrate/substrate/internal/ocispec"
 	"github.com/agent-substrate/substrate/internal/proto/ateompb"
+	"github.com/agent-substrate/substrate/internal/tarutil"
 )
 
 // durableTarFile is the snapshot file holding the tar of the actor's durable-dir

cmd/ateom-microvm/rootfsupper.go

@@ -50,8 +50,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/tarutil"
 	"github.com/agent-substrate/substrate/internal/ateompath"
+	"github.com/agent-substrate/substrate/internal/tarutil"
 )
 
 // rootfsUpperTarFile is the snapshot file holding the tar of the actor's

internal/ateompath/ateompath.go

@@ -191,7 +191,7 @@ func LocalSnapshotDir(actorUID, snapshotName string) string {
 	return filepath.Join(LocalCheckpointsDir(actorUID), snapshotName)
 }
 
-// DurableDirTarFile is the snapshot file holding the tar of a micro-VM
+// DurableDirTarFile is the snapshot file holding the tar of an
 // actor's durable-dir volumes (entries are <volumeName>/... relative to
 // DurableDirVolumeMountsDir). Written by ateom-microvm at checkpoint; a DATA
 // snapshot consists of this file alone, so atelet uses the name to carve the