diff --git a/cmd/atelet/deviceplugin.go b/cmd/atelet/deviceplugin.go index 1cb74dc8b7..d4b3cce8b0 100644 --- a/cmd/atelet/deviceplugin.go +++ b/cmd/atelet/deviceplugin.go @@ -30,6 +30,27 @@ import ( // exist; workers are handed the real host paths, which kubelet resolves. const hostDevRoot = "/host/dev" +// microvmNodeCapable reports whether this node can host micro-VM workers: +// cloud-hypervisor needs /dev/kvm (VmCreate fails with EPERM without it), and +// worker pods request the matching extended resource, so they only schedule to +// nodes where the device exists. Device presence is therefore the earliest +// reliable eligibility signal — known at atelet startup, before any WorkerPool +// schedules here. +// +// TODO(https://github.com/agent-substrate/substrate/pull/1207): /dev/kvm is +// not the only micro-VM hypervisor device; once /dev/mshv support lands, an +// mshv-only node would be wrongly reported incapable here. Treat presence of +// any micro-VM hypervisor device in SandboxDevices as capability, not KVM +// alone. +func microvmNodeCapable(devRoot string) bool { + for _, d := range deviceplugin.SandboxDevices { + if d.ResourceName == deviceplugin.ResourceKVM { + return d.Present(devRoot) + } + } + return false +} + // startDevicePlugins advertises the sandbox host devices present on this node to // kubelet as extended resources, in the background for the lifetime of ctx. This // is what lets a worker be granted /dev/kvm without running privileged. atelet diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index c1994865b3..911d659feb 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -308,6 +308,24 @@ func main() { csiDriverConfigLister, clusterTrustBundleLister, ) + // Pre-download sandbox assets as SandboxConfigs appear/change so the first + // Run/Restore on this node hits the cache. Best-effort: on failure the + // on-demand fetch in ensureSandboxAssets still covers correctness. + // + // The informer is requested only now, after the factory's blocking + // WaitForCacheSync above, so it cannot hold up atelet startup when its + // list/watch fails (e.g. Forbidden while the ClusterRole rollout lags the + // binary): the reflector retries in the background and prewarm stays cold + // until it recovers. + sandboxConfigInformer := ateFactory.Api().V1alpha1().SandboxConfigs().Informer() + if err := startSandboxAssetPrewarm(ctx, sandboxConfigInformer, wmService, imageCache, microvmNodeCapable(hostDevRoot)); err != nil { + slog.ErrorContext(ctx, "Sandbox asset prewarm disabled", slog.Any("err", err)) + } + // The factory only runs informers that exist when Start is called: the + // Start above predates the SandboxConfigs informer, so without this call + // it would never list or watch. Start is idempotent per informer — this + // launches the new one and leaves the already-running ones untouched. + ateFactory.Start(stopCh) dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ K8sClient: k8sClient, CAFile: *ateapiCAFile, diff --git a/cmd/atelet/sandbox_prewarm.go b/cmd/atelet/sandbox_prewarm.go new file mode 100644 index 0000000000..5b360f7b19 --- /dev/null +++ b/cmd/atelet/sandbox_prewarm.go @@ -0,0 +1,311 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "math/rand/v2" + "runtime" + "sync" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/agent-substrate/substrate/internal/imagecache" + "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" +) + +// prewarmMaxJitter spreads the fleet's asset downloads after a SandboxConfig +// change. Every atelet observes a create/update within about a second, and +// without jitter they would all open the same bucket objects at once. The +// jitter is applied as the enqueue delay, so duplicate events arriving inside +// the window collapse into one queue entry instead of stacking waits in the +// worker. A var so tests can zero it. +var prewarmMaxJitter = 30 * time.Second + +// prewarmTimeout bounds a single prewarm attempt. The queue is drained by one +// worker, so without a deadline a download that hangs without failing (a +// registry that accepts the connection and stalls, a wedged bucket read) +// would block every other config forever — prewarm runs on the daemon's +// never-cancelled context. Generous enough for multi-hundred-MiB micro-VM +// guest images on a busy node; a timed-out attempt requeues with backoff like +// any other failure. A var so tests can shorten it. +var prewarmTimeout = 5 * time.Minute + +// prewarmMaxRetries bounds how often one config's failed prewarm is retried +// before it is dropped until the next config event (or first use, which stays +// the correctness path). The backoff below caps at 5 minutes, so this covers +// transient bucket or registry outages of several minutes without hammering a +// permanently broken config forever. +const prewarmMaxRetries = 8 + +// sandboxAssetFetcher is the one slice of AteomHerder the prewarmer needs. +// Prewarming through the same method as the Run/Restore path keeps the two +// fetches from ever diverging on cache layout or validation. +type sandboxAssetFetcher interface { + ensureSandboxAssets(ctx context.Context, rec *sandboxAssetsRecord) (map[string]string, error) +} + +// sandboxPrewarmer downloads SandboxConfig assets into the node's +// content-addressed static-files cache before any actor asks for them, so the +// fetch inside the first Run/Restore on the node is a cache hit instead of a +// download+extract on the critical path. +type sandboxPrewarmer struct { + assets sandboxAssetFetcher + images *imagecache.Store + // lister resolves a queued config name to its latest revision at + // processing time, so coalesced events never prewarm a stale spec. + lister listersv1alpha1.SandboxConfigLister + // queue decouples informer event handlers (which must not block) from the + // downloads, dedupes by config name so relists cannot stack duplicate + // work, and rate-limits retries after failures. A single worker drains + // it, which also serializes downloads so concurrent prewarms never + // compete for node bandwidth. + queue workqueue.TypedRateLimitingInterface[string] + // microvmCapable gates micro-VM configs: their guest images run to + // hundreds of MiB, and a node without /dev/kvm can never run that class + // (workers request the ate.dev/kvm extended resource, so they only + // schedule where the device exists). See microvmNodeCapable. + microvmCapable bool +} + +func newSandboxPrewarmer(assets sandboxAssetFetcher, images *imagecache.Store, lister listersv1alpha1.SandboxConfigLister, microvmCapable bool) *sandboxPrewarmer { + return &sandboxPrewarmer{ + assets: assets, + images: images, + lister: lister, + // Downloads fail on the scale of network timeouts, not API conflicts, + // so back off in seconds and cap in minutes rather than the + // millisecond-based controller default. + queue: workqueue.NewTypedRateLimitingQueue( + workqueue.NewTypedItemExponentialFailureRateLimiter[string](time.Second, 5*time.Minute)), + microvmCapable: microvmCapable, + } +} + +// startSandboxAssetPrewarm registers an event handler on the SandboxConfig +// informer and starts a background worker that pre-downloads each config's +// sandbox assets for this node's architecture, and pulls its pause image into +// the image cache. Prewarming is purely a latency optimization: every failure +// is logged and left to the on-demand fetch in ensureSandboxAssets and the +// pull inside prepareOCIBundles, which remain the correctness path. +// +// TODO: the static-files cache is never pruned, and prewarming every config +// revision makes stale releases accumulate faster. Add a GC that removes +// assets referenced by no current SandboxConfig and no on-node actor record. +func startSandboxAssetPrewarm(ctx context.Context, informer cache.SharedIndexInformer, assets sandboxAssetFetcher, images *imagecache.Store, microvmCapable bool) error { + p := newSandboxPrewarmer(assets, images, listersv1alpha1.NewSandboxConfigLister(informer.GetIndexer()), microvmCapable) + // Atelet startup never waits for this informer to sync: prewarm is + // best-effort, so a failing list/watch (e.g. Forbidden while an RBAC + // rollout lags the binary) must degrade prewarm, not hang the node. The + // handler makes that degradation visible in the log; the reflector keeps + // retrying and prewarm recovers with it. Setting it fails only on an + // already-started informer, where the default reflector logging applies. + if err := informer.SetWatchErrorHandler(func(_ *cache.Reflector, err error) { + slog.WarnContext(ctx, "SandboxConfig list/watch failed; sandbox asset prewarm degraded until it recovers", slog.Any("err", err)) + }); err != nil { + slog.InfoContext(ctx, "Could not set sandbox config watch error handler", slog.Any("err", err)) + } + // The initial List replays every existing SandboxConfig into the handler + // as an Add: a freshly booted node prewarms the current configs, not only + // future changes. + if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { p.enqueue(ctx, obj) }, + UpdateFunc: func(_, obj any) { p.enqueue(ctx, obj) }, + }); err != nil { + return fmt.Errorf("while registering sandbox config prewarm handler: %w", err) + } + go p.run(ctx) + slog.InfoContext(ctx, "Sandbox asset prewarm started", slog.Bool("microvmCapable", microvmCapable)) + return nil +} + +// skipConfig reports whether this node has nothing to prewarm for cfg, +// logging why. Both enqueue and process apply it: sandboxClass is mutable, +// so the class observed at enqueue time can be stale by the time the worker +// resolves the name after jitter or backoff, and the gate must hold for the +// revision actually prewarmed. +func (p *sandboxPrewarmer) skipConfig(ctx context.Context, cfg *v1alpha1.SandboxConfig) bool { + switch cfg.Spec.SandboxClass { + case v1alpha1.SandboxClassGvisor: + // Every node runs gVisor workers; always prewarm. + return false + case v1alpha1.SandboxClassMicroVM: + if !p.microvmCapable { + slog.DebugContext(ctx, "Skipping sandbox asset prewarm: node has no /dev/kvm, cannot run micro-VM workers", + slog.String("config", cfg.Name)) + return true + } + return false + default: + // An unknown class has no backend in this atelet (likely version skew + // with a newer control plane); nothing to prewarm. + slog.InfoContext(ctx, "Skipping sandbox asset prewarm: unknown sandbox class", + slog.String("config", cfg.Name), + slog.String("sandboxClass", string(cfg.Spec.SandboxClass))) + return true + } +} + +func (p *sandboxPrewarmer) enqueue(ctx context.Context, obj any) { + cfg, ok := obj.(*v1alpha1.SandboxConfig) + if !ok { + return + } + if p.skipConfig(ctx, cfg) { + return + } + if prewarmMaxJitter > 0 { + p.queue.AddAfter(cfg.Name, rand.N(prewarmMaxJitter)) + return + } + p.queue.Add(cfg.Name) +} + +func (p *sandboxPrewarmer) run(ctx context.Context) { + go func() { + <-ctx.Done() + p.queue.ShutDown() + }() + for { + name, shutdown := p.queue.Get() + if shutdown { + return + } + p.process(ctx, name) + } +} + +// process prewarms the named config's current revision, requeueing with +// backoff on failure. The queue holds names, not objects, so an event that +// arrives while its config is being processed is simply requeued by the +// workqueue and prewarms the newer revision afterwards. +func (p *sandboxPrewarmer) process(ctx context.Context, name string) { + defer p.queue.Done(name) + cfg, err := p.lister.Get(name) + if apierrors.IsNotFound(err) { + // Deleted since it was enqueued; nothing to prewarm anymore. + p.queue.Forget(name) + return + } + if err == nil && p.skipConfig(ctx, cfg) { + // The class gate re-applies to the revision resolved now, which may + // differ from the one that passed enqueue's filter. + p.queue.Forget(name) + return + } + if err == nil { + err = p.prewarm(ctx, cfg) + } + if err == nil { + p.queue.Forget(name) + return + } + if ctx.Err() != nil { + // Shutting down, not a prewarm failure; drop without retry noise. + return + } + if retries := p.queue.NumRequeues(name); retries < prewarmMaxRetries { + slog.WarnContext(ctx, "Sandbox asset prewarm failed; will retry", + slog.String("config", name), slog.Int("retries", retries), slog.Any("err", err)) + p.queue.AddRateLimited(name) + return + } + // Best-effort: give up until the next config event or first use. + slog.WarnContext(ctx, "Sandbox asset prewarm failed; giving up", + slog.String("config", name), slog.Int("retries", prewarmMaxRetries), slog.Any("err", err)) + p.queue.Forget(name) +} + +// prewarm fetches every asset of one SandboxConfig into the static-files +// cache and its pause image into the image cache. Racing an on-demand +// ensureSandboxAssets for the same assets is safe: both paths install +// content-addressed files via atomic rename, and the image cache collapses +// concurrent pulls of one digest. +func (p *sandboxPrewarmer) prewarm(ctx context.Context, cfg *v1alpha1.SandboxConfig) error { + ctx, cancel := context.WithTimeout(ctx, prewarmTimeout) + defer cancel() + t := time.Now() + + var imageErr error + var wg sync.WaitGroup + // schedule prewarm pause image if provided + if cfg.Spec.PauseImage != "" { + wg.Go(func() { + if _, err := p.images.EnsureImage(ctx, cfg.Spec.PauseImage); err != nil { + imageErr = fmt.Errorf("while prewarming pause image %q: %w", cfg.Spec.PauseImage, err) + } + }) + } + // schedule prewarm sandbox assets if provided + rec, assetErr := recordFromSandboxConfig(cfg) + switch { + case errors.Is(assetErr, errNoAssetsForArch): + // Permanent until the config changes, and a change re-enqueues: + // retrying would burn every attempt on a non-failure and redo the + // pause-image pull with it. Nothing to do for this node. + slog.DebugContext(ctx, "No sandbox assets to prewarm for this node's architecture", + slog.String("config", cfg.Name), slog.String("arch", runtime.GOARCH)) + assetErr = nil + case assetErr == nil: + wg.Go(func() { + _, assetErr = p.assets.ensureSandboxAssets(ctx, rec) + }) + } + wg.Wait() + if err := errors.Join(assetErr, imageErr); err != nil { + return err + } + assets := 0 + if rec != nil { + assets = len(rec.Assets) + } + slog.InfoContext(ctx, "Sandbox assets prewarmed", + slog.String("config", cfg.Name), + slog.Int("assets", assets), + slog.String("pauseImage", cfg.Spec.PauseImage), + slog.Duration("duration", time.Since(t))) + return nil +} + +// errNoAssetsForArch reports that a SandboxConfig lists no assets for this +// node's architecture. Unlike a failed download it is permanent until the +// config changes, so prewarm treats it as nothing-to-do rather than retrying. +var errNoAssetsForArch = errors.New("no sandbox assets for this architecture") + +// recordFromSandboxConfig projects a SandboxConfig's per-architecture assets +// onto the local node's architecture, mirroring recordFromRequest. +func recordFromSandboxConfig(cfg *v1alpha1.SandboxConfig) (*sandboxAssetsRecord, error) { + arch := runtime.GOARCH + files := cfg.Spec.Assets[arch] + if len(files) == 0 { + return nil, fmt.Errorf("sandbox config %q, architecture %q: %w", cfg.Name, arch, errNoAssetsForArch) + } + rec := &sandboxAssetsRecord{ + SandboxClass: string(cfg.Spec.SandboxClass), + PauseImage: cfg.Spec.PauseImage, + Assets: make(map[string]assetEntry, len(files)), + } + for name, f := range files { + rec.Assets[name] = assetEntry{URL: f.URL, SHA256: f.SHA256} + } + return rec, nil +} diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go new file mode 100644 index 0000000000..71030d34d3 --- /dev/null +++ b/cmd/atelet/sandbox_prewarm_test.go @@ -0,0 +1,403 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "log" + "net/http/httptest" + "net/url" + "os" + "runtime" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/imagecache" + "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + "github.com/agent-substrate/substrate/pkg/client/clientset/versioned/fake" + "github.com/agent-substrate/substrate/pkg/client/informers/externalversions" + listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// pushPauseImage pushes a tiny image to ref and returns its manifest digest, +// so tests can later assert a cache hit by digest with the registry gone. +func pushPauseImage(t *testing.T, ref string) v1.Hash { + t.Helper() + img, err := mutate.AppendLayers(empty.Image, singleFileLayer(t, "pause", "pause bytes")) + if err != nil { + t.Fatalf("mutate.AppendLayers: %v", err) + } + digest, err := img.Digest() + if err != nil { + t.Fatalf("img.Digest: %v", err) + } + tag, err := name.ParseReference(ref, name.Insecure) + if err != nil { + t.Fatalf("name.ParseReference(%q): %v", ref, err) + } + if err := remote.Write(tag, img); err != nil { + t.Fatalf("remote.Write(%q): %v", ref, err) + } + return digest +} + +func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { + return &v1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.SandboxConfigSpec{ + SandboxClass: v1alpha1.SandboxClassGvisor, + PauseImage: "registry.k8s.io/pause@sha256:abc", + Assets: map[string]map[string]v1alpha1.AssetFile{ + runtime.GOARCH: { + runscAssetName: {URL: url, SHA256: sha}, + }, + }, + }, + } +} + +func TestRecordFromSandboxConfig(t *testing.T) { + sha := fmt.Sprintf("%x", sha256.Sum256([]byte("runsc"))) + cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) + + rec, err := recordFromSandboxConfig(cfg) + if err != nil { + t.Fatalf("recordFromSandboxConfig: %v", err) + } + if rec.SandboxClass != string(v1alpha1.SandboxClassGvisor) { + t.Errorf("SandboxClass = %q, want %q", rec.SandboxClass, v1alpha1.SandboxClassGvisor) + } + if rec.PauseImage != cfg.Spec.PauseImage { + t.Errorf("PauseImage = %q, want %q", rec.PauseImage, cfg.Spec.PauseImage) + } + want := assetEntry{URL: "gs://bucket/runsc", SHA256: sha} + if got := rec.Assets[runscAssetName]; got != want { + t.Errorf("Assets[%q] = %+v, want %+v", runscAssetName, got, want) + } + + // A config with no assets for this node's architecture cannot be + // projected, and the error carries the sentinel that keeps prewarm from + // retrying a condition only a config change can clear. + cfg.Spec.Assets = map[string]map[string]v1alpha1.AssetFile{ + "other-arch": {runscAssetName: {URL: "gs://bucket/runsc", SHA256: sha}}, + } + if _, err := recordFromSandboxConfig(cfg); !errors.Is(err, errNoAssetsForArch) { + t.Errorf("recordFromSandboxConfig with no assets for the local architecture = %v, want errNoAssetsForArch", err) + } +} + +func TestPrewarmEnqueueFilters(t *testing.T) { + origJitter := prewarmMaxJitter + prewarmMaxJitter = 0 // enqueue synchronously so Len is observable + t.Cleanup(func() { prewarmMaxJitter = origJitter }) + + ctx := context.Background() + microvm := &v1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "microvm-default"}, + Spec: v1alpha1.SandboxConfigSpec{SandboxClass: v1alpha1.SandboxClassMicroVM}, + } + gvisor := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("runsc")))) + + t.Run("node without KVM", func(t *testing.T) { + p := newSandboxPrewarmer(nil, nil, nil, false) + + p.enqueue(ctx, "not a sandbox config") + p.enqueue(ctx, microvm) + p.enqueue(ctx, &v1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "future-class"}, + Spec: v1alpha1.SandboxConfigSpec{SandboxClass: "future-class"}, + }) + if p.queue.Len() != 0 { + t.Fatalf("queue holds %d configs after filtered enqueues, want 0", p.queue.Len()) + } + + p.enqueue(ctx, gvisor) + if p.queue.Len() != 1 { + t.Fatalf("queue holds %d configs after gvisor enqueue, want 1", p.queue.Len()) + } + // Duplicate events (e.g. a relist) must coalesce, not stack work. + p.enqueue(ctx, gvisor) + if p.queue.Len() != 1 { + t.Errorf("queue holds %d configs after duplicate enqueue, want 1", p.queue.Len()) + } + }) + + t.Run("node with KVM", func(t *testing.T) { + p := newSandboxPrewarmer(nil, nil, nil, true) + p.enqueue(ctx, microvm) + p.enqueue(ctx, gvisor) + if p.queue.Len() != 2 { + t.Errorf("queue holds %d configs, want both microvm and gvisor queued", p.queue.Len()) + } + }) +} + +// TestPrewarmProcessRetries covers the worker's failure handling: a failed +// prewarm is requeued with backoff, and a config deleted between enqueue and +// processing is forgotten without retries. +func TestPrewarmProcessRetries(t *testing.T) { + origDir := ateompath.StaticFilesDir + ateompath.StaticFilesDir = t.TempDir() + t.Cleanup(func() { ateompath.StaticFilesDir = origDir }) + + ctx := context.Background() + cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("runsc")))) + // No pause image, so a failing prewarm exercises only the asset path. + cfg.Spec.PauseImage = "" + + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + if err := indexer.Add(cfg); err != nil { + t.Fatalf("indexer.Add: %v", err) + } + herder := &AteomHerder{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} + p := newSandboxPrewarmer(herder, nil, listersv1alpha1.NewSandboxConfigLister(indexer), false) + + p.queue.Add(cfg.Name) + name, _ := p.queue.Get() + p.process(ctx, name) + if got := p.queue.NumRequeues(cfg.Name); got != 1 { + t.Errorf("NumRequeues after failed prewarm = %d, want 1 (requeued with backoff)", got) + } + + p.process(ctx, "deleted-config") + if got := p.queue.NumRequeues("deleted-config"); got != 0 { + t.Errorf("NumRequeues for a deleted config = %d, want 0 (forgotten)", got) + } +} + +// TestPrewarmProcessReappliesClassGate covers the gap between enqueue and +// processing: sandboxClass is mutable, so a config that passed the enqueue +// filter as gVisor may be micro-VM by the time the worker resolves it, and a +// node without KVM must skip it rather than download guest assets. +func TestPrewarmProcessReappliesClassGate(t *testing.T) { + origDir := ateompath.StaticFilesDir + ateompath.StaticFilesDir = t.TempDir() + t.Cleanup(func() { ateompath.StaticFilesDir = origDir }) + + ctx := context.Background() + // Enqueued while gVisor, edited to micro-VM before the worker ran. + cfg := gvisorConfig("mutating-config", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("runsc")))) + cfg.Spec.SandboxClass = v1alpha1.SandboxClassMicroVM + + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + if err := indexer.Add(cfg); err != nil { + t.Fatalf("indexer.Add: %v", err) + } + // Any fetch would fail and requeue, so NumRequeues distinguishes + // "skipped" from "attempted and failed". + herder := &AteomHerder{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} + p := newSandboxPrewarmer(herder, nil, listersv1alpha1.NewSandboxConfigLister(indexer), false) + + p.queue.Add(cfg.Name) + name, _ := p.queue.Get() + p.process(ctx, name) + if got := p.queue.NumRequeues(cfg.Name); got != 0 { + t.Errorf("NumRequeues after a class change to micro-VM on a non-KVM node = %d, want 0 (skipped, not retried)", got) + } +} + +// TestMicrovmNodeCapable covers the detectable negative cases; the positive +// case needs a /dev/kvm character device, which a test cannot mknod. +func TestMicrovmNodeCapable(t *testing.T) { + devRoot := t.TempDir() + if microvmNodeCapable(devRoot) { + t.Error("microvmNodeCapable = true for a dev root without kvm") + } + // A plain file named kvm is not a character device and must not count. + if err := os.WriteFile(devRoot+"/kvm", []byte("not a device"), 0o600); err != nil { + t.Fatal(err) + } + if microvmNodeCapable(devRoot) { + t.Error("microvmNodeCapable = true for a regular file named kvm") + } +} + +// TestPrewarmPauseImage covers the pause-image half of prewarm: the image +// lands in the image cache, and an asset fetch failure does not stop the +// pull (the two live in different backends). +func TestPrewarmPauseImage(t *testing.T) { + origDir := ateompath.StaticFilesDir + ateompath.StaticFilesDir = t.TempDir() + t.Cleanup(func() { ateompath.StaticFilesDir = origDir }) + + srv := httptest.NewServer(registry.New(registry.Logger(log.New(io.Discard, "", 0)))) + defer srv.Close() + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parsing registry URL: %v", err) + } + pauseRef := u.Host + "/pause:3.10" + pauseDigest := pushPauseImage(t, pauseRef) + + newStore := func() *imagecache.Store { + s, err := imagecache.New(t.TempDir()) + if err != nil { + t.Fatalf("imagecache.New: %v", err) + } + return s + } + okStore, failStore, archStore := newStore(), newStore(), newStore() + + ctx := context.Background() + content := []byte("runsc binary bytes") + cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) + cfg.Spec.PauseImage = pauseRef + + p := &sandboxPrewarmer{ + assets: &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}}, + images: okStore, + } + if err := p.prewarm(ctx, cfg); err != nil { + t.Fatalf("prewarm: %v", err) + } + + // A different asset hash misses the shared static-files cache, so the + // failing object storage is actually consulted — and must not keep the + // pause image from being pulled. + failCfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("other runsc")))) + failCfg.Spec.PauseImage = pauseRef + p = &sandboxPrewarmer{ + assets: &AteomHerder{anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, + images: failStore, + } + if err := p.prewarm(ctx, failCfg); err == nil { + t.Error("prewarm returned nil despite the asset fetch failing") + } + + // A config with no assets for this node's architecture still gets its + // pause image pulled, and prewarm succeeds: the missing assets are + // permanent until the config changes, not a retryable failure. + archCfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256(content))) + archCfg.Spec.Assets = map[string]map[string]v1alpha1.AssetFile{ + "other-arch": {runscAssetName: archCfg.Spec.Assets[runtime.GOARCH][runscAssetName]}, + } + archCfg.Spec.PauseImage = pauseRef + p = &sandboxPrewarmer{images: archStore} + if err := p.prewarm(ctx, archCfg); err != nil { + t.Errorf("prewarm with no assets for the local architecture: %v", err) + } + + // With the registry gone, only a cache hit can satisfy a digest ref. + srv.Close() + digestRef := u.Host + "/pause@" + pauseDigest.String() + for name, store := range map[string]*imagecache.Store{"ok": okStore, "asset-failure": failStore, "no-local-arch": archStore} { + if _, err := store.EnsureImage(ctx, digestRef); err != nil { + t.Errorf("pause image not prewarmed into the %s store: %v", name, err) + } + } +} + +// hangingObjectStorage blocks GetObject until the caller's context ends, +// simulating a bucket read that stalls without failing. +type hangingObjectStorage struct{} + +func (hangingObjectStorage) GetObject(ctx context.Context, _, _ string) (io.ReadCloser, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func (hangingObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) error { return nil } + +// TestPrewarmTimeout verifies a single prewarm attempt is bounded by +// prewarmTimeout: the queue has one worker, so an attempt that never returned +// would block every other config's prewarm. +func TestPrewarmTimeout(t *testing.T) { + origDir, origTimeout := ateompath.StaticFilesDir, prewarmTimeout + ateompath.StaticFilesDir = t.TempDir() + prewarmTimeout = 50 * time.Millisecond + t.Cleanup(func() { ateompath.StaticFilesDir, prewarmTimeout = origDir, origTimeout }) + + cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", fmt.Sprintf("%x", sha256.Sum256([]byte("hung runsc")))) + cfg.Spec.PauseImage = "" + p := &sandboxPrewarmer{assets: &AteomHerder{anonGCSClient: hangingObjectStorage{}}} + + done := make(chan error, 1) + go func() { done <- p.prewarm(context.Background(), cfg) }() + select { + case err := <-done: + if err == nil { + t.Error("prewarm returned nil despite the download hanging") + } + case <-time.After(10 * time.Second): + t.Fatal("prewarm never returned; a hung download would block the worker forever") + } +} + +// TestSandboxAssetPrewarmDownloads runs the whole path: a SandboxConfig in a +// fake clientset flows through the informer into the prewarm worker, which +// lands the asset in the static-files cache without any Run/Restore request. +func TestSandboxAssetPrewarmDownloads(t *testing.T) { + origDir, origJitter := ateompath.StaticFilesDir, prewarmMaxJitter + ateompath.StaticFilesDir = t.TempDir() + prewarmMaxJitter = 0 + t.Cleanup(func() { ateompath.StaticFilesDir, prewarmMaxJitter = origDir, origJitter }) + + host := imageVolumeTestRegistry(t) + pauseRef := host + "/pause:3.10" + pushPauseImage(t, pauseRef) + + content := []byte("runsc binary bytes") + sha := fmt.Sprintf("%x", sha256.Sum256(content)) + cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) + cfg.Spec.PauseImage = pauseRef + + ctx := t.Context() + client := fake.NewSimpleClientset(cfg) + factory := externalversions.NewSharedInformerFactory(client, 0) + informer := factory.Api().V1alpha1().SandboxConfigs().Informer() + + store, err := imagecache.New(t.TempDir()) + if err != nil { + t.Fatalf("imagecache.New: %v", err) + } + herder := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}} + // Handler first, informer start second, mirroring main: atelet startup + // must never wait on this informer's sync, and the initial List replays + // the pre-existing config into the handler as an Add. + if err := startSandboxAssetPrewarm(ctx, informer, herder, store, false); err != nil { + t.Fatalf("startSandboxAssetPrewarm: %v", err) + } + stopCh := make(chan struct{}) + defer close(stopCh) + factory.Start(stopCh) + + wantPath := ateompath.RunSCBinaryPath(sha) + deadline := time.Now().Add(10 * time.Second) + for { + if _, err := os.Stat(wantPath); err == nil { + return + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("stat %s: %v", wantPath, err) + } + if time.Now().After(deadline) { + t.Fatalf("asset never prewarmed to %s", wantPath) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index 5ac985fd81..7ba99e9d10 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -94,6 +94,10 @@ const ( // image pull. Memory use is O(stream buffers) per slot, independent of // layer size. layerPullConcurrency = 4 + + // defaultPullTimeout is the default per-pull bound (see WithPullTimeout). + // Generous enough for multi-GiB images on a busy node. + defaultPullTimeout = 10 * time.Minute ) // Store is atelet's handle to the on-disk layer pool. It is safe for @@ -121,6 +125,11 @@ type Store struct { // spec write / ateom mount that roots it. minAge time.Duration + // pullTimeout bounds each pull. Pulls run detached from the contexts of + // the callers waiting on them (see EnsureImage), so this is the only + // bound on how long one can run. + pullTimeout time.Duration + // meter, when set, is the meter the store reports on. See WithMeter. meter metric.Meter @@ -177,6 +186,11 @@ func WithMinAge(d time.Duration) Option { return func(s *Store) { s.minAge = d } } +// WithPullTimeout overrides the per-pull timeout (default 10m). +func WithPullTimeout(d time.Duration) Option { + return func(s *Store) { s.pullTimeout = d } +} + // WithMeter attaches the meter the store reports ate.imagecache.requests on. // Without it the store records nothing, so a caller with no metrics pipeline // needs no meter provider. @@ -209,7 +223,7 @@ type imageRecord struct { // startup recovery: verifying the layout version and sweeping temp dirs left // by unpacks that were in flight when a previous atelet died. func New(root string, opts ...Option) (*Store, error) { - s := &Store{root: root, minAge: defaultMinAge} + s := &Store{root: root, minAge: defaultMinAge, pullTimeout: defaultPullTimeout} for _, o := range opts { o(s) } @@ -366,16 +380,28 @@ func (s *Store) EnsureImage(ctx context.Context, ref string) (_ *Image, err erro slog.InfoContext(ctx, "Image cache miss", slog.String("ref", ref), slog.String("digest", digest.String())) // Collapse concurrent pulls of the same digest (e.g. several containers of - // one actor, or several actors landing at once). The winning call's ctx - // governs the pull; if it is cancelled the waiters fail too and retry at - // the RPC level. - v, err, _ := s.imageSF.Do(digest.String(), func() (any, error) { - return s.pull(ctx, parsedRef, digest) + // one actor, or several actors landing at once). Callers with very + // different lifetimes share these flights — a serving Restore, a + // best-effort prewarm bounded by its own deadline, a caller whose daemon + // is draining — so the pull runs on a context detached from whichever + // caller happened to start the flight, bounded only by the store's pull + // timeout: a waiter must only ever see a real pull failure, never another + // caller's cancellation. Each caller stops waiting when its own ctx ends, + // while the pull runs on to warm the cache for the next attempt. + ch := s.imageSF.DoChan(digest.String(), func() (any, error) { + pullCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.pullTimeout) + defer cancel() + return s.pull(pullCtx, parsedRef, digest) }) - if err != nil { - return nil, err + select { + case res := <-ch: + if res.Err != nil { + return nil, res.Err + } + return res.Val.(*Image), nil + case <-ctx.Done(): + return nil, fmt.Errorf("while waiting for pull of %s: %w", digest, context.Cause(ctx)) } - return v.(*Image), nil } // cachedImageHit is the hit side of the hitMu contract: it verifies the diff --git a/internal/imagecache/pull_gated_test.go b/internal/imagecache/pull_gated_test.go index e605cd8736..639b4fe03e 100644 --- a/internal/imagecache/pull_gated_test.go +++ b/internal/imagecache/pull_gated_test.go @@ -22,6 +22,7 @@ import ( "archive/tar" "context" "encoding/json" + "errors" "io" "log" "net/http" @@ -303,8 +304,9 @@ func TestPullReverifyFailsCleanlyOnYankedLayer(t *testing.T) { } } -// The one behavior change visible without GC: an interrupted pull leaves a -// valid partial record — resumable progress — rather than nothing. +// A mid-flight pull holds a valid partial record — resumable progress — +// rather than nothing, and a caller that stops waiting (cancellation) does +// not interrupt the pull itself, which runs on a detached context. func TestInterruptedPullLeavesResumableRecord(t *testing.T) { reg := newGatedRegistry(t) free, gatedLayer, _ := gatedTestLayers(t) @@ -328,12 +330,12 @@ func TestInterruptedPullLeavesResumableRecord(t *testing.T) { }) cancel() - if err := <-done; err == nil { - t.Fatal("EnsureImage succeeded despite cancellation") + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("EnsureImage with a cancelled caller = %v, want context.Canceled", err) } - release() - // The pre-written record survives the failure, referencing both layers. + // The caller is gone but the pull is still in flight, blocked on the + // gate. The pre-written record already references both layers. var recs []string entries, err := os.ReadDir(store.manifestsDir()) if err != nil { @@ -367,7 +369,10 @@ func TestInterruptedPullLeavesResumableRecord(t *testing.T) { t.Errorf("gated layer present after interrupted pull: %v", err) } - // A retry completes the image and yields a full cache hit. + // A retry joins the still-running flight and completes the image once + // the gate opens (which also drains the detached pull before the test's + // temp dirs are removed). + release() img, err := store.EnsureImage(context.Background(), ref) if err != nil { t.Fatalf("EnsureImage retry: %v", err) @@ -377,3 +382,20 @@ func TestInterruptedPullLeavesResumableRecord(t *testing.T) { t.Fatalf("no complete cachedImage after retry: %v, %v", cached, err) } } + +// A detached pull is not unbounded: pulls no longer end with the caller +// that started them, so the store's own pull timeout is what reclaims a +// wedged pull's flight slot. +func TestDetachedPullBoundedByPullTimeout(t *testing.T) { + reg := newGatedRegistry(t) + free, gatedLayer, _ := gatedTestLayers(t) + ref := reg.host + "/test/wedged:latest" + pushImage(t, ref, v1.Config{}, free, gatedLayer) + + store := newTestStore(t, WithPullTimeout(200*time.Millisecond)) + reg.gate(t, gatedLayer) // released only at test cleanup: the pull is wedged + + if _, err := store.EnsureImage(context.Background(), ref); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("EnsureImage on a wedged pull = %v, want context.DeadlineExceeded", err) + } +} diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 573501a0d3..9dcaa07fc0 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -31,6 +31,11 @@ rules: - apiGroups: ["ate.dev"] resources: ["csidriverconfigs"] verbs: ["get", "list", "watch"] +# SandboxConfigs are watched to pre-download sandbox assets into the node's +# cache before the first actor needs them (see cmd/atelet/sandbox_prewarm.go). +- apiGroups: ["ate.dev"] + resources: ["sandboxconfigs"] + verbs: ["get", "list", "watch"] # ClusterTrustBundles referenced by SystemInfo trustBundle data sources are # resolved on the node: atelet reads them through an informer and projects # the sanitized PEM into actors (see cmd/atelet/trustbundle.go).