diff --git a/cmd/silo/main.go b/cmd/silo/main.go index cb0b918ad4..ab22692525 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -1917,6 +1917,7 @@ func main() { }, ) pluginHost = pluginhost.NewHost(pluginhost.Config{ + IdleStopAfter: pluginhost.IdleStopAfter, EventPublisher: eventsHub, LibraryLister: pluginhost.NewLibraryLister(libDataSource), CatalogPresence: catalogPresence, diff --git a/docs/admin-api.md b/docs/admin-api.md index 64d1c6ca6c..22c4cca6a7 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -3440,8 +3440,10 @@ exponential backoff from 1 s to 60 s, and is parked as `failed` after ten consec failures until restarted or reconfigured), `state` (`stopped`, `starting`, `running`, `backoff`, `failed`), `restart_count`, `last_error`, `last_started_at` and `next_restart_at`. Plugins declaring `network_access_provider.v1` are resident; every other plugin starts on -first use and reports `running`, `stopped`, or `failed` when its last launch on the answering -server failed, with `last_error` saying why. A later successful launch, an administrator +first use, is stopped again after 30 minutes +without use, and reports `running`, `stopped`, or `failed` when its last launch on the answering +server failed, with `last_error` saying why. A plugin stopped for being idle reports `stopped` +and starts on its next use. A later successful launch, an administrator restart, an update or a configuration change clears that failure. A plugin that isn't running also reports `failed` when the answering server has to restore its files from the stored archive and the archive can't be read (it won't open or extract, or doesn't match the diff --git a/docs/architecture/plugin-idle-stop.md b/docs/architecture/plugin-idle-stop.md new file mode 100644 index 0000000000..105bd5721c --- /dev/null +++ b/docs/architecture/plugin-idle-stop.md @@ -0,0 +1,76 @@ +# Stopping idle plugin processes + +A plugin that is not resident starts on its first RPC. The host stops it again +once nothing has used it for 30 minutes, and the next RPC starts it as the first +one did. A server whose libraries never call a plugin (an audiobook metadata +provider on a Movies and Series server) does not keep that process in memory. + +The period is fixed (`pluginhost.IdleStopAfter`). Thirty minutes covers a burst +of calls, and a plugin that caches responses for up to half an hour keeps that +cache through a quiet spell. Proxy nodes build their plugin host without an idle +period, so nothing there is stopped for being idle. + +## What counts as use + +`internal/pluginhost/idle.go` keeps, per process, the time of the last use and +the number of calls in flight: + +- A lookup through `Host.Client`, which every RPC to a lazily started plugin + makes through `Service.ensureClient`. +- A host call to the plugin on the `silo.plugin.v1` services, from the moment + it is sent until it returns. Scheduled task runs, event deliveries, plugin + HTTP routes, scan source polls and sign-in exchanges are all such calls. The + idle period counts from the end of the last call. +- A plugin call back to the host (`RuntimeHost`), for the same span. A callback + with no host call in flight is the only sign of work a plugin does on its + own. + +The health probe, go-plugin's broker, stdio and controller traffic, and status +reads (`Host.Running`, which backs the admin `runtime.state`) are not use. + +## What is never stopped + +- Resident plugins (`network_access_provider.v1`). The supervisor owns them and + their RPCs do not start a process. `Service.start` asks for an idle stop only + when the manifest is not resident; the host never decides this by itself. +- Config test runs (negative installation IDs), which their caller stops. +- A process that opened a `silo.plugin.v1` stream. Every capability RPC is + unary today. A streaming one has to count as in flight until it closes + before its plugin can be stopped. + +## Invariants + +- The idle check and the removal from the host's instance map happen under the + lock `Host.Client` takes to record a use. A caller either finds the process + and keeps it alive for another idle period, or finds none and starts one. + The same decision marks the process as stopping under the lock that admits + plugin callbacks, so a callback the plugin starts afterwards is refused with + `Unavailable` instead of being cut off mid-flight. +- The watcher holds the installation's start slot until the old process has + exited, so a restart never overlaps the process it replaces. A `Start` + already in flight wins: it cancels the watcher. `Stop` and `Shutdown` also + wait for that exit. The one exception is a process that still has not + exited `DefaultIdleStopTimeout` after being killed (stuck in uninterruptible + I/O): the watcher logs a warning and releases the slot, so a restart can + overlap it and `Stop` or `Shutdown` can return before it exits. +- The first call after an idle stop waits for a cold start before it is sent: + the binary is verified again, the process launches, and the host reads its + manifest and configures it. A start that fails is recorded as any failed + lazy start is. +- An idle stop is a deliberate stop. The client is not marked unhealthy, the + exit handler is not called, and no lazy start failure is recorded. The + installation reports `stopped`, and the admin UI shows it as **Active**, like + any other healthy plugin that is not running. The stop is logged at info. +- The process is asked to shut down and is killed if it has not exited after + two seconds. go-plugin's shutdown request has no deadline, so a process that + is frozen when its idle period ends is killed after `DefaultIdleStopTimeout` + instead of holding the start slot, and every caller waiting on it, for good. +- State a plugin keeps in memory does not survive an idle stop. Host-held + state (configuration, instance state, OAuth and device-code flow state, scan + markers, sync cursors) is sent again with each call. A plugin must not rely + on a response cache or a background timer outliving its last call by more + than the idle period. + +A caller that resolves a capability client and waits longer than the idle +period before calling it gets a connection error, not a restart. No caller +holds a client across operations; keep it that way. diff --git a/internal/pluginhost/host.go b/internal/pluginhost/host.go index 80fed2ecf3..c02cd37989 100644 --- a/internal/pluginhost/host.go +++ b/internal/pluginhost/host.go @@ -30,6 +30,15 @@ type Config struct { // ExitCheckInterval is how often the exit watcher polls the plugin // process for an exit. Zero takes DefaultExitCheckInterval. ExitCheckInterval time.Duration + // IdleStopAfter is how long a process started with StopWhenIdle may go + // unused before the host stops it. Zero never stops an idle process. + IdleStopAfter time.Duration + // IdleCheckInterval is how often such a process is checked. Zero takes + // DefaultIdleCheckInterval. + IdleCheckInterval time.Duration + // IdleStopTimeout is how long an idle stop waits for the plugin to shut + // down before killing its process. Zero takes DefaultIdleStopTimeout. + IdleStopTimeout time.Duration // EventPublisher receives events that plugins publish via RuntimeHost. // Typically silo's *events.Hub. When nil, plugins that try to @@ -65,6 +74,11 @@ type StartRequest struct { BinaryPath string Manifest *pluginv1.PluginManifest Config []*pluginv1.ConfigEntry + // StopWhenIdle lets the host stop the process after Config.IdleStopAfter + // without use. Only set it for a plugin whose callers go back through + // Client or Start before every call, so the next one starts it again. + // It is ignored for config test runs (negative installation ids). + StopWhenIdle bool } type Host struct { @@ -72,6 +86,11 @@ type Host struct { healthCheckInterval time.Duration healthFailureLimit int exitCheckInterval time.Duration + idleStopAfter time.Duration + idleCheckInterval time.Duration + idleStopTimeout time.Duration + // now is a test seam for the idle clock. + now func() time.Time // exitHandler is told the installation id of a plugin whose process went // away on its own (exit or failed health), never one the host stopped on @@ -93,7 +112,11 @@ type Host struct { mu sync.RWMutex instances map[int]*instance starting map[int]chan struct{} - startSeq atomic.Uint64 + // idleStopping holds, per installation, a channel closed once the idle + // watcher has finished stopping a process it already removed from + // instances. + idleStopping map[int]chan struct{} + startSeq atomic.Uint64 } type instance struct { @@ -102,6 +125,7 @@ type instance struct { usageOnce sync.Once protocol plugin.ClientProtocol client *Client + activity *activity // installationID lets stopInstance revoke the ingress token of a network // access provider without a map lookup. installationID int @@ -133,8 +157,20 @@ func NewHost(cfg Config) *Host { if exitInterval <= 0 { exitInterval = DefaultExitCheckInterval } + idleInterval := cfg.IdleCheckInterval + if idleInterval <= 0 { + idleInterval = DefaultIdleCheckInterval + } + idleStopTimeout := cfg.IdleStopTimeout + if idleStopTimeout <= 0 { + idleStopTimeout = DefaultIdleStopTimeout + } return &Host{ + idleStopAfter: max(cfg.IdleStopAfter, 0), + idleCheckInterval: idleInterval, + idleStopTimeout: idleStopTimeout, + now: time.Now, logger: logger, healthCheckInterval: interval, healthFailureLimit: failureLimit, @@ -150,6 +186,7 @@ func NewHost(cfg Config) *Host { networkAccess: cfg.NetworkAccess, instances: make(map[int]*instance), starting: make(map[int]chan struct{}), + idleStopping: make(map[int]chan struct{}), } } @@ -206,11 +243,15 @@ func (h *Host) Start(ctx context.Context, req StartRequest) (*Client, error) { ingressToken = token } + used := newActivity(h.now) command := exec.Command(req.BinaryPath) command.Env = ProcessEnv() process := plugin.NewClient(&plugin.ClientConfig{ HandshakeConfig: HandshakeConfig(), - GRPCDialOptions: []grpc.DialOption{grpc.WithChainUnaryInterceptor(observePluginRPC)}, + GRPCDialOptions: []grpc.DialOption{ + grpc.WithChainUnaryInterceptor(used.unaryClient, observePluginRPC), + grpc.WithChainStreamInterceptor(used.streamClient), + }, AllowedProtocols: []plugin.Protocol{ plugin.ProtocolGRPC, }, @@ -257,7 +298,7 @@ func (h *Host) Start(ctx context.Context, req StartRequest) (*Client, error) { return nil, fmt.Errorf("unexpected plugin runtime client type %T", rawClient) } - if err := h.bindRuntimeHost(ctx, rpcClient, req.Manifest.GetPluginId(), req.InstallationID, provider, ingressToken, hostInfo, instanceState); err != nil { + if err := h.bindRuntimeHost(ctx, rpcClient, used, req.Manifest.GetPluginId(), req.InstallationID, provider, ingressToken, hostInfo, instanceState); err != nil { _ = protocol.Close() process.Kill() return nil, fmt.Errorf("bind runtime host: %w", err) @@ -307,6 +348,7 @@ func (h *Host) Start(ctx context.Context, req StartRequest) (*Client, error) { command: command, protocol: protocol, client: client, + activity: used, installationID: req.InstallationID, provider: provider, ingressToken: ingressToken, @@ -320,6 +362,9 @@ func (h *Host) Start(ctx context.Context, req StartRequest) (*Client, error) { go h.monitorHealth(monitorCtx, req.InstallationID, instance) go h.watchExit(monitorCtx, req.InstallationID, instance) + if req.StopWhenIdle && req.InstallationID > 0 && h.idleStopAfter > 0 { + go h.watchIdle(monitorCtx, req.InstallationID, instance) + } return client, nil } @@ -362,9 +407,15 @@ func (h *Host) SetExitHandler(handler func(installationID int)) { h.exitMu.Unlock() } +// Client returns the running client for a caller about to use it, which +// counts as a use: the idle watcher takes the write side of h.mu to remove an +// instance, so one returned here is not stopped for another idle period. func (h *Host) Client(installationID int) (*Client, error) { h.mu.RLock() instance, ok := h.instances[installationID] + if ok && instance.activity != nil { + instance.activity.touch() + } h.mu.RUnlock() if !ok { return nil, ErrClientNotFound @@ -380,6 +431,20 @@ func (h *Host) Client(installationID int) (*Client, error) { return instance.client, nil } +// Running reports whether a healthy process is up for the installation +// without counting as a use of it, for status reads. +func (h *Host) Running(installationID int) bool { + h.mu.RLock() + instance, ok := h.instances[installationID] + h.mu.RUnlock() + if !ok { + return false + } + instance.client.mu.RLock() + defer instance.client.mu.RUnlock() + return !instance.client.unhealthy +} + func (h *Host) NextStartSeq() uint64 { return h.startSeq.Load() } func (h *Host) Stop(installationID int) error { @@ -388,9 +453,15 @@ func (h *Host) Stop(installationID int) error { if ok { delete(h.instances, installationID) } + idleStop := h.idleStopping[installationID] h.mu.Unlock() if !ok { + // The process may still be exiting after an idle stop. Callers treat + // a return from Stop as the process being gone. + if idleStop != nil { + <-idleStop + } return ErrClientNotFound } h.stopInstance(instance) @@ -406,11 +477,18 @@ func (h *Host) Shutdown(ctx context.Context) error { delete(h.instances, installationID) instances = append(instances, instance) } + idleStops := make([]chan struct{}, 0, len(h.idleStopping)) + for _, idleStop := range h.idleStopping { + idleStops = append(idleStops, idleStop) + } h.mu.Unlock() for _, instance := range instances { h.stopInstance(instance) } + for _, idleStop := range idleStops { + <-idleStop + } close(done) }() @@ -509,13 +587,21 @@ func (h *Host) retireInstance(ctx context.Context, installationID int, instance } func (h *Host) stopInstance(instance *instance) { + h.teardown(instance, false) +} + +// teardown stops an instance already removed from the host. A graceful one +// leaves closing the connection to Kill, which then asks the plugin to shut +// down and waits briefly for it to exit by itself, so a routine idle stop is +// not logged and counted as a killed process. +func (h *Host) teardown(instance *instance, graceful bool) { if instance == nil { return } if instance.cancelMonitors != nil { instance.cancelMonitors() } - if instance.protocol != nil { + if instance.protocol != nil && !graceful { _ = instance.protocol.Close() } if instance.process != nil { @@ -539,7 +625,7 @@ func (h *Host) stopInstance(instance *instance) { // tears it down. // // Skipped when no RuntimeHost services are configured. -func (h *Host) bindRuntimeHost(ctx context.Context, sdkClient *sdkruntime.Client, pluginID string, installationID int, provider, ingressToken string, hostInfo HostInfoFunc, instanceState InstanceStateStore) error { +func (h *Host) bindRuntimeHost(ctx context.Context, sdkClient *sdkruntime.Client, used *activity, pluginID string, installationID int, provider, ingressToken string, hostInfo HostInfoFunc, instanceState InstanceStateStore) error { if h.eventPublisher == nil && h.libraryLister == nil && h.catalogPresence == nil && h.installedPlugins == nil && h.globalConfigSetter == nil && hostInfo == nil && instanceState == nil && h.networkAccess == nil { return nil @@ -554,7 +640,7 @@ func (h *Host) bindRuntimeHost(ctx context.Context, sdkClient *sdkruntime.Client streamID := broker.NextId() go broker.AcceptAndServe(streamID, func(opts []grpc.ServerOption) *grpc.Server { - s := grpc.NewServer(append(opts, grpc.ChainUnaryInterceptor(observePluginCallback))...) + s := grpc.NewServer(append(opts, grpc.ChainUnaryInterceptor(used.unaryServer, observePluginCallback))...) srv := NewRuntimeHostServerWithOptions(RuntimeHostOptions{ Publisher: h.eventPublisher, Libraries: h.libraryLister, diff --git a/internal/pluginhost/idle.go b/internal/pluginhost/idle.go new file mode 100644 index 0000000000..2a2249fbcb --- /dev/null +++ b/internal/pluginhost/idle.go @@ -0,0 +1,223 @@ +package pluginhost + +import ( + "context" + "strings" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + // IdleStopAfter is how long a lazily started plugin process may go unused + // before the host stops it. It is long enough that a plugin called in + // bursts (a scan, a metadata refresh, someone browsing) stays up across the + // burst, and a plugin that caches responses for up to half an hour keeps + // that cache through a quiet spell. It is also far above the longest gap between a caller + // resolving a plugin client and making its call. + IdleStopAfter = 30 * time.Minute + // DefaultIdleCheckInterval is how often a running plugin is checked. + DefaultIdleCheckInterval = 30 * time.Second + // DefaultIdleStopTimeout bounds an idle stop. go-plugin asks the plugin to + // shut down with no deadline, so a frozen process would otherwise hold the + // installation's start slot, and every caller waiting on it, forever. + DefaultIdleStopTimeout = 10 * time.Second + + // pluginServicePrefix selects the Silo plugin protocol. The health probe + // and go-plugin's own broker, stdio and controller calls share the + // connection and say nothing about whether the plugin is in use. + pluginServicePrefix = "/silo.plugin.v1." +) + +// activity records when one plugin process was last used and how many calls +// are in flight, in either direction. The idle watcher reads it to decide +// whether the process can be stopped. +type activity struct { + now func() time.Time + + mu sync.Mutex + inFlight int + last time.Time + // pinned marks a process that opened a plugin protocol stream. Every + // capability RPC is unary today; a stream has no end this type can see, + // so a process that opens one is never stopped for being idle. + pinned bool + // stopping is set when the idle stop commits to stopping the process. + // Callbacks the plugin starts after that are refused rather than cut + // off by the teardown. + stopping bool +} + +func newActivity(now func() time.Time) *activity { + return &activity{now: now, last: now()} +} + +func (a *activity) touch() { + a.mu.Lock() + a.last = a.now() + a.mu.Unlock() +} + +func (a *activity) begin() { + a.mu.Lock() + a.inFlight++ + a.last = a.now() + a.mu.Unlock() +} + +func (a *activity) end() { + a.mu.Lock() + a.inFlight-- + a.last = a.now() + a.mu.Unlock() +} + +// idleFor reports whether nothing is in flight and nothing has used the +// process for at least period. +func (a *activity) idleFor(period time.Duration) bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.idleForLocked(period) +} + +func (a *activity) idleForLocked(period time.Duration) bool { + return !a.pinned && a.inFlight == 0 && a.now().Sub(a.last) >= period +} + +// commitStop marks the process as stopping if it is still idle for period. +// The check and the mark share a.mu with callback admission, so no callback +// can start between them. +func (a *activity) commitStop(period time.Duration) bool { + a.mu.Lock() + defer a.mu.Unlock() + if !a.idleForLocked(period) { + return false + } + a.stopping = true + return true +} + +// unaryClient counts host calls to the plugin for as long as they run. +func (a *activity) unaryClient(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if !strings.HasPrefix(method, pluginServicePrefix) { + return invoker(ctx, method, req, reply, cc, opts...) + } + a.begin() + defer a.end() + return invoker(ctx, method, req, reply, cc, opts...) +} + +func (a *activity) streamClient(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + if strings.HasPrefix(method, pluginServicePrefix) { + a.mu.Lock() + a.pinned = true + a.mu.Unlock() + } + return streamer(ctx, desc, cc, method, opts...) +} + +// unaryServer counts the plugin's calls back to the host. A callback with no +// host call in flight is the only sign of work the plugin does on its own. +func (a *activity) unaryServer(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + a.mu.Lock() + if a.stopping { + a.mu.Unlock() + return nil, status.Error(codes.Unavailable, "plugin process is stopping after being idle") + } + a.inFlight++ + a.last = a.now() + a.mu.Unlock() + defer a.end() + return handler(ctx, req) +} + +// watchIdle stops the instance once it has been idle for the host's idle +// period. It runs only for instances started with StopWhenIdle. +func (h *Host) watchIdle(ctx context.Context, installationID int, instance *instance) { + ticker := time.NewTicker(h.idleCheckInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if h.stopIfIdle(ctx, installationID, instance) { + return + } + } + } +} + +// stopIfIdle stops the instance when it is still the one the host maps and +// has been idle for the idle period. This is a deliberate stop: the instance +// is not marked unhealthy and the exit handler is not told. +// +// The idle check and the removal happen under h.mu, the lock Client takes to +// record a use, so a caller either finds the instance and keeps it alive or +// finds none and starts a new one. The start slot is held until the old +// process has exited, so that new start never overlaps it, and Stop and +// Shutdown wait for the same exit through h.idleStopping. +func (h *Host) stopIfIdle(ctx context.Context, installationID int, instance *instance) bool { + if !instance.activity.idleFor(h.idleStopAfter) { + return false + } + // A Start already in flight for the installation replaces this instance + // and cancels ctx, which ends the wait. + if err := h.acquireStart(ctx, installationID); err != nil { + return false + } + defer h.releaseStart(installationID) + + h.mu.Lock() + current, ok := h.instances[installationID] + if !ok || current != instance || !instance.activity.commitStop(h.idleStopAfter) { + h.mu.Unlock() + return false + } + delete(h.instances, installationID) + exited := make(chan struct{}) + h.idleStopping[installationID] = exited + h.mu.Unlock() + + h.logger.Info("stopping idle plugin", "installation_id", installationID, "idle_for", h.idleStopAfter) + h.teardownIdle(installationID, instance) + + h.mu.Lock() + delete(h.idleStopping, installationID) + h.mu.Unlock() + close(exited) + return true +} + +// teardownIdle stops an idle instance gracefully, and kills its process when +// it has not shut down within the idle stop timeout (see +// DefaultIdleStopTimeout). +func (h *Host) teardownIdle(installationID int, instance *instance) { + done := make(chan struct{}) + go func() { + defer close(done) + h.teardown(instance, true) + }() + select { + case <-done: + return + case <-time.After(h.idleStopTimeout): + } + + h.logger.Warn("idle plugin did not shut down in time, killing it", "installation_id", installationID, "timeout", h.idleStopTimeout) + if instance.command != nil && instance.command.Process != nil { + _ = instance.command.Process.Kill() + } + select { + case <-done: + case <-time.After(h.idleStopTimeout): + // The process is stuck in the kernel (uninterruptible I/O) and cannot + // die yet. Release the start slot anyway: a new process is better than + // a plugin no caller can reach until the server restarts. + h.logger.Warn("idle plugin process has not exited after being killed", "installation_id", installationID) + } +} diff --git a/internal/pluginhost/idle_frozen_linux_test.go b/internal/pluginhost/idle_frozen_linux_test.go new file mode 100644 index 0000000000..495b3afb19 --- /dev/null +++ b/internal/pluginhost/idle_frozen_linux_test.go @@ -0,0 +1,84 @@ +//go:build linux + +package pluginhost_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "syscall" + "testing" + "time" + + "github.com/hashicorp/go-hclog" + + "github.com/Silo-Server/silo-server/internal/pluginhost" +) + +// pidOf finds the process running bin, which is unique to the test. +func pidOf(t *testing.T, bin string) int { + t.Helper() + entries, err := os.ReadDir("/proc") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + if exe, err := os.Readlink(filepath.Join("/proc", entry.Name(), "exe")); err == nil && exe == bin { + return pid + } + } + t.Fatalf("no process runs %s", bin) + return 0 +} + +// A process that is frozen when its idle period ends cannot answer go-plugin's +// shutdown request, which has no deadline. The idle stop kills it instead of +// holding the installation's start slot, so Stop returns and the plugin can +// start again. +func TestHostIdleStopKillsFrozenProcess(t *testing.T) { + bin, manifest := buildExitingPlugin(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + host := pluginhost.NewHost(pluginhost.Config{ + Logger: hclog.NewNullLogger(), + IdleStopAfter: 100 * time.Millisecond, + IdleCheckInterval: 10 * time.Millisecond, + IdleStopTimeout: 200 * time.Millisecond, + }) + t.Cleanup(func() { _ = host.Shutdown(context.Background()) }) + + if _, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 7, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}); err != nil { + t.Fatalf("Start: %v", err) + } + pid := pidOf(t, bin) + if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil { + t.Fatal(err) + } + // Without the fix the frozen process outlives the test; don't leave it. + t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGKILL) }) + + waitUntilStopped(ctx, t, host, 7) + stopped := make(chan error, 1) + go func() { stopped <- host.Stop(7) }() + select { + case err := <-stopped: + if !errors.Is(err, pluginhost.ErrClientNotFound) { + t.Fatalf("Stop after idle stop = %v, want ErrClientNotFound", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Stop is still waiting for the idle stop of a frozen process") + } + if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) { + t.Fatalf("frozen process %d still exists after the idle stop (kill 0: %v)", pid, err) + } + if _, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 7, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}); err != nil { + t.Fatalf("Start after the idle stop of a frozen process: %v", err) + } +} diff --git a/internal/pluginhost/idle_process_test.go b/internal/pluginhost/idle_process_test.go new file mode 100644 index 0000000000..f40171c183 --- /dev/null +++ b/internal/pluginhost/idle_process_test.go @@ -0,0 +1,166 @@ +package pluginhost_test + +import ( + "bytes" + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/hashicorp/go-hclog" + + pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" + + "github.com/Silo-Server/silo-server/internal/pluginhost" +) + +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func waitUntilStopped(ctx context.Context, t *testing.T, host *pluginhost.Host, installationID int) { + t.Helper() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for host.Running(installationID) { + select { + case <-ctx.Done(): + t.Fatalf("installation %d was not stopped for being idle", installationID) + case <-ticker.C: + } + } +} + +// keepCalling sends a capability RPC on client every 10 ms until the +// returned stop is called. The fixture answers Unimplemented, which still +// counts as use. +func keepCalling(t *testing.T, client *pluginhost.Client) (stop func()) { + t.Helper() + metadata, err := client.MetadataProvider("exiting") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + finished := make(chan struct{}) + go func() { + defer close(finished) + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + _, _ = metadata.Search(context.Background(), &pluginv1.SearchMetadataRequest{}) + } + } + }() + return func() { + close(done) + <-finished + } +} + +// Only a process started with StopWhenIdle on a host with an idle period is +// stopped; the stop is not a crash, and the next Start brings it back. +// Calls made on the client keep its process up, so the activity interceptors +// are wired into the real connection. It runs on the real clock, with +// several plugin starts and two idle periods, so it takes about a second. +func TestHostStopsIdleProcessAndStartsItAgain(t *testing.T) { + bin, manifest := buildExitingPlugin(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + disabled := pluginhost.NewHost(pluginhost.Config{Logger: hclog.NewNullLogger(), IdleCheckInterval: 10 * time.Millisecond}) + // A graceful stop logs nothing at warn or above; a killed process does. + var warnings lockedBuffer + host := pluginhost.NewHost(pluginhost.Config{ + Logger: hclog.New(&hclog.LoggerOptions{Level: hclog.Warn, Output: &warnings}), + IdleStopAfter: 200 * time.Millisecond, + IdleCheckInterval: 10 * time.Millisecond, + }) + exited := make(chan int, 4) + host.SetExitHandler(func(id int) { exited <- id }) + t.Cleanup(func() { + _ = disabled.Shutdown(context.Background()) + _ = host.Shutdown(context.Background()) + }) + + // Started first, so each has been up for longer than installation 3 by + // the time that one is stopped. + busy, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 5, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}) + if err != nil { + t.Fatalf("Start of a plugin in use: %v", err) + } + stopCalling := keepCalling(t, busy) + if _, err := disabled.Start(ctx, pluginhost.StartRequest{InstallationID: 1, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}); err != nil { + t.Fatalf("Start on a host with no idle period: %v", err) + } + if _, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 2, BinaryPath: bin, Manifest: manifest}); err != nil { + t.Fatalf("Start without StopWhenIdle: %v", err) + } + if _, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: -4, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}); err != nil { + t.Fatalf("Start of a config test run: %v", err) + } + client, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 3, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}) + if err != nil { + t.Fatalf("Start with StopWhenIdle: %v", err) + } + + waitUntilStopped(ctx, t, host, 3) + if _, err := host.Client(3); !errors.Is(err, pluginhost.ErrClientNotFound) { + t.Fatalf("Client after idle stop = %v, want ErrClientNotFound", err) + } + if _, err := client.MetadataProvider("exiting"); errors.Is(err, pluginhost.ErrPluginUnhealthy) { + t.Fatal("idle stop marked the client unhealthy") + } + if !disabled.Running(1) { + t.Fatal("a host with no idle period stopped an idle process") + } + if !host.Running(2) { + t.Fatal("a process started without StopWhenIdle was stopped") + } + if !host.Running(-4) { + t.Fatal("a config test run was stopped for being idle") + } + + restarted, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 3, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}) + if err != nil { + t.Fatalf("Start after idle stop: %v", err) + } + if restarted.StartSeq() == client.StartSeq() { + t.Fatal("Start after idle stop returned the stopped client") + } + waitUntilStopped(ctx, t, host, 3) + stopCalling() + if !host.Running(5) { + t.Fatal("a plugin was stopped while it was being called") + } + // Stop returns once the idle stop has finished, so nothing is logged later. + if err := host.Stop(3); !errors.Is(err, pluginhost.ErrClientNotFound) { + t.Fatalf("Stop after idle stop = %v, want ErrClientNotFound", err) + } + + select { + case id := <-exited: + t.Fatalf("idle stop reported installation %d to the exit handler", id) + default: + } + if logged := warnings.String(); logged != "" { + t.Fatalf("idle stops logged at warn or above:\n%s", logged) + } +} diff --git a/internal/pluginhost/idle_test.go b/internal/pluginhost/idle_test.go new file mode 100644 index 0000000000..85c21d7ead --- /dev/null +++ b/internal/pluginhost/idle_test.go @@ -0,0 +1,353 @@ +package pluginhost + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type idleClock struct { + mu sync.Mutex + now time.Time +} + +func (c *idleClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *idleClock) Advance(d time.Duration) { + c.mu.Lock() + c.now = c.now.Add(d) + c.mu.Unlock() +} + +const ( + testIdlePeriod = 15 * time.Minute + testIdleCapability = "/silo.plugin.v1.MetadataProvider/Search" +) + +// idleTestHost returns a host mapping one instance with no process behind +// it, and the channel its exit handler reports to. +func idleTestHost(t *testing.T, clock *idleClock, installationID int) (*Host, *instance, chan int) { + t.Helper() + host := NewHost(Config{IdleStopAfter: testIdlePeriod}) + host.now = clock.Now + exited := make(chan int, 1) + host.SetExitHandler(func(id int) { exited <- id }) + inst := &instance{ + client: makeTestClient(t, nil), + activity: newActivity(clock.Now), + installationID: installationID, + } + host.instances[installationID] = inst + return host, inst, exited +} + +func TestStopIfIdleStopsAnIdleInstanceQuietly(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, exited := idleTestHost(t, clock, 7) + + clock.Advance(testIdlePeriod - time.Second) + if host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("stopped before the idle period passed") + } + clock.Advance(time.Second) + if !host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("did not stop an instance idle for the whole period") + } + + if _, err := host.Client(7); !errors.Is(err, ErrClientNotFound) { + t.Fatalf("Client after idle stop = %v, want ErrClientNotFound so the next call starts it", err) + } + inst.client.mu.RLock() + unhealthy := inst.client.unhealthy + inst.client.mu.RUnlock() + if unhealthy { + t.Fatal("idle stop marked the client unhealthy") + } + select { + case id := <-exited: + t.Fatalf("idle stop reported installation %d to the exit handler", id) + default: + } + // The start slot is free again for the next launch. + if err := host.acquireStart(context.Background(), 7); err != nil { + t.Fatalf("acquireStart after idle stop: %v", err) + } + host.releaseStart(7) +} + +func TestStopIfIdleKeepsABusyInstance(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + + inCall := make(chan struct{}) + release := make(chan struct{}) + invoker := func(context.Context, string, any, any, *grpc.ClientConn, ...grpc.CallOption) error { + close(inCall) + <-release + return nil + } + done := make(chan error, 1) + go func() { + done <- inst.activity.unaryClient(context.Background(), testIdleCapability, nil, nil, nil, invoker) + }() + <-inCall + + clock.Advance(4 * testIdlePeriod) + if host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("stopped an instance with a call in flight") + } + + close(release) + if err := <-done; err != nil { + t.Fatalf("call: %v", err) + } + // The idle period counts from the end of the call, not its start. + if host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("stopped as soon as a long call returned") + } + clock.Advance(testIdlePeriod) + if !host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("did not stop once idle after the call") + } +} + +func TestIdleActivitySources(t *testing.T) { + noop := func(context.Context, string, any, any, *grpc.ClientConn, ...grpc.CallOption) error { return nil } + tests := []struct { + name string + use func(t *testing.T, host *Host, inst *instance) + wantIdle bool + }{ + {name: "nothing", use: func(*testing.T, *Host, *instance) {}, wantIdle: true}, + {name: "capability call", use: func(t *testing.T, _ *Host, inst *instance) { + if err := inst.activity.unaryClient(context.Background(), testIdleCapability, nil, nil, nil, noop); err != nil { + t.Fatal(err) + } + }}, + {name: "host callback", use: func(t *testing.T, _ *Host, inst *instance) { + handler := func(context.Context, any) (any, error) { return struct{}{}, nil } + if _, err := inst.activity.unaryServer(context.Background(), nil, &grpc.UnaryServerInfo{}, handler); err != nil { + t.Fatal(err) + } + }}, + {name: "client lookup", use: func(t *testing.T, host *Host, _ *instance) { + if _, err := host.Client(7); err != nil { + t.Fatal(err) + } + }}, + {name: "health probe", wantIdle: true, use: func(t *testing.T, _ *Host, inst *instance) { + if err := inst.activity.unaryClient(context.Background(), "/grpc.health.v1.Health/Check", nil, nil, nil, noop); err != nil { + t.Fatal(err) + } + }}, + {name: "status read", wantIdle: true, use: func(t *testing.T, host *Host, _ *instance) { + if !host.Running(7) { + t.Fatal("Running = false for a mapped instance") + } + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + clock.Advance(testIdlePeriod - time.Second) + tt.use(t, host, inst) + clock.Advance(time.Second) + if got := host.stopIfIdle(context.Background(), 7, inst); got != tt.wantIdle { + t.Fatalf("stopped = %v, want %v", got, tt.wantIdle) + } + }) + } +} + +func TestIdleStreamPinsTheInstance(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + streamer := func(context.Context, *grpc.StreamDesc, *grpc.ClientConn, string, ...grpc.CallOption) (grpc.ClientStream, error) { + return nil, errors.New("no stream in this test") + } + // go-plugin's own broker and stdio streams stay open for the life of + // every process and must not pin it. + _, _ = inst.activity.streamClient(context.Background(), nil, nil, "/plugin.GRPCBroker/StartStream", streamer) + clock.Advance(testIdlePeriod) + if !inst.activity.idleFor(testIdlePeriod) { + t.Fatal("a go-plugin stream pinned the instance") + } + _, _ = inst.activity.streamClient(context.Background(), nil, nil, "/silo.plugin.v1.Future/Watch", streamer) + clock.Advance(4 * testIdlePeriod) + if host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("stopped an instance that opened a plugin protocol stream") + } +} + +// A caller that looks the client up while the idle watcher runs either keeps +// the instance alive or is told to start a new one. It is never handed a +// client whose process the watcher then stops. +func TestStopIfIdleRacingClientLookup(t *testing.T) { + for range 300 { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + clock.Advance(testIdlePeriod) + + const callers = 4 + var found atomic.Int32 + var stopped atomic.Bool + release := make(chan struct{}) + var wg sync.WaitGroup + for range callers { + wg.Add(1) + go func() { + defer wg.Done() + <-release + if _, err := host.Client(7); err == nil { + found.Add(1) + } else if !errors.Is(err, ErrClientNotFound) { + t.Errorf("Client = %v", err) + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + <-release + stopped.Store(host.stopIfIdle(context.Background(), 7, inst)) + }() + close(release) + wg.Wait() + + if stopped.Load() && found.Load() != 0 { + t.Fatalf("instance stopped after %d callers were handed its client", found.Load()) + } + if !stopped.Load() && found.Load() != callers { + t.Fatalf("instance kept but only %d of %d callers found it", found.Load(), callers) + } + } +} + +// A replacing Start owns the installation's start slot. The idle watcher +// waits for it and then leaves the instance that Start mapped alone. +func TestStopIfIdleYieldsToAStartInFlight(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + clock.Advance(testIdlePeriod) + + if err := host.acquireStart(context.Background(), 7); err != nil { + t.Fatal(err) + } + result := make(chan bool, 1) + go func() { result <- host.stopIfIdle(context.Background(), 7, inst) }() + + replacement := &instance{client: makeTestClient(t, nil), activity: newActivity(clock.Now), installationID: 7} + host.mu.Lock() + host.instances[7] = replacement + host.mu.Unlock() + host.releaseStart(7) + + if <-result { + t.Fatal("idle watcher stopped an installation a Start had just replaced") + } + if client, err := host.Client(7); err != nil || client != replacement.client { + t.Fatalf("Client = %v, %v; want the replacement instance", client, err) + } + + // A canceled watcher gives up on a held slot instead of waiting for it. + if err := host.acquireStart(context.Background(), 7); err != nil { + t.Fatal(err) + } + defer host.releaseStart(7) + clock.Advance(testIdlePeriod) + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if host.stopIfIdle(canceled, 7, replacement) { + t.Fatal("canceled idle watcher stopped the instance") + } +} + +// Callers of Stop and Shutdown treat a return as the process being gone, so +// both wait for an idle stop that has already taken the instance. +func TestStopAndShutdownWaitForAnIdleStopInProgress(t *testing.T) { + waiters := map[string]func(*Host) error{ + "Stop": func(host *Host) error { + if err := host.Stop(7); !errors.Is(err, ErrClientNotFound) { + return err + } + return nil + }, + "Shutdown": func(host *Host) error { return host.Shutdown(context.Background()) }, + } + for name, wait := range waiters { + t.Run(name, func(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + clock.Advance(testIdlePeriod) + + synctest.Test(t, func(t *testing.T) { + // Stands in for a process that takes a while to exit. + exiting := make(chan struct{}) + exit := make(chan struct{}) + inst.cancelMonitors = func() { + close(exiting) + <-exit + } + go host.stopIfIdle(context.Background(), 7, inst) + <-exiting + + returned := make(chan error, 1) + go func() { returned <- wait(host) }() + synctest.Wait() + select { + case err := <-returned: + t.Fatalf("returned (%v) while the idle process was still exiting", err) + default: + } + close(exit) + if err := <-returned; err != nil { + t.Fatal(err) + } + }) + }) + } +} + +// Once the idle stop has decided, a callback the plugin starts on its own is +// refused instead of being cut off mid-flight by the teardown. +func TestIdleStopRefusesACallbackAfterTheDecision(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + + // Hold the teardown open: the callback lands while the process is being + // stopped, the window the decision has to cover. + inTeardown := make(chan struct{}) + release := make(chan struct{}) + inst.cancelMonitors = func() { + close(inTeardown) + <-release + } + clock.Advance(testIdlePeriod) + stopped := make(chan bool, 1) + go func() { stopped <- host.stopIfIdle(context.Background(), 7, inst) }() + <-inTeardown + + ran := false + handler := func(context.Context, any) (any, error) { ran = true; return nil, nil } + _, err := inst.activity.unaryServer(context.Background(), nil, &grpc.UnaryServerInfo{}, handler) + close(release) + if !<-stopped { + t.Fatal("did not stop an instance idle for the whole period") + } + if status.Code(err) != codes.Unavailable || ran { + t.Fatalf("callback during the idle stop: err = %v, handler ran = %v; want Unavailable without running it", err, ran) + } +} diff --git a/internal/plugins/host_adapter.go b/internal/plugins/host_adapter.go index 3cec17061e..0821de986a 100644 --- a/internal/plugins/host_adapter.go +++ b/internal/plugins/host_adapter.go @@ -39,3 +39,7 @@ func (h *hostAdapter) Shutdown(ctx context.Context) error { func (h *hostAdapter) NextStartSeq() uint64 { return h.host.NextStartSeq() } + +func (h *hostAdapter) Running(installationID int) bool { + return h.host.Running(installationID) +} diff --git a/internal/plugins/resident.go b/internal/plugins/resident.go index 12970f407c..e667209246 100644 --- a/internal/plugins/resident.go +++ b/internal/plugins/resident.go @@ -794,7 +794,7 @@ func (s *Service) RuntimeState(installationID int) RuntimeState { return tracked } if s.host != nil { - if _, err := s.host.Client(installationID); err == nil { + if s.host.Running(installationID) { state.State = ResidentRunning } } diff --git a/internal/plugins/resident_generation_test.go b/internal/plugins/resident_generation_test.go index cce4a10557..4723a4a6f0 100644 --- a/internal/plugins/resident_generation_test.go +++ b/internal/plugins/resident_generation_test.go @@ -156,3 +156,8 @@ func TestResidentSupervisorNewGenerationNeverAdoptsInFlightOlderLaunch(t *testin t.Fatalf("stopped seqs = %v, want the adopted generation 1 process (seq 1) stopped exactly once", stopped) } } + +func (h *seqFakeHost) Running(id int) bool { + _, err := h.Client(id) + return err == nil +} diff --git a/internal/plugins/service.go b/internal/plugins/service.go index 4486ccc0a1..6e83c6fff9 100644 --- a/internal/plugins/service.go +++ b/internal/plugins/service.go @@ -52,6 +52,10 @@ type Host interface { // StartSeq is at or below the value read before a launch was issued by // an earlier launch that the singleflight joined. NextStartSeq() uint64 + // Running reports whether the installation's process is up. Unlike + // Client it does not count as a use, so a status read cannot keep an + // idle plugin running. + Running(installationID int) bool } type serviceInstallationStore interface { @@ -601,6 +605,10 @@ func (s *Service) start(ctx context.Context, installationID int, allowResident b BinaryPath: binaryPath, Manifest: manifest, Config: configEntries, + // Every RPC to a lazily started plugin resolves its client through + // ensureClient, which starts it again. A resident is the + // supervisor's to keep up. + StopWhenIdle: !isResidentManifest(manifest), }) } diff --git a/internal/plugins/service_hot_reload_test.go b/internal/plugins/service_hot_reload_test.go index 8acb59d30e..17530c26f4 100644 --- a/internal/plugins/service_hot_reload_test.go +++ b/internal/plugins/service_hot_reload_test.go @@ -495,3 +495,8 @@ func recordTestEvent(events *[]string, event string) { } *events = append(*events, event) } + +func (f *fakeServiceHost) Running(_ int) bool { + _, err := f.Client(0) + return err == nil +} diff --git a/internal/plugins/service_singleflight_test.go b/internal/plugins/service_singleflight_test.go index ea4e012b41..d8d52e3fd5 100644 --- a/internal/plugins/service_singleflight_test.go +++ b/internal/plugins/service_singleflight_test.go @@ -7,6 +7,9 @@ import ( "testing" "time" + pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" + "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginsdk/capability" + "github.com/Silo-Server/silo-server/internal/pluginhost" ) @@ -281,3 +284,53 @@ func TestEnsureClientFailedLaunchPropagatesToAllCallers(t *testing.T) { t.Fatalf("failed launch host.Start calls = %d, want 1 (still single-flighted)", got) } } + +func (h *countingHerdHost) Running(id int) bool { + _, err := h.Client(id) + return err == nil +} + +func (h *ctxCaptureHost) Running(id int) bool { + _, err := h.Client(id) + return err == nil +} + +// The host stops an idle plugin only when its start asked for that. An RPC +// that finds no client, as after an idle stop, starts a lazy plugin with the +// request set and reports the stopped plugin without an error; a resident is +// the supervisor's and is never started with it. +func TestOnlyLazyPluginsAreStartedWithStopWhenIdle(t *testing.T) { + manifest := testPluginManifest(t, "silo.metadb", "0.0.36") + store := installedPluginStore(t, 7, manifest) + host := &fakeServiceHost{clientErr: pluginhost.ErrClientNotFound} + service := &Service{installations: store, host: host} + + // No process, as after an idle stop: reported stopped, not failed. + if state := service.RuntimeStateOf(store.byID[7]); state.State != ResidentStopped || state.LastError != "" { + t.Fatalf("runtime state of a stopped idle plugin = %+v, want stopped with no error", state) + } + if _, err := service.ensureClient(context.Background(), 7); err != nil { + t.Fatalf("ensureClient after an idle stop: %v", err) + } + if len(host.started) != 1 || len(host.stopped) != 0 { + t.Fatalf("starts = %d, stops = %d; want 1 start and no stop", len(host.started), len(host.stopped)) + } + if !host.started[0].StopWhenIdle { + t.Fatal("a lazily started plugin was not started with StopWhenIdle") + } + + resident := testPluginManifest(t, "silo.overlay", "1.0.0") + resident.Capabilities = []*pluginv1.CapabilityDescriptor{{ + Type: capability.NetworkAccessProvider, Id: "overlay", + NetworkAccessProvider: &pluginv1.NetworkAccessProviderDescriptor{Provider: "tailscale"}, + }} + residentStore := installedPluginStore(t, 8, resident) + residentHost := &fakeServiceHost{} + residentService := &Service{installations: residentStore, host: residentHost} + if _, err := residentService.start(context.Background(), 8, true); err != nil { + t.Fatalf("start resident: %v", err) + } + if len(residentHost.started) != 1 || residentHost.started[0].StopWhenIdle { + t.Fatalf("resident start requests = %+v, want one without StopWhenIdle", residentHost.started) + } +}