From dc2766d28838597b1a69880a57d6c3a07a271a3c Mon Sep 17 00:00:00 2001 From: blurbery <271863866+blurbery@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:53:10 +1100 Subject: [PATCH 1/7] perf(plugins): stop plugin processes that have been idle A plugin that is not resident starts on its first call and then stayed up until it was updated, disabled or crashed. A server with nine plugins installed kept nine processes in memory, including metadata providers for library types it never scans. The plugin host now stops a lazily started plugin once nothing has used it for SILO_PLUGIN_IDLE_STOP_AFTER (default 15m, 0 keeps every plugin running). The next call starts it again through the existing lazy start path. Use is a client lookup, a host call on the plugin protocol while it is in flight, or a callback from the plugin to the host. The health probe and admin status reads do not count. Resident plugins (network access providers) and config test runs are never stopped this way. The idle check and the removal happen under the lock a client lookup takes, so a caller either keeps the process alive or starts a new one. The watcher holds the installation's start slot until the old process has exited, and Stop and Shutdown wait for that exit. An idle stop asks the plugin to shut down instead of killing it, is logged at debug, and does not mark the plugin unhealthy or reach the resident supervisor. --- cmd/silo/main.go | 6 + docs/admin-api.md | 6 +- docs/architecture/plugin-idle-stop.md | 68 ++++ internal/pluginhost/host.go | 89 ++++- internal/pluginhost/idle.go | 194 ++++++++++ internal/pluginhost/idle_process_test.go | 122 +++++++ internal/pluginhost/idle_test.go | 344 ++++++++++++++++++ internal/plugins/host_adapter.go | 4 + internal/plugins/resident.go | 2 +- internal/plugins/resident_generation_test.go | 5 + internal/plugins/service.go | 8 + internal/plugins/service_hot_reload_test.go | 5 + internal/plugins/service_singleflight_test.go | 52 +++ 13 files changed, 896 insertions(+), 9 deletions(-) create mode 100644 docs/architecture/plugin-idle-stop.md create mode 100644 internal/pluginhost/idle.go create mode 100644 internal/pluginhost/idle_process_test.go create mode 100644 internal/pluginhost/idle_test.go diff --git a/cmd/silo/main.go b/cmd/silo/main.go index cb0b918ad4..0505f2c4db 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -1916,7 +1916,13 @@ func main() { return out, nil }, ) + pluginIdleStopAfter, idleErr := pluginhost.ParseIdleStopAfter(os.Getenv(pluginhost.EnvIdleStopAfter)) + if idleErr != nil { + slog.Warn("invalid plugin idle period; using the default", "component", "plugins", + "env", pluginhost.EnvIdleStopAfter, "error", idleErr, "default", pluginIdleStopAfter) + } pluginHost = pluginhost.NewHost(pluginhost.Config{ + IdleStopAfter: pluginIdleStopAfter, EventPublisher: eventsHub, LibraryLister: pluginhost.NewLibraryLister(libDataSource), CatalogPresence: catalogPresence, diff --git a/docs/admin-api.md b/docs/admin-api.md index 64d1c6ca6c..9b2616a1b9 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -3440,8 +3440,10 @@ exponential backoff from 1 s to 60 s, and is parked as `failed` after ten consec failures until restarted or reconfigured), `state` (`stopped`, `starting`, `running`, `backoff`, `failed`), `restart_count`, `last_error`, `last_started_at` and `next_restart_at`. Plugins declaring `network_access_provider.v1` are resident; every other plugin starts on -first use and reports `running`, `stopped`, or `failed` when its last launch on the answering -server failed, with `last_error` saying why. A later successful launch, an administrator +first use, is stopped again after `SILO_PLUGIN_IDLE_STOP_AFTER` (default 15 minutes, `0` never) +without use, and reports `running`, `stopped`, or `failed` when its last launch on the answering +server failed, with `last_error` saying why. A plugin stopped for being idle reports `stopped` +and starts on its next use. A later successful launch, an administrator restart, an update or a configuration change clears that failure. A plugin that isn't running also reports `failed` when the answering server has to restore its files from the stored archive and the archive can't be read (it won't open or extract, or doesn't match the diff --git a/docs/architecture/plugin-idle-stop.md b/docs/architecture/plugin-idle-stop.md new file mode 100644 index 0000000000..83f38d3db7 --- /dev/null +++ b/docs/architecture/plugin-idle-stop.md @@ -0,0 +1,68 @@ +# Stopping idle plugin processes + +A plugin that is not resident starts on its first RPC. The host stops it again +once nothing has used it for `SILO_PLUGIN_IDLE_STOP_AFTER`, and the next RPC +starts it as the first one did. A server whose libraries never call a plugin +(an audiobook metadata provider on a Movies and Series server) does not keep +that process in memory. + +`SILO_PLUGIN_IDLE_STOP_AFTER` is a Go duration or whole seconds. The default +is `15m`, `0` keeps every plugin running, and a shorter period than one minute +is raised to one minute. It is read once at start-up on the API server. Proxy +nodes run only resident plugins and ignore it. + +## What counts as use + +`internal/pluginhost/idle.go` keeps, per process, the time of the last use and +the number of calls in flight: + +- A lookup through `Host.Client`, which every RPC to a lazily started plugin + makes through `Service.ensureClient`. +- A host call to the plugin on the `silo.plugin.v1` services, from the moment + it is sent until it returns. Scheduled task runs, event deliveries, plugin + HTTP routes, scan source polls and sign-in exchanges are all such calls. The + idle period counts from the end of the last call. +- A plugin call back to the host (`RuntimeHost`), for the same span. A callback + with no host call in flight is the only sign of work a plugin does on its + own. + +The health probe, go-plugin's broker, stdio and controller traffic, and status +reads (`Host.Running`, which backs the admin `runtime.state`) are not use. + +## What is never stopped + +- Resident plugins (`network_access_provider.v1`). The supervisor owns them and + their RPCs do not start a process. `Service.start` asks for an idle stop only + when the manifest is not resident; the host never decides this by itself. +- Config test runs (negative installation IDs), which their caller stops. +- A process that opened a `silo.plugin.v1` stream. Every capability RPC is + unary today. A streaming one has to count as in flight until it closes + before its plugin can be stopped. + +## Invariants + +- The idle check and the removal from the host's instance map happen under the + lock `Host.Client` takes to record a use. A caller either finds the process + and keeps it alive for another idle period, or finds none and starts one. +- The watcher holds the installation's start slot until the old process has + exited, so a restart never overlaps the process it replaces. A `Start` + already in flight wins: it cancels the watcher. `Stop` and `Shutdown` also + wait for that exit. +- The first call after an idle stop waits for a cold start before it is sent: + the binary is verified again, the process launches, and the host reads its + manifest and configures it. A start that fails is recorded as any failed + lazy start is. +- An idle stop is a deliberate stop. The client is not marked unhealthy, the + exit handler is not called, no lazy start failure is recorded, and the + installation reports `stopped`, which the admin UI shows as a healthy plugin + that is not running. The process is asked to shut down and is only killed if + it has not exited after two seconds. +- State a plugin keeps in memory does not survive an idle stop. Host-held + state (configuration, instance state, OAuth and device-code flow state, scan + markers, sync cursors) is sent again with each call. A plugin must not rely + on a response cache or a background timer outliving its last call by more + than the idle period. + +A caller that resolves a capability client and waits longer than the idle +period before calling it gets a connection error, not a restart. No caller +holds a client across operations; keep it that way. diff --git a/internal/pluginhost/host.go b/internal/pluginhost/host.go index 80fed2ecf3..28c2b9f2d6 100644 --- a/internal/pluginhost/host.go +++ b/internal/pluginhost/host.go @@ -30,6 +30,12 @@ type Config struct { // ExitCheckInterval is how often the exit watcher polls the plugin // process for an exit. Zero takes DefaultExitCheckInterval. ExitCheckInterval time.Duration + // IdleStopAfter is how long a process started with StopWhenIdle may go + // unused before the host stops it. Zero never stops an idle process. + IdleStopAfter time.Duration + // IdleCheckInterval is how often such a process is checked. Zero takes + // DefaultIdleCheckInterval. + IdleCheckInterval time.Duration // EventPublisher receives events that plugins publish via RuntimeHost. // Typically silo's *events.Hub. When nil, plugins that try to @@ -65,6 +71,11 @@ type StartRequest struct { BinaryPath string Manifest *pluginv1.PluginManifest Config []*pluginv1.ConfigEntry + // StopWhenIdle lets the host stop the process after Config.IdleStopAfter + // without use. Only set it for a plugin whose callers go back through + // Client or Start before every call, so the next one starts it again. + // It is ignored for config test runs (negative installation ids). + StopWhenIdle bool } type Host struct { @@ -72,6 +83,10 @@ type Host struct { healthCheckInterval time.Duration healthFailureLimit int exitCheckInterval time.Duration + idleStopAfter time.Duration + idleCheckInterval time.Duration + // now is a test seam for the idle clock. + now func() time.Time // exitHandler is told the installation id of a plugin whose process went // away on its own (exit or failed health), never one the host stopped on @@ -93,7 +108,11 @@ type Host struct { mu sync.RWMutex instances map[int]*instance starting map[int]chan struct{} - startSeq atomic.Uint64 + // idleStopping holds, per installation, a channel closed once the idle + // watcher has finished stopping a process it already removed from + // instances. + idleStopping map[int]chan struct{} + startSeq atomic.Uint64 } type instance struct { @@ -102,6 +121,7 @@ type instance struct { usageOnce sync.Once protocol plugin.ClientProtocol client *Client + activity *activity // installationID lets stopInstance revoke the ingress token of a network // access provider without a map lookup. installationID int @@ -133,8 +153,15 @@ func NewHost(cfg Config) *Host { if exitInterval <= 0 { exitInterval = DefaultExitCheckInterval } + idleInterval := cfg.IdleCheckInterval + if idleInterval <= 0 { + idleInterval = DefaultIdleCheckInterval + } return &Host{ + idleStopAfter: max(cfg.IdleStopAfter, 0), + idleCheckInterval: idleInterval, + now: time.Now, logger: logger, healthCheckInterval: interval, healthFailureLimit: failureLimit, @@ -150,6 +177,7 @@ func NewHost(cfg Config) *Host { networkAccess: cfg.NetworkAccess, instances: make(map[int]*instance), starting: make(map[int]chan struct{}), + idleStopping: make(map[int]chan struct{}), } } @@ -206,11 +234,15 @@ func (h *Host) Start(ctx context.Context, req StartRequest) (*Client, error) { ingressToken = token } + used := newActivity(h.now) command := exec.Command(req.BinaryPath) command.Env = ProcessEnv() process := plugin.NewClient(&plugin.ClientConfig{ HandshakeConfig: HandshakeConfig(), - GRPCDialOptions: []grpc.DialOption{grpc.WithChainUnaryInterceptor(observePluginRPC)}, + GRPCDialOptions: []grpc.DialOption{ + grpc.WithChainUnaryInterceptor(used.unaryClient, observePluginRPC), + grpc.WithChainStreamInterceptor(used.streamClient), + }, AllowedProtocols: []plugin.Protocol{ plugin.ProtocolGRPC, }, @@ -257,7 +289,7 @@ func (h *Host) Start(ctx context.Context, req StartRequest) (*Client, error) { return nil, fmt.Errorf("unexpected plugin runtime client type %T", rawClient) } - if err := h.bindRuntimeHost(ctx, rpcClient, req.Manifest.GetPluginId(), req.InstallationID, provider, ingressToken, hostInfo, instanceState); err != nil { + if err := h.bindRuntimeHost(ctx, rpcClient, used, req.Manifest.GetPluginId(), req.InstallationID, provider, ingressToken, hostInfo, instanceState); err != nil { _ = protocol.Close() process.Kill() return nil, fmt.Errorf("bind runtime host: %w", err) @@ -307,6 +339,7 @@ func (h *Host) Start(ctx context.Context, req StartRequest) (*Client, error) { command: command, protocol: protocol, client: client, + activity: used, installationID: req.InstallationID, provider: provider, ingressToken: ingressToken, @@ -320,6 +353,9 @@ func (h *Host) Start(ctx context.Context, req StartRequest) (*Client, error) { go h.monitorHealth(monitorCtx, req.InstallationID, instance) go h.watchExit(monitorCtx, req.InstallationID, instance) + if req.StopWhenIdle && req.InstallationID > 0 && h.idleStopAfter > 0 { + go h.watchIdle(monitorCtx, req.InstallationID, instance) + } return client, nil } @@ -362,9 +398,15 @@ func (h *Host) SetExitHandler(handler func(installationID int)) { h.exitMu.Unlock() } +// Client returns the running client for a caller about to use it, which +// counts as a use: the idle watcher takes the write side of h.mu to remove an +// instance, so one returned here is not stopped for another idle period. func (h *Host) Client(installationID int) (*Client, error) { h.mu.RLock() instance, ok := h.instances[installationID] + if ok && instance.activity != nil { + instance.activity.touch() + } h.mu.RUnlock() if !ok { return nil, ErrClientNotFound @@ -380,6 +422,20 @@ func (h *Host) Client(installationID int) (*Client, error) { return instance.client, nil } +// Running reports whether a healthy process is up for the installation +// without counting as a use of it, for status reads. +func (h *Host) Running(installationID int) bool { + h.mu.RLock() + instance, ok := h.instances[installationID] + h.mu.RUnlock() + if !ok { + return false + } + instance.client.mu.RLock() + defer instance.client.mu.RUnlock() + return !instance.client.unhealthy +} + func (h *Host) NextStartSeq() uint64 { return h.startSeq.Load() } func (h *Host) Stop(installationID int) error { @@ -388,9 +444,15 @@ func (h *Host) Stop(installationID int) error { if ok { delete(h.instances, installationID) } + idleStop := h.idleStopping[installationID] h.mu.Unlock() if !ok { + // The process may still be exiting after an idle stop. Callers treat + // a return from Stop as the process being gone. + if idleStop != nil { + <-idleStop + } return ErrClientNotFound } h.stopInstance(instance) @@ -406,11 +468,18 @@ func (h *Host) Shutdown(ctx context.Context) error { delete(h.instances, installationID) instances = append(instances, instance) } + idleStops := make([]chan struct{}, 0, len(h.idleStopping)) + for _, idleStop := range h.idleStopping { + idleStops = append(idleStops, idleStop) + } h.mu.Unlock() for _, instance := range instances { h.stopInstance(instance) } + for _, idleStop := range idleStops { + <-idleStop + } close(done) }() @@ -509,13 +578,21 @@ func (h *Host) retireInstance(ctx context.Context, installationID int, instance } func (h *Host) stopInstance(instance *instance) { + h.teardown(instance, false) +} + +// teardown stops an instance already removed from the host. A graceful one +// leaves closing the connection to Kill, which then asks the plugin to shut +// down and waits briefly for it to exit by itself, so a routine idle stop is +// not logged and counted as a killed process. +func (h *Host) teardown(instance *instance, graceful bool) { if instance == nil { return } if instance.cancelMonitors != nil { instance.cancelMonitors() } - if instance.protocol != nil { + if instance.protocol != nil && !graceful { _ = instance.protocol.Close() } if instance.process != nil { @@ -539,7 +616,7 @@ func (h *Host) stopInstance(instance *instance) { // tears it down. // // Skipped when no RuntimeHost services are configured. -func (h *Host) bindRuntimeHost(ctx context.Context, sdkClient *sdkruntime.Client, pluginID string, installationID int, provider, ingressToken string, hostInfo HostInfoFunc, instanceState InstanceStateStore) error { +func (h *Host) bindRuntimeHost(ctx context.Context, sdkClient *sdkruntime.Client, used *activity, pluginID string, installationID int, provider, ingressToken string, hostInfo HostInfoFunc, instanceState InstanceStateStore) error { if h.eventPublisher == nil && h.libraryLister == nil && h.catalogPresence == nil && h.installedPlugins == nil && h.globalConfigSetter == nil && hostInfo == nil && instanceState == nil && h.networkAccess == nil { return nil @@ -554,7 +631,7 @@ func (h *Host) bindRuntimeHost(ctx context.Context, sdkClient *sdkruntime.Client streamID := broker.NextId() go broker.AcceptAndServe(streamID, func(opts []grpc.ServerOption) *grpc.Server { - s := grpc.NewServer(append(opts, grpc.ChainUnaryInterceptor(observePluginCallback))...) + s := grpc.NewServer(append(opts, grpc.ChainUnaryInterceptor(used.unaryServer, observePluginCallback))...) srv := NewRuntimeHostServerWithOptions(RuntimeHostOptions{ Publisher: h.eventPublisher, Libraries: h.libraryLister, diff --git a/internal/pluginhost/idle.go b/internal/pluginhost/idle.go new file mode 100644 index 0000000000..f21e8cc581 --- /dev/null +++ b/internal/pluginhost/idle.go @@ -0,0 +1,194 @@ +package pluginhost + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "google.golang.org/grpc" +) + +const ( + // EnvIdleStopAfter sets how long a lazily started plugin process may sit + // unused before the host stops it: a Go duration or whole seconds. Zero + // keeps every plugin running. + EnvIdleStopAfter = "SILO_PLUGIN_IDLE_STOP_AFTER" + // DefaultIdleStopAfter is long enough that a plugin called in bursts (a + // scan, a metadata refresh, someone browsing) stays up across the burst. + DefaultIdleStopAfter = 15 * time.Minute + // MinIdleStopAfter keeps a configured period well above the longest gap + // between a caller resolving a plugin client and making its call. + MinIdleStopAfter = time.Minute + // DefaultIdleCheckInterval is how often a running plugin is checked. + DefaultIdleCheckInterval = 30 * time.Second + + maxIdleStopAfter = time.Duration(1<<63 - 1) + + // pluginServicePrefix selects the Silo plugin protocol. The health probe + // and go-plugin's own broker, stdio and controller calls share the + // connection and say nothing about whether the plugin is in use. + pluginServicePrefix = "/silo.plugin.v1." +) + +// ParseIdleStopAfter reads EnvIdleStopAfter's value. Empty takes the default +// and zero disables idle stops. A shorter period than MinIdleStopAfter is +// raised to it. An unreadable value is an error and yields the default. +func ParseIdleStopAfter(value string) (time.Duration, error) { + value = strings.TrimSpace(value) + if value == "" { + return DefaultIdleStopAfter, nil + } + period, err := time.ParseDuration(value) + if err != nil { + seconds, secondsErr := strconv.ParseInt(value, 10, 64) + if secondsErr != nil || seconds > int64(maxIdleStopAfter/time.Second) { + return DefaultIdleStopAfter, fmt.Errorf("invalid duration %q: %w", value, err) + } + period = time.Duration(seconds) * time.Second + } + switch { + case period < 0: + return DefaultIdleStopAfter, fmt.Errorf("negative duration %q", value) + case period == 0: + return 0, nil + case period < MinIdleStopAfter: + return MinIdleStopAfter, nil + } + return period, nil +} + +// activity records when one plugin process was last used and how many calls +// are in flight, in either direction. The idle watcher reads it to decide +// whether the process can be stopped. +type activity struct { + now func() time.Time + + mu sync.Mutex + inFlight int + last time.Time + // pinned marks a process that opened a plugin protocol stream. Every + // capability RPC is unary today; a stream has no end this type can see, + // so a process that opens one is never stopped for being idle. + pinned bool +} + +func newActivity(now func() time.Time) *activity { + return &activity{now: now, last: now()} +} + +func (a *activity) touch() { + a.mu.Lock() + a.last = a.now() + a.mu.Unlock() +} + +func (a *activity) begin() { + a.mu.Lock() + a.inFlight++ + a.last = a.now() + a.mu.Unlock() +} + +func (a *activity) end() { + a.mu.Lock() + a.inFlight-- + a.last = a.now() + a.mu.Unlock() +} + +// idleFor reports whether nothing is in flight and nothing has used the +// process for at least period. +func (a *activity) idleFor(period time.Duration) bool { + a.mu.Lock() + defer a.mu.Unlock() + return !a.pinned && a.inFlight == 0 && a.now().Sub(a.last) >= period +} + +// unaryClient counts host calls to the plugin for as long as they run. +func (a *activity) unaryClient(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if !strings.HasPrefix(method, pluginServicePrefix) { + return invoker(ctx, method, req, reply, cc, opts...) + } + a.begin() + defer a.end() + return invoker(ctx, method, req, reply, cc, opts...) +} + +func (a *activity) streamClient(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + if strings.HasPrefix(method, pluginServicePrefix) { + a.mu.Lock() + a.pinned = true + a.mu.Unlock() + } + return streamer(ctx, desc, cc, method, opts...) +} + +// unaryServer counts the plugin's calls back to the host. A callback with no +// host call in flight is the only sign of work the plugin does on its own. +func (a *activity) unaryServer(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + a.begin() + defer a.end() + return handler(ctx, req) +} + +// watchIdle stops the instance once it has been idle for the host's idle +// period. It runs only for instances started with StopWhenIdle. +func (h *Host) watchIdle(ctx context.Context, installationID int, instance *instance) { + ticker := time.NewTicker(h.idleCheckInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if h.stopIfIdle(ctx, installationID, instance) { + return + } + } + } +} + +// stopIfIdle stops the instance when it is still the one the host maps and +// has been idle for the idle period. This is a deliberate stop: the instance +// is not marked unhealthy and the exit handler is not told. +// +// The idle check and the removal happen under h.mu, the lock Client takes to +// record a use, so a caller either finds the instance and keeps it alive or +// finds none and starts a new one. The start slot is held until the old +// process has exited, so that new start never overlaps it, and Stop and +// Shutdown wait for the same exit through h.idleStopping. +func (h *Host) stopIfIdle(ctx context.Context, installationID int, instance *instance) bool { + if !instance.activity.idleFor(h.idleStopAfter) { + return false + } + // A Start already in flight for the installation replaces this instance + // and cancels ctx, which ends the wait. + if err := h.acquireStart(ctx, installationID); err != nil { + return false + } + defer h.releaseStart(installationID) + + h.mu.Lock() + current, ok := h.instances[installationID] + if !ok || current != instance || !instance.activity.idleFor(h.idleStopAfter) { + h.mu.Unlock() + return false + } + delete(h.instances, installationID) + exited := make(chan struct{}) + h.idleStopping[installationID] = exited + h.mu.Unlock() + + h.logger.Debug("stopping idle plugin", "installation_id", installationID, "idle_for", h.idleStopAfter) + h.teardown(instance, true) + + h.mu.Lock() + delete(h.idleStopping, installationID) + h.mu.Unlock() + close(exited) + return true +} diff --git a/internal/pluginhost/idle_process_test.go b/internal/pluginhost/idle_process_test.go new file mode 100644 index 0000000000..cbefb1496e --- /dev/null +++ b/internal/pluginhost/idle_process_test.go @@ -0,0 +1,122 @@ +package pluginhost_test + +import ( + "bytes" + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/hashicorp/go-hclog" + + "github.com/Silo-Server/silo-server/internal/pluginhost" +) + +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func waitUntilStopped(ctx context.Context, t *testing.T, host *pluginhost.Host, installationID int) { + t.Helper() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for host.Running(installationID) { + select { + case <-ctx.Done(): + t.Fatalf("installation %d was not stopped for being idle", installationID) + case <-ticker.C: + } + } +} + +// Only a process started with StopWhenIdle on a host with an idle period is +// stopped; the stop is not a crash, and the next Start brings it back. +func TestHostStopsIdleProcessAndStartsItAgain(t *testing.T) { + bin, manifest := buildExitingPlugin(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + disabled := pluginhost.NewHost(pluginhost.Config{Logger: hclog.NewNullLogger(), IdleCheckInterval: 10 * time.Millisecond}) + // A graceful stop logs nothing at warn or above; a killed process does. + var warnings lockedBuffer + host := pluginhost.NewHost(pluginhost.Config{ + Logger: hclog.New(&hclog.LoggerOptions{Level: hclog.Warn, Output: &warnings}), + IdleStopAfter: 200 * time.Millisecond, + IdleCheckInterval: 10 * time.Millisecond, + }) + exited := make(chan int, 4) + host.SetExitHandler(func(id int) { exited <- id }) + t.Cleanup(func() { + _ = disabled.Shutdown(context.Background()) + _ = host.Shutdown(context.Background()) + }) + + // Started first, so each has been idle for longer than installation 3 by + // the time that one is stopped. + if _, err := disabled.Start(ctx, pluginhost.StartRequest{InstallationID: 1, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}); err != nil { + t.Fatalf("Start on a host with no idle period: %v", err) + } + if _, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 2, BinaryPath: bin, Manifest: manifest}); err != nil { + t.Fatalf("Start without StopWhenIdle: %v", err) + } + if _, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: -4, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}); err != nil { + t.Fatalf("Start of a config test run: %v", err) + } + client, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 3, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}) + if err != nil { + t.Fatalf("Start with StopWhenIdle: %v", err) + } + + waitUntilStopped(ctx, t, host, 3) + if _, err := host.Client(3); !errors.Is(err, pluginhost.ErrClientNotFound) { + t.Fatalf("Client after idle stop = %v, want ErrClientNotFound", err) + } + if _, err := client.MetadataProvider("exiting"); errors.Is(err, pluginhost.ErrPluginUnhealthy) { + t.Fatal("idle stop marked the client unhealthy") + } + if !disabled.Running(1) { + t.Fatal("a host with no idle period stopped an idle process") + } + if !host.Running(2) { + t.Fatal("a process started without StopWhenIdle was stopped") + } + if !host.Running(-4) { + t.Fatal("a config test run was stopped for being idle") + } + + restarted, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 3, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}) + if err != nil { + t.Fatalf("Start after idle stop: %v", err) + } + if restarted.StartSeq() == client.StartSeq() { + t.Fatal("Start after idle stop returned the stopped client") + } + waitUntilStopped(ctx, t, host, 3) + // Stop returns once the idle stop has finished, so nothing is logged later. + if err := host.Stop(3); !errors.Is(err, pluginhost.ErrClientNotFound) { + t.Fatalf("Stop after idle stop = %v, want ErrClientNotFound", err) + } + + select { + case id := <-exited: + t.Fatalf("idle stop reported installation %d to the exit handler", id) + default: + } + if logged := warnings.String(); logged != "" { + t.Fatalf("idle stops logged at warn or above:\n%s", logged) + } +} diff --git a/internal/pluginhost/idle_test.go b/internal/pluginhost/idle_test.go new file mode 100644 index 0000000000..a6d8f79437 --- /dev/null +++ b/internal/pluginhost/idle_test.go @@ -0,0 +1,344 @@ +package pluginhost + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" +) + +type idleClock struct { + mu sync.Mutex + now time.Time +} + +func (c *idleClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *idleClock) Advance(d time.Duration) { + c.mu.Lock() + c.now = c.now.Add(d) + c.mu.Unlock() +} + +const ( + testIdlePeriod = 15 * time.Minute + testIdleCapability = "/silo.plugin.v1.MetadataProvider/Search" +) + +// idleTestHost returns a host mapping one instance with no process behind +// it, and the channel its exit handler reports to. +func idleTestHost(t *testing.T, clock *idleClock, installationID int) (*Host, *instance, chan int) { + t.Helper() + host := NewHost(Config{IdleStopAfter: testIdlePeriod}) + host.now = clock.Now + exited := make(chan int, 1) + host.SetExitHandler(func(id int) { exited <- id }) + inst := &instance{ + client: makeTestClient(t, nil), + activity: newActivity(clock.Now), + installationID: installationID, + } + host.instances[installationID] = inst + return host, inst, exited +} + +func TestParseIdleStopAfter(t *testing.T) { + tests := []struct { + value string + want time.Duration + wantErr bool + }{ + {value: "", want: DefaultIdleStopAfter}, + {value: " 30m ", want: 30 * time.Minute}, + {value: "600", want: 10 * time.Minute}, + {value: "0", want: 0}, + {value: "0s", want: 0}, + {value: "5s", want: MinIdleStopAfter}, + {value: "-1m", want: DefaultIdleStopAfter, wantErr: true}, + {value: "-5", want: DefaultIdleStopAfter, wantErr: true}, + {value: "soon", want: DefaultIdleStopAfter, wantErr: true}, + {value: "99999999999999999999", want: DefaultIdleStopAfter, wantErr: true}, + } + for _, tt := range tests { + got, err := ParseIdleStopAfter(tt.value) + if got != tt.want || (err != nil) != tt.wantErr { + t.Errorf("ParseIdleStopAfter(%q) = %v, %v; want %v, error %v", tt.value, got, err, tt.want, tt.wantErr) + } + } +} + +func TestStopIfIdleStopsAnIdleInstanceQuietly(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, exited := idleTestHost(t, clock, 7) + + clock.Advance(testIdlePeriod - time.Second) + if host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("stopped before the idle period passed") + } + clock.Advance(time.Second) + if !host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("did not stop an instance idle for the whole period") + } + + if _, err := host.Client(7); !errors.Is(err, ErrClientNotFound) { + t.Fatalf("Client after idle stop = %v, want ErrClientNotFound so the next call starts it", err) + } + inst.client.mu.RLock() + unhealthy := inst.client.unhealthy + inst.client.mu.RUnlock() + if unhealthy { + t.Fatal("idle stop marked the client unhealthy") + } + select { + case id := <-exited: + t.Fatalf("idle stop reported installation %d to the exit handler", id) + default: + } + // The start slot is free again for the next launch. + if err := host.acquireStart(context.Background(), 7); err != nil { + t.Fatalf("acquireStart after idle stop: %v", err) + } + host.releaseStart(7) +} + +func TestStopIfIdleKeepsABusyInstance(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + + inCall := make(chan struct{}) + release := make(chan struct{}) + invoker := func(context.Context, string, any, any, *grpc.ClientConn, ...grpc.CallOption) error { + close(inCall) + <-release + return nil + } + done := make(chan error, 1) + go func() { + done <- inst.activity.unaryClient(context.Background(), testIdleCapability, nil, nil, nil, invoker) + }() + <-inCall + + clock.Advance(4 * testIdlePeriod) + if host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("stopped an instance with a call in flight") + } + + close(release) + if err := <-done; err != nil { + t.Fatalf("call: %v", err) + } + // The idle period counts from the end of the call, not its start. + if host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("stopped as soon as a long call returned") + } + clock.Advance(testIdlePeriod) + if !host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("did not stop once idle after the call") + } +} + +func TestIdleActivitySources(t *testing.T) { + noop := func(context.Context, string, any, any, *grpc.ClientConn, ...grpc.CallOption) error { return nil } + tests := []struct { + name string + use func(t *testing.T, host *Host, inst *instance) + wantIdle bool + }{ + {name: "nothing", use: func(*testing.T, *Host, *instance) {}, wantIdle: true}, + {name: "capability call", use: func(t *testing.T, _ *Host, inst *instance) { + if err := inst.activity.unaryClient(context.Background(), testIdleCapability, nil, nil, nil, noop); err != nil { + t.Fatal(err) + } + }}, + {name: "host callback", use: func(t *testing.T, _ *Host, inst *instance) { + handler := func(context.Context, any) (any, error) { return struct{}{}, nil } + if _, err := inst.activity.unaryServer(context.Background(), nil, &grpc.UnaryServerInfo{}, handler); err != nil { + t.Fatal(err) + } + }}, + {name: "client lookup", use: func(t *testing.T, host *Host, _ *instance) { + if _, err := host.Client(7); err != nil { + t.Fatal(err) + } + }}, + {name: "health probe", wantIdle: true, use: func(t *testing.T, _ *Host, inst *instance) { + if err := inst.activity.unaryClient(context.Background(), "/grpc.health.v1.Health/Check", nil, nil, nil, noop); err != nil { + t.Fatal(err) + } + }}, + {name: "status read", wantIdle: true, use: func(t *testing.T, host *Host, _ *instance) { + if !host.Running(7) { + t.Fatal("Running = false for a mapped instance") + } + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + clock.Advance(testIdlePeriod - time.Second) + tt.use(t, host, inst) + clock.Advance(time.Second) + if got := host.stopIfIdle(context.Background(), 7, inst); got != tt.wantIdle { + t.Fatalf("stopped = %v, want %v", got, tt.wantIdle) + } + }) + } +} + +func TestIdleStreamPinsTheInstance(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + streamer := func(context.Context, *grpc.StreamDesc, *grpc.ClientConn, string, ...grpc.CallOption) (grpc.ClientStream, error) { + return nil, errors.New("no stream in this test") + } + // go-plugin's own broker and stdio streams stay open for the life of + // every process and must not pin it. + _, _ = inst.activity.streamClient(context.Background(), nil, nil, "/plugin.GRPCBroker/StartStream", streamer) + clock.Advance(testIdlePeriod) + if !inst.activity.idleFor(testIdlePeriod) { + t.Fatal("a go-plugin stream pinned the instance") + } + _, _ = inst.activity.streamClient(context.Background(), nil, nil, "/silo.plugin.v1.Future/Watch", streamer) + clock.Advance(4 * testIdlePeriod) + if host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("stopped an instance that opened a plugin protocol stream") + } +} + +// A caller that looks the client up while the idle watcher runs either keeps +// the instance alive or is told to start a new one. It is never handed a +// client whose process the watcher then stops. +func TestStopIfIdleRacingClientLookup(t *testing.T) { + for range 300 { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + clock.Advance(testIdlePeriod) + + const callers = 4 + var found atomic.Int32 + var stopped atomic.Bool + release := make(chan struct{}) + var wg sync.WaitGroup + for range callers { + wg.Add(1) + go func() { + defer wg.Done() + <-release + if _, err := host.Client(7); err == nil { + found.Add(1) + } else if !errors.Is(err, ErrClientNotFound) { + t.Errorf("Client = %v", err) + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + <-release + stopped.Store(host.stopIfIdle(context.Background(), 7, inst)) + }() + close(release) + wg.Wait() + + if stopped.Load() && found.Load() != 0 { + t.Fatalf("instance stopped after %d callers were handed its client", found.Load()) + } + if !stopped.Load() && found.Load() != callers { + t.Fatalf("instance kept but only %d of %d callers found it", found.Load(), callers) + } + } +} + +// A replacing Start owns the installation's start slot. The idle watcher +// waits for it and then leaves the instance that Start mapped alone. +func TestStopIfIdleYieldsToAStartInFlight(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + clock.Advance(testIdlePeriod) + + if err := host.acquireStart(context.Background(), 7); err != nil { + t.Fatal(err) + } + result := make(chan bool, 1) + go func() { result <- host.stopIfIdle(context.Background(), 7, inst) }() + + replacement := &instance{client: makeTestClient(t, nil), activity: newActivity(clock.Now), installationID: 7} + host.mu.Lock() + host.instances[7] = replacement + host.mu.Unlock() + host.releaseStart(7) + + if <-result { + t.Fatal("idle watcher stopped an installation a Start had just replaced") + } + if client, err := host.Client(7); err != nil || client != replacement.client { + t.Fatalf("Client = %v, %v; want the replacement instance", client, err) + } + + // A canceled watcher gives up on a held slot instead of waiting for it. + if err := host.acquireStart(context.Background(), 7); err != nil { + t.Fatal(err) + } + defer host.releaseStart(7) + clock.Advance(testIdlePeriod) + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if host.stopIfIdle(canceled, 7, replacement) { + t.Fatal("canceled idle watcher stopped the instance") + } +} + +// Callers of Stop and Shutdown treat a return as the process being gone, so +// both wait for an idle stop that has already taken the instance. +func TestStopAndShutdownWaitForAnIdleStopInProgress(t *testing.T) { + waiters := map[string]func(*Host) error{ + "Stop": func(host *Host) error { + if err := host.Stop(7); !errors.Is(err, ErrClientNotFound) { + return err + } + return nil + }, + "Shutdown": func(host *Host) error { return host.Shutdown(context.Background()) }, + } + for name, wait := range waiters { + t.Run(name, func(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + clock.Advance(testIdlePeriod) + + // Stands in for a process that takes a while to exit. + exiting := make(chan struct{}) + exit := make(chan struct{}) + inst.cancelMonitors = func() { + close(exiting) + <-exit + } + go host.stopIfIdle(context.Background(), 7, inst) + <-exiting + + var exitedFirst atomic.Bool + returned := make(chan error, 1) + go func() { + err := wait(host) + if err == nil && !exitedFirst.Load() { + err = errors.New("returned while the idle process was still exiting") + } + returned <- err + }() + exitedFirst.Store(true) + close(exit) + if err := <-returned; err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/plugins/host_adapter.go b/internal/plugins/host_adapter.go index 3cec17061e..0821de986a 100644 --- a/internal/plugins/host_adapter.go +++ b/internal/plugins/host_adapter.go @@ -39,3 +39,7 @@ func (h *hostAdapter) Shutdown(ctx context.Context) error { func (h *hostAdapter) NextStartSeq() uint64 { return h.host.NextStartSeq() } + +func (h *hostAdapter) Running(installationID int) bool { + return h.host.Running(installationID) +} diff --git a/internal/plugins/resident.go b/internal/plugins/resident.go index 12970f407c..e667209246 100644 --- a/internal/plugins/resident.go +++ b/internal/plugins/resident.go @@ -794,7 +794,7 @@ func (s *Service) RuntimeState(installationID int) RuntimeState { return tracked } if s.host != nil { - if _, err := s.host.Client(installationID); err == nil { + if s.host.Running(installationID) { state.State = ResidentRunning } } diff --git a/internal/plugins/resident_generation_test.go b/internal/plugins/resident_generation_test.go index cce4a10557..4723a4a6f0 100644 --- a/internal/plugins/resident_generation_test.go +++ b/internal/plugins/resident_generation_test.go @@ -156,3 +156,8 @@ func TestResidentSupervisorNewGenerationNeverAdoptsInFlightOlderLaunch(t *testin t.Fatalf("stopped seqs = %v, want the adopted generation 1 process (seq 1) stopped exactly once", stopped) } } + +func (h *seqFakeHost) Running(id int) bool { + _, err := h.Client(id) + return err == nil +} diff --git a/internal/plugins/service.go b/internal/plugins/service.go index 4486ccc0a1..6e83c6fff9 100644 --- a/internal/plugins/service.go +++ b/internal/plugins/service.go @@ -52,6 +52,10 @@ type Host interface { // StartSeq is at or below the value read before a launch was issued by // an earlier launch that the singleflight joined. NextStartSeq() uint64 + // Running reports whether the installation's process is up. Unlike + // Client it does not count as a use, so a status read cannot keep an + // idle plugin running. + Running(installationID int) bool } type serviceInstallationStore interface { @@ -601,6 +605,10 @@ func (s *Service) start(ctx context.Context, installationID int, allowResident b BinaryPath: binaryPath, Manifest: manifest, Config: configEntries, + // Every RPC to a lazily started plugin resolves its client through + // ensureClient, which starts it again. A resident is the + // supervisor's to keep up. + StopWhenIdle: !isResidentManifest(manifest), }) } diff --git a/internal/plugins/service_hot_reload_test.go b/internal/plugins/service_hot_reload_test.go index 8acb59d30e..17530c26f4 100644 --- a/internal/plugins/service_hot_reload_test.go +++ b/internal/plugins/service_hot_reload_test.go @@ -495,3 +495,8 @@ func recordTestEvent(events *[]string, event string) { } *events = append(*events, event) } + +func (f *fakeServiceHost) Running(_ int) bool { + _, err := f.Client(0) + return err == nil +} diff --git a/internal/plugins/service_singleflight_test.go b/internal/plugins/service_singleflight_test.go index ea4e012b41..44ca587b1d 100644 --- a/internal/plugins/service_singleflight_test.go +++ b/internal/plugins/service_singleflight_test.go @@ -7,6 +7,9 @@ import ( "testing" "time" + pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" + "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginsdk/capability" + "github.com/Silo-Server/silo-server/internal/pluginhost" ) @@ -281,3 +284,52 @@ func TestEnsureClientFailedLaunchPropagatesToAllCallers(t *testing.T) { t.Fatalf("failed launch host.Start calls = %d, want 1 (still single-flighted)", got) } } + +func (h *countingHerdHost) Running(id int) bool { + _, err := h.Client(id) + return err == nil +} + +func (h *ctxCaptureHost) Running(id int) bool { + _, err := h.Client(id) + return err == nil +} + +// The host stops an idle plugin only when its start asked for that. An RPC +// that finds no client, as after an idle stop, starts a lazy plugin with the +// request set and reports the stopped plugin without an error; a resident is +// the supervisor's and is never started with it. +func TestOnlyLazyPluginsAreStartedWithStopWhenIdle(t *testing.T) { + manifest := testPluginManifest(t, "silo.metadb", "0.0.36") + store := installedPluginStore(t, 7, manifest) + host := &fakeServiceHost{clientErr: pluginhost.ErrClientNotFound} + service := &Service{installations: store, host: host} + + if _, err := service.ensureClient(context.Background(), 7); err != nil { + t.Fatalf("ensureClient after an idle stop: %v", err) + } + if len(host.started) != 1 || len(host.stopped) != 0 { + t.Fatalf("starts = %d, stops = %d; want 1 start and no stop", len(host.started), len(host.stopped)) + } + if !host.started[0].StopWhenIdle { + t.Fatal("a lazily started plugin was not started with StopWhenIdle") + } + if state := service.RuntimeStateOf(store.byID[7]); state.State != ResidentStopped || state.LastError != "" { + t.Fatalf("runtime state of a stopped idle plugin = %+v, want stopped with no error", state) + } + + resident := testPluginManifest(t, "silo.overlay", "1.0.0") + resident.Capabilities = []*pluginv1.CapabilityDescriptor{{ + Type: capability.NetworkAccessProvider, Id: "overlay", + NetworkAccessProvider: &pluginv1.NetworkAccessProviderDescriptor{Provider: "tailscale"}, + }} + residentStore := installedPluginStore(t, 8, resident) + residentHost := &fakeServiceHost{} + residentService := &Service{installations: residentStore, host: residentHost} + if _, err := residentService.start(context.Background(), 8, true); err != nil { + t.Fatalf("start resident: %v", err) + } + if len(residentHost.started) != 1 || residentHost.started[0].StopWhenIdle { + t.Fatalf("resident start requests = %+v, want one without StopWhenIdle", residentHost.started) + } +} From 5d58f9751a29e3682339af7186791cfd4f663b8d Mon Sep 17 00:00:00 2001 From: blurbery <271863866+blurbery@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:52:33 +1100 Subject: [PATCH 2/7] perf(plugins): fix the idle stop period at 30 minutes Drop the SILO_PLUGIN_IDLE_STOP_AFTER setting: idle stop is the only behaviour, after a fixed 30 minutes. That is no shorter than the response caches plugins keep (typically half an hour), so a quiet spell does not throw a live cache away with the process. --- cmd/silo/main.go | 7 +--- docs/admin-api.md | 2 +- docs/architecture/plugin-idle-stop.md | 12 +++---- internal/pluginhost/idle.go | 49 +++++---------------------- internal/pluginhost/idle_test.go | 25 -------------- 5 files changed, 16 insertions(+), 79 deletions(-) diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 0505f2c4db..ab22692525 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -1916,13 +1916,8 @@ func main() { return out, nil }, ) - pluginIdleStopAfter, idleErr := pluginhost.ParseIdleStopAfter(os.Getenv(pluginhost.EnvIdleStopAfter)) - if idleErr != nil { - slog.Warn("invalid plugin idle period; using the default", "component", "plugins", - "env", pluginhost.EnvIdleStopAfter, "error", idleErr, "default", pluginIdleStopAfter) - } pluginHost = pluginhost.NewHost(pluginhost.Config{ - IdleStopAfter: pluginIdleStopAfter, + IdleStopAfter: pluginhost.IdleStopAfter, EventPublisher: eventsHub, LibraryLister: pluginhost.NewLibraryLister(libDataSource), CatalogPresence: catalogPresence, diff --git a/docs/admin-api.md b/docs/admin-api.md index 9b2616a1b9..22c4cca6a7 100644 --- a/docs/admin-api.md +++ b/docs/admin-api.md @@ -3440,7 +3440,7 @@ exponential backoff from 1 s to 60 s, and is parked as `failed` after ten consec failures until restarted or reconfigured), `state` (`stopped`, `starting`, `running`, `backoff`, `failed`), `restart_count`, `last_error`, `last_started_at` and `next_restart_at`. Plugins declaring `network_access_provider.v1` are resident; every other plugin starts on -first use, is stopped again after `SILO_PLUGIN_IDLE_STOP_AFTER` (default 15 minutes, `0` never) +first use, is stopped again after 30 minutes without use, and reports `running`, `stopped`, or `failed` when its last launch on the answering server failed, with `last_error` saying why. A plugin stopped for being idle reports `stopped` and starts on its next use. A later successful launch, an administrator diff --git a/docs/architecture/plugin-idle-stop.md b/docs/architecture/plugin-idle-stop.md index 83f38d3db7..4b34f3138d 100644 --- a/docs/architecture/plugin-idle-stop.md +++ b/docs/architecture/plugin-idle-stop.md @@ -1,15 +1,15 @@ # Stopping idle plugin processes A plugin that is not resident starts on its first RPC. The host stops it again -once nothing has used it for `SILO_PLUGIN_IDLE_STOP_AFTER`, and the next RPC -starts it as the first one did. A server whose libraries never call a plugin +once nothing has used it for 30 minutes, and the next RPC starts it as the first +one did. A server whose libraries never call a plugin (an audiobook metadata provider on a Movies and Series server) does not keep that process in memory. -`SILO_PLUGIN_IDLE_STOP_AFTER` is a Go duration or whole seconds. The default -is `15m`, `0` keeps every plugin running, and a shorter period than one minute -is raised to one minute. It is read once at start-up on the API server. Proxy -nodes run only resident plugins and ignore it. +The period is fixed (`pluginhost.IdleStopAfter`). Thirty minutes covers a burst +of calls and is no shorter than the response caches plugins keep, typically half +an hour, so a quiet spell does not throw a live cache away. Proxy nodes run only resident +plugins, so nothing there is stopped for being idle. ## What counts as use diff --git a/internal/pluginhost/idle.go b/internal/pluginhost/idle.go index f21e8cc581..3975649825 100644 --- a/internal/pluginhost/idle.go +++ b/internal/pluginhost/idle.go @@ -2,8 +2,6 @@ package pluginhost import ( "context" - "fmt" - "strconv" "strings" "sync" "time" @@ -12,54 +10,23 @@ import ( ) const ( - // EnvIdleStopAfter sets how long a lazily started plugin process may sit - // unused before the host stops it: a Go duration or whole seconds. Zero - // keeps every plugin running. - EnvIdleStopAfter = "SILO_PLUGIN_IDLE_STOP_AFTER" - // DefaultIdleStopAfter is long enough that a plugin called in bursts (a - // scan, a metadata refresh, someone browsing) stays up across the burst. - DefaultIdleStopAfter = 15 * time.Minute - // MinIdleStopAfter keeps a configured period well above the longest gap - // between a caller resolving a plugin client and making its call. - MinIdleStopAfter = time.Minute + // IdleStopAfter is how long a lazily started plugin process may go unused + // before the host stops it. It is long enough that a plugin called in + // bursts (a scan, a metadata refresh, someone browsing) stays up across the + // burst, and no shorter than the response cache a plugin keeps, typically + // half an hour, so a quiet spell does not throw a live cache away with + // the process. It is also far above the longest gap between a caller + // resolving a plugin client and making its call. + IdleStopAfter = 30 * time.Minute // DefaultIdleCheckInterval is how often a running plugin is checked. DefaultIdleCheckInterval = 30 * time.Second - maxIdleStopAfter = time.Duration(1<<63 - 1) - // pluginServicePrefix selects the Silo plugin protocol. The health probe // and go-plugin's own broker, stdio and controller calls share the // connection and say nothing about whether the plugin is in use. pluginServicePrefix = "/silo.plugin.v1." ) -// ParseIdleStopAfter reads EnvIdleStopAfter's value. Empty takes the default -// and zero disables idle stops. A shorter period than MinIdleStopAfter is -// raised to it. An unreadable value is an error and yields the default. -func ParseIdleStopAfter(value string) (time.Duration, error) { - value = strings.TrimSpace(value) - if value == "" { - return DefaultIdleStopAfter, nil - } - period, err := time.ParseDuration(value) - if err != nil { - seconds, secondsErr := strconv.ParseInt(value, 10, 64) - if secondsErr != nil || seconds > int64(maxIdleStopAfter/time.Second) { - return DefaultIdleStopAfter, fmt.Errorf("invalid duration %q: %w", value, err) - } - period = time.Duration(seconds) * time.Second - } - switch { - case period < 0: - return DefaultIdleStopAfter, fmt.Errorf("negative duration %q", value) - case period == 0: - return 0, nil - case period < MinIdleStopAfter: - return MinIdleStopAfter, nil - } - return period, nil -} - // activity records when one plugin process was last used and how many calls // are in flight, in either direction. The idle watcher reads it to decide // whether the process can be stopped. diff --git a/internal/pluginhost/idle_test.go b/internal/pluginhost/idle_test.go index a6d8f79437..9bea5501b9 100644 --- a/internal/pluginhost/idle_test.go +++ b/internal/pluginhost/idle_test.go @@ -50,31 +50,6 @@ func idleTestHost(t *testing.T, clock *idleClock, installationID int) (*Host, *i return host, inst, exited } -func TestParseIdleStopAfter(t *testing.T) { - tests := []struct { - value string - want time.Duration - wantErr bool - }{ - {value: "", want: DefaultIdleStopAfter}, - {value: " 30m ", want: 30 * time.Minute}, - {value: "600", want: 10 * time.Minute}, - {value: "0", want: 0}, - {value: "0s", want: 0}, - {value: "5s", want: MinIdleStopAfter}, - {value: "-1m", want: DefaultIdleStopAfter, wantErr: true}, - {value: "-5", want: DefaultIdleStopAfter, wantErr: true}, - {value: "soon", want: DefaultIdleStopAfter, wantErr: true}, - {value: "99999999999999999999", want: DefaultIdleStopAfter, wantErr: true}, - } - for _, tt := range tests { - got, err := ParseIdleStopAfter(tt.value) - if got != tt.want || (err != nil) != tt.wantErr { - t.Errorf("ParseIdleStopAfter(%q) = %v, %v; want %v, error %v", tt.value, got, err, tt.want, tt.wantErr) - } - } -} - func TestStopIfIdleStopsAnIdleInstanceQuietly(t *testing.T) { clock := &idleClock{now: time.Unix(1000, 0)} host, inst, exited := idleTestHost(t, clock, 7) From 14a6d154fe2b58079b37b8f1afab31bddaf96e49 Mon Sep 17 00:00:00 2001 From: blurbery <271863866+blurbery@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:09:54 +1100 Subject: [PATCH 3/7] test(plugins): prove idle stop waits and counts real calls The Stop and Shutdown wait test now runs in a synctest bubble and checks that neither returns while the idle process is still exiting; the old check passed even when they did not wait. The process test keeps calling one plugin through its client for two idle periods and checks it stays up, so the activity interceptors have to be on the real connection. Correct the design doc: the admin UI shows an idle-stopped plugin as Active, proxy nodes are safe because their host has no idle period, and the half-hour figure is a cache the period is sized for, not a claim about every plugin. --- docs/architecture/plugin-idle-stop.md | 19 +++++----- internal/pluginhost/idle.go | 5 ++- internal/pluginhost/idle_process_test.go | 46 +++++++++++++++++++++++- internal/pluginhost/idle_test.go | 45 +++++++++++------------ 4 files changed, 79 insertions(+), 36 deletions(-) diff --git a/docs/architecture/plugin-idle-stop.md b/docs/architecture/plugin-idle-stop.md index 4b34f3138d..fb247fca31 100644 --- a/docs/architecture/plugin-idle-stop.md +++ b/docs/architecture/plugin-idle-stop.md @@ -2,14 +2,13 @@ A plugin that is not resident starts on its first RPC. The host stops it again once nothing has used it for 30 minutes, and the next RPC starts it as the first -one did. A server whose libraries never call a plugin -(an audiobook metadata provider on a Movies and Series server) does not keep -that process in memory. +one did. A server whose libraries never call a plugin (an audiobook metadata +provider on a Movies and Series server) does not keep that process in memory. The period is fixed (`pluginhost.IdleStopAfter`). Thirty minutes covers a burst -of calls and is no shorter than the response caches plugins keep, typically half -an hour, so a quiet spell does not throw a live cache away. Proxy nodes run only resident -plugins, so nothing there is stopped for being idle. +of calls, and a plugin that caches responses for up to half an hour keeps that +cache through a quiet spell. Proxy nodes build their plugin host without an idle +period, so nothing there is stopped for being idle. ## What counts as use @@ -53,10 +52,10 @@ reads (`Host.Running`, which backs the admin `runtime.state`) are not use. manifest and configures it. A start that fails is recorded as any failed lazy start is. - An idle stop is a deliberate stop. The client is not marked unhealthy, the - exit handler is not called, no lazy start failure is recorded, and the - installation reports `stopped`, which the admin UI shows as a healthy plugin - that is not running. The process is asked to shut down and is only killed if - it has not exited after two seconds. + exit handler is not called, and no lazy start failure is recorded. The + installation reports `stopped`, and the admin UI shows it as **Active**, like + any other healthy plugin that is not running. The process is asked to shut + down and is only killed if it has not exited after two seconds. - State a plugin keeps in memory does not survive an idle stop. Host-held state (configuration, instance state, OAuth and device-code flow state, scan markers, sync cursors) is sent again with each call. A plugin must not rely diff --git a/internal/pluginhost/idle.go b/internal/pluginhost/idle.go index 3975649825..af679a22f1 100644 --- a/internal/pluginhost/idle.go +++ b/internal/pluginhost/idle.go @@ -13,9 +13,8 @@ const ( // IdleStopAfter is how long a lazily started plugin process may go unused // before the host stops it. It is long enough that a plugin called in // bursts (a scan, a metadata refresh, someone browsing) stays up across the - // burst, and no shorter than the response cache a plugin keeps, typically - // half an hour, so a quiet spell does not throw a live cache away with - // the process. It is also far above the longest gap between a caller + // burst, and a plugin that caches responses for up to half an hour keeps + // that cache through a quiet spell. It is also far above the longest gap between a caller // resolving a plugin client and making its call. IdleStopAfter = 30 * time.Minute // DefaultIdleCheckInterval is how often a running plugin is checked. diff --git a/internal/pluginhost/idle_process_test.go b/internal/pluginhost/idle_process_test.go index cbefb1496e..f40171c183 100644 --- a/internal/pluginhost/idle_process_test.go +++ b/internal/pluginhost/idle_process_test.go @@ -10,6 +10,8 @@ import ( "github.com/hashicorp/go-hclog" + pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" + "github.com/Silo-Server/silo-server/internal/pluginhost" ) @@ -43,8 +45,41 @@ func waitUntilStopped(ctx context.Context, t *testing.T, host *pluginhost.Host, } } +// keepCalling sends a capability RPC on client every 10 ms until the +// returned stop is called. The fixture answers Unimplemented, which still +// counts as use. +func keepCalling(t *testing.T, client *pluginhost.Client) (stop func()) { + t.Helper() + metadata, err := client.MetadataProvider("exiting") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + finished := make(chan struct{}) + go func() { + defer close(finished) + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + _, _ = metadata.Search(context.Background(), &pluginv1.SearchMetadataRequest{}) + } + } + }() + return func() { + close(done) + <-finished + } +} + // Only a process started with StopWhenIdle on a host with an idle period is // stopped; the stop is not a crash, and the next Start brings it back. +// Calls made on the client keep its process up, so the activity interceptors +// are wired into the real connection. It runs on the real clock, with +// several plugin starts and two idle periods, so it takes about a second. func TestHostStopsIdleProcessAndStartsItAgain(t *testing.T) { bin, manifest := buildExitingPlugin(t) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) @@ -65,8 +100,13 @@ func TestHostStopsIdleProcessAndStartsItAgain(t *testing.T) { _ = host.Shutdown(context.Background()) }) - // Started first, so each has been idle for longer than installation 3 by + // Started first, so each has been up for longer than installation 3 by // the time that one is stopped. + busy, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 5, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}) + if err != nil { + t.Fatalf("Start of a plugin in use: %v", err) + } + stopCalling := keepCalling(t, busy) if _, err := disabled.Start(ctx, pluginhost.StartRequest{InstallationID: 1, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}); err != nil { t.Fatalf("Start on a host with no idle period: %v", err) } @@ -106,6 +146,10 @@ func TestHostStopsIdleProcessAndStartsItAgain(t *testing.T) { t.Fatal("Start after idle stop returned the stopped client") } waitUntilStopped(ctx, t, host, 3) + stopCalling() + if !host.Running(5) { + t.Fatal("a plugin was stopped while it was being called") + } // Stop returns once the idle stop has finished, so nothing is logged later. if err := host.Stop(3); !errors.Is(err, pluginhost.ErrClientNotFound) { t.Fatalf("Stop after idle stop = %v, want ErrClientNotFound", err) diff --git a/internal/pluginhost/idle_test.go b/internal/pluginhost/idle_test.go index 9bea5501b9..b9e63dda64 100644 --- a/internal/pluginhost/idle_test.go +++ b/internal/pluginhost/idle_test.go @@ -6,6 +6,7 @@ import ( "sync" "sync/atomic" "testing" + "testing/synctest" "time" "google.golang.org/grpc" @@ -290,30 +291,30 @@ func TestStopAndShutdownWaitForAnIdleStopInProgress(t *testing.T) { host, inst, _ := idleTestHost(t, clock, 7) clock.Advance(testIdlePeriod) - // Stands in for a process that takes a while to exit. - exiting := make(chan struct{}) - exit := make(chan struct{}) - inst.cancelMonitors = func() { - close(exiting) - <-exit - } - go host.stopIfIdle(context.Background(), 7, inst) - <-exiting + synctest.Test(t, func(t *testing.T) { + // Stands in for a process that takes a while to exit. + exiting := make(chan struct{}) + exit := make(chan struct{}) + inst.cancelMonitors = func() { + close(exiting) + <-exit + } + go host.stopIfIdle(context.Background(), 7, inst) + <-exiting - var exitedFirst atomic.Bool - returned := make(chan error, 1) - go func() { - err := wait(host) - if err == nil && !exitedFirst.Load() { - err = errors.New("returned while the idle process was still exiting") + returned := make(chan error, 1) + go func() { returned <- wait(host) }() + synctest.Wait() + select { + case err := <-returned: + t.Fatalf("returned (%v) while the idle process was still exiting", err) + default: } - returned <- err - }() - exitedFirst.Store(true) - close(exit) - if err := <-returned; err != nil { - t.Fatal(err) - } + close(exit) + if err := <-returned; err != nil { + t.Fatal(err) + } + }) }) } } From 7b46556828e1d281fbf1166256c96fe0f80a60ce Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:50:23 -0500 Subject: [PATCH 4/7] fix(plugins): bound the idle stop of a frozen plugin process go-plugin's shutdown request has no deadline, so an idle stop of a process that was frozen when its idle period ended held the installation's start slot for good: Stop, Shutdown and every later start of that plugin waited forever. Kill the process after DefaultIdleStopTimeout, and release the slot with a warning if it still cannot exit. Log idle stops at info. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/plugin-idle-stop.md | 7 +- internal/pluginhost/host.go | 9 ++ internal/pluginhost/idle.go | 37 +++++++- internal/pluginhost/idle_frozen_linux_test.go | 84 +++++++++++++++++++ 4 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 internal/pluginhost/idle_frozen_linux_test.go diff --git a/docs/architecture/plugin-idle-stop.md b/docs/architecture/plugin-idle-stop.md index fb247fca31..c7341c3003 100644 --- a/docs/architecture/plugin-idle-stop.md +++ b/docs/architecture/plugin-idle-stop.md @@ -54,8 +54,11 @@ reads (`Host.Running`, which backs the admin `runtime.state`) are not use. - An idle stop is a deliberate stop. The client is not marked unhealthy, the exit handler is not called, and no lazy start failure is recorded. The installation reports `stopped`, and the admin UI shows it as **Active**, like - any other healthy plugin that is not running. The process is asked to shut - down and is only killed if it has not exited after two seconds. + any other healthy plugin that is not running. The stop is logged at info. +- The process is asked to shut down and is killed if it has not exited after + two seconds. go-plugin's shutdown request has no deadline, so a process that + is frozen when its idle period ends is killed after `DefaultIdleStopTimeout` + instead of holding the start slot, and every caller waiting on it, for good. - State a plugin keeps in memory does not survive an idle stop. Host-held state (configuration, instance state, OAuth and device-code flow state, scan markers, sync cursors) is sent again with each call. A plugin must not rely diff --git a/internal/pluginhost/host.go b/internal/pluginhost/host.go index 28c2b9f2d6..c02cd37989 100644 --- a/internal/pluginhost/host.go +++ b/internal/pluginhost/host.go @@ -36,6 +36,9 @@ type Config struct { // IdleCheckInterval is how often such a process is checked. Zero takes // DefaultIdleCheckInterval. IdleCheckInterval time.Duration + // IdleStopTimeout is how long an idle stop waits for the plugin to shut + // down before killing its process. Zero takes DefaultIdleStopTimeout. + IdleStopTimeout time.Duration // EventPublisher receives events that plugins publish via RuntimeHost. // Typically silo's *events.Hub. When nil, plugins that try to @@ -85,6 +88,7 @@ type Host struct { exitCheckInterval time.Duration idleStopAfter time.Duration idleCheckInterval time.Duration + idleStopTimeout time.Duration // now is a test seam for the idle clock. now func() time.Time @@ -157,10 +161,15 @@ func NewHost(cfg Config) *Host { if idleInterval <= 0 { idleInterval = DefaultIdleCheckInterval } + idleStopTimeout := cfg.IdleStopTimeout + if idleStopTimeout <= 0 { + idleStopTimeout = DefaultIdleStopTimeout + } return &Host{ idleStopAfter: max(cfg.IdleStopAfter, 0), idleCheckInterval: idleInterval, + idleStopTimeout: idleStopTimeout, now: time.Now, logger: logger, healthCheckInterval: interval, diff --git a/internal/pluginhost/idle.go b/internal/pluginhost/idle.go index af679a22f1..c776377c37 100644 --- a/internal/pluginhost/idle.go +++ b/internal/pluginhost/idle.go @@ -19,6 +19,10 @@ const ( IdleStopAfter = 30 * time.Minute // DefaultIdleCheckInterval is how often a running plugin is checked. DefaultIdleCheckInterval = 30 * time.Second + // DefaultIdleStopTimeout bounds an idle stop. go-plugin asks the plugin to + // shut down with no deadline, so a frozen process would otherwise hold the + // installation's start slot, and every caller waiting on it, forever. + DefaultIdleStopTimeout = 10 * time.Second // pluginServicePrefix selects the Silo plugin protocol. The health probe // and go-plugin's own broker, stdio and controller calls share the @@ -149,8 +153,8 @@ func (h *Host) stopIfIdle(ctx context.Context, installationID int, instance *ins h.idleStopping[installationID] = exited h.mu.Unlock() - h.logger.Debug("stopping idle plugin", "installation_id", installationID, "idle_for", h.idleStopAfter) - h.teardown(instance, true) + h.logger.Info("stopping idle plugin", "installation_id", installationID, "idle_for", h.idleStopAfter) + h.teardownIdle(installationID, instance) h.mu.Lock() delete(h.idleStopping, installationID) @@ -158,3 +162,32 @@ func (h *Host) stopIfIdle(ctx context.Context, installationID int, instance *ins close(exited) return true } + +// teardownIdle stops an idle instance gracefully, and kills its process when +// it has not shut down within the idle stop timeout (see +// DefaultIdleStopTimeout). +func (h *Host) teardownIdle(installationID int, instance *instance) { + done := make(chan struct{}) + go func() { + defer close(done) + h.teardown(instance, true) + }() + select { + case <-done: + return + case <-time.After(h.idleStopTimeout): + } + + h.logger.Warn("idle plugin did not shut down in time, killing it", "installation_id", installationID, "timeout", h.idleStopTimeout) + if instance.command != nil && instance.command.Process != nil { + _ = instance.command.Process.Kill() + } + select { + case <-done: + case <-time.After(h.idleStopTimeout): + // The process is stuck in the kernel (uninterruptible I/O) and cannot + // die yet. Release the start slot anyway: a new process is better than + // a plugin no caller can reach until the server restarts. + h.logger.Warn("idle plugin process has not exited after being killed", "installation_id", installationID) + } +} diff --git a/internal/pluginhost/idle_frozen_linux_test.go b/internal/pluginhost/idle_frozen_linux_test.go new file mode 100644 index 0000000000..495b3afb19 --- /dev/null +++ b/internal/pluginhost/idle_frozen_linux_test.go @@ -0,0 +1,84 @@ +//go:build linux + +package pluginhost_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "syscall" + "testing" + "time" + + "github.com/hashicorp/go-hclog" + + "github.com/Silo-Server/silo-server/internal/pluginhost" +) + +// pidOf finds the process running bin, which is unique to the test. +func pidOf(t *testing.T, bin string) int { + t.Helper() + entries, err := os.ReadDir("/proc") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + if exe, err := os.Readlink(filepath.Join("/proc", entry.Name(), "exe")); err == nil && exe == bin { + return pid + } + } + t.Fatalf("no process runs %s", bin) + return 0 +} + +// A process that is frozen when its idle period ends cannot answer go-plugin's +// shutdown request, which has no deadline. The idle stop kills it instead of +// holding the installation's start slot, so Stop returns and the plugin can +// start again. +func TestHostIdleStopKillsFrozenProcess(t *testing.T) { + bin, manifest := buildExitingPlugin(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + host := pluginhost.NewHost(pluginhost.Config{ + Logger: hclog.NewNullLogger(), + IdleStopAfter: 100 * time.Millisecond, + IdleCheckInterval: 10 * time.Millisecond, + IdleStopTimeout: 200 * time.Millisecond, + }) + t.Cleanup(func() { _ = host.Shutdown(context.Background()) }) + + if _, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 7, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}); err != nil { + t.Fatalf("Start: %v", err) + } + pid := pidOf(t, bin) + if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil { + t.Fatal(err) + } + // Without the fix the frozen process outlives the test; don't leave it. + t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGKILL) }) + + waitUntilStopped(ctx, t, host, 7) + stopped := make(chan error, 1) + go func() { stopped <- host.Stop(7) }() + select { + case err := <-stopped: + if !errors.Is(err, pluginhost.ErrClientNotFound) { + t.Fatalf("Stop after idle stop = %v, want ErrClientNotFound", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Stop is still waiting for the idle stop of a frozen process") + } + if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) { + t.Fatalf("frozen process %d still exists after the idle stop (kill 0: %v)", pid, err) + } + if _, err := host.Start(ctx, pluginhost.StartRequest{InstallationID: 7, BinaryPath: bin, Manifest: manifest, StopWhenIdle: true}); err != nil { + t.Fatalf("Start after the idle stop of a frozen process: %v", err) + } +} From 0164bb7d65aa5fff2d7e6c33256c0537d3bd135e Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:55:54 -0500 Subject: [PATCH 5/7] fix(plugins): refuse plugin callbacks once an idle stop has decided A RuntimeHost callback the plugin started on its own after the idle check ran until the teardown cut it off. The idle stop now marks the process as stopping under the same lock that admits callbacks, and later callbacks get Unavailable. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/plugin-idle-stop.md | 3 +++ internal/pluginhost/idle.go | 34 +++++++++++++++++++++++++-- internal/pluginhost/idle_test.go | 20 ++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/docs/architecture/plugin-idle-stop.md b/docs/architecture/plugin-idle-stop.md index c7341c3003..521e93b6b5 100644 --- a/docs/architecture/plugin-idle-stop.md +++ b/docs/architecture/plugin-idle-stop.md @@ -43,6 +43,9 @@ reads (`Host.Running`, which backs the admin `runtime.state`) are not use. - The idle check and the removal from the host's instance map happen under the lock `Host.Client` takes to record a use. A caller either finds the process and keeps it alive for another idle period, or finds none and starts one. + The same decision marks the process as stopping under the lock that admits + plugin callbacks, so a callback the plugin starts afterwards is refused with + `Unavailable` instead of being cut off mid-flight. - The watcher holds the installation's start slot until the old process has exited, so a restart never overlaps the process it replaces. A `Start` already in flight wins: it cancels the watcher. `Stop` and `Shutdown` also diff --git a/internal/pluginhost/idle.go b/internal/pluginhost/idle.go index c776377c37..2a2249fbcb 100644 --- a/internal/pluginhost/idle.go +++ b/internal/pluginhost/idle.go @@ -7,6 +7,8 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -43,6 +45,10 @@ type activity struct { // capability RPC is unary today; a stream has no end this type can see, // so a process that opens one is never stopped for being idle. pinned bool + // stopping is set when the idle stop commits to stopping the process. + // Callbacks the plugin starts after that are refused rather than cut + // off by the teardown. + stopping bool } func newActivity(now func() time.Time) *activity { @@ -74,9 +80,26 @@ func (a *activity) end() { func (a *activity) idleFor(period time.Duration) bool { a.mu.Lock() defer a.mu.Unlock() + return a.idleForLocked(period) +} + +func (a *activity) idleForLocked(period time.Duration) bool { return !a.pinned && a.inFlight == 0 && a.now().Sub(a.last) >= period } +// commitStop marks the process as stopping if it is still idle for period. +// The check and the mark share a.mu with callback admission, so no callback +// can start between them. +func (a *activity) commitStop(period time.Duration) bool { + a.mu.Lock() + defer a.mu.Unlock() + if !a.idleForLocked(period) { + return false + } + a.stopping = true + return true +} + // unaryClient counts host calls to the plugin for as long as they run. func (a *activity) unaryClient(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { if !strings.HasPrefix(method, pluginServicePrefix) { @@ -99,7 +122,14 @@ func (a *activity) streamClient(ctx context.Context, desc *grpc.StreamDesc, cc * // unaryServer counts the plugin's calls back to the host. A callback with no // host call in flight is the only sign of work the plugin does on its own. func (a *activity) unaryServer(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { - a.begin() + a.mu.Lock() + if a.stopping { + a.mu.Unlock() + return nil, status.Error(codes.Unavailable, "plugin process is stopping after being idle") + } + a.inFlight++ + a.last = a.now() + a.mu.Unlock() defer a.end() return handler(ctx, req) } @@ -144,7 +174,7 @@ func (h *Host) stopIfIdle(ctx context.Context, installationID int, instance *ins h.mu.Lock() current, ok := h.instances[installationID] - if !ok || current != instance || !instance.activity.idleFor(h.idleStopAfter) { + if !ok || current != instance || !instance.activity.commitStop(h.idleStopAfter) { h.mu.Unlock() return false } diff --git a/internal/pluginhost/idle_test.go b/internal/pluginhost/idle_test.go index b9e63dda64..d7ada69c80 100644 --- a/internal/pluginhost/idle_test.go +++ b/internal/pluginhost/idle_test.go @@ -10,6 +10,8 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type idleClock struct { @@ -318,3 +320,21 @@ func TestStopAndShutdownWaitForAnIdleStopInProgress(t *testing.T) { }) } } + +// Once the idle stop has decided, a callback the plugin starts on its own is +// refused instead of being cut off mid-flight by the teardown. +func TestIdleStopRefusesACallbackAfterTheDecision(t *testing.T) { + clock := &idleClock{now: time.Unix(1000, 0)} + host, inst, _ := idleTestHost(t, clock, 7) + + clock.Advance(testIdlePeriod) + if !host.stopIfIdle(context.Background(), 7, inst) { + t.Fatal("did not stop an instance idle for the whole period") + } + ran := false + handler := func(context.Context, any) (any, error) { ran = true; return nil, nil } + _, err := inst.activity.unaryServer(context.Background(), nil, &grpc.UnaryServerInfo{}, handler) + if status.Code(err) != codes.Unavailable || ran { + t.Fatalf("callback after the idle stop decision: err = %v, handler ran = %v; want Unavailable without running it", err, ran) + } +} From da5e67cc97e91541369f0cb529334d5908e69858 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:03:17 -0500 Subject: [PATCH 6/7] docs(plugins): note when an idle stop can overlap a restart The start slot is released without an exit when a killed process is stuck in uninterruptible I/O. Also check the stopped runtime state before the test's start, where no process exists, instead of after it. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/plugin-idle-stop.md | 5 ++++- internal/plugins/service_singleflight_test.go | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/architecture/plugin-idle-stop.md b/docs/architecture/plugin-idle-stop.md index 521e93b6b5..105bd5721c 100644 --- a/docs/architecture/plugin-idle-stop.md +++ b/docs/architecture/plugin-idle-stop.md @@ -49,7 +49,10 @@ reads (`Host.Running`, which backs the admin `runtime.state`) are not use. - The watcher holds the installation's start slot until the old process has exited, so a restart never overlaps the process it replaces. A `Start` already in flight wins: it cancels the watcher. `Stop` and `Shutdown` also - wait for that exit. + wait for that exit. The one exception is a process that still has not + exited `DefaultIdleStopTimeout` after being killed (stuck in uninterruptible + I/O): the watcher logs a warning and releases the slot, so a restart can + overlap it and `Stop` or `Shutdown` can return before it exits. - The first call after an idle stop waits for a cold start before it is sent: the binary is verified again, the process launches, and the host reads its manifest and configures it. A start that fails is recorded as any failed diff --git a/internal/plugins/service_singleflight_test.go b/internal/plugins/service_singleflight_test.go index 44ca587b1d..d8d52e3fd5 100644 --- a/internal/plugins/service_singleflight_test.go +++ b/internal/plugins/service_singleflight_test.go @@ -305,6 +305,10 @@ func TestOnlyLazyPluginsAreStartedWithStopWhenIdle(t *testing.T) { host := &fakeServiceHost{clientErr: pluginhost.ErrClientNotFound} service := &Service{installations: store, host: host} + // No process, as after an idle stop: reported stopped, not failed. + if state := service.RuntimeStateOf(store.byID[7]); state.State != ResidentStopped || state.LastError != "" { + t.Fatalf("runtime state of a stopped idle plugin = %+v, want stopped with no error", state) + } if _, err := service.ensureClient(context.Background(), 7); err != nil { t.Fatalf("ensureClient after an idle stop: %v", err) } @@ -314,9 +318,6 @@ func TestOnlyLazyPluginsAreStartedWithStopWhenIdle(t *testing.T) { if !host.started[0].StopWhenIdle { t.Fatal("a lazily started plugin was not started with StopWhenIdle") } - if state := service.RuntimeStateOf(store.byID[7]); state.State != ResidentStopped || state.LastError != "" { - t.Fatalf("runtime state of a stopped idle plugin = %+v, want stopped with no error", state) - } resident := testPluginManifest(t, "silo.overlay", "1.0.0") resident.Capabilities = []*pluginv1.CapabilityDescriptor{{ From b93a5b61a4781ddfb09296407e81b5f79a9a1516 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:13:20 -0500 Subject: [PATCH 7/7] test(plugins): send the refused callback while the idle teardown runs Co-Authored-By: Claude Opus 5.5 --- internal/pluginhost/idle_test.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/internal/pluginhost/idle_test.go b/internal/pluginhost/idle_test.go index d7ada69c80..85c21d7ead 100644 --- a/internal/pluginhost/idle_test.go +++ b/internal/pluginhost/idle_test.go @@ -327,14 +327,27 @@ func TestIdleStopRefusesACallbackAfterTheDecision(t *testing.T) { clock := &idleClock{now: time.Unix(1000, 0)} host, inst, _ := idleTestHost(t, clock, 7) - clock.Advance(testIdlePeriod) - if !host.stopIfIdle(context.Background(), 7, inst) { - t.Fatal("did not stop an instance idle for the whole period") + // Hold the teardown open: the callback lands while the process is being + // stopped, the window the decision has to cover. + inTeardown := make(chan struct{}) + release := make(chan struct{}) + inst.cancelMonitors = func() { + close(inTeardown) + <-release } + clock.Advance(testIdlePeriod) + stopped := make(chan bool, 1) + go func() { stopped <- host.stopIfIdle(context.Background(), 7, inst) }() + <-inTeardown + ran := false handler := func(context.Context, any) (any, error) { ran = true; return nil, nil } _, err := inst.activity.unaryServer(context.Background(), nil, &grpc.UnaryServerInfo{}, handler) + close(release) + if !<-stopped { + t.Fatal("did not stop an instance idle for the whole period") + } if status.Code(err) != codes.Unavailable || ran { - t.Fatalf("callback after the idle stop decision: err = %v, handler ran = %v; want Unavailable without running it", err, ran) + t.Fatalf("callback during the idle stop: err = %v, handler ran = %v; want Unavailable without running it", err, ran) } }