From 73b734b4bea519e8015e20221fa284479753bd3f Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:04:34 -0500 Subject: [PATCH 1/3] feat(runtimehost): make instance state writes conditional Every API server runs its own process of a plugin against one shared instance state, so two read-modify-write cycles can lose a change. Hosts now return a version with each read and honor expected_version on a write, failing with ABORTED when the key changed. The client adds ReadInstanceStateVersion, WriteInstanceStateIf and UpdateInstanceState, which repeats the change after a conflict. On a host that doesn't keep versions they return ErrInstanceStateUnversioned, and UpdateInstanceState writes nothing. Co-Authored-By: Claude Opus 5.5 --- docs/runtime-host.md | 23 +++- .../silo/plugin/v1/runtime_host.pb.go | 67 ++++++++++-- .../runtimehost/instance_state_version.go | 103 ++++++++++++++++++ .../instance_state_version_test.go | 91 ++++++++++++++++ proto/silo/plugin/v1/runtime_host.proto | 18 ++- 5 files changed, 291 insertions(+), 11 deletions(-) create mode 100644 pkg/pluginsdk/runtimehost/instance_state_version.go create mode 100644 pkg/pluginsdk/runtimehost/instance_state_version_test.go diff --git a/docs/runtime-host.md b/docs/runtime-host.md index 2ccad0c..e681113 100644 --- a/docs/runtime-host.md +++ b/docs/runtime-host.md @@ -22,7 +22,7 @@ plugin, the plugin calls back into the host. | `MintScopedStream(request)` | Mint a short-lived stream grant for plugin-owned public access workflows. | | `CallPluginHTTP(request)` | Invoke another installed plugin's HTTP route through the host control plane. | | `ReadInstanceState(key)` | Read one key of the calling instance's encrypted, host-scoped state. | -| `WriteInstanceState(key, value)` | Write one key of that state. Key ≤ 256 bytes, value ≤ 256 KiB, ≤ 256 keys per scope. | +| `WriteInstanceState(key, value, expected_version?)` | Write one key of that state. Key ≤ 256 bytes, value ≤ 256 KiB, ≤ 256 keys per scope. With `expected_version`, write only if no one else wrote the key since it was read; see [Shared state](#shared-state). | | `ReportNetworkAccessStatus(status)` | Push a `network_access_provider.v1` status change so the host does not poll. | ## Using it from a plugin @@ -177,6 +177,27 @@ body, decodes the response into `Response`, and returns `*runtimehost.HTTPStatus for HTTP status codes 400 and above. The lower-level `CallPluginHTTP` remains available when a plugin needs custom content types or raw bytes. +### Shared state + +Every API server runs its own process of a plugin, and all of them read and +write the same instance state on the API host. Two processes that read a +key, change it, and write it back can lose one of the changes. Use +`UpdateInstanceState` for such a key: it writes only if the key is unchanged +since the read, and otherwise reads again and repeats your change. + +```go +err := host.UpdateInstanceState(ctx, "profiles", func(current []byte, found bool) ([]byte, error) { + settings := decode(current, found) + settings[profileID] = updated + return encode(settings) +}) +``` + +The function can run more than once, so it must not have side effects. +`ReadInstanceStateVersion` and `WriteInstanceStateIf` are the two steps on +their own. A host that doesn't keep versions answers +`ErrInstanceStateUnversioned`; `UpdateInstanceState` then writes nothing. + ### Addressing events `PublishEvent` broadcasts to every matching `event_consumer.v1` subscription. diff --git a/pkg/pluginproto/silo/plugin/v1/runtime_host.pb.go b/pkg/pluginproto/silo/plugin/v1/runtime_host.pb.go index 8569828..56434b0 100644 --- a/pkg/pluginproto/silo/plugin/v1/runtime_host.pb.go +++ b/pkg/pluginproto/silo/plugin/v1/runtime_host.pb.go @@ -2148,7 +2148,16 @@ type ReadInstanceStateResponse struct { // Stored value; empty when found is false. Value []byte `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` // False when no value is stored under key in this instance's scope. - Found bool `protobuf:"varint,2,opt,name=found,proto3" json:"found,omitempty"` + Found bool `protobuf:"varint,2,opt,name=found,proto3" json:"found,omitempty"` + // Version of the stored value: 1 after the first write, one more after + // each later write. 0 when found is false. Pass it as + // WriteInstanceStateRequest.expected_version to write only if no one else + // wrote the key since. + Version int64 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + // True when the host keeps versions and honors expected_version. Older + // hosts leave it false and ignore expected_version, writing + // unconditionally. + Versioned bool `protobuf:"varint,4,opt,name=versioned,proto3" json:"versioned,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2197,14 +2206,32 @@ func (x *ReadInstanceStateResponse) GetFound() bool { return false } +func (x *ReadInstanceStateResponse) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *ReadInstanceStateResponse) GetVersioned() bool { + if x != nil { + return x.Versioned + } + return false +} + type WriteInstanceStateRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // State key, <= 256 bytes. Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // Value to store, <= 256 KiB. - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + // When set, the host writes only if the key's current version equals it, + // and 0 means the key must not exist yet. Otherwise it fails with ABORTED + // and changes nothing. Unset writes unconditionally. + ExpectedVersion *int64 `protobuf:"varint,3,opt,name=expected_version,json=expectedVersion,proto3,oneof" json:"expected_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WriteInstanceStateRequest) Reset() { @@ -2251,8 +2278,17 @@ func (x *WriteInstanceStateRequest) GetValue() []byte { return nil } +func (x *WriteInstanceStateRequest) GetExpectedVersion() int64 { + if x != nil && x.ExpectedVersion != nil { + return *x.ExpectedVersion + } + return 0 +} + type WriteInstanceStateResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + // Version of the value just written. Older hosts leave it 0. + Version int64 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2287,6 +2323,13 @@ func (*WriteInstanceStateResponse) Descriptor() ([]byte, []int) { return file_silo_plugin_v1_runtime_host_proto_rawDescGZIP(), []int{36} } +func (x *WriteInstanceStateResponse) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + type ReportNetworkAccessStatusRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Status *NetworkAccessStatus `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` @@ -2549,14 +2592,19 @@ const file_silo_plugin_v1_runtime_host_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\",\n" + "\x18ReadInstanceStateRequest\x12\x10\n" + - "\x03key\x18\x01 \x01(\tR\x03key\"G\n" + + "\x03key\x18\x01 \x01(\tR\x03key\"\x7f\n" + "\x19ReadInstanceStateResponse\x12\x14\n" + "\x05value\x18\x01 \x01(\fR\x05value\x12\x14\n" + - "\x05found\x18\x02 \x01(\bR\x05found\"C\n" + + "\x05found\x18\x02 \x01(\bR\x05found\x12\x18\n" + + "\aversion\x18\x03 \x01(\x03R\aversion\x12\x1c\n" + + "\tversioned\x18\x04 \x01(\bR\tversioned\"\x88\x01\n" + "\x19WriteInstanceStateRequest\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\fR\x05value\"\x1c\n" + - "\x1aWriteInstanceStateResponse\"_\n" + + "\x05value\x18\x02 \x01(\fR\x05value\x12.\n" + + "\x10expected_version\x18\x03 \x01(\x03H\x00R\x0fexpectedVersion\x88\x01\x01B\x13\n" + + "\x11_expected_version\"6\n" + + "\x1aWriteInstanceStateResponse\x12\x18\n" + + "\aversion\x18\x01 \x01(\x03R\aversion\"_\n" + " ReportNetworkAccessStatusRequest\x12;\n" + "\x06status\x18\x01 \x01(\v2#.silo.plugin.v1.NetworkAccessStatusR\x06status\"#\n" + "!ReportNetworkAccessStatusResponse2\xc1\r\n" + @@ -2702,6 +2750,7 @@ func file_silo_plugin_v1_runtime_host_proto_init() { } file_silo_plugin_v1_common_proto_init() file_silo_plugin_v1_network_access_provider_proto_init() + file_silo_plugin_v1_runtime_host_proto_msgTypes[35].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/pkg/pluginsdk/runtimehost/instance_state_version.go b/pkg/pluginsdk/runtimehost/instance_state_version.go new file mode 100644 index 0000000..3b2e59d --- /dev/null +++ b/pkg/pluginsdk/runtimehost/instance_state_version.go @@ -0,0 +1,103 @@ +package runtimehost + +import ( + "context" + "errors" + "fmt" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" +) + +// ErrInstanceStateConflict means another write changed the key after the +// version the caller expected. Read the key again and redo the change. +var ErrInstanceStateConflict = errors.New("runtimehost: instance state key changed since it was read") + +// ErrInstanceStateUnversioned means the host doesn't keep versions, so it +// can't make a write conditional. +var ErrInstanceStateUnversioned = errors.New("runtimehost: host does not keep instance state versions") + +// maxUpdateAttempts bounds UpdateInstanceState's retries on conflict. +const maxUpdateAttempts = 8 + +// VersionedState is one key's value and the version it was read at. Version +// is 0 when Found is false. +type VersionedState struct { + Value []byte + Found bool + Version int64 +} + +// ReadInstanceStateVersion reads a key with its version. It returns +// ErrInstanceStateUnversioned on a host that doesn't keep versions. +func (c *Client) ReadInstanceStateVersion(ctx context.Context, key string) (VersionedState, error) { + if err := validateInstanceStateKey(key); err != nil { + return VersionedState{}, err + } + resp, err := c.rpc.ReadInstanceState(ctx, &pluginv1.ReadInstanceStateRequest{Key: key}) + if err != nil { + return VersionedState{}, err + } + if !resp.GetVersioned() { + return VersionedState{}, ErrInstanceStateUnversioned + } + if !resp.GetFound() { + return VersionedState{}, nil + } + return VersionedState{Value: append([]byte(nil), resp.GetValue()...), Found: true, Version: resp.GetVersion()}, nil +} + +// WriteInstanceStateIf writes value only if the key is still at +// expectedVersion (0: the key must not exist), and returns the new version. +// It returns ErrInstanceStateConflict when another write got there first. +// +// Pass a version from ReadInstanceStateVersion, which fails on hosts that +// don't keep versions: such a host ignores the condition, writes anyway, and +// this method then returns ErrInstanceStateUnversioned. +func (c *Client) WriteInstanceStateIf(ctx context.Context, key string, value []byte, expectedVersion int64) (int64, error) { + if err := validateInstanceStateKey(key); err != nil { + return 0, err + } + if len(value) > MaxInstanceStateValueBytes { + return 0, fmt.Errorf("runtimehost: instance state value for %q is %d bytes, limit %d", key, len(value), MaxInstanceStateValueBytes) + } + resp, err := c.rpc.WriteInstanceState(ctx, &pluginv1.WriteInstanceStateRequest{Key: key, Value: value, ExpectedVersion: &expectedVersion}) + if status.Code(err) == codes.Aborted { + return 0, ErrInstanceStateConflict + } + if err != nil { + return 0, err + } + if resp.GetVersion() == 0 { + return 0, ErrInstanceStateUnversioned + } + return resp.GetVersion(), nil +} + +// UpdateInstanceState reads key, passes its value to update, and writes what +// update returns only if no other write changed the key in between. On a +// conflict it reads again and calls update again, a few times at most, so +// update must be safe to repeat and should not have side effects. An error +// from update stops the loop and is returned as is. +// +// Use it when several API servers can change the same key, such as one +// record shared by every process of the plugin. +func (c *Client) UpdateInstanceState(ctx context.Context, key string, update func(current []byte, found bool) ([]byte, error)) error { + for range maxUpdateAttempts { + current, err := c.ReadInstanceStateVersion(ctx, key) + if err != nil { + return err + } + next, err := update(current.Value, current.Found) + if err != nil { + return err + } + _, err = c.WriteInstanceStateIf(ctx, key, next, current.Version) + if !errors.Is(err, ErrInstanceStateConflict) { + return err + } + } + return fmt.Errorf("%w: gave up on %q after %d attempts", ErrInstanceStateConflict, key, maxUpdateAttempts) +} diff --git a/pkg/pluginsdk/runtimehost/instance_state_version_test.go b/pkg/pluginsdk/runtimehost/instance_state_version_test.go new file mode 100644 index 0000000..5e3c2de --- /dev/null +++ b/pkg/pluginsdk/runtimehost/instance_state_version_test.go @@ -0,0 +1,91 @@ +package runtimehost_test + +import ( + "context" + "errors" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" + "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginsdk/runtimehost" +) + +type versionedEntry struct { + value []byte + version int64 +} + +// versionedStateServer keeps versions like a current host. beforeWrite, when +// set, runs once before the next conditional write, to simulate another +// server's write landing in between. +type versionedStateServer struct { + pluginv1.UnimplementedRuntimeHostServer + values map[string]versionedEntry + beforeWrite func() +} + +func (s *versionedStateServer) ReadInstanceState(_ context.Context, req *pluginv1.ReadInstanceStateRequest) (*pluginv1.ReadInstanceStateResponse, error) { + entry, ok := s.values[req.GetKey()] + return &pluginv1.ReadInstanceStateResponse{Value: entry.value, Found: ok, Version: entry.version, Versioned: true}, nil +} + +func (s *versionedStateServer) WriteInstanceState(_ context.Context, req *pluginv1.WriteInstanceStateRequest) (*pluginv1.WriteInstanceStateResponse, error) { + if hook := s.beforeWrite; hook != nil { + s.beforeWrite = nil + hook() + } + current := s.values[req.GetKey()] + if req.ExpectedVersion != nil && req.GetExpectedVersion() != current.version { + return nil, status.Error(codes.Aborted, "version changed") + } + next := versionedEntry{value: req.GetValue(), version: current.version + 1} + s.values[req.GetKey()] = next + return &pluginv1.WriteInstanceStateResponse{Version: next.version}, nil +} + +func TestUpdateInstanceStateRedoesTheChangeAfterAConflict(t *testing.T) { + srv := &versionedStateServer{values: map[string]versionedEntry{"count": {value: []byte("1"), version: 1}}} + // Another server bumps the count between this client's read and write. + srv.beforeWrite = func() { srv.values["count"] = versionedEntry{value: []byte("5"), version: 2} } + c := runtimehost.NewClient(dialServer(t, srv)) + + calls := 0 + err := c.UpdateInstanceState(context.Background(), "count", func(current []byte, _ bool) ([]byte, error) { + calls++ + return append(current, '+'), nil + }) + if err != nil { + t.Fatalf("UpdateInstanceState: %v", err) + } + if got := srv.values["count"]; string(got.value) != "5+" || got.version != 3 || calls != 2 { + t.Fatalf("stored %q at version %d after %d calls; want the other write kept, \"5+\" at 3, 2 calls", got.value, got.version, calls) + } +} + +func TestWriteInstanceStateIfConflicts(t *testing.T) { + srv := &versionedStateServer{values: map[string]versionedEntry{"k": {value: []byte("a"), version: 4}}} + c := runtimehost.NewClient(dialServer(t, srv)) + // 0 means "create": the key exists, so it conflicts. + if _, err := c.WriteInstanceStateIf(context.Background(), "k", []byte("b"), 0); !errors.Is(err, runtimehost.ErrInstanceStateConflict) { + t.Fatalf("WriteInstanceStateIf = %v, want ErrInstanceStateConflict", err) + } + if string(srv.values["k"].value) != "a" { + t.Fatal("a conflicting write changed the value") + } +} + +// An older host would ignore expected_version, so the update must stop at +// the read instead of writing unconditionally. +func TestUpdateInstanceStateOnAnUnversionedHostWritesNothing(t *testing.T) { + srv := newInstanceStateServer() + c := dialInstanceState(t, srv) + err := c.UpdateInstanceState(context.Background(), "k", func([]byte, bool) ([]byte, error) { return []byte("x"), nil }) + if !errors.Is(err, runtimehost.ErrInstanceStateUnversioned) { + t.Fatalf("UpdateInstanceState = %v, want ErrInstanceStateUnversioned", err) + } + if _, wrote := srv.values["k"]; wrote { + t.Fatal("UpdateInstanceState wrote on a host that can't make writes conditional") + } +} diff --git a/proto/silo/plugin/v1/runtime_host.proto b/proto/silo/plugin/v1/runtime_host.proto index 80165d7..1c22aaa 100644 --- a/proto/silo/plugin/v1/runtime_host.proto +++ b/proto/silo/plugin/v1/runtime_host.proto @@ -338,6 +338,15 @@ message ReadInstanceStateResponse { bytes value = 1; // False when no value is stored under key in this instance's scope. bool found = 2; + // Version of the stored value: 1 after the first write, one more after + // each later write. 0 when found is false. Pass it as + // WriteInstanceStateRequest.expected_version to write only if no one else + // wrote the key since. + int64 version = 3; + // True when the host keeps versions and honors expected_version. Older + // hosts leave it false and ignore expected_version, writing + // unconditionally. + bool versioned = 4; } message WriteInstanceStateRequest { @@ -345,9 +354,16 @@ message WriteInstanceStateRequest { string key = 1; // Value to store, <= 256 KiB. bytes value = 2; + // When set, the host writes only if the key's current version equals it, + // and 0 means the key must not exist yet. Otherwise it fails with ABORTED + // and changes nothing. Unset writes unconditionally. + optional int64 expected_version = 3; } -message WriteInstanceStateResponse {} +message WriteInstanceStateResponse { + // Version of the value just written. Older hosts leave it 0. + int64 version = 1; +} message ReportNetworkAccessStatusRequest { NetworkAccessStatus status = 1; From 379318059f54824e15141646e80a3d9124ddc643 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:36:53 -0500 Subject: [PATCH 2/3] fix(runtimehost): never send a conditional write to a host that ignores it WriteInstanceStateIf sent the write first and only then noticed the host keeps no versions, so on an older host it overwrote the value unconditionally. It now learns whether the host keeps versions before the first conditional write, from a read it makes once per client or one the plugin already made, and writes nothing on such a host. Co-Authored-By: Claude Opus 5.5 --- pkg/pluginsdk/runtimehost/client.go | 4 ++ .../runtimehost/instance_state_version.go | 42 ++++++++++++++++--- .../instance_state_version_test.go | 14 +++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/pkg/pluginsdk/runtimehost/client.go b/pkg/pluginsdk/runtimehost/client.go index 9f0b415..1e36e25 100644 --- a/pkg/pluginsdk/runtimehost/client.go +++ b/pkg/pluginsdk/runtimehost/client.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "net/http" + "sync/atomic" "time" "google.golang.org/grpc" @@ -17,6 +18,9 @@ import ( type Client struct { rpc pluginv1.RuntimeHostClient + // stateVersions caches whether the host keeps instance state versions: + // 0 not known yet, 1 yes, 2 no. + stateVersions atomic.Int32 } type CallPluginHTTPRequest struct { diff --git a/pkg/pluginsdk/runtimehost/instance_state_version.go b/pkg/pluginsdk/runtimehost/instance_state_version.go index 3b2e59d..1707ac6 100644 --- a/pkg/pluginsdk/runtimehost/instance_state_version.go +++ b/pkg/pluginsdk/runtimehost/instance_state_version.go @@ -40,6 +40,7 @@ func (c *Client) ReadInstanceStateVersion(ctx context.Context, key string) (Vers if err != nil { return VersionedState{}, err } + c.recordStateVersions(resp.GetVersioned()) if !resp.GetVersioned() { return VersionedState{}, ErrInstanceStateUnversioned } @@ -51,11 +52,10 @@ func (c *Client) ReadInstanceStateVersion(ctx context.Context, key string) (Vers // WriteInstanceStateIf writes value only if the key is still at // expectedVersion (0: the key must not exist), and returns the new version. -// It returns ErrInstanceStateConflict when another write got there first. -// -// Pass a version from ReadInstanceStateVersion, which fails on hosts that -// don't keep versions: such a host ignores the condition, writes anyway, and -// this method then returns ErrInstanceStateUnversioned. +// It returns ErrInstanceStateConflict when another write got there first, +// and ErrInstanceStateUnversioned, without writing, on a host that doesn't +// keep versions: such a host would ignore the condition. The first call on +// a client reads the key once to find out, unless a read already did. func (c *Client) WriteInstanceStateIf(ctx context.Context, key string, value []byte, expectedVersion int64) (int64, error) { if err := validateInstanceStateKey(key); err != nil { return 0, err @@ -63,6 +63,13 @@ func (c *Client) WriteInstanceStateIf(ctx context.Context, key string, value []b if len(value) > MaxInstanceStateValueBytes { return 0, fmt.Errorf("runtimehost: instance state value for %q is %d bytes, limit %d", key, len(value), MaxInstanceStateValueBytes) } + versioned, err := c.hostKeepsStateVersions(ctx, key) + if err != nil { + return 0, err + } + if !versioned { + return 0, ErrInstanceStateUnversioned + } resp, err := c.rpc.WriteInstanceState(ctx, &pluginv1.WriteInstanceStateRequest{Key: key, Value: value, ExpectedVersion: &expectedVersion}) if status.Code(err) == codes.Aborted { return 0, ErrInstanceStateConflict @@ -76,6 +83,31 @@ func (c *Client) WriteInstanceStateIf(ctx context.Context, key string, value []b return resp.GetVersion(), nil } +// hostKeepsStateVersions reports whether the host honors expected_version, +// reading key to find out the first time. +func (c *Client) hostKeepsStateVersions(ctx context.Context, key string) (bool, error) { + switch c.stateVersions.Load() { + case 1: + return true, nil + case 2: + return false, nil + } + resp, err := c.rpc.ReadInstanceState(ctx, &pluginv1.ReadInstanceStateRequest{Key: key}) + if err != nil { + return false, err + } + c.recordStateVersions(resp.GetVersioned()) + return resp.GetVersioned(), nil +} + +func (c *Client) recordStateVersions(versioned bool) { + if versioned { + c.stateVersions.Store(1) + } else { + c.stateVersions.Store(2) + } +} + // UpdateInstanceState reads key, passes its value to update, and writes what // update returns only if no other write changed the key in between. On a // conflict it reads again and calls update again, a few times at most, so diff --git a/pkg/pluginsdk/runtimehost/instance_state_version_test.go b/pkg/pluginsdk/runtimehost/instance_state_version_test.go index 5e3c2de..82f08f8 100644 --- a/pkg/pluginsdk/runtimehost/instance_state_version_test.go +++ b/pkg/pluginsdk/runtimehost/instance_state_version_test.go @@ -76,6 +76,20 @@ func TestWriteInstanceStateIfConflicts(t *testing.T) { } } +// An older host would ignore expected_version, so a conditional write must +// write nothing there: called directly, or through UpdateInstanceState. +func TestWriteInstanceStateIfOnAnUnversionedHostWritesNothing(t *testing.T) { + srv := newInstanceStateServer() + srv.values["k"] = []byte("keep") + c := dialInstanceState(t, srv) + if _, err := c.WriteInstanceStateIf(context.Background(), "k", []byte("overwrite"), 0); !errors.Is(err, runtimehost.ErrInstanceStateUnversioned) { + t.Fatalf("WriteInstanceStateIf = %v, want ErrInstanceStateUnversioned", err) + } + if string(srv.values["k"]) != "keep" { + t.Fatal("WriteInstanceStateIf overwrote a value on a host that can't make writes conditional") + } +} + // An older host would ignore expected_version, so the update must stop at // the read instead of writing unconditionally. func TestUpdateInstanceStateOnAnUnversionedHostWritesNothing(t *testing.T) { From 03b2f1a80f0cebaa6e79092b99a14e4c28751bda Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:46:49 +0000 Subject: [PATCH 3/3] fix(runtimehost): back off with jitter between conditional write retries UpdateInstanceState retried a conflicting write at once, so servers that collided on a key tended to collide again. It now waits a random time in a window that doubles from 10 ms to 500 ms between attempts, and stops when the context ends. Co-Authored-By: Claude Opus 5.5 --- .../runtimehost/instance_state_version.go | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/pkg/pluginsdk/runtimehost/instance_state_version.go b/pkg/pluginsdk/runtimehost/instance_state_version.go index 1707ac6..d4be0f4 100644 --- a/pkg/pluginsdk/runtimehost/instance_state_version.go +++ b/pkg/pluginsdk/runtimehost/instance_state_version.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "math/rand/v2" + "time" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -22,6 +24,23 @@ var ErrInstanceStateUnversioned = errors.New("runtimehost: host does not keep in // maxUpdateAttempts bounds UpdateInstanceState's retries on conflict. const maxUpdateAttempts = 8 +// Conflict backoff: full jitter over a window that doubles from +// conflictBackoffBase up to conflictBackoffMax, so writers that collided +// spread out instead of colliding again. +const ( + conflictBackoffBase = 10 * time.Millisecond + conflictBackoffMax = 500 * time.Millisecond +) + +// conflictBackoff is the wait before attempt n+1 after n conflicts (n >= 1). +func conflictBackoff(n int) time.Duration { + window := conflictBackoffBase << (n - 1) + if window <= 0 || window > conflictBackoffMax { + window = conflictBackoffMax + } + return rand.N(window) + 1 +} + // VersionedState is one key's value and the version it was read at. Version // is 0 when Found is false. type VersionedState struct { @@ -110,14 +129,24 @@ func (c *Client) recordStateVersions(versioned bool) { // UpdateInstanceState reads key, passes its value to update, and writes what // update returns only if no other write changed the key in between. On a -// conflict it reads again and calls update again, a few times at most, so +// conflict it waits a short, random, growing time, then reads again and +// calls update again, a few times at most, so // update must be safe to repeat and should not have side effects. An error // from update stops the loop and is returned as is. // // Use it when several API servers can change the same key, such as one // record shared by every process of the plugin. func (c *Client) UpdateInstanceState(ctx context.Context, key string, update func(current []byte, found bool) ([]byte, error)) error { - for range maxUpdateAttempts { + for attempt := range maxUpdateAttempts { + if attempt > 0 { + timer := time.NewTimer(conflictBackoff(attempt)) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } current, err := c.ReadInstanceStateVersion(ctx, key) if err != nil { return err