diff --git a/docs/runtime-host.md b/docs/runtime-host.md index 2ccad0c..e681113 100644 --- a/docs/runtime-host.md +++ b/docs/runtime-host.md @@ -22,7 +22,7 @@ plugin, the plugin calls back into the host. | `MintScopedStream(request)` | Mint a short-lived stream grant for plugin-owned public access workflows. | | `CallPluginHTTP(request)` | Invoke another installed plugin's HTTP route through the host control plane. | | `ReadInstanceState(key)` | Read one key of the calling instance's encrypted, host-scoped state. | -| `WriteInstanceState(key, value)` | Write one key of that state. Key ≤ 256 bytes, value ≤ 256 KiB, ≤ 256 keys per scope. | +| `WriteInstanceState(key, value, expected_version?)` | Write one key of that state. Key ≤ 256 bytes, value ≤ 256 KiB, ≤ 256 keys per scope. With `expected_version`, write only if no one else wrote the key since it was read; see [Shared state](#shared-state). | | `ReportNetworkAccessStatus(status)` | Push a `network_access_provider.v1` status change so the host does not poll. | ## Using it from a plugin @@ -177,6 +177,27 @@ body, decodes the response into `Response`, and returns `*runtimehost.HTTPStatus for HTTP status codes 400 and above. The lower-level `CallPluginHTTP` remains available when a plugin needs custom content types or raw bytes. +### Shared state + +Every API server runs its own process of a plugin, and all of them read and +write the same instance state on the API host. Two processes that read a +key, change it, and write it back can lose one of the changes. Use +`UpdateInstanceState` for such a key: it writes only if the key is unchanged +since the read, and otherwise reads again and repeats your change. + +```go +err := host.UpdateInstanceState(ctx, "profiles", func(current []byte, found bool) ([]byte, error) { + settings := decode(current, found) + settings[profileID] = updated + return encode(settings) +}) +``` + +The function can run more than once, so it must not have side effects. +`ReadInstanceStateVersion` and `WriteInstanceStateIf` are the two steps on +their own. A host that doesn't keep versions answers +`ErrInstanceStateUnversioned`; `UpdateInstanceState` then writes nothing. + ### Addressing events `PublishEvent` broadcasts to every matching `event_consumer.v1` subscription. diff --git a/pkg/pluginproto/silo/plugin/v1/runtime_host.pb.go b/pkg/pluginproto/silo/plugin/v1/runtime_host.pb.go index 8569828..56434b0 100644 --- a/pkg/pluginproto/silo/plugin/v1/runtime_host.pb.go +++ b/pkg/pluginproto/silo/plugin/v1/runtime_host.pb.go @@ -2148,7 +2148,16 @@ type ReadInstanceStateResponse struct { // Stored value; empty when found is false. Value []byte `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` // False when no value is stored under key in this instance's scope. - Found bool `protobuf:"varint,2,opt,name=found,proto3" json:"found,omitempty"` + Found bool `protobuf:"varint,2,opt,name=found,proto3" json:"found,omitempty"` + // Version of the stored value: 1 after the first write, one more after + // each later write. 0 when found is false. Pass it as + // WriteInstanceStateRequest.expected_version to write only if no one else + // wrote the key since. + Version int64 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + // True when the host keeps versions and honors expected_version. Older + // hosts leave it false and ignore expected_version, writing + // unconditionally. + Versioned bool `protobuf:"varint,4,opt,name=versioned,proto3" json:"versioned,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2197,14 +2206,32 @@ func (x *ReadInstanceStateResponse) GetFound() bool { return false } +func (x *ReadInstanceStateResponse) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *ReadInstanceStateResponse) GetVersioned() bool { + if x != nil { + return x.Versioned + } + return false +} + type WriteInstanceStateRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // State key, <= 256 bytes. Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // Value to store, <= 256 KiB. - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + // When set, the host writes only if the key's current version equals it, + // and 0 means the key must not exist yet. Otherwise it fails with ABORTED + // and changes nothing. Unset writes unconditionally. + ExpectedVersion *int64 `protobuf:"varint,3,opt,name=expected_version,json=expectedVersion,proto3,oneof" json:"expected_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WriteInstanceStateRequest) Reset() { @@ -2251,8 +2278,17 @@ func (x *WriteInstanceStateRequest) GetValue() []byte { return nil } +func (x *WriteInstanceStateRequest) GetExpectedVersion() int64 { + if x != nil && x.ExpectedVersion != nil { + return *x.ExpectedVersion + } + return 0 +} + type WriteInstanceStateResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState `protogen:"open.v1"` + // Version of the value just written. Older hosts leave it 0. + Version int64 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2287,6 +2323,13 @@ func (*WriteInstanceStateResponse) Descriptor() ([]byte, []int) { return file_silo_plugin_v1_runtime_host_proto_rawDescGZIP(), []int{36} } +func (x *WriteInstanceStateResponse) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + type ReportNetworkAccessStatusRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Status *NetworkAccessStatus `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` @@ -2549,14 +2592,19 @@ const file_silo_plugin_v1_runtime_host_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\",\n" + "\x18ReadInstanceStateRequest\x12\x10\n" + - "\x03key\x18\x01 \x01(\tR\x03key\"G\n" + + "\x03key\x18\x01 \x01(\tR\x03key\"\x7f\n" + "\x19ReadInstanceStateResponse\x12\x14\n" + "\x05value\x18\x01 \x01(\fR\x05value\x12\x14\n" + - "\x05found\x18\x02 \x01(\bR\x05found\"C\n" + + "\x05found\x18\x02 \x01(\bR\x05found\x12\x18\n" + + "\aversion\x18\x03 \x01(\x03R\aversion\x12\x1c\n" + + "\tversioned\x18\x04 \x01(\bR\tversioned\"\x88\x01\n" + "\x19WriteInstanceStateRequest\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\fR\x05value\"\x1c\n" + - "\x1aWriteInstanceStateResponse\"_\n" + + "\x05value\x18\x02 \x01(\fR\x05value\x12.\n" + + "\x10expected_version\x18\x03 \x01(\x03H\x00R\x0fexpectedVersion\x88\x01\x01B\x13\n" + + "\x11_expected_version\"6\n" + + "\x1aWriteInstanceStateResponse\x12\x18\n" + + "\aversion\x18\x01 \x01(\x03R\aversion\"_\n" + " ReportNetworkAccessStatusRequest\x12;\n" + "\x06status\x18\x01 \x01(\v2#.silo.plugin.v1.NetworkAccessStatusR\x06status\"#\n" + "!ReportNetworkAccessStatusResponse2\xc1\r\n" + @@ -2702,6 +2750,7 @@ func file_silo_plugin_v1_runtime_host_proto_init() { } file_silo_plugin_v1_common_proto_init() file_silo_plugin_v1_network_access_provider_proto_init() + file_silo_plugin_v1_runtime_host_proto_msgTypes[35].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/pkg/pluginsdk/runtimehost/client.go b/pkg/pluginsdk/runtimehost/client.go index 9f0b415..1e36e25 100644 --- a/pkg/pluginsdk/runtimehost/client.go +++ b/pkg/pluginsdk/runtimehost/client.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "net/http" + "sync/atomic" "time" "google.golang.org/grpc" @@ -17,6 +18,9 @@ import ( type Client struct { rpc pluginv1.RuntimeHostClient + // stateVersions caches whether the host keeps instance state versions: + // 0 not known yet, 1 yes, 2 no. + stateVersions atomic.Int32 } type CallPluginHTTPRequest struct { diff --git a/pkg/pluginsdk/runtimehost/instance_state_version.go b/pkg/pluginsdk/runtimehost/instance_state_version.go new file mode 100644 index 0000000..d4be0f4 --- /dev/null +++ b/pkg/pluginsdk/runtimehost/instance_state_version.go @@ -0,0 +1,164 @@ +package runtimehost + +import ( + "context" + "errors" + "fmt" + "math/rand/v2" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" +) + +// ErrInstanceStateConflict means another write changed the key after the +// version the caller expected. Read the key again and redo the change. +var ErrInstanceStateConflict = errors.New("runtimehost: instance state key changed since it was read") + +// ErrInstanceStateUnversioned means the host doesn't keep versions, so it +// can't make a write conditional. +var ErrInstanceStateUnversioned = errors.New("runtimehost: host does not keep instance state versions") + +// maxUpdateAttempts bounds UpdateInstanceState's retries on conflict. +const maxUpdateAttempts = 8 + +// Conflict backoff: full jitter over a window that doubles from +// conflictBackoffBase up to conflictBackoffMax, so writers that collided +// spread out instead of colliding again. +const ( + conflictBackoffBase = 10 * time.Millisecond + conflictBackoffMax = 500 * time.Millisecond +) + +// conflictBackoff is the wait before attempt n+1 after n conflicts (n >= 1). +func conflictBackoff(n int) time.Duration { + window := conflictBackoffBase << (n - 1) + if window <= 0 || window > conflictBackoffMax { + window = conflictBackoffMax + } + return rand.N(window) + 1 +} + +// VersionedState is one key's value and the version it was read at. Version +// is 0 when Found is false. +type VersionedState struct { + Value []byte + Found bool + Version int64 +} + +// ReadInstanceStateVersion reads a key with its version. It returns +// ErrInstanceStateUnversioned on a host that doesn't keep versions. +func (c *Client) ReadInstanceStateVersion(ctx context.Context, key string) (VersionedState, error) { + if err := validateInstanceStateKey(key); err != nil { + return VersionedState{}, err + } + resp, err := c.rpc.ReadInstanceState(ctx, &pluginv1.ReadInstanceStateRequest{Key: key}) + if err != nil { + return VersionedState{}, err + } + c.recordStateVersions(resp.GetVersioned()) + if !resp.GetVersioned() { + return VersionedState{}, ErrInstanceStateUnversioned + } + if !resp.GetFound() { + return VersionedState{}, nil + } + return VersionedState{Value: append([]byte(nil), resp.GetValue()...), Found: true, Version: resp.GetVersion()}, nil +} + +// WriteInstanceStateIf writes value only if the key is still at +// expectedVersion (0: the key must not exist), and returns the new version. +// It returns ErrInstanceStateConflict when another write got there first, +// and ErrInstanceStateUnversioned, without writing, on a host that doesn't +// keep versions: such a host would ignore the condition. The first call on +// a client reads the key once to find out, unless a read already did. +func (c *Client) WriteInstanceStateIf(ctx context.Context, key string, value []byte, expectedVersion int64) (int64, error) { + if err := validateInstanceStateKey(key); err != nil { + return 0, err + } + if len(value) > MaxInstanceStateValueBytes { + return 0, fmt.Errorf("runtimehost: instance state value for %q is %d bytes, limit %d", key, len(value), MaxInstanceStateValueBytes) + } + versioned, err := c.hostKeepsStateVersions(ctx, key) + if err != nil { + return 0, err + } + if !versioned { + return 0, ErrInstanceStateUnversioned + } + resp, err := c.rpc.WriteInstanceState(ctx, &pluginv1.WriteInstanceStateRequest{Key: key, Value: value, ExpectedVersion: &expectedVersion}) + if status.Code(err) == codes.Aborted { + return 0, ErrInstanceStateConflict + } + if err != nil { + return 0, err + } + if resp.GetVersion() == 0 { + return 0, ErrInstanceStateUnversioned + } + return resp.GetVersion(), nil +} + +// hostKeepsStateVersions reports whether the host honors expected_version, +// reading key to find out the first time. +func (c *Client) hostKeepsStateVersions(ctx context.Context, key string) (bool, error) { + switch c.stateVersions.Load() { + case 1: + return true, nil + case 2: + return false, nil + } + resp, err := c.rpc.ReadInstanceState(ctx, &pluginv1.ReadInstanceStateRequest{Key: key}) + if err != nil { + return false, err + } + c.recordStateVersions(resp.GetVersioned()) + return resp.GetVersioned(), nil +} + +func (c *Client) recordStateVersions(versioned bool) { + if versioned { + c.stateVersions.Store(1) + } else { + c.stateVersions.Store(2) + } +} + +// UpdateInstanceState reads key, passes its value to update, and writes what +// update returns only if no other write changed the key in between. On a +// conflict it waits a short, random, growing time, then reads again and +// calls update again, a few times at most, so +// update must be safe to repeat and should not have side effects. An error +// from update stops the loop and is returned as is. +// +// Use it when several API servers can change the same key, such as one +// record shared by every process of the plugin. +func (c *Client) UpdateInstanceState(ctx context.Context, key string, update func(current []byte, found bool) ([]byte, error)) error { + for attempt := range maxUpdateAttempts { + if attempt > 0 { + timer := time.NewTimer(conflictBackoff(attempt)) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + current, err := c.ReadInstanceStateVersion(ctx, key) + if err != nil { + return err + } + next, err := update(current.Value, current.Found) + if err != nil { + return err + } + _, err = c.WriteInstanceStateIf(ctx, key, next, current.Version) + if !errors.Is(err, ErrInstanceStateConflict) { + return err + } + } + return fmt.Errorf("%w: gave up on %q after %d attempts", ErrInstanceStateConflict, key, maxUpdateAttempts) +} diff --git a/pkg/pluginsdk/runtimehost/instance_state_version_test.go b/pkg/pluginsdk/runtimehost/instance_state_version_test.go new file mode 100644 index 0000000..82f08f8 --- /dev/null +++ b/pkg/pluginsdk/runtimehost/instance_state_version_test.go @@ -0,0 +1,105 @@ +package runtimehost_test + +import ( + "context" + "errors" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" + "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginsdk/runtimehost" +) + +type versionedEntry struct { + value []byte + version int64 +} + +// versionedStateServer keeps versions like a current host. beforeWrite, when +// set, runs once before the next conditional write, to simulate another +// server's write landing in between. +type versionedStateServer struct { + pluginv1.UnimplementedRuntimeHostServer + values map[string]versionedEntry + beforeWrite func() +} + +func (s *versionedStateServer) ReadInstanceState(_ context.Context, req *pluginv1.ReadInstanceStateRequest) (*pluginv1.ReadInstanceStateResponse, error) { + entry, ok := s.values[req.GetKey()] + return &pluginv1.ReadInstanceStateResponse{Value: entry.value, Found: ok, Version: entry.version, Versioned: true}, nil +} + +func (s *versionedStateServer) WriteInstanceState(_ context.Context, req *pluginv1.WriteInstanceStateRequest) (*pluginv1.WriteInstanceStateResponse, error) { + if hook := s.beforeWrite; hook != nil { + s.beforeWrite = nil + hook() + } + current := s.values[req.GetKey()] + if req.ExpectedVersion != nil && req.GetExpectedVersion() != current.version { + return nil, status.Error(codes.Aborted, "version changed") + } + next := versionedEntry{value: req.GetValue(), version: current.version + 1} + s.values[req.GetKey()] = next + return &pluginv1.WriteInstanceStateResponse{Version: next.version}, nil +} + +func TestUpdateInstanceStateRedoesTheChangeAfterAConflict(t *testing.T) { + srv := &versionedStateServer{values: map[string]versionedEntry{"count": {value: []byte("1"), version: 1}}} + // Another server bumps the count between this client's read and write. + srv.beforeWrite = func() { srv.values["count"] = versionedEntry{value: []byte("5"), version: 2} } + c := runtimehost.NewClient(dialServer(t, srv)) + + calls := 0 + err := c.UpdateInstanceState(context.Background(), "count", func(current []byte, _ bool) ([]byte, error) { + calls++ + return append(current, '+'), nil + }) + if err != nil { + t.Fatalf("UpdateInstanceState: %v", err) + } + if got := srv.values["count"]; string(got.value) != "5+" || got.version != 3 || calls != 2 { + t.Fatalf("stored %q at version %d after %d calls; want the other write kept, \"5+\" at 3, 2 calls", got.value, got.version, calls) + } +} + +func TestWriteInstanceStateIfConflicts(t *testing.T) { + srv := &versionedStateServer{values: map[string]versionedEntry{"k": {value: []byte("a"), version: 4}}} + c := runtimehost.NewClient(dialServer(t, srv)) + // 0 means "create": the key exists, so it conflicts. + if _, err := c.WriteInstanceStateIf(context.Background(), "k", []byte("b"), 0); !errors.Is(err, runtimehost.ErrInstanceStateConflict) { + t.Fatalf("WriteInstanceStateIf = %v, want ErrInstanceStateConflict", err) + } + if string(srv.values["k"].value) != "a" { + t.Fatal("a conflicting write changed the value") + } +} + +// An older host would ignore expected_version, so a conditional write must +// write nothing there: called directly, or through UpdateInstanceState. +func TestWriteInstanceStateIfOnAnUnversionedHostWritesNothing(t *testing.T) { + srv := newInstanceStateServer() + srv.values["k"] = []byte("keep") + c := dialInstanceState(t, srv) + if _, err := c.WriteInstanceStateIf(context.Background(), "k", []byte("overwrite"), 0); !errors.Is(err, runtimehost.ErrInstanceStateUnversioned) { + t.Fatalf("WriteInstanceStateIf = %v, want ErrInstanceStateUnversioned", err) + } + if string(srv.values["k"]) != "keep" { + t.Fatal("WriteInstanceStateIf overwrote a value on a host that can't make writes conditional") + } +} + +// An older host would ignore expected_version, so the update must stop at +// the read instead of writing unconditionally. +func TestUpdateInstanceStateOnAnUnversionedHostWritesNothing(t *testing.T) { + srv := newInstanceStateServer() + c := dialInstanceState(t, srv) + err := c.UpdateInstanceState(context.Background(), "k", func([]byte, bool) ([]byte, error) { return []byte("x"), nil }) + if !errors.Is(err, runtimehost.ErrInstanceStateUnversioned) { + t.Fatalf("UpdateInstanceState = %v, want ErrInstanceStateUnversioned", err) + } + if _, wrote := srv.values["k"]; wrote { + t.Fatal("UpdateInstanceState wrote on a host that can't make writes conditional") + } +} diff --git a/proto/silo/plugin/v1/runtime_host.proto b/proto/silo/plugin/v1/runtime_host.proto index 80165d7..1c22aaa 100644 --- a/proto/silo/plugin/v1/runtime_host.proto +++ b/proto/silo/plugin/v1/runtime_host.proto @@ -338,6 +338,15 @@ message ReadInstanceStateResponse { bytes value = 1; // False when no value is stored under key in this instance's scope. bool found = 2; + // Version of the stored value: 1 after the first write, one more after + // each later write. 0 when found is false. Pass it as + // WriteInstanceStateRequest.expected_version to write only if no one else + // wrote the key since. + int64 version = 3; + // True when the host keeps versions and honors expected_version. Older + // hosts leave it false and ignore expected_version, writing + // unconditionally. + bool versioned = 4; } message WriteInstanceStateRequest { @@ -345,9 +354,16 @@ message WriteInstanceStateRequest { string key = 1; // Value to store, <= 256 KiB. bytes value = 2; + // When set, the host writes only if the key's current version equals it, + // and 0 means the key must not exist yet. Otherwise it fails with ABORTED + // and changes nothing. Unset writes unconditionally. + optional int64 expected_version = 3; } -message WriteInstanceStateResponse {} +message WriteInstanceStateResponse { + // Version of the value just written. Older hosts leave it 0. + int64 version = 1; +} message ReportNetworkAccessStatusRequest { NetworkAccessStatus status = 1;