From 6f2115b11690bc8fd4dfba3acee1d3c8815a13e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Dec 2025 20:12:13 +0000 Subject: [PATCH] Bump github.com/conductorone/baton-sdk from 0.5.25 to 0.6.5 Bumps [github.com/conductorone/baton-sdk](https://github.com/conductorone/baton-sdk) from 0.5.25 to 0.6.5. - [Release notes](https://github.com/conductorone/baton-sdk/releases) - [Commits](https://github.com/conductorone/baton-sdk/compare/v0.5.25...v0.6.5) --- updated-dependencies: - dependency-name: github.com/conductorone/baton-sdk dependency-version: 0.6.5 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 +- .../connector/connector_server_unix.go | 1 - .../baton-sdk/pb/c1/connector/v2/action.pb.go | 53 ++++--- .../pb/c1/connector/v2/action.pb.validate.go | 2 + .../c1/connector/v2/action_protoopaque.pb.go | 53 ++++--- .../v2/annotation_resource_tree.pb.go | 51 ++++++- .../annotation_resource_tree.pb.validate.go | 100 +++++++++++++ ...annotation_resource_tree_protoopaque.pb.go | 51 ++++++- .../pb/c1/connectorapi/baton/v1/session.pb.go | 41 ++++-- .../baton/v1/session.pb.validate.go | 8 +- .../baton/v1/session_protoopaque.pb.go | 41 ++++-- .../baton-sdk/pkg/cli/commands.go | 7 +- .../baton-sdk/pkg/cli/lazy_session.go | 4 +- .../baton-sdk/pkg/config/config.go | 39 +++++- .../pkg/connectorbuilder/session_store.go | 2 +- .../baton-sdk/pkg/crypto/client_secret.go | 2 +- .../baton-sdk/pkg/crypto/crypto.go | 2 +- .../baton-sdk/pkg/crypto/password.go | 2 +- .../baton-sdk/pkg/dotc1z/c1file.go | 31 ++++- .../baton-sdk/pkg/dotc1z/decoder.go | 2 +- .../baton-sdk/pkg/dotc1z/grants.go | 23 +++ .../baton-sdk/pkg/dotc1z/session_store.go | 131 ++++++++++++++---- .../baton-sdk/pkg/field/defaults.go | 1 + .../conductorone/baton-sdk/pkg/sdk/version.go | 2 +- .../baton-sdk/pkg/session/json_session.go | 2 +- .../baton-sdk/pkg/session/memory_cache.go | 13 +- .../baton-sdk/pkg/session/noop_session.go | 4 +- .../baton-sdk/pkg/session/session.go | 31 ++++- .../baton-sdk/pkg/session/session_client.go | 14 +- .../baton-sdk/pkg/session/session_server.go | 5 +- .../baton-sdk/pkg/session/typed_session.go | 10 +- .../baton-sdk/pkg/sync/expand/scc/bitset.go | 2 +- .../conductorone/baton-sdk/pkg/sync/state.go | 13 ++ .../conductorone/baton-sdk/pkg/sync/syncer.go | 63 +++++++-- .../pkg/synccompactor/attached/attached.go | 12 ++ .../baton-sdk/pkg/synccompactor/compactor.go | 63 +++++++-- .../baton-sdk/pkg/tasks/c1api/manager.go | 2 +- .../baton-sdk/pkg/types/sessions/sessions.go | 2 +- vendor/modules.txt | 2 +- 40 files changed, 715 insertions(+), 178 deletions(-) diff --git a/go.mod b/go.mod index bd34a0ba..b0087ba7 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.2 require ( github.com/SAP/go-hdb v1.14.5 - github.com/conductorone/baton-sdk v0.5.25 + github.com/conductorone/baton-sdk v0.6.5 github.com/elliotchance/phpserialize v1.4.0 github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/go-sql-driver/mysql v1.9.2 diff --git a/go.sum b/go.sum index 114256f4..8cc6a414 100644 --- a/go.sum +++ b/go.sum @@ -78,8 +78,8 @@ github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/conductorone/baton-sdk v0.5.25 h1:G0+MVkqwB4lGF0FbHDFh1Zt1fPsqn7gGGQWJx88w58Q= -github.com/conductorone/baton-sdk v0.5.25/go.mod h1:TQuaFYyonGW1Xblxjm+qQE3XsJGAZvTMPGaO4c4Yv6E= +github.com/conductorone/baton-sdk v0.6.5 h1:qy8KtlyfddSR99fauP1X+HDXfiCPW03E/3v29Xg1yzs= +github.com/conductorone/baton-sdk v0.6.5/go.mod h1:TQuaFYyonGW1Xblxjm+qQE3XsJGAZvTMPGaO4c4Yv6E= github.com/conductorone/dpop v0.2.4 h1:PaiDOX1gAIXtOJPxXf08GsGkpCuT/iECEjSJzLpi0zU= github.com/conductorone/dpop v0.2.4/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4 h1:lYxYi9/WTSL9sE96CO0QF2BY3kehs8dTTApI134TGCA= diff --git a/vendor/github.com/conductorone/baton-sdk/internal/connector/connector_server_unix.go b/vendor/github.com/conductorone/baton-sdk/internal/connector/connector_server_unix.go index bac831a5..830ba052 100644 --- a/vendor/github.com/conductorone/baton-sdk/internal/connector/connector_server_unix.go +++ b/vendor/github.com/conductorone/baton-sdk/internal/connector/connector_server_unix.go @@ -15,7 +15,6 @@ import ( ) func getPort(listener net.Listener) uint32 { - //nolint:gosec // No risk of overflow because `Port` is 16-bit. return uint32(listener.Addr().(*net.TCPAddr).Port) } diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.go index 1801c93c..1b79424b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.go @@ -133,16 +133,17 @@ func (x ActionType) Number() protoreflect.EnumNumber { } type BatonActionSchema struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Arguments []*v1.Field `protobuf:"bytes,2,rep,name=arguments,proto3" json:"arguments,omitempty"` - Constraints []*v1.Constraint `protobuf:"bytes,3,rep,name=constraints,proto3" json:"constraints,omitempty"` - ReturnTypes []*v1.Field `protobuf:"bytes,4,rep,name=return_types,json=returnTypes,proto3" json:"return_types,omitempty"` - DisplayName string `protobuf:"bytes,5,opt,name=display_name,json=displayName,proto3" json:"display_name,omitempty"` - Description string `protobuf:"bytes,6,opt,name=description,proto3" json:"description,omitempty"` - ActionType []ActionType `protobuf:"varint,7,rep,packed,name=action_type,json=actionType,proto3,enum=c1.connector.v2.ActionType" json:"action_type,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"hybrid.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Arguments []*v1.Field `protobuf:"bytes,2,rep,name=arguments,proto3" json:"arguments,omitempty"` + Constraints []*v1.Constraint `protobuf:"bytes,3,rep,name=constraints,proto3" json:"constraints,omitempty"` + ReturnTypes []*v1.Field `protobuf:"bytes,4,rep,name=return_types,json=returnTypes,proto3" json:"return_types,omitempty"` + DisplayName string `protobuf:"bytes,5,opt,name=display_name,json=displayName,proto3" json:"display_name,omitempty"` + Description string `protobuf:"bytes,6,opt,name=description,proto3" json:"description,omitempty"` + ActionType []ActionType `protobuf:"varint,7,rep,packed,name=action_type,json=actionType,proto3,enum=c1.connector.v2.ActionType" json:"action_type,omitempty"` + ResourceTypeId string `protobuf:"bytes,8,opt,name=resource_type_id,json=resourceTypeId,proto3" json:"resource_type_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BatonActionSchema) Reset() { @@ -219,6 +220,13 @@ func (x *BatonActionSchema) GetActionType() []ActionType { return nil } +func (x *BatonActionSchema) GetResourceTypeId() string { + if x != nil { + return x.ResourceTypeId + } + return "" +} + func (x *BatonActionSchema) SetName(v string) { x.Name = v } @@ -247,16 +255,21 @@ func (x *BatonActionSchema) SetActionType(v []ActionType) { x.ActionType = v } +func (x *BatonActionSchema) SetResourceTypeId(v string) { + x.ResourceTypeId = v +} + type BatonActionSchema_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - Name string - Arguments []*v1.Field - Constraints []*v1.Constraint - ReturnTypes []*v1.Field - DisplayName string - Description string - ActionType []ActionType + Name string + Arguments []*v1.Field + Constraints []*v1.Constraint + ReturnTypes []*v1.Field + DisplayName string + Description string + ActionType []ActionType + ResourceTypeId string } func (b0 BatonActionSchema_builder) Build() *BatonActionSchema { @@ -270,6 +283,7 @@ func (b0 BatonActionSchema_builder) Build() *BatonActionSchema { x.DisplayName = b.DisplayName x.Description = b.Description x.ActionType = b.ActionType + x.ResourceTypeId = b.ResourceTypeId return m0 } @@ -991,7 +1005,7 @@ var File_c1_connector_v2_action_proto protoreflect.FileDescriptor const file_c1_connector_v2_action_proto_rawDesc = "" + "\n" + - "\x1cc1/connector/v2/action.proto\x12\x0fc1.connector.v2\x1a\x19c1/config/v1/config.proto\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xd1\x02\n" + + "\x1cc1/connector/v2/action.proto\x12\x0fc1.connector.v2\x1a\x19c1/config/v1/config.proto\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xfb\x02\n" + "\x11BatonActionSchema\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x121\n" + "\targuments\x18\x02 \x03(\v2\x13.c1.config.v1.FieldR\targuments\x12:\n" + @@ -1000,7 +1014,8 @@ const file_c1_connector_v2_action_proto_rawDesc = "" + "\fdisplay_name\x18\x05 \x01(\tR\vdisplayName\x12 \n" + "\vdescription\x18\x06 \x01(\tR\vdescription\x12<\n" + "\vaction_type\x18\a \x03(\x0e2\x1b.c1.connector.v2.ActionTypeR\n" + - "actionType\"\x8e\x01\n" + + "actionType\x12(\n" + + "\x10resource_type_id\x18\b \x01(\tR\x0eresourceTypeId\"\x8e\x01\n" + "\x13InvokeActionRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12+\n" + "\x04args\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x04args\x126\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.validate.go index 590b0616..c8eb3b53 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.validate.go @@ -165,6 +165,8 @@ func (m *BatonActionSchema) validate(all bool) error { // no validation rules for Description + // no validation rules for ResourceTypeId + if len(errors) > 0 { return BatonActionSchemaMultiError(errors) } diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action_protoopaque.pb.go index e8a0e89d..5fcd7ab5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action_protoopaque.pb.go @@ -133,16 +133,17 @@ func (x ActionType) Number() protoreflect.EnumNumber { } type BatonActionSchema struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name,proto3"` - xxx_hidden_Arguments *[]*v1.Field `protobuf:"bytes,2,rep,name=arguments,proto3"` - xxx_hidden_Constraints *[]*v1.Constraint `protobuf:"bytes,3,rep,name=constraints,proto3"` - xxx_hidden_ReturnTypes *[]*v1.Field `protobuf:"bytes,4,rep,name=return_types,json=returnTypes,proto3"` - xxx_hidden_DisplayName string `protobuf:"bytes,5,opt,name=display_name,json=displayName,proto3"` - xxx_hidden_Description string `protobuf:"bytes,6,opt,name=description,proto3"` - xxx_hidden_ActionType []ActionType `protobuf:"varint,7,rep,packed,name=action_type,json=actionType,proto3,enum=c1.connector.v2.ActionType"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name,proto3"` + xxx_hidden_Arguments *[]*v1.Field `protobuf:"bytes,2,rep,name=arguments,proto3"` + xxx_hidden_Constraints *[]*v1.Constraint `protobuf:"bytes,3,rep,name=constraints,proto3"` + xxx_hidden_ReturnTypes *[]*v1.Field `protobuf:"bytes,4,rep,name=return_types,json=returnTypes,proto3"` + xxx_hidden_DisplayName string `protobuf:"bytes,5,opt,name=display_name,json=displayName,proto3"` + xxx_hidden_Description string `protobuf:"bytes,6,opt,name=description,proto3"` + xxx_hidden_ActionType []ActionType `protobuf:"varint,7,rep,packed,name=action_type,json=actionType,proto3,enum=c1.connector.v2.ActionType"` + xxx_hidden_ResourceTypeId string `protobuf:"bytes,8,opt,name=resource_type_id,json=resourceTypeId,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BatonActionSchema) Reset() { @@ -225,6 +226,13 @@ func (x *BatonActionSchema) GetActionType() []ActionType { return nil } +func (x *BatonActionSchema) GetResourceTypeId() string { + if x != nil { + return x.xxx_hidden_ResourceTypeId + } + return "" +} + func (x *BatonActionSchema) SetName(v string) { x.xxx_hidden_Name = v } @@ -253,16 +261,21 @@ func (x *BatonActionSchema) SetActionType(v []ActionType) { x.xxx_hidden_ActionType = v } +func (x *BatonActionSchema) SetResourceTypeId(v string) { + x.xxx_hidden_ResourceTypeId = v +} + type BatonActionSchema_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - Name string - Arguments []*v1.Field - Constraints []*v1.Constraint - ReturnTypes []*v1.Field - DisplayName string - Description string - ActionType []ActionType + Name string + Arguments []*v1.Field + Constraints []*v1.Constraint + ReturnTypes []*v1.Field + DisplayName string + Description string + ActionType []ActionType + ResourceTypeId string } func (b0 BatonActionSchema_builder) Build() *BatonActionSchema { @@ -276,6 +289,7 @@ func (b0 BatonActionSchema_builder) Build() *BatonActionSchema { x.xxx_hidden_DisplayName = b.DisplayName x.xxx_hidden_Description = b.Description x.xxx_hidden_ActionType = b.ActionType + x.xxx_hidden_ResourceTypeId = b.ResourceTypeId return m0 } @@ -1014,7 +1028,7 @@ var File_c1_connector_v2_action_proto protoreflect.FileDescriptor const file_c1_connector_v2_action_proto_rawDesc = "" + "\n" + - "\x1cc1/connector/v2/action.proto\x12\x0fc1.connector.v2\x1a\x19c1/config/v1/config.proto\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xd1\x02\n" + + "\x1cc1/connector/v2/action.proto\x12\x0fc1.connector.v2\x1a\x19c1/config/v1/config.proto\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xfb\x02\n" + "\x11BatonActionSchema\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x121\n" + "\targuments\x18\x02 \x03(\v2\x13.c1.config.v1.FieldR\targuments\x12:\n" + @@ -1023,7 +1037,8 @@ const file_c1_connector_v2_action_proto_rawDesc = "" + "\fdisplay_name\x18\x05 \x01(\tR\vdisplayName\x12 \n" + "\vdescription\x18\x06 \x01(\tR\vdescription\x12<\n" + "\vaction_type\x18\a \x03(\x0e2\x1b.c1.connector.v2.ActionTypeR\n" + - "actionType\"\x8e\x01\n" + + "actionType\x12(\n" + + "\x10resource_type_id\x18\b \x01(\tR\x0eresourceTypeId\"\x8e\x01\n" + "\x13InvokeActionRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12+\n" + "\x04args\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x04args\x126\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree.pb.go index b14ee19a..407721f2 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree.pb.go @@ -165,6 +165,49 @@ func (b0 SkipGrants_builder) Build() *SkipGrants { return m0 } +type SkipEntitlements struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SkipEntitlements) Reset() { + *x = SkipEntitlements{} + mi := &file_c1_connector_v2_annotation_resource_tree_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SkipEntitlements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SkipEntitlements) ProtoMessage() {} + +func (x *SkipEntitlements) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_resource_tree_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SkipEntitlements_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SkipEntitlements_builder) Build() *SkipEntitlements { + m0 := &SkipEntitlements{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + var File_c1_connector_v2_annotation_resource_tree_proto protoreflect.FileDescriptor const file_c1_connector_v2_annotation_resource_tree_proto_rawDesc = "" + @@ -174,13 +217,15 @@ const file_c1_connector_v2_annotation_resource_tree_proto_rawDesc = "" + "\x10resource_type_id\x18\x01 \x01(\tR\x0eresourceTypeId\"\x1b\n" + "\x19SkipEntitlementsAndGrants\"\f\n" + "\n" + - "SkipGrantsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + "SkipGrants\"\x12\n" + + "\x10SkipEntitlementsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" -var file_c1_connector_v2_annotation_resource_tree_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_c1_connector_v2_annotation_resource_tree_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_c1_connector_v2_annotation_resource_tree_proto_goTypes = []any{ (*ChildResourceType)(nil), // 0: c1.connector.v2.ChildResourceType (*SkipEntitlementsAndGrants)(nil), // 1: c1.connector.v2.SkipEntitlementsAndGrants (*SkipGrants)(nil), // 2: c1.connector.v2.SkipGrants + (*SkipEntitlements)(nil), // 3: c1.connector.v2.SkipEntitlements } var file_c1_connector_v2_annotation_resource_tree_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type @@ -201,7 +246,7 @@ func file_c1_connector_v2_annotation_resource_tree_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_resource_tree_proto_rawDesc), len(file_c1_connector_v2_annotation_resource_tree_proto_rawDesc)), NumEnums: 0, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree.pb.validate.go index 36e768bd..8cff9db3 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree.pb.validate.go @@ -339,3 +339,103 @@ var _ interface { Cause() error ErrorName() string } = SkipGrantsValidationError{} + +// Validate checks the field values on SkipEntitlements with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SkipEntitlements) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SkipEntitlements with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SkipEntitlementsMultiError, or nil if none found. +func (m *SkipEntitlements) ValidateAll() error { + return m.validate(true) +} + +func (m *SkipEntitlements) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SkipEntitlementsMultiError(errors) + } + + return nil +} + +// SkipEntitlementsMultiError is an error wrapping multiple validation errors +// returned by SkipEntitlements.ValidateAll() if the designated constraints +// aren't met. +type SkipEntitlementsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SkipEntitlementsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SkipEntitlementsMultiError) AllErrors() []error { return m } + +// SkipEntitlementsValidationError is the validation error returned by +// SkipEntitlements.Validate if the designated constraints aren't met. +type SkipEntitlementsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SkipEntitlementsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SkipEntitlementsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SkipEntitlementsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SkipEntitlementsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SkipEntitlementsValidationError) ErrorName() string { return "SkipEntitlementsValidationError" } + +// Error satisfies the builtin error interface +func (e SkipEntitlementsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSkipEntitlements.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SkipEntitlementsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SkipEntitlementsValidationError{} diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree_protoopaque.pb.go index e382fb0d..187165f9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_resource_tree_protoopaque.pb.go @@ -165,6 +165,49 @@ func (b0 SkipGrants_builder) Build() *SkipGrants { return m0 } +type SkipEntitlements struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SkipEntitlements) Reset() { + *x = SkipEntitlements{} + mi := &file_c1_connector_v2_annotation_resource_tree_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SkipEntitlements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SkipEntitlements) ProtoMessage() {} + +func (x *SkipEntitlements) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_resource_tree_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SkipEntitlements_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SkipEntitlements_builder) Build() *SkipEntitlements { + m0 := &SkipEntitlements{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + var File_c1_connector_v2_annotation_resource_tree_proto protoreflect.FileDescriptor const file_c1_connector_v2_annotation_resource_tree_proto_rawDesc = "" + @@ -174,13 +217,15 @@ const file_c1_connector_v2_annotation_resource_tree_proto_rawDesc = "" + "\x10resource_type_id\x18\x01 \x01(\tR\x0eresourceTypeId\"\x1b\n" + "\x19SkipEntitlementsAndGrants\"\f\n" + "\n" + - "SkipGrantsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + "SkipGrants\"\x12\n" + + "\x10SkipEntitlementsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" -var file_c1_connector_v2_annotation_resource_tree_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_c1_connector_v2_annotation_resource_tree_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_c1_connector_v2_annotation_resource_tree_proto_goTypes = []any{ (*ChildResourceType)(nil), // 0: c1.connector.v2.ChildResourceType (*SkipEntitlementsAndGrants)(nil), // 1: c1.connector.v2.SkipEntitlementsAndGrants (*SkipGrants)(nil), // 2: c1.connector.v2.SkipGrants + (*SkipEntitlements)(nil), // 3: c1.connector.v2.SkipEntitlements } var file_c1_connector_v2_annotation_resource_tree_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type @@ -201,7 +246,7 @@ func file_c1_connector_v2_annotation_resource_tree_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_resource_tree_proto_rawDesc), len(file_c1_connector_v2_annotation_resource_tree_proto_rawDesc)), NumEnums: 0, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session.pb.go index dd64fa42..18537268 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session.pb.go @@ -268,10 +268,11 @@ func (b0 GetManyRequest_builder) Build() *GetManyRequest { } type GetManyResponse struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - Items []*GetManyItem `protobuf:"bytes,1,rep,name=items,proto3" json:"items,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"hybrid.v1"` + Items []*GetManyItem `protobuf:"bytes,1,rep,name=items,proto3" json:"items,omitempty"` + UnprocessedKeys []string `protobuf:"bytes,2,rep,name=unprocessed_keys,json=unprocessedKeys,proto3" json:"unprocessed_keys,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetManyResponse) Reset() { @@ -306,14 +307,26 @@ func (x *GetManyResponse) GetItems() []*GetManyItem { return nil } +func (x *GetManyResponse) GetUnprocessedKeys() []string { + if x != nil { + return x.UnprocessedKeys + } + return nil +} + func (x *GetManyResponse) SetItems(v []*GetManyItem) { x.Items = v } +func (x *GetManyResponse) SetUnprocessedKeys(v []string) { + x.UnprocessedKeys = v +} + type GetManyResponse_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - Items []*GetManyItem + Items []*GetManyItem + UnprocessedKeys []string } func (b0 GetManyResponse_builder) Build() *GetManyResponse { @@ -321,6 +334,7 @@ func (b0 GetManyResponse_builder) Build() *GetManyResponse { b, x := &b0, m0 _, _ = b, x x.Items = b.Items + x.UnprocessedKeys = b.UnprocessedKeys return m0 } @@ -1289,9 +1303,10 @@ const file_c1_connectorapi_baton_v1_session_proto_rawDesc = "" + "\x0eGetManyRequest\x121\n" + "\async_id\x18\x01 \x01(\tB\x18\xfaB\x15r\x132\x11^[a-zA-Z0-9]{27}$R\x06syncId\x12)\n" + "\x04keys\x18\x02 \x03(\tB\x15\xfaB\x12\x92\x01\x0f\b\x01\x10d\x18\x01\"\ar\x05\x10\x01\x18\x80\x02R\x04keys\x12 \n" + - "\x06prefix\x18\x03 \x01(\tB\b\xfaB\x05r\x03\x18\x80\x02R\x06prefix\"N\n" + + "\x06prefix\x18\x03 \x01(\tB\b\xfaB\x05r\x03\x18\x80\x02R\x06prefix\"y\n" + "\x0fGetManyResponse\x12;\n" + - "\x05items\x18\x01 \x03(\v2%.c1.connectorapi.baton.v1.GetManyItemR\x05items\"5\n" + + "\x05items\x18\x01 \x03(\v2%.c1.connectorapi.baton.v1.GetManyItemR\x05items\x12)\n" + + "\x10unprocessed_keys\x18\x02 \x03(\tR\x0funprocessedKeys\"5\n" + "\vGetManyItem\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\fR\x05value\"\x83\x01\n" + @@ -1307,18 +1322,18 @@ const file_c1_connectorapi_baton_v1_session_proto_rawDesc = "" + "\n" + "GetAllItem\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\fR\x05value\"\xa2\x01\n" + + "\x05value\x18\x02 \x01(\fR\x05value\"\xa3\x01\n" + "\n" + "SetRequest\x121\n" + "\async_id\x18\x01 \x01(\tB\x18\xfaB\x15r\x132\x11^[a-zA-Z0-9]{27}$R\x06syncId\x12\x1c\n" + "\x03key\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05\x10\x01\x18\x80\x02R\x03key\x12!\n" + - "\x05value\x18\x03 \x01(\fB\v\xfaB\bz\x06\x10\x01\x18\x80\x98\x02R\x05value\x12 \n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\x03key\x12\"\n" + + "\x05value\x18\x03 \x01(\fB\f\xfaB\tz\a\x10\x00\x18\x80\xe8\xfd\x01R\x05value\x12 \n" + "\x06prefix\x18\x04 \x01(\tB\b\xfaB\x05r\x03\x18\x80\x02R\x06prefix\"\r\n" + - "\vSetResponse\"\x8d\x02\n" + + "\vSetResponse\"\x8e\x02\n" + "\x0eSetManyRequest\x121\n" + - "\async_id\x18\x01 \x01(\tB\x18\xfaB\x15r\x132\x11^[a-zA-Z0-9]{27}$R\x06syncId\x12k\n" + - "\x06values\x18\x02 \x03(\v24.c1.connectorapi.baton.v1.SetManyRequest.ValuesEntryB\x1d\xfaB\x1a\x9a\x01\x17\b\x01\x10d\"\ar\x05\x10\x01\x18\x80\x02*\bz\x06\x10\x01\x18\x80\x98\x02R\x06values\x12 \n" + + "\async_id\x18\x01 \x01(\tB\x18\xfaB\x15r\x132\x11^[a-zA-Z0-9]{27}$R\x06syncId\x12l\n" + + "\x06values\x18\x02 \x03(\v24.c1.connectorapi.baton.v1.SetManyRequest.ValuesEntryB\x1e\xfaB\x1b\x9a\x01\x18\b\x01\x10d\"\ar\x05\x10\x01\x18\x80\x02*\tz\a\x10\x00\x18\x80\xe8\xfd\x01R\x06values\x12 \n" + "\x06prefix\x18\x03 \x01(\tB\b\xfaB\x05r\x03\x18\x80\x02R\x06prefix\x1a9\n" + "\vValuesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session.pb.validate.go index 87696b1f..7f0c234b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session.pb.validate.go @@ -1084,10 +1084,10 @@ func (m *SetRequest) validate(all bool) error { errors = append(errors, err) } - if l := len(m.GetValue()); l < 1 || l > 35840 { + if l := len(m.GetValue()); l < 0 || l > 4158464 { err := SetRequestValidationError{ field: "Value", - reason: "value length must be between 1 and 35840 bytes, inclusive", + reason: "value length must be between 0 and 4158464 bytes, inclusive", } if !all { return err @@ -1351,10 +1351,10 @@ func (m *SetManyRequest) validate(all bool) error { errors = append(errors, err) } - if l := len(val); l < 1 || l > 35840 { + if l := len(val); l < 0 || l > 4158464 { err := SetManyRequestValidationError{ field: fmt.Sprintf("Values[%v]", key), - reason: "value length must be between 1 and 35840 bytes, inclusive", + reason: "value length must be between 0 and 4158464 bytes, inclusive", } if !all { return err diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session_protoopaque.pb.go index 0285f149..870613bb 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/session_protoopaque.pb.go @@ -268,10 +268,11 @@ func (b0 GetManyRequest_builder) Build() *GetManyRequest { } type GetManyResponse struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Items *[]*GetManyItem `protobuf:"bytes,1,rep,name=items,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Items *[]*GetManyItem `protobuf:"bytes,1,rep,name=items,proto3"` + xxx_hidden_UnprocessedKeys []string `protobuf:"bytes,2,rep,name=unprocessed_keys,json=unprocessedKeys,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetManyResponse) Reset() { @@ -308,14 +309,26 @@ func (x *GetManyResponse) GetItems() []*GetManyItem { return nil } +func (x *GetManyResponse) GetUnprocessedKeys() []string { + if x != nil { + return x.xxx_hidden_UnprocessedKeys + } + return nil +} + func (x *GetManyResponse) SetItems(v []*GetManyItem) { x.xxx_hidden_Items = &v } +func (x *GetManyResponse) SetUnprocessedKeys(v []string) { + x.xxx_hidden_UnprocessedKeys = v +} + type GetManyResponse_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - Items []*GetManyItem + Items []*GetManyItem + UnprocessedKeys []string } func (b0 GetManyResponse_builder) Build() *GetManyResponse { @@ -323,6 +336,7 @@ func (b0 GetManyResponse_builder) Build() *GetManyResponse { b, x := &b0, m0 _, _ = b, x x.xxx_hidden_Items = &b.Items + x.xxx_hidden_UnprocessedKeys = b.UnprocessedKeys return m0 } @@ -1293,9 +1307,10 @@ const file_c1_connectorapi_baton_v1_session_proto_rawDesc = "" + "\x0eGetManyRequest\x121\n" + "\async_id\x18\x01 \x01(\tB\x18\xfaB\x15r\x132\x11^[a-zA-Z0-9]{27}$R\x06syncId\x12)\n" + "\x04keys\x18\x02 \x03(\tB\x15\xfaB\x12\x92\x01\x0f\b\x01\x10d\x18\x01\"\ar\x05\x10\x01\x18\x80\x02R\x04keys\x12 \n" + - "\x06prefix\x18\x03 \x01(\tB\b\xfaB\x05r\x03\x18\x80\x02R\x06prefix\"N\n" + + "\x06prefix\x18\x03 \x01(\tB\b\xfaB\x05r\x03\x18\x80\x02R\x06prefix\"y\n" + "\x0fGetManyResponse\x12;\n" + - "\x05items\x18\x01 \x03(\v2%.c1.connectorapi.baton.v1.GetManyItemR\x05items\"5\n" + + "\x05items\x18\x01 \x03(\v2%.c1.connectorapi.baton.v1.GetManyItemR\x05items\x12)\n" + + "\x10unprocessed_keys\x18\x02 \x03(\tR\x0funprocessedKeys\"5\n" + "\vGetManyItem\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\fR\x05value\"\x83\x01\n" + @@ -1311,18 +1326,18 @@ const file_c1_connectorapi_baton_v1_session_proto_rawDesc = "" + "\n" + "GetAllItem\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\fR\x05value\"\xa2\x01\n" + + "\x05value\x18\x02 \x01(\fR\x05value\"\xa3\x01\n" + "\n" + "SetRequest\x121\n" + "\async_id\x18\x01 \x01(\tB\x18\xfaB\x15r\x132\x11^[a-zA-Z0-9]{27}$R\x06syncId\x12\x1c\n" + "\x03key\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05\x10\x01\x18\x80\x02R\x03key\x12!\n" + - "\x05value\x18\x03 \x01(\fB\v\xfaB\bz\x06\x10\x01\x18\x80\x98\x02R\x05value\x12 \n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\x03key\x12\"\n" + + "\x05value\x18\x03 \x01(\fB\f\xfaB\tz\a\x10\x00\x18\x80\xe8\xfd\x01R\x05value\x12 \n" + "\x06prefix\x18\x04 \x01(\tB\b\xfaB\x05r\x03\x18\x80\x02R\x06prefix\"\r\n" + - "\vSetResponse\"\x8d\x02\n" + + "\vSetResponse\"\x8e\x02\n" + "\x0eSetManyRequest\x121\n" + - "\async_id\x18\x01 \x01(\tB\x18\xfaB\x15r\x132\x11^[a-zA-Z0-9]{27}$R\x06syncId\x12k\n" + - "\x06values\x18\x02 \x03(\v24.c1.connectorapi.baton.v1.SetManyRequest.ValuesEntryB\x1d\xfaB\x1a\x9a\x01\x17\b\x01\x10d\"\ar\x05\x10\x01\x18\x80\x02*\bz\x06\x10\x01\x18\x80\x98\x02R\x06values\x12 \n" + + "\async_id\x18\x01 \x01(\tB\x18\xfaB\x15r\x132\x11^[a-zA-Z0-9]{27}$R\x06syncId\x12l\n" + + "\x06values\x18\x02 \x03(\v24.c1.connectorapi.baton.v1.SetManyRequest.ValuesEntryB\x1e\xfaB\x1b\x9a\x01\x18\b\x01\x10d\"\ar\x05\x10\x01\x18\x80\x02*\tz\a\x10\x00\x18\x80\xe8\xfd\x01R\x06values\x12 \n" + "\x06prefix\x18\x03 \x01(\tB\b\xfaB\x05r\x03\x18\x80\x02R\x06prefix\x1a9\n" + "\vValuesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go index eea84255..218878d3 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "os" - "sort" "time" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -357,6 +356,7 @@ func MakeMainCommand[T field.Configurable]( } opts = append(opts, connectorrunner.WithSkipEntitlementsAndGrants(v.GetBool("skip-entitlements-and-grants"))) + if v.GetBool("skip-grants") { opts = append(opts, connectorrunner.WithSkipGrants(v.GetBool("skip-grants"))) } @@ -674,11 +674,6 @@ func MakeConfigSchemaCommand[T field.Configurable]( getconnector GetConnectorFunc2[T], ) func(*cobra.Command, []string) error { return func(cmd *cobra.Command, args []string) error { - // Sort fields by FieldName - sort.Slice(confschema.Fields, func(i, j int) bool { - return confschema.Fields[i].FieldName < confschema.Fields[j].FieldName - }) - // Use MarshalIndent for pretty printing pb, err := json.MarshalIndent(&confschema, "", " ") if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/lazy_session.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/lazy_session.go index 6e03e0e7..224ff248 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/lazy_session.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/lazy_session.go @@ -81,9 +81,9 @@ func (l *lazyCachingSessionStore) Get(ctx context.Context, key string, opt ...se } // GetMany implements types.SessionStore. -func (l *lazyCachingSessionStore) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, error) { +func (l *lazyCachingSessionStore) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, []string, error) { if err := l.ensureSession(ctx); err != nil { - return nil, err + return nil, nil, err } return l.session.GetMany(ctx, keys, opt...) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go index c018e940..d9304524 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go @@ -61,6 +61,8 @@ func RunConnector[T field.Configurable]( } } +var ErrDuplicateField = errors.New("multiple fields with the same name") + // GetConnectorFunc is a function type that creates a connector instance. // It takes a context and configuration. The session cache constructor is retrieved from the context. // deprecated - prefer RunConnector. @@ -111,23 +113,48 @@ func DefineConfigurationV2[T field.Configurable]( v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) v.AutomaticEnv() + defaultFieldsByName := make(map[string]field.SchemaField) + for _, f := range field.DefaultFields { + if _, ok := defaultFieldsByName[f.FieldName]; ok { + return nil, nil, fmt.Errorf("multiple default fields with the same name: %s", f.FieldName) + } + defaultFieldsByName[f.FieldName] = f + } + confschema := schema confschema.Fields = append(field.DefaultFields, confschema.Fields...) // Ensure unique fields uniqueFields := make(map[string]field.SchemaField) + fieldsToDelete := make(map[string]bool) for _, f := range confschema.Fields { - if s, ok := uniqueFields[f.FieldName]; ok { - if !f.WasReExported && !s.WasReExported { - return nil, nil, fmt.Errorf("multiple fields with the same name: %s.If you want to use a default field in the SDK, use ExportAs on the connector schema field", f.FieldName) + if existingField, ok := uniqueFields[f.FieldName]; ok { + // If the duplicate field is not a default field, error. + if _, ok := defaultFieldsByName[f.FieldName]; !ok { + return nil, nil, fmt.Errorf("%w: %s", ErrDuplicateField, f.FieldName) + } + // If redeclaring a default field and not reexporting it, error. + if !f.WasReExported { + return nil, nil, fmt.Errorf("%w: %s. If you want to use a default field in the SDK, use ExportAs on the connector schema field", ErrDuplicateField, f.FieldName) } + if existingField.WasReExported { + return nil, nil, fmt.Errorf("%w: %s. If you want to use a default field in the SDK, use ExportAs on the connector schema field", ErrDuplicateField, f.FieldName) + } + + fieldsToDelete[existingField.FieldName] = true } uniqueFields[f.FieldName] = f } - confschema.Fields = make([]field.SchemaField, 0, len(uniqueFields)) - for _, f := range uniqueFields { - confschema.Fields = append(confschema.Fields, f) + + // Filter out fields that were not reexported and were in the fieldsToDelete list. + fields := make([]field.SchemaField, 0, len(confschema.Fields)) + for _, f := range confschema.Fields { + if !f.WasReExported && fieldsToDelete[f.FieldName] { + continue + } + fields = append(fields, f) } + confschema.Fields = fields // setup CLI with cobra mainCMD := &cobra.Command{ diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/session_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/session_store.go index 1bf097b0..0362404d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/session_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/session_store.go @@ -27,7 +27,7 @@ func (w *SessionStoreWithSyncID) Get(ctx context.Context, key string, opt ...ses return w.ss.Get(ctx, key, opts...) } -func (w *SessionStoreWithSyncID) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, error) { +func (w *SessionStoreWithSyncID) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, []string, error) { opts := append([]sessions.SessionStoreOption{sessions.WithSyncID(w.syncID)}, opt...) return w.ss.GetMany(ctx, keys, opts...) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/client_secret.go b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/client_secret.go index 6126db43..f5fd9e9d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/client_secret.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/client_secret.go @@ -1,4 +1,4 @@ -package crypto +package crypto //nolint:revive,nolintlint // we can't change the package name for backwards compatibility import ( "bytes" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/crypto.go b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/crypto.go index 407209db..7a287bce 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/crypto.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/crypto.go @@ -1,4 +1,4 @@ -package crypto +package crypto //nolint:revive,nolintlint // we can't change the package name for backwards compatibility import ( "context" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/password.go b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/password.go index 98bc11af..cfd16ac0 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/password.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/password.go @@ -1,4 +1,4 @@ -package crypto +package crypto //nolint:revive,nolintlint // we can't change the package name for backwards compatibility import ( "context" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go index 3e922ec6..59c7f036 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go @@ -204,6 +204,30 @@ func (c *C1File) init(ctx context.Context) error { return err } + err = c.InitTables(ctx) + if err != nil { + return err + } + + for _, pragma := range c.pragmas { + _, err := c.db.ExecContext(ctx, fmt.Sprintf("PRAGMA %s = %s", pragma.name, pragma.value)) + if err != nil { + return err + } + } + + return nil +} + +func (c *C1File) InitTables(ctx context.Context) error { + ctx, span := tracer.Start(ctx, "C1File.InitTables") + defer span.End() + + err := c.validateDb(ctx) + if err != nil { + return err + } + for _, t := range allTableDescriptors { query, args := t.Schema() _, err = c.db.ExecContext(ctx, fmt.Sprintf(query, args...)) @@ -216,13 +240,6 @@ func (c *C1File) init(ctx context.Context) error { } } - for _, pragma := range c.pragmas { - _, err := c.db.ExecContext(ctx, fmt.Sprintf("PRAGMA %s = %s", pragma.name, pragma.value)) - if err != nil { - return err - } - } - return nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go index 68480acc..e6c20b7b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go @@ -185,7 +185,7 @@ func (d *decoder) Read(p []byte) (int, error) { // Do underlying read n, err := d.zd.Read(p) - //nolint:gosec // No risk of overflow/underflow because n is always >= 0. + d.decodedBytes += uint64(n) if err != nil { // NOTE(morgabra) This happens if you set a small DecoderMaxMemory diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants.go index c34d49d7..3d79597c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/grants.go @@ -62,6 +62,29 @@ func (r *grantsTable) Migrations(ctx context.Context, db *goqu.Database) error { return nil } +// DropGrantIndexes drops the indexes on the grants table. +// This should only be called when compacting the grants table. +// These indexes are re-created when we open the database again. +func (c *C1File) DropGrantIndexes(ctx context.Context) error { + ctx, span := tracer.Start(ctx, "C1File.DropGrantsIndexes") + defer span.End() + + indexes := []string{ + fmt.Sprintf("idx_grants_resource_type_id_resource_id_v%s", grants.Version()), + fmt.Sprintf("idx_grants_principal_id_v%s", grants.Version()), + fmt.Sprintf("idx_grants_entitlement_id_principal_id_v%s", grants.Version()), + fmt.Sprintf("idx_grants_external_sync_v%s", grants.Version()), + } + + for _, index := range indexes { + _, err := c.db.ExecContext(ctx, fmt.Sprintf("DROP INDEX IF EXISTS %s", index)) + if err != nil { + return err + } + } + return nil +} + func (c *C1File) ListGrants(ctx context.Context, request *v2.GrantsServiceListGrantsRequest) (*v2.GrantsServiceListGrantsResponse, error) { ctx, span := tracer.Start(ctx, "C1File.ListGrants") defer span.End() diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/session_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/session_store.go index d4507790..4fc850ed 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/session_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/session_store.go @@ -3,6 +3,7 @@ package dotc1z import ( "context" "fmt" + "maps" "strings" "github.com/doug-martin/goqu/v9" @@ -16,6 +17,10 @@ type SessionStore interface { var _ sessions.SessionStore = (*C1File)(nil) +// The default gRPC message size limit is 4MB (we subtract 30KB for general overhead, which is overkill). +// Unfortunately, this layer has to be aware of the size limit to avoid exceeding the size limit +// because the client does not know the size of the items it requests. +const sessionStoreSizeLimit = 4163584 const sessionStoreTableVersion = "1" const sessionStoreTableName = "connector_sessions" const sessionStoreTableSchema = ` @@ -232,14 +237,14 @@ func (c *C1File) Clear(ctx context.Context, opt ...sessions.SessionStoreOption) } // GetMany implements types.SessionStore. -func (c *C1File) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, error) { +func (c *C1File) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, []string, error) { bag, err := applyBag(ctx, opt...) if err != nil { - return nil, fmt.Errorf("error applying session option: %w", err) + return nil, nil, fmt.Errorf("session-get-many: error applying session option: %w", err) } if len(keys) == 0 { - return make(map[string][]byte), nil + return make(map[string][]byte), nil, nil } prefixedKeys := make([]string, len(keys)) if bag.Prefix == "" { @@ -254,98 +259,174 @@ func (c *C1File) GetMany(ctx context.Context, keys []string, opt ...sessions.Ses q = q.Select("key", "value") q = q.Where(goqu.C("sync_id").Eq(bag.SyncID)) q = q.Where(goqu.C("key").In(prefixedKeys)) + q = q.Order(goqu.C("key").Asc()) sql, params, err := q.ToSQL() if err != nil { - return nil, fmt.Errorf("error getting many sessions: %w", err) + return nil, nil, fmt.Errorf("session-get-many: error generating SQL: %w", err) } rows, err := c.db.QueryContext(ctx, sql, params...) if err != nil { - return nil, fmt.Errorf("error getting many sessions: %w", err) + return nil, nil, fmt.Errorf("session-get-many: error executing SQL: %w", err) } defer rows.Close() - result := make(map[string][]byte) + unprocessedKeys := make(map[string]struct{}, len(keys)) + // Initialize unprocessedKeys with all keys - we'll remove them as we process results + // Start by calculating size of all unprocessed keys (they'll be in the return slice) + + type item struct { + key string + value []byte + } + results := make([]item, 0, len(keys)) + messageSize := 0 for rows.Next() { var key string var value []byte err = rows.Scan(&key, &value) if err != nil { - return nil, fmt.Errorf("error scanning session: %w", err) + return nil, nil, fmt.Errorf("session-get-many: error scanning row: %w", err) } // Remove prefix from key to return original key if bag.Prefix != "" && len(key) >= len(bag.Prefix) && key[:len(bag.Prefix)] == bag.Prefix { key = key[len(bag.Prefix):] } - result[key] = value + results = append(results, item{key: key, value: value}) + // 10 is extra padding. The key goes into the response unconditionally. + messageSize += len(key) + 10 } if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error getting data from session: %w", err) + return nil, nil, fmt.Errorf("session-get-many: error getting data from session: %w", err) + } + + ret := make(map[string][]byte) + for _, r := range results { + value := r.value + key := r.key + + netItemSize := len(value) + 10 // 10 is extra padding for overhead. + if messageSize+netItemSize <= sessionStoreSizeLimit { + messageSize += netItemSize + ret[key] = value + } else { + unprocessedKeys[key] = struct{}{} + } } - return result, nil + unprocessedKeysSlice := make([]string, 0, len(unprocessedKeys)) + for key := range unprocessedKeys { + unprocessedKeysSlice = append(unprocessedKeysSlice, key) + } + return ret, unprocessedKeysSlice, nil } // GetAll implements types.SessionStore. func (c *C1File) GetAll(ctx context.Context, pageToken string, opt ...sessions.SessionStoreOption) (map[string][]byte, string, error) { bag, err := applyBag(ctx, opt...) if err != nil { - return nil, "", fmt.Errorf("error applying session option: %w", err) + return nil, "", fmt.Errorf("session-get-all: error applying session option: %w", err) + } + + result := make(map[string][]byte) + messageSizeRemaining := sessionStoreSizeLimit + for { + items, nextPageToken, itemsSize, err := c.getAllChunk(ctx, pageToken, messageSizeRemaining, bag) + if err != nil { + return nil, "", fmt.Errorf("session-get-all: error getting all data from session: %w", err) + } + maps.Copy(result, items) + + if len(items) == 0 { + break + } + + if nextPageToken == "" { + pageToken = "" + break + } + + if pageToken == nextPageToken { + return nil, "", fmt.Errorf("page token is the same as the next page token: %s", pageToken) + } + pageToken = nextPageToken + + messageSizeRemaining -= itemsSize + if messageSizeRemaining <= 0 { + break + } } + return result, pageToken, nil +} + +func (c *C1File) getAllChunk(ctx context.Context, pageToken string, sizeLimit int, bag *sessions.SessionStoreBag) (map[string][]byte, string, int, error) { q := c.db.From(sessionStore.Name()).Prepared(true). Select("key", "value"). Where(goqu.C("sync_id").Eq(bag.SyncID)). Order(goqu.C("key").Asc()). - Limit(101) + Limit(100) if bag.Prefix != "" { q = q.Where(goqu.C("key").Like(escapeLike(bag.Prefix) + "%")) } if pageToken != "" { - q = q.Where(goqu.C("key").Gte(pageToken)) + q = q.Where(goqu.C("key").Gte(bag.Prefix + pageToken)) } sql, params, err := q.ToSQL() - if err != nil { - return nil, "", fmt.Errorf("error getting all sessions: %w", err) + return nil, "", 0, fmt.Errorf("session-get-all: error generating SQL: %w", err) } rows, err := c.db.QueryContext(ctx, sql, params...) if err != nil { - return nil, "", fmt.Errorf("error getting all sessions: %w", err) + return nil, "", 0, fmt.Errorf("session-get-all: error executing SQL: %w", err) } defer rows.Close() result := make(map[string][]byte) nextPageToken := "" - i := 0 + messageSize := 0 + tooBig := false for rows.Next() { var key string var value []byte err = rows.Scan(&key, &value) if err != nil { - return nil, "", fmt.Errorf("error scanning session: %w", err) - } - i++ - if i > 100 { - nextPageToken = key - break + return nil, "", 0, fmt.Errorf("session-get-all: error scanning row: %w", err) } // Remove prefix from key to return original key if bag.Prefix != "" && len(key) >= len(bag.Prefix) && key[:len(bag.Prefix)] == bag.Prefix { key = key[len(bag.Prefix):] } + nextPageToken = key + itemSize := len(key) + len(value) + 20 + if messageSize+itemSize > sizeLimit { + tooBig = true + break + } + if len(result) >= 100 { + break + } result[key] = value + messageSize += itemSize } if err := rows.Err(); err != nil { - return nil, "", fmt.Errorf("error getting data from session: %w", err) + return nil, "", 0, fmt.Errorf("session-get-all: error getting data from session: %w", err) + } + + if tooBig { + return result, nextPageToken, messageSize, nil + } + + if len(result) < 100 { + return result, "", messageSize, nil } - return result, nextPageToken, nil + return result, nextPageToken, messageSize, nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go index 8a05c793..bdaabe96 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go @@ -109,6 +109,7 @@ var ( WithExportTarget(ExportTargetNone), WithHidden(true), ) + syncResourceTypeIDs = StringSliceField("sync-resource-types", WithDescription("The resource type IDs to sync"), WithPersistent(true), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index 84105768..2bd4c768 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.5.24" +const Version = "v0.6.4" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/session/json_session.go b/vendor/github.com/conductorone/baton-sdk/pkg/session/json_session.go index 9f0f4903..1d943e1e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/session/json_session.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/session/json_session.go @@ -11,7 +11,7 @@ import ( // See GRPC validation rules for eg GetManyRequest. func GetManyJSON[T any](ctx context.Context, ss sessions.SessionStore, keys []string, opt ...sessions.SessionStoreOption) (map[string]T, error) { - allBytes, err := UnrollGetMany(ctx, ss, keys, opt...) + allBytes, err := UnrollGetMany[[]byte](ctx, ss, keys, opt...) if err != nil { return nil, err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/session/memory_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/session/memory_cache.go index 3881ded4..b5e70c5a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/session/memory_cache.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/session/memory_cache.go @@ -153,16 +153,19 @@ func (m *MemorySessionCache) GetAll(ctx context.Context, pageToken string, opt . return values, nextPageToken, nil } -func (m *MemorySessionCache) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, error) { +func (m *MemorySessionCache) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, []string, error) { bag, err := applyOptions(ctx, opt...) if err != nil { - return nil, err + return nil, nil, err } values, err := m.cache.BulkGet(ctx, cacheKeys(bag, keys), otter.BulkLoaderFunc[string, []byte](func(ctx context.Context, cacheKeys []string) (map[string][]byte, error) { - backingValues, err := m.ss.GetMany(ctx, stripPrefixes(bag, cacheKeys), opt...) + backingValues, unprocessedKeys, err := m.ss.GetMany(ctx, stripPrefixes(bag, cacheKeys), opt...) if err != nil { return nil, err } + if len(unprocessedKeys) > 0 { + return nil, fmt.Errorf("get many returned unprocessed keys") + } cacheKeyValues := make(map[string][]byte, len(backingValues)) for k, v := range backingValues { cacheKeyValues[cacheKey(bag, k)] = v @@ -172,7 +175,7 @@ func (m *MemorySessionCache) GetMany(ctx context.Context, keys []string, opt ... })) if err != nil { - return nil, err + return nil, nil, err } unprefixedValues := make(map[string][]byte) for k, v := range values { @@ -183,7 +186,7 @@ func (m *MemorySessionCache) GetMany(ctx context.Context, keys []string, opt ... } unprefixedValues[stripPrefix(bag, k)] = v } - return unprefixedValues, nil + return unprefixedValues, nil, nil } func (m *MemorySessionCache) Set(ctx context.Context, key string, value []byte, opt ...sessions.SessionStoreOption) error { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/session/noop_session.go b/vendor/github.com/conductorone/baton-sdk/pkg/session/noop_session.go index 11300026..901c96bc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/session/noop_session.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/session/noop_session.go @@ -26,8 +26,8 @@ func (n *NoOpSessionStore) Get(ctx context.Context, key string, opt ...sessions. return nil, false, n.logAndError(ctx, "Get") } -func (n *NoOpSessionStore) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, error) { - return nil, n.logAndError(ctx, "GetMany") +func (n *NoOpSessionStore) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, []string, error) { + return nil, nil, n.logAndError(ctx, "GetMany") } func (n *NoOpSessionStore) Set(ctx context.Context, key string, value []byte, opt ...sessions.SessionStoreOption) error { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/session/session.go b/vendor/github.com/conductorone/baton-sdk/pkg/session/session.go index 97b05678..868fa5ac 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/session/session.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/session/session.go @@ -23,7 +23,7 @@ func Chunk[T any](items []T, chunkSize int) iter.Seq[[]T] { } type GetManyable[T any] interface { - GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string]T, error) + GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string]T, []string, error) } func UnrollGetMany[T any](ctx context.Context, ss GetManyable[T], keys []string, opt ...sessions.SessionStoreOption) (map[string]T, error) { @@ -33,12 +33,31 @@ func UnrollGetMany[T any](ctx context.Context, ss GetManyable[T], keys []string, } // TODO(Kans): parallelize this? - for keys := range Chunk(keys, MaxKeysPerRequest) { - some, err := ss.GetMany(ctx, keys, opt...) - if err != nil { - return nil, err + for keyChunk := range Chunk(keys, MaxKeysPerRequest) { + // For each chunk, unroll any unprocessed keys until all are processed + remainingKeys := keyChunk + for { + some, unprocessedKeys, err := ss.GetMany(ctx, remainingKeys, opt...) + if err != nil { + return nil, err + } + + // Accumulate results + maps.Copy(all, some) + + // If no unprocessed keys, we're done with this chunk + if len(unprocessedKeys) == 0 { + break + } + + // Check for infinite loop: if unprocessed keys haven't been reduced, something is wrong + if len(unprocessedKeys) == len(remainingKeys) { + return nil, fmt.Errorf("unprocessed keys not reduced: %d unprocessed out of %d requested", len(unprocessedKeys), len(remainingKeys)) + } + + // Continue with unprocessed keys + remainingKeys = unprocessedKeys } - maps.Copy(all, some) } return all, nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/session/session_client.go b/vendor/github.com/conductorone/baton-sdk/pkg/session/session_client.go index 6b84736e..f75d920a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/session/session_client.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/session/session_client.go @@ -7,6 +7,7 @@ import ( "fmt" "net" "os" + "slices" "time" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" @@ -171,27 +172,30 @@ func (g *GRPCSessionStoreClient) Get(ctx context.Context, key string, opt ...ses } // GetMany retrieves multiple values from the cache by keys. -func (g *GRPCSessionStoreClient) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, error) { +func (g *GRPCSessionStoreClient) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string][]byte, []string, error) { bag, err := applyOptions(ctx, opt...) if err != nil { - return nil, err + return nil, nil, err } - results := make(map[string][]byte) + slices.Sort(keys) + keys = slices.Compact(keys) + resp, err := g.client.GetMany(ctx, v1.GetManyRequest_builder{ SyncId: bag.SyncID, Keys: keys, Prefix: bag.Prefix, }.Build()) if err != nil { - return nil, fmt.Errorf("failed to get many values from gRPC session cache: %w", err) + return nil, nil, fmt.Errorf("failed to get many values from gRPC session cache: %w", err) } + results := make(map[string][]byte, len(resp.Items)) for _, item := range resp.Items { results[item.Key] = item.Value } - return results, nil + return results, resp.UnprocessedKeys, nil } // Set stores a value in the cache with the given key. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/session/session_server.go b/vendor/github.com/conductorone/baton-sdk/pkg/session/session_server.go index e817808c..622ec1ee 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/session/session_server.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/session/session_server.go @@ -60,7 +60,7 @@ func (s *GRPCSessionServer) GetMany(ctx context.Context, req *v1.GetManyRequest) return nil, err } - values, err := s.store.GetMany( + values, unprocessedKeys, err := s.store.GetMany( ctx, req.GetKeys(), sessions.WithSyncID(req.GetSyncId()), @@ -80,7 +80,8 @@ func (s *GRPCSessionServer) GetMany(ctx context.Context, req *v1.GetManyRequest) } return v1.GetManyResponse_builder{ - Items: items, + Items: items, + UnprocessedKeys: unprocessedKeys, }.Build(), nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/session/typed_session.go b/vendor/github.com/conductorone/baton-sdk/pkg/session/typed_session.go index 43a2f4f8..8933b555 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/session/typed_session.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/session/typed_session.go @@ -52,22 +52,22 @@ func (t *TypedSessionCache[T]) Set(ctx context.Context, key string, value T, opt return t.cache.Set(ctx, key, data, opt...) } -func (t *TypedSessionCache[T]) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string]T, error) { - dataMap, err := t.cache.GetMany(ctx, keys, opt...) +func (t *TypedSessionCache[T]) GetMany(ctx context.Context, keys []string, opt ...sessions.SessionStoreOption) (map[string]T, []string, error) { + dataMap, unprocessedKeys, err := t.cache.GetMany(ctx, keys, opt...) if err != nil { - return nil, err + return nil, nil, err } result := make(map[string]T) for key, data := range dataMap { value, err := t.codec.Decode(data) if err != nil { - return nil, fmt.Errorf("failed to decode value for key %s: %w", key, err) + return nil, nil, fmt.Errorf("failed to decode value for key %s: %w", key, err) } result[key] = value } - return result, nil + return result, unprocessedKeys, nil } func (t *TypedSessionCache[T]) SetMany(ctx context.Context, values map[string]T, opt ...sessions.SessionStoreOption) error { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/bitset.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/bitset.go index 33c910f8..586d232d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/bitset.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/bitset.go @@ -125,7 +125,7 @@ func (b *bitset) forEachSet(fn func(i int)) { tz := bits.TrailingZeros64(w) i := (wi << 6) + tz fn(i) - w &^= 1 << uint(tz) //nolint:gosec // trailing zeros is non-negative + w &^= 1 << uint(tz) } } } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go index 71e14911..81eeace7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go @@ -36,6 +36,7 @@ type State interface { SetShouldSkipEntitlementsAndGrants() ShouldSkipGrants() bool SetShouldSkipGrants() + GetCompletedActionsCount() uint64 } // ActionOp represents a sync operation. @@ -156,6 +157,7 @@ type state struct { shouldFetchRelatedResources bool shouldSkipEntitlementsAndGrants bool shouldSkipGrants bool + completedActionsCount uint64 } // serializedToken is used to serialize the token to JSON. This separate object is used to avoid having exported fields @@ -169,6 +171,7 @@ type serializedToken struct { ShouldFetchRelatedResources bool `json:"should_fetch_related_resources,omitempty"` ShouldSkipEntitlementsAndGrants bool `json:"should_skip_entitlements_and_grants,omitempty"` ShouldSkipGrants bool `json:"should_skip_grants,omitempty"` + CompletedActionsCount uint64 `json:"completed_actions_count,omitempty"` } // push adds a new action to the stack. If there is no current state, the action is directly set to current, else @@ -196,6 +199,7 @@ func (st *state) pop() *Action { } ret := *st.currentAction + st.completedActionsCount++ if len(st.actions) > 0 { st.currentAction = &st.actions[len(st.actions)-1] @@ -242,10 +246,12 @@ func (st *state) Unmarshal(input string) error { st.shouldSkipEntitlementsAndGrants = token.ShouldSkipEntitlementsAndGrants st.shouldSkipGrants = token.ShouldSkipGrants st.shouldFetchRelatedResources = token.ShouldFetchRelatedResources + st.completedActionsCount = token.CompletedActionsCount } else { st.actions = nil st.entitlementGraph = nil st.currentAction = &Action{Op: InitOp} + st.completedActionsCount = 0 } return nil @@ -265,6 +271,7 @@ func (st *state) Marshal() (string, error) { ShouldFetchRelatedResources: st.shouldFetchRelatedResources, ShouldSkipEntitlementsAndGrants: st.shouldSkipEntitlementsAndGrants, ShouldSkipGrants: st.shouldSkipGrants, + CompletedActionsCount: st.completedActionsCount, }) if err != nil { return "", err @@ -405,3 +412,9 @@ func (st *state) ParentResourceTypeID(ctx context.Context) string { return c.ParentResourceTypeID } + +func (st *state) GetCompletedActionsCount() uint64 { + st.mtx.RLock() + defer st.mtx.RUnlock() + return st.completedActionsCount +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index 48fb33b7..a0fd2aa9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -215,6 +215,7 @@ type syncer struct { dontExpandGrants bool syncID string skipEGForResourceType map[string]bool + skipEntitlementsForResourceType map[string]bool skipEntitlementsAndGrants bool skipGrants bool resourceTypeTraits map[string][]v2.ResourceType_Trait @@ -255,7 +256,6 @@ func (s *syncer) handleInitialActionForStep(ctx context.Context, a Action) { func (s *syncer) handleProgress(ctx context.Context, a *Action, c int) { if s.progressHandler != nil { - //nolint:gosec // No risk of overflow because `c` is a slice length. count := uint32(c) s.progressHandler(NewProgress(a, count)) } @@ -475,9 +475,12 @@ func (s *syncer) Sync(ctx context.Context) error { return err } - // TODO: count actions divided by warnings and error if warning percentage is too high + // If we have more than 10 warnings and more than 10% of actions ended in a warning, exit the sync. if len(warnings) > 10 { - return fmt.Errorf("too many warnings, exiting sync. warnings: %v", warnings) + completedActionsCount := s.state.GetCompletedActionsCount() + if completedActionsCount > 0 && float64(len(warnings))/float64(completedActionsCount) > 0.1 { + return fmt.Errorf("too many warnings, exiting sync. warnings: %v completed actions: %d", warnings, completedActionsCount) + } } select { case <-runCtx.Done(): @@ -550,7 +553,9 @@ func (s *syncer) Sync(ctx context.Context) error { if !s.state.ShouldSkipGrants() { s.state.PushAction(ctx, Action{Op: SyncGrantsOp}) } + s.state.PushAction(ctx, Action{Op: SyncEntitlementsOp}) + s.state.PushAction(ctx, Action{Op: SyncStaticEntitlementsOp}) } s.state.PushAction(ctx, Action{Op: SyncResourcesOp}) @@ -950,7 +955,7 @@ func (s *syncer) SyncTargetedResource(ctx context.Context) error { }) } - shouldSkipEnts, err := s.shouldSkipEntitlementsAndGrants(ctx, resource) + shouldSkipEnts, err := s.shouldSkipEntitlements(ctx, resource) if err != nil { return err } @@ -1190,6 +1195,43 @@ func (s *syncer) shouldSkipGrants(ctx context.Context, r *v2.Resource) (bool, er return s.shouldSkipEntitlementsAndGrants(ctx, r) } +func (s *syncer) shouldSkipEntitlements(ctx context.Context, r *v2.Resource) (bool, error) { + ctx, span := tracer.Start(ctx, "syncer.shouldSkipEntitlements") + defer span.End() + + ok, err := s.shouldSkipEntitlementsAndGrants(ctx, r) + if err != nil { + return false, err + } + + if ok { + return true, nil + } + + rAnnos := annotations.Annotations(r.GetAnnotations()) + if rAnnos.Contains(&v2.SkipEntitlements{}) || rAnnos.Contains(&v2.SkipEntitlementsAndGrants{}) { + return true, nil + } + + if skip, ok := s.skipEntitlementsForResourceType[r.GetId().GetResourceType()]; ok { + return skip, nil + } + + rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: r.GetId().GetResourceType(), + }.Build()) + if err != nil { + return false, err + } + + rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) + + skipEntitlements := rtAnnos.Contains(&v2.SkipEntitlements{}) || rtAnnos.Contains(&v2.SkipEntitlementsAndGrants{}) + s.skipEntitlementsForResourceType[r.GetId().GetResourceType()] = skipEntitlements + + return skipEntitlements, nil +} + // SyncEntitlements fetches the entitlements from the connector. It first lists each resource from the datastore, // and pushes an action to fetch the entitlements for each resource. func (s *syncer) SyncEntitlements(ctx context.Context) error { @@ -1220,7 +1262,7 @@ func (s *syncer) SyncEntitlements(ctx context.Context) error { } for _, r := range resp.GetList() { - shouldSkipEntitlements, err := s.shouldSkipEntitlementsAndGrants(ctx, r) + shouldSkipEntitlements, err := s.shouldSkipEntitlements(ctx, r) if err != nil { return err } @@ -3166,11 +3208,12 @@ func WithSkipGrants(skip bool) SyncOpt { // NewSyncer returns a new syncer object. func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (Syncer, error) { s := &syncer{ - connector: c, - skipEGForResourceType: make(map[string]bool), - resourceTypeTraits: make(map[string][]v2.ResourceType_Trait), - counts: NewProgressCounts(), - syncType: connectorstore.SyncTypeFull, + connector: c, + skipEGForResourceType: make(map[string]bool), + skipEntitlementsForResourceType: make(map[string]bool), + resourceTypeTraits: make(map[string][]v2.ResourceType_Trait), + counts: NewProgressCounts(), + syncType: connectorstore.SyncTypeFull, } for _, o := range opts { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go index e3e9fa4b..cb675996 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go @@ -66,10 +66,22 @@ func (c *Compactor) CompactWithSyncID(ctx context.Context, destSyncID string) er } }() + // Drop grants indexes to improve performance. + err = c.dest.DropGrantIndexes(ctx) + if err != nil { + return fmt.Errorf("failed to drop grants indexes: %w", err) + } + if err := c.processRecords(ctx, attached, destSyncID, baseSyncID, appliedSyncID); err != nil { return fmt.Errorf("failed to process records: %w", err) } + // Re-create the destination database to re-create the grant indexes. + err = c.dest.InitTables(ctx) + if err != nil { + return fmt.Errorf("failed to re-create destination database: %w", err) + } + return nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go index aa588251..790841a5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go @@ -36,9 +36,10 @@ type Compactor struct { compactorType CompactorType entries []*CompactableSync - tmpDir string - destDir string - runDuration time.Duration + tmpDir string + destDir string + runDuration time.Duration + optimizeInserts bool // TODO: Remove this option once we're confident it's stable. } type CompactableSync struct { @@ -70,6 +71,12 @@ func WithRunDuration(runDuration time.Duration) Option { } } +func WithOptimizeInserts(optimizeInserts bool) Option { + return func(c *Compactor) { + c.optimizeInserts = optimizeInserts + } +} + func NewCompactor(ctx context.Context, outputDir string, compactableSyncs []*CompactableSync, opts ...Option) (*Compactor, func() error, error) { if len(compactableSyncs) < 2 { return nil, nil, ErrNotEnoughFilesToCompact @@ -107,12 +114,36 @@ func NewCompactor(ctx context.Context, outputDir string, compactableSyncs []*Com func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { ctx, span := tracer.Start(ctx, "Compactor.Compact") defer span.End() - now := time.Now() if len(c.entries) < 2 { return nil, nil } + compactionStart := time.Now() + runCtx := ctx + var runCanc context.CancelFunc + if c.runDuration > 0 { + runCtx, runCanc = context.WithTimeout(ctx, c.runDuration) + } + if runCanc != nil { + defer runCanc() + } + + l := ctxzap.Extract(ctx) var err error + select { + case <-runCtx.Done(): + err = context.Cause(runCtx) + switch { + case errors.Is(err, context.DeadlineExceeded): + l.Info("compaction run duration has expired, exiting compaction early") + return nil, fmt.Errorf("compaction run duration has expired: %w", err) + default: + l.Error("compaction context cancelled", zap.Error(err)) + return nil, err + } + default: + } + // Base sync is c.entries[0], so compact all incrementals first, then apply that onto the base. applied := c.entries[len(c.entries)-1] for i := len(c.entries) - 2; i >= 0; i-- { @@ -122,7 +153,6 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { } } - l := ctxzap.Extract(ctx) // Grant expansion doesn't use the connector interface at all, so giving syncer an empty connector is safe... for now. // If that ever changes, we should implement a file connector that is a wrapper around the reader. emptyConnector, err := sdk.NewEmptyConnector() @@ -140,7 +170,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { sync.WithOnlyExpandGrants(), } - compactionDuration := time.Since(now) + compactionDuration := time.Since(compactionStart) runDuration := c.runDuration - compactionDuration l.Debug("finished compaction", zap.Duration("compaction_duration", compactionDuration)) @@ -262,12 +292,29 @@ func (c *Compactor) doOneCompaction(ctx context.Context, base *CompactableSync, zap.String("applied_sync", applied.SyncID), zap.String("tmp_dir", c.tmpDir), ) - opts := []dotc1z.C1ZOption{ - dotc1z.WithPragma("journal_mode", "WAL"), dotc1z.WithTmpDir(c.tmpDir), } + if c.optimizeInserts { + opts = append(opts, + // Performance improvements: + // Disable journaling. + dotc1z.WithPragma("journal_mode", "OFF"), + // Disable synchronous writes + dotc1z.WithPragma("synchronous", "OFF"), + // Use exclusive locking. + dotc1z.WithPragma("main.locking_mode", "EXCLUSIVE"), + // Use memory for temporary storage. + dotc1z.WithPragma("temp_store", "MEMORY"), + // We close this c1z after compaction, so syncer won't have these pragmas when expanding grants. + ) + } else { + opts = append(opts, + dotc1z.WithPragma("journal_mode", "WAL"), + ) + } + fileName := fmt.Sprintf("compacted-%s-%s.c1z", base.SyncID, applied.SyncID) newFile, err := dotc1z.NewC1ZFile(ctx, path.Join(c.tmpDir, fileName), opts...) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go index 97a036a1..194c16b5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go @@ -186,7 +186,7 @@ func (c *c1ApiTaskManager) finishTask(ctx context.Context, task *v1.Task, resp p _, rpcErr := c.serviceClient.FinishTask(finishCtx, v1.BatonServiceFinishTaskRequest_builder{ TaskId: task.GetId(), Status: &pbstatus.Status{ - //nolint:gosec // No risk of overflow because `Code` is a small enum. + Code: int32(statusErr.Code()), Message: statusErr.Message(), }, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/sessions/sessions.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/sessions/sessions.go index 7b5ca3ca..d2a87fec 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/sessions/sessions.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/sessions/sessions.go @@ -8,7 +8,7 @@ type SessionStoreKey struct{} type SessionStore interface { Get(ctx context.Context, key string, opt ...SessionStoreOption) ([]byte, bool, error) - GetMany(ctx context.Context, keys []string, opt ...SessionStoreOption) (map[string][]byte, error) + GetMany(ctx context.Context, keys []string, opt ...SessionStoreOption) (map[string][]byte, []string, error) Set(ctx context.Context, key string, value []byte, opt ...SessionStoreOption) error SetMany(ctx context.Context, values map[string][]byte, opt ...SessionStoreOption) error Delete(ctx context.Context, key string, opt ...SessionStoreOption) error diff --git a/vendor/modules.txt b/vendor/modules.txt index 1877051d..9800d376 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -178,7 +178,7 @@ github.com/benbjohnson/clock # github.com/cenkalti/backoff/v4 v4.3.0 ## explicit; go 1.18 github.com/cenkalti/backoff/v4 -# github.com/conductorone/baton-sdk v0.5.25 +# github.com/conductorone/baton-sdk v0.6.5 ## explicit; go 1.25.2 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1