From b4ca62d05ff519e55f95576355c848662e0251f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 21:02:58 +0000 Subject: [PATCH] Bump github.com/conductorone/baton-sdk from 0.3.48 to 0.3.51 Bumps [github.com/conductorone/baton-sdk](https://github.com/conductorone/baton-sdk) from 0.3.48 to 0.3.51. - [Release notes](https://github.com/conductorone/baton-sdk/releases) - [Commits](https://github.com/conductorone/baton-sdk/compare/v0.3.48...v0.3.51) --- updated-dependencies: - dependency-name: github.com/conductorone/baton-sdk dependency-version: 0.3.51 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 +- .../baton-sdk/pkg/config/config.go | 4 +- .../baton-sdk/pkg/connectorrunner/runner.go | 2 +- .../pkg/crypto/providers/registry.go | 8 +- .../baton-sdk/pkg/dotc1z/c1file.go | 13 +- .../baton-sdk/pkg/dotc1z/decoder.go | 36 +- .../baton-sdk/pkg/dotc1z/dotc1z.go | 6 +- .../conductorone/baton-sdk/pkg/dotc1z/file.go | 4 +- .../pkg/dotc1z/manager/local/local.go | 22 +- .../baton-sdk/pkg/dotc1z/manager/manager.go | 15 +- .../baton-sdk/pkg/dotc1z/manager/s3/s3.go | 24 +- .../baton-sdk/pkg/field/validation.go | 2 +- .../pkg/ratelimit/mem_ratelimiter.go | 13 +- .../conductorone/baton-sdk/pkg/sdk/version.go | 2 +- .../baton-sdk/pkg/sync/expand/cycle.go | 201 +++--- .../baton-sdk/pkg/sync/expand/graph.go | 56 ++ .../baton-sdk/pkg/sync/expand/scc/bitset.go | 126 ++++ .../baton-sdk/pkg/sync/expand/scc/scc.go | 649 ++++++++++++++++++ .../pkg/sync/expand/scc/test_source.go | 24 + .../conductorone/baton-sdk/pkg/sync/syncer.go | 13 +- .../pkg/synccompactor/attached/attached.go | 13 +- .../baton-sdk/pkg/tasks/c1api/full_sync.go | 42 +- .../baton-sdk/pkg/tasks/c1api/manager.go | 2 +- .../conductorone/baton-sdk/pkg/types/types.go | 2 +- .../baton-sdk/pkg/uhttp/dbcache.go | 2 +- .../baton-sdk/pkg/uhttp/wrapper.go | 2 +- .../conductorone/baton-sdk/pkg/us3/s3.go | 1 - vendor/modules.txt | 3 +- 29 files changed, 1114 insertions(+), 179 deletions(-) create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/bitset.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/scc.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/test_source.go diff --git a/go.mod b/go.mod index 99bd44b9..da1ace5b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/conductorone/baton-sql go 1.24 require ( - github.com/conductorone/baton-sdk v0.3.48 + github.com/conductorone/baton-sdk v0.3.51 github.com/elliotchance/phpserialize v1.4.0 github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/go-sql-driver/mysql v1.9.2 diff --git a/go.sum b/go.sum index 176f16c7..46dde8ea 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,8 @@ github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/conductorone/baton-sdk v0.3.48 h1:qHaQx6JG3JaZuTIuTLyHacWSPBHvU4tq28VHoiTM6ds= -github.com/conductorone/baton-sdk v0.3.48/go.mod h1:L55WO3ERMx1mfpjDgwK3jWNRGRF2E76WrQHmW6ev8VY= +github.com/conductorone/baton-sdk v0.3.51 h1:l652STIrHRmBpq8I1FyMsNxsq3ORii2LyguN8/4v3ak= +github.com/conductorone/baton-sdk v0.3.51/go.mod h1:V3X2R0LnXmcH4yZps9/ZdT6rqWwgn7fIWImSiGriXXw= github.com/conductorone/dpop v0.2.4 h1:PaiDOX1gAIXtOJPxXf08GsGkpCuT/iECEjSJzLpi0zU= github.com/conductorone/dpop v0.2.4/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4 h1:lYxYi9/WTSL9sE96CO0QF2BY3kehs8dTTApI134TGCA= diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go index 4149a39b..cf2fab9b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go @@ -52,7 +52,7 @@ func DefineConfiguration[T field.Configurable]( uniqueFields := make(map[string]field.SchemaField) for _, f := range confschema.Fields { if s, ok := uniqueFields[f.FieldName]; ok { - if !(f.WasReExported || s.WasReExported) { + if !f.WasReExported && !s.WasReExported { return nil, nil, fmt.Errorf("multiple fields with the same name: %s.If you want to use a default field in the SDK, use ExportAs on the connector schema field", f.FieldName) } } @@ -140,7 +140,7 @@ func verifyStructFields[T field.Configurable](schema field.Configuration) error configType = configType.Elem() } if configType.Kind() != reflect.Struct { - return fmt.Errorf("T must be a struct type, got %v", configType.Kind()) + return fmt.Errorf("T must be a struct type, got %v", configType.Kind()) //nolint:staticcheck // we want to capital letter here } for _, field := range schema.Fields { fieldFound := false diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go index 313a1ac1..a0b76df3 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go @@ -228,7 +228,7 @@ func (c *connectorRunner) run(ctx context.Context) error { } if stopForLoop { - return fmt.Errorf("Unable to communicate with gRPC server") + return fmt.Errorf("unable to communicate with gRPC server") } return nil diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/providers/registry.go b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/providers/registry.go index 399be670..4dba8f41 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/providers/registry.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/providers/registry.go @@ -9,7 +9,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/crypto/providers/jwk" ) -var EncryptionProviderNotRegisteredError = fmt.Errorf("crypto/providers: encryption provider not registered") +var ErrEncryptionProviderNotRegistered = fmt.Errorf("crypto/providers: encryption provider not registered") type EncryptionProvider interface { Encrypt(ctx context.Context, conf *v2.EncryptionConfig, plainText *v2.PlaintextData) (*v2.EncryptedData, error) @@ -29,7 +29,7 @@ func normalizeProviderName(name string) string { func GetEncryptionProvider(name string) (EncryptionProvider, error) { provider, ok := providerRegistry[normalizeProviderName(name)] if !ok { - return nil, fmt.Errorf("%w (%s)", EncryptionProviderNotRegisteredError, name) + return nil, fmt.Errorf("%w (%s)", ErrEncryptionProviderNotRegistered, name) } return provider, nil } @@ -37,7 +37,7 @@ func GetEncryptionProvider(name string) (EncryptionProvider, error) { // GetEncryptionProviderForConfig returns the encryption provider for the given config. // If the config specifies a provider, we will fetch it directly by name and return an error if it's not found. // If the config contains a non-nil well-known configuration (like JWKPublicKeyConfig), we will return the provider for that by name. -// If we can't find a provider, we return an EncryptionProviderNotRegisteredError. +// If we can't find a provider, we return an ErrEncryptionProviderNotRegistered. func GetEncryptionProviderForConfig(ctx context.Context, conf *v2.EncryptionConfig) (EncryptionProvider, error) { providerName := normalizeProviderName(conf.GetProvider()) @@ -51,7 +51,7 @@ func GetEncryptionProviderForConfig(ctx context.Context, conf *v2.EncryptionConf // If we don't have a provider by now, bail. if providerName == "" { - return nil, EncryptionProviderNotRegisteredError + return nil, ErrEncryptionProviderNotRegistered } return GetEncryptionProvider(providerName) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go index a424bfc9..ba6f8a13 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go @@ -103,8 +103,9 @@ func NewC1File(ctx context.Context, dbFilePath string, opts ...C1FOption) (*C1Fi } type c1zOptions struct { - tmpDir string - pragmas []pragma + tmpDir string + pragmas []pragma + decoderOptions []DecoderOption } type C1ZOption func(*c1zOptions) @@ -120,6 +121,12 @@ func WithPragma(name string, value string) C1ZOption { } } +func WithDecoderOptions(opts ...DecoderOption) C1ZOption { + return func(o *c1zOptions) { + o.decoderOptions = opts + } +} + // Returns a new C1File instance with its state stored at the provided filename. func NewC1ZFile(ctx context.Context, outputFilePath string, opts ...C1ZOption) (*C1File, error) { ctx, span := tracer.Start(ctx, "NewC1ZFile") @@ -130,7 +137,7 @@ func NewC1ZFile(ctx context.Context, outputFilePath string, opts ...C1ZOption) ( opt(options) } - dbFilePath, err := loadC1z(outputFilePath, options.tmpDir) + dbFilePath, err := loadC1z(outputFilePath, options.tmpDir, options.decoderOptions...) if err != nil { return nil, err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go index e2360c68..68480acc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/decoder.go @@ -58,9 +58,10 @@ type DecoderOption func(*decoderOptions) error // options retains accumulated state of multiple options. type decoderOptions struct { - ctx context.Context - maxDecodedSize uint64 - maxMemorySize uint64 + ctx context.Context + maxDecodedSize uint64 + maxMemorySize uint64 + decoderConcurrency int } // WithContext sets a context, when cancelled, will cause subequent calls to Read() to return ctx.Error(). @@ -73,7 +74,7 @@ func WithContext(ctx context.Context) DecoderOption { // WithDecoderMaxMemory sets the maximum window size for streaming operations. // This can be used to control memory usage of potentially hostile content. -// Maximum is 1 << 63 bytes. Default is 32MiB. +// Maximum is 1 << 63 bytes. Default is 128MiB. func WithDecoderMaxMemory(n uint64) DecoderOption { return func(o *decoderOptions) error { if n == 0 { @@ -103,6 +104,16 @@ func WithDecoderMaxDecodedSize(n uint64) DecoderOption { } } +// WithDecoderConcurrency sets the number of created decoders. +// Default is 1, which disables async decoding/concurrency. +// 0 uses GOMAXPROCS. +func WithDecoderConcurrency(n int) DecoderOption { + return func(o *decoderOptions) error { + o.decoderConcurrency = n + return nil + } +} + type decoder struct { o *decoderOptions f io.Reader @@ -128,11 +139,18 @@ func (d *decoder) Read(p []byte) (int, error) { if maxMemSize == 0 { maxMemSize = defaultDecoderMaxMemory } - zd, err := zstd.NewReader( - d.f, - zstd.WithDecoderConcurrency(1), // disables async decoding/concurrency + + zstdOpts := []zstd.DOption{ zstd.WithDecoderLowmem(true), // uses lower memory, trading potentially more allocations zstd.WithDecoderMaxMemory(maxMemSize), // sets limit on maximum memory used when decoding stream + } + if d.o.decoderConcurrency >= 0 { + zstdOpts = append(zstdOpts, zstd.WithDecoderConcurrency(d.o.decoderConcurrency)) + } + + zd, err := zstd.NewReader( + d.f, + zstdOpts..., ) if err != nil { d.decoderInitErr = err @@ -206,7 +224,9 @@ func NewDecoder(f io.Reader, opts ...DecoderOption) (*decoder, error) { } } - o := &decoderOptions{} + o := &decoderOptions{ + decoderConcurrency: 1, + } for _, opt := range opts { err := opt(o) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/dotc1z.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/dotc1z.go index cc46a12f..835a2958 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/dotc1z.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/dotc1z.go @@ -14,12 +14,12 @@ import ( var tracer = otel.Tracer("baton-sdk/pkg.dotc1z") // NewC1FileReader returns a connectorstore.Reader implementation for the given sqlite db file path. -func NewC1FileReader(ctx context.Context, dbFilePath string) (connectorstore.Reader, error) { - return NewC1File(ctx, dbFilePath) +func NewC1FileReader(ctx context.Context, dbFilePath string, opts ...C1FOption) (connectorstore.Reader, error) { + return NewC1File(ctx, dbFilePath, opts...) } // NewC1ZFileDecoder wraps a given .c1z io.Reader that validates the .c1z and decompresses/decodes the underlying file. -// Defaults: 32MiB max memory and 2GiB max decoded size +// Defaults: 128MiB max memory and 3GiB max decoded size // You must close the resulting io.ReadCloser when you are done, do not forget to close the given io.Reader if necessary. func NewC1ZFileDecoder(f io.Reader, opts ...DecoderOption) (io.ReadCloser, error) { return NewDecoder(f, opts...) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file.go index 9d3db37c..8e0b296e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/file.go @@ -11,7 +11,7 @@ import ( "go.uber.org/zap" ) -func loadC1z(filePath string, tmpDir string) (string, error) { +func loadC1z(filePath string, tmpDir string, opts ...DecoderOption) (string, error) { var err error workingDir, err := os.MkdirTemp(tmpDir, "c1z") if err != nil { @@ -38,7 +38,7 @@ func loadC1z(filePath string, tmpDir string) (string, error) { } defer c1zFile.Close() - r, err := NewDecoder(c1zFile) + r, err := NewDecoder(c1zFile, opts...) if err != nil { return "", err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/local/local.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/local/local.go index 79e8195a..f07f6b6e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/local/local.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/local/local.go @@ -16,9 +16,10 @@ import ( var tracer = otel.Tracer("baton-sdk/pkg.dotc1z.manager.local") type localManager struct { - filePath string - tmpPath string - tmpDir string + filePath string + tmpPath string + tmpDir string + decoderOptions []dotc1z.DecoderOption } type Option func(*localManager) @@ -29,6 +30,12 @@ func WithTmpDir(tmpDir string) Option { } } +func WithDecoderOptions(opts ...dotc1z.DecoderOption) Option { + return func(o *localManager) { + o.decoderOptions = opts + } +} + func (l *localManager) copyFileToTmp(ctx context.Context) error { _, span := tracer.Start(ctx, "localManager.copyFileToTmp") defer span.End() @@ -97,7 +104,14 @@ func (l *localManager) LoadC1Z(ctx context.Context) (*dotc1z.C1File, error) { zap.String("temp_path", l.tmpPath), ) - return dotc1z.NewC1ZFile(ctx, l.tmpPath, dotc1z.WithTmpDir(l.tmpDir), dotc1z.WithPragma("journal_mode", "WAL")) + opts := []dotc1z.C1ZOption{ + dotc1z.WithTmpDir(l.tmpDir), + dotc1z.WithPragma("journal_mode", "WAL"), + } + if len(l.decoderOptions) > 0 { + opts = append(opts, dotc1z.WithDecoderOptions(l.decoderOptions...)) + } + return dotc1z.NewC1ZFile(ctx, l.tmpPath, opts...) } // SaveC1Z saves the C1Z file to the local file system. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/manager.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/manager.go index a3b0bfb1..e9466525 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/manager.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/manager.go @@ -18,7 +18,8 @@ type Manager interface { } type managerOptions struct { - tmpDir string + tmpDir string + decoderOptions []dotc1z.DecoderOption } type ManagerOption func(*managerOptions) @@ -29,6 +30,12 @@ func WithTmpDir(tmpDir string) ManagerOption { } } +func WithDecoderOptions(opts ...dotc1z.DecoderOption) ManagerOption { + return func(o *managerOptions) { + o.decoderOptions = opts + } +} + // Given a file path, return a Manager that can read and write files to that path. // // The first thing we do is check if the file path starts with "s3://". If it does, we return a new @@ -46,12 +53,18 @@ func New(ctx context.Context, filePath string, opts ...ManagerOption) (Manager, if options.tmpDir != "" { s3Opts = append(s3Opts, s3.WithTmpDir(options.tmpDir)) } + if len(options.decoderOptions) > 0 { + s3Opts = append(s3Opts, s3.WithDecoderOptions(options.decoderOptions...)) + } return s3.NewS3Manager(ctx, filePath, s3Opts...) default: var localOpts []local.Option if options.tmpDir != "" { localOpts = append(localOpts, local.WithTmpDir(options.tmpDir)) } + if len(options.decoderOptions) > 0 { + localOpts = append(localOpts, local.WithDecoderOptions(options.decoderOptions...)) + } return local.New(ctx, filePath, localOpts...) } } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/s3/s3.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/s3/s3.go index bca6b1ff..385b1bc4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/s3/s3.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/manager/s3/s3.go @@ -19,10 +19,11 @@ import ( var tracer = otel.Tracer("baton-sdk/pkg.dotc1z.manager.s3") type s3Manager struct { - client *us3.S3Client - fileName string - tmpFile string - tmpDir string + client *us3.S3Client + fileName string + tmpFile string + tmpDir string + decoderOptions []dotc1z.DecoderOption } type Option func(*s3Manager) @@ -33,6 +34,12 @@ func WithTmpDir(tmpDir string) Option { } } +func WithDecoderOptions(opts ...dotc1z.DecoderOption) Option { + return func(o *s3Manager) { + o.decoderOptions = opts + } +} + func (s *s3Manager) copyToTempFile(ctx context.Context, r io.Reader) error { _, span := tracer.Start(ctx, "s3Manager.copyToTempFile") defer span.End() @@ -116,7 +123,14 @@ func (s *s3Manager) LoadC1Z(ctx context.Context) (*dotc1z.C1File, error) { return nil, err } - return dotc1z.NewC1ZFile(ctx, s.tmpFile, dotc1z.WithTmpDir(s.tmpDir), dotc1z.WithPragma("journal_mode", "WAL")) + opts := []dotc1z.C1ZOption{ + dotc1z.WithTmpDir(s.tmpDir), + dotc1z.WithPragma("journal_mode", "WAL"), + } + if len(s.decoderOptions) > 0 { + opts = append(opts, dotc1z.WithDecoderOptions(s.decoderOptions...)) + } + return dotc1z.NewC1ZFile(ctx, s.tmpFile, opts...) } // SaveC1Z saves a file to the AWS S3 bucket. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/validation.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/validation.go index 87474f52..aeebb3e6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/validation.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/validation.go @@ -115,7 +115,7 @@ func isValidDomain(hostname string) bool { // Labels can only contain letters, digits, or hyphens for i := range label { - if !(isAlphaNumeric(label[i]) || label[i] == '-') { + if !isAlphaNumeric(label[i]) && label[i] != '-' { return false } } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/ratelimit/mem_ratelimiter.go b/vendor/github.com/conductorone/baton-sdk/pkg/ratelimit/mem_ratelimiter.go index 9aa60cb5..ec27d7f2 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/ratelimit/mem_ratelimiter.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/ratelimit/mem_ratelimiter.go @@ -6,7 +6,6 @@ import ( "time" ratelimitV1 "github.com/conductorone/baton-sdk/pb/c1/ratelimit/v1" - v1 "github.com/conductorone/baton-sdk/pb/c1/ratelimit/v1" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" rl "go.uber.org/ratelimit" "go.uber.org/zap" @@ -22,20 +21,20 @@ type MemRateLimiter struct { // TODO func (m *MemRateLimiter) Do(ctx context.Context, req *ratelimitV1.DoRequest) (*ratelimitV1.DoResponse, error) { if m.limiter == nil { - return &v1.DoResponse{ + return &ratelimitV1.DoResponse{ RequestToken: req.RequestToken, - Description: &v1.RateLimitDescription{ - Status: v1.RateLimitDescription_STATUS_EMPTY, + Description: &ratelimitV1.RateLimitDescription{ + Status: ratelimitV1.RateLimitDescription_STATUS_EMPTY, }, }, nil } m.limiter.Take() - return &v1.DoResponse{ + return &ratelimitV1.DoResponse{ RequestToken: req.RequestToken, - Description: &v1.RateLimitDescription{ - Status: v1.RateLimitDescription_STATUS_EMPTY, + Description: &ratelimitV1.RateLimitDescription{ + Status: ratelimitV1.RateLimitDescription_STATUS_EMPTY, }, }, nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index 53c0e04c..dc3984ad 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.3.47" +const Version = "v0.3.50" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/cycle.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/cycle.go index 292d0261..52e2ba4c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/cycle.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/cycle.go @@ -1,114 +1,114 @@ package expand import ( - mapset "github.com/deckarep/golang-set/v2" -) + "context" -const ( - colorWhite uint8 = iota - colorGray - colorBlack + "github.com/conductorone/baton-sdk/pkg/sync/expand/scc" + mapset "github.com/deckarep/golang-set/v2" ) -// cycleDetector encapsulates coloring state for cycle detection on an -// EntitlementGraph. Node IDs are dense (1..NextNodeID), so slices are used for -// O(1) access and zero per-op allocations. -type cycleDetector struct { - g *EntitlementGraph - state []uint8 - parent []int -} - -func newCycleDetector(g *EntitlementGraph) *cycleDetector { - cd := &cycleDetector{ - g: g, - state: make([]uint8, g.NextNodeID+1), - parent: make([]int, g.NextNodeID+1), +// GetFirstCycle given an entitlements graph, return a cycle by node ID if it +// exists. Returns nil if no cycle exists. If there is a single +// node pointing to itself, that will count as a cycle. +func (g *EntitlementGraph) GetFirstCycle(ctx context.Context) []int { + if g.HasNoCycles { + return nil } - for i := range cd.parent { - cd.parent[i] = -1 + comps, _ := g.ComputeCyclicComponents(ctx) + if len(comps) == 0 { + return nil } - return cd + return comps[0] } -// dfs performs a coloring-based DFS from u, returning the first detected cycle -// as a slice of node IDs or nil if no cycle is reachable from u. -func (cd *cycleDetector) dfs(u int) ([]int, bool) { - // Self-loop fast path. - if nbrs, ok := cd.g.SourcesToDestinations[u]; ok { - if _, ok := nbrs[u]; ok { - return []int{u}, true - } +// HasCycles returns true if the graph contains any cycle. +func (g *EntitlementGraph) HasCycles(ctx context.Context) bool { + if g.HasNoCycles { + return false } + comps, _ := g.ComputeCyclicComponents(ctx) + return len(comps) > 0 +} - cd.state[u] = colorGray - if nbrs, ok := cd.g.SourcesToDestinations[u]; ok { - for v := range nbrs { - switch cd.state[v] { - case colorWhite: - cd.parent[v] = u - if cyc, ok := cd.dfs(v); ok { - return cyc, true - } - case colorGray: - // Back-edge to a node on the current recursion stack. - // Reconstruct cycle by walking parents from u back to v (inclusive), then reverse. - cycle := make([]int, 0, 8) - for x := u; ; x = cd.parent[x] { - cycle = append(cycle, x) - if x == v || cd.parent[x] == -1 { - break - } - } - for i, j := 0, len(cycle)-1; i < j; i, j = i+1, j-1 { - cycle[i], cycle[j] = cycle[j], cycle[i] - } - return cycle, true - } +func (g *EntitlementGraph) cycleDetectionHelper( + ctx context.Context, + nodeID int, +) ([]int, bool) { + reach := g.reachableFrom(nodeID) + if len(reach) == 0 { + return nil, false + } + fg := filteredGraph{g: g, include: func(id int) bool { _, ok := reach[id]; return ok }} + groups, _ := scc.CondenseFWBW(ctx, fg, scc.DefaultOptions()) + for _, comp := range groups { + if len(comp) > 1 || (len(comp) == 1 && g.hasSelfLoop(comp[0])) { + return comp, true } } - cd.state[u] = colorBlack return nil, false } -// FindAny scans all nodes and returns the first detected cycle or nil if none exist. -func (cd *cycleDetector) FindAny() []int { - for nodeID := range cd.g.Nodes { - if cd.state[nodeID] != colorWhite { - continue - } - if cyc, ok := cd.dfs(nodeID); ok { - return cyc +func (g *EntitlementGraph) FixCycles(ctx context.Context) error { + comps, _ := g.ComputeCyclicComponents(ctx) + return g.FixCyclesFromComponents(ctx, comps) +} + +// ComputeCyclicComponents runs SCC once and returns only cyclic components. +// A component is cyclic if len>1 or a singleton with a self-loop. +func (g *EntitlementGraph) ComputeCyclicComponents(ctx context.Context) ([][]int, *scc.Metrics) { + if g.HasNoCycles { + return nil, nil + } + groups, metrics := scc.CondenseFWBW(ctx, g, scc.DefaultOptions()) + cyclic := make([][]int, 0) + for _, comp := range groups { + if len(comp) > 1 || (len(comp) == 1 && g.hasSelfLoop(comp[0])) { + cyclic = append(cyclic, comp) } } - return nil + return cyclic, metrics } -// FindFrom starts cycle detection from a specific node and returns the first -// cycle reachable from that node, or nil,false if none. -func (cd *cycleDetector) FindFrom(start int) ([]int, bool) { - return cd.dfs(start) +// hasSelfLoop reports whether a node has a self-edge. +func (g *EntitlementGraph) hasSelfLoop(id int) bool { + if row, ok := g.SourcesToDestinations[id]; ok { + _, ok := row[id] + return ok + } + return false } -// GetFirstCycle given an entitlements graph, return a cycle by node ID if it -// exists. Returns nil if no cycle exists. If there is a single -// node pointing to itself, that will count as a cycle. -func (g *EntitlementGraph) GetFirstCycle() []int { - if g.HasNoCycles { - return nil +// filteredGraph restricts EntitlementGraph iteration to nodes for which include(id) is true. +type filteredGraph struct { + g *EntitlementGraph + include func(int) bool +} + +func (fg filteredGraph) ForEachNode(fn func(id int) bool) { + for id := range fg.g.Nodes { + if fg.include != nil && !fg.include(id) { + continue + } + if !fn(id) { + return + } } - cd := newCycleDetector(g) - return cd.FindAny() } -func (g *EntitlementGraph) cycleDetectionHelper( - nodeID int, -) ([]int, bool) { - // Thin wrapper around the coloring-based DFS, starting from a specific node. - // The provided visited/currentCycle are ignored here; coloring provides the - // necessary state for correctness and performance. - cd := newCycleDetector(g) - return cd.FindFrom(nodeID) +func (fg filteredGraph) ForEachEdgeFrom(src int, fn func(dst int) bool) { + if fg.include != nil && !fg.include(src) { + return + } + if dsts, ok := fg.g.SourcesToDestinations[src]; ok { + for dst := range dsts { + if fg.include != nil && !fg.include(dst) { + continue + } + if !fn(dst) { + return + } + } + } } // removeNode obliterates a node and all incoming/outgoing edges. @@ -145,30 +145,33 @@ func (g *EntitlementGraph) removeNode(nodeID int) { delete(g.SourcesToDestinations, nodeID) } -// FixCycles if any cycles of nodes exist, merge all nodes in that cycle into a -// single node and then repeat. Iteration ends when there are no more cycles. -func (g *EntitlementGraph) FixCycles() error { +// FixCyclesFromComponents merges all provided cyclic components in one pass. +func (g *EntitlementGraph) FixCyclesFromComponents(ctx context.Context, cyclic [][]int) error { if g.HasNoCycles { return nil } - cycle := g.GetFirstCycle() - if cycle == nil { + if len(cyclic) == 0 { g.HasNoCycles = true return nil } - - if err := g.fixCycle(cycle); err != nil { - return err + for _, comp := range cyclic { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if err := g.fixCycle(comp); err != nil { + return err + } } - - // Recurse! - return g.FixCycles() + g.HasNoCycles = true + return nil } // fixCycle takes a list of Node IDs that form a cycle and merges them into a // single, new node. func (g *EntitlementGraph) fixCycle(nodeIDs []int) error { - entitlementIDs := mapset.NewSet[string]() + entitlementIDs := mapset.NewThreadUnsafeSet[string]() outgoingEdgesToResourceTypeIDs := map[int]mapset.Set[string]{} incomingEdgesToResourceTypeIDs := map[int]mapset.Set[string]{} for _, nodeID := range nodeIDs { @@ -184,7 +187,7 @@ func (g *EntitlementGraph) fixCycle(nodeIDs []int) error { if edge, ok := g.Edges[edgeID]; ok { resourceTypeIDs, ok := incomingEdgesToResourceTypeIDs[sourceNodeID] if !ok { - resourceTypeIDs = mapset.NewSet[string]() + resourceTypeIDs = mapset.NewThreadUnsafeSet[string]() } for _, resourceTypeID := range edge.ResourceTypeIDs { resourceTypeIDs.Add(resourceTypeID) @@ -200,7 +203,7 @@ func (g *EntitlementGraph) fixCycle(nodeIDs []int) error { if edge, ok := g.Edges[edgeID]; ok { resourceTypeIDs, ok := outgoingEdgesToResourceTypeIDs[destinationNodeID] if !ok { - resourceTypeIDs = mapset.NewSet[string]() + resourceTypeIDs = mapset.NewThreadUnsafeSet[string]() } for _, resourceTypeID := range edge.ResourceTypeIDs { resourceTypeIDs.Add(resourceTypeID) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go index 8696b6d4..ef7d9850 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go @@ -4,6 +4,7 @@ import ( "context" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/sync/expand/scc" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" ) @@ -309,3 +310,58 @@ func (g *EntitlementGraph) DeleteEdge(ctx context.Context, srcEntitlementID stri } return nil } + +// toAdjacency builds an adjacency map for SCC. If nodesSubset is non-nil, only +// include those nodes (and edges between them). Always include all nodes in the +// subset as keys, even if they have zero outgoing edges. +// toAdjacency removed: use SCC via scc.Source on EntitlementGraph + +var _ scc.Source = (*EntitlementGraph)(nil) + +// ForEachNode implements scc.Source iteration over nodes (including isolated nodes). +// It does not import scc; matching the method names/signatures is sufficient. +func (g *EntitlementGraph) ForEachNode(fn func(id int) bool) { + for id := range g.Nodes { + if !fn(id) { + return + } + } +} + +// ForEachEdgeFrom implements scc.Source iteration of outgoing edges for src. +// It enumerates unique destination node IDs. +func (g *EntitlementGraph) ForEachEdgeFrom(src int, fn func(dst int) bool) { + if dsts, ok := g.SourcesToDestinations[src]; ok { + for dst := range dsts { + if !fn(dst) { + return + } + } + } +} + +// reachableFrom computes the set of node IDs reachable from start over +// SourcesToDestinations using an iterative BFS. +func (g *EntitlementGraph) reachableFrom(start int) map[int]struct{} { + if _, ok := g.Nodes[start]; !ok { + return nil + } + visited := make(map[int]struct{}, 16) + queue := make([]int, 0, 16) + queue = append(queue, start) + visited[start] = struct{}{} + for len(queue) > 0 { + u := queue[0] + queue = queue[1:] + if nbrs, ok := g.SourcesToDestinations[u]; ok { + for v := range nbrs { + if _, seen := visited[v]; seen { + continue + } + visited[v] = struct{}{} + queue = append(queue, v) + } + } + } + return visited +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/bitset.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/bitset.go new file mode 100644 index 00000000..e017d891 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/bitset.go @@ -0,0 +1,126 @@ +package scc + +import ( + "math/bits" + "sync/atomic" +) + +// bitset is a packed, atomically updatable bitset. +// +// Concurrency notes: +// - Only testAndSetAtomic and clearAtomic are safe concurrently. +// - All other methods must not race with writers. +// - Slice storage aligns on 64-bit boundaries for atomic ops. +type bitset struct{ w []uint64 } + +func newBitset(n int) *bitset { + if n <= 0 { + return &bitset{} + } + return &bitset{w: make([]uint64, (n+63)>>6)} +} + +func (b *bitset) test(i int) bool { + if i < 0 { + return false + } + w := i >> 6 + if w >= len(b.w) { + return false + } + return (b.w[w] & (1 << (uint(i) & 63))) != 0 +} + +func (b *bitset) set(i int) { + if i < 0 { + return + } + w := i >> 6 + b.w[w] |= 1 << (uint(i) & 63) +} + +func (b *bitset) testAndSetAtomic(i int) bool { + if i < 0 { + return false + } + w := i >> 6 + if w >= len(b.w) { + return false + } + mask := uint64(1) << (uint(i) & 63) + addr := &b.w[w] + for { + old := atomic.LoadUint64(addr) + if old&mask != 0 { + return true + } + if atomic.CompareAndSwapUint64(addr, old, old|mask) { + return false + } + } +} + +func (b *bitset) clearAtomic(i int) { + if i < 0 { + return + } + w := i >> 6 + if w >= len(b.w) { + return + } + mask := ^(uint64(1) << (uint(i) & 63)) + addr := &b.w[w] + for { + old := atomic.LoadUint64(addr) + if atomic.CompareAndSwapUint64(addr, old, old&mask) { + return + } + } +} + +func (b *bitset) clone() *bitset { + cp := &bitset{w: make([]uint64, len(b.w))} + copy(cp.w, b.w) + return cp +} + +func (b *bitset) and(x *bitset) *bitset { + for i := range b.w { + b.w[i] &= x.w[i] + } + return b +} + +func (b *bitset) or(x *bitset) *bitset { + for i := range b.w { + b.w[i] |= x.w[i] + } + return b +} + +func (b *bitset) andNot(x *bitset) *bitset { + for i := range b.w { + b.w[i] &^= x.w[i] + } + return b +} + +func (b *bitset) isEmpty() bool { + for _, w := range b.w { + if w != 0 { + return false + } + } + return true +} + +func (b *bitset) forEachSet(fn func(i int)) { + for wi, w := range b.w { + for w != 0 { + tz := bits.TrailingZeros64(w) + i := (wi << 6) + tz + fn(i) + w &^= 1 << uint(tz) //nolint:gosec // trailing zeros is non-negative + } + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/scc.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/scc.go new file mode 100644 index 00000000..5059aa72 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/scc.go @@ -0,0 +1,649 @@ +// Package scc provides an iterative FW–BW SCC condensation for directed graphs, +// adapted for Baton’s entitlement graph. It builds an immutable CSR + transpose, +// runs reachability-based SCC with a stack-based driver (no recursion, BFS may +// run in parallel), and returns components as groups of your original node IDs. +package scc + +// Iterative FW–BW SCC condensation for directed graphs. +// +// High-level algorithm: Build CSR + transpose; maintain a LIFO stack of +// subproblems (bitset masks). For each mask: +// 1) Trim sources/sinks repeatedly; each peeled vertex is a singleton SCC. +// 2) Pick a pivot (lowest-index active vertex), run forward/backward BFS +// restricted to the mask to get F and B. +// 3) The SCC is C = F ∩ B. Assign its component id and clear those bits +// from the mask. +// 4) Partition the remaining mask into F\C, B\C, and U = mask \ (F ∪ B), +// and push the non-empty masks onto the stack in a deterministic order. +// +// Parallelism is contained inside BFS (bounded by Options.MaxWorkers) and no +// recursive goroutines are spawned by the driver. Determinism is achieved via +// deterministic CSR construction (sorted ids and neighbors) and by using the +// lowest-index active pivot with a fixed child-push order. + +import ( + "context" + "runtime" + "sort" + "sync" + "time" +) + +// Options controls SCC execution. +// +// MaxWorkers bounds BFS concurrency per level. Deterministic toggles stable +// CSR index assignment and neighbor ordering. +type Options struct { + MaxWorkers int + Deterministic bool +} + +// DefaultOptions returns a sensible baseline. +func DefaultOptions() Options { + return Options{ + MaxWorkers: runtime.GOMAXPROCS(0), + Deterministic: false, + } +} + +// CSR is a compact adjacency for G and its transpose Gᵗ. +// Indices are 0..N-1; IdxToNodeID maps back to the original node IDs. +// +// Invariants (validated by validateCSR): +// - len(Row) == N+1; Row[0] == 0; Row is non-decreasing; Row[N] == len(Col) +// - len(TRow) == N+1; TRow[0] == 0; TRow is non-decreasing; TRow[N] == len(TCol) +// - 0 <= Col[p] < N for all p; 0 <= TCol[p] < N for all p +// - len(IdxToNodeID) == N; NodeIDToIdx[IdxToNodeID[i]] == i for all i +// - For each v, (TRow[v+1]-TRow[v]) equals the number of occurrences of v in Col +// (transpose degree matches inbound counts) +type CSR struct { + N int + Row []int // len N+1 + Col []int // len = m, m = Row[N] + TRow []int // len N+1 + TCol []int // len = m, m = TRow[N] + IdxToNodeID []int // len N +} + +// Source is a minimal read-only graph provider used to build CSR without +// materializing an intermediate adjacency map. It must enumerate all nodes +// (including isolated nodes) and for each node provide its unique outgoing +// destinations. +type Source interface { + ForEachNode(fn func(id int) bool) + ForEachEdgeFrom(src int, fn func(dst int) bool) +} + +// Metrics captures a few summary counters for a condense run. +type Metrics struct { + Nodes int + Edges int + Components int + Peeled int + MasksProcessed int + MasksPushed int + BFScalls int + Duration time.Duration + MaxWorkers int + Deterministic bool +} + +// CondenseFWBW runs SCC directly from a streaming Source. Preferred entry point. +func CondenseFWBW(ctx context.Context, src Source, opts Options) ([][]int, *Metrics) { + if opts.MaxWorkers <= 0 { + opts.MaxWorkers = runtime.GOMAXPROCS(0) + } + start := time.Now() + csr := buildCSRFromSource(src, opts) + metrics := Metrics{ + Nodes: csr.N, + Edges: len(csr.Col), + MaxWorkers: opts.MaxWorkers, + Deterministic: opts.Deterministic, + } + comp := make([]int, csr.N) + for i := range comp { + comp[i] = -1 + } + nextID := sccFWBWIterative(ctx, csr, comp, opts, &metrics) + + groups := make([][]int, nextID) + for idx := range csr.N { + cid := comp[idx] + if cid < 0 { + cid = nextID + nextID++ + comp[idx] = cid + if cid >= len(groups) { + tmp := make([][]int, cid+1) + copy(tmp, groups) + groups = tmp + } + } + groups[cid] = append(groups[cid], csr.IdxToNodeID[idx]) + } + metrics.Components = nextID + metrics.Duration = time.Since(start) + return groups, &metrics +} + +// buildCSRFromSource constructs CSR/transpose from a Source without +// materializing an intermediate adjacency map. If opts.Deterministic, node IDs +// are sorted and per-row neighbors are written in ascending order. +func buildCSRFromSource(src Source, opts Options) *CSR { + // 1) Collect nodes + nodes := make([]int, 0, 1024) + src.ForEachNode(func(id int) bool { + nodes = append(nodes, id) + return true + }) + if opts.Deterministic { + sort.Ints(nodes) + } + id2idx := make(map[int]int, len(nodes)) + for i, id := range nodes { + id2idx[id] = i + } + n := len(nodes) + + // 2) Count out-degrees and total edges + outDeg := make([]int, n) + m := 0 + for i := range n { + srcID := nodes[i] + src.ForEachEdgeFrom(srcID, func(dst int) bool { + j, ok := id2idx[dst] + if !ok { + return true + } + _ = j // only used to validate membership + outDeg[i]++ + m++ + return true + }) + } + + // 3) Allocate Row/Col + row := make([]int, n+1) + for i := range n { + row[i+1] = row[i] + outDeg[i] + } + col := make([]int, m) + cur := make([]int, n) + copy(cur, row) + + // 4) Fill rows + if opts.Deterministic { + for i := range n { + srcID := nodes[i] + neighbors := make([]int, 0, outDeg[i]) + src.ForEachEdgeFrom(srcID, func(dst int) bool { + if j, ok := id2idx[dst]; ok { + neighbors = append(neighbors, j) + } + return true + }) + sort.Ints(neighbors) + off := cur[i] + copy(col[off:off+len(neighbors)], neighbors) + cur[i] += len(neighbors) + } + } else { + for i := range n { + srcID := nodes[i] + src.ForEachEdgeFrom(srcID, func(dst int) bool { + if j, ok := id2idx[dst]; ok { + pos := cur[i] + col[pos] = j + cur[i] = pos + 1 + } + return true + }) + } + } + + // 5) Transpose + inDeg := make([]int, n) + for _, v := range col { + inDeg[v]++ + } + trow := make([]int, n+1) + for i := range n { + trow[i+1] = trow[i] + inDeg[i] + } + tcol := make([]int, m) + tcur := make([]int, n) + copy(tcur, trow) + for u := range n { + start, end := row[u], row[u+1] + for p := start; p < end; p++ { + v := col[p] + pos := tcur[v] + tcol[pos] = u + tcur[v] = pos + 1 + } + } + + csr := &CSR{ + N: n, + Row: row, + Col: col, + TRow: trow, + TCol: tcol, + IdxToNodeID: nodes, + } + validateCSR(csr) + return csr +} + +// validateCSR performs internal consistency checks on CSR and panics +// with a descriptive message when a violation is found. This is intended to +// catch programmer errors at build time and in tests; it runs unconditionally. +func validateCSR(csr *CSR) { + if csr == nil { + panic("scc: CSR is nil") + } + n := csr.N + if n < 0 { + panic("scc: CSR.N is negative") + } + if len(csr.Row) != n+1 { + panic("scc: len(Row) != N+1") + } + if len(csr.TRow) != n+1 { + panic("scc: len(TRow) != N+1") + } + if len(csr.IdxToNodeID) != n { + panic("scc: len(IdxToNodeID) != N") + } + // Row invariants and degree sums + if csr.Row[0] != 0 { + panic("scc: Row[0] != 0") + } + for i := range len(csr.Row) - 1 { + if csr.Row[i] > csr.Row[i+1] { + panic("scc: Row is not non-decreasing") + } + } + m := csr.Row[n] + if m != len(csr.Col) { + panic("scc: Row[N] != len(Col)") + } + // TRow invariants + if csr.TRow[0] != 0 { + panic("scc: TRow[0] != 0") + } + for i := range len(csr.TRow) - 1 { + if csr.TRow[i] > csr.TRow[i+1] { + panic("scc: TRow is not non-decreasing") + } + } + mt := csr.TRow[n] + if mt != len(csr.TCol) { + panic("scc: TRow[N] != len(TCol)") + } + // Col bounds + for p := range len(csr.Col) { + v := csr.Col[p] + if v < 0 || v >= n { + panic("scc: Col index out of range") + } + } + for p := range len(csr.TCol) { + v := csr.TCol[p] + if v < 0 || v >= n { + panic("scc: TCol index out of range") + } + } + // NodeID mapping bijection check removed: CSR does not store NodeIDToIdx. + // Transpose degree equals inbound counts + inDeg := make([]int, n) + for _, v := range csr.Col { + inDeg[v]++ + } + for v := range n { + expected := inDeg[v] + span := csr.TRow[v+1] - csr.TRow[v] + if span != expected { + panic("scc: transpose degree mismatch") + } + } +} + +// bitset moved to bitset.go + +// sccFWBWIterative implements the driver loop described at the top. +func sccFWBWIterative(ctx context.Context, csr *CSR, comp []int, opts Options, metrics *Metrics) int { + nextID := 0 + + // Initialize root mask with all vertices. + root := newBitset(csr.N) + for i := range csr.N { + root.set(i) + } + + type item struct{ mask *bitset } + stack := make([]item, 0, 64) + stack = append(stack, item{mask: root}) + + for len(stack) > 0 { + select { + case <-ctx.Done(): + return nextID + default: + } + + it := stack[len(stack)-1] + stack = stack[:len(stack)-1] + active := it.mask + if metrics != nil { + metrics.MasksProcessed++ + } + + // Trim loop: peel sources/sinks; each peeled vertex becomes its own SCC. + for { + if n := trimSingletons(csr, active, comp, &nextID); n == 0 { + break + } else if metrics != nil { + metrics.Peeled += n + } + if active.isEmpty() { + break + } + } + if active.isEmpty() { + continue + } + + // Pivot and BFS (restricted to active mask). + pivot := firstActive(active) + f := bfsMultiSource(ctx, csr, []int{pivot}, active, false, opts.MaxWorkers) + b := bfsMultiSource(ctx, csr, []int{pivot}, active, true, opts.MaxWorkers) + if metrics != nil { + metrics.BFScalls += 2 + } + + // Component and partition masks. + c := f.clone().and(b) + assignComponent(c, comp, &nextID, active) + + fNotC := f.clone().andNot(c) + bNotC := b.clone().andNot(c) + fOrB := f.clone().or(b) + u := active.clone().andNot(fOrB) + + // assignComponent cleared C from 'active'; child masks are disjoint subsets + // of the original mask. Push children in a fixed order for determinism. + pushes := 0 + if !u.isEmpty() { + stack = append(stack, item{mask: u}) + pushes++ + } + if !bNotC.isEmpty() { + stack = append(stack, item{mask: bNotC}) + pushes++ + } + if !fNotC.isEmpty() { + stack = append(stack, item{mask: fNotC}) + pushes++ + } + if metrics != nil { + metrics.MasksPushed += pushes + } + } + + return nextID +} + +// bfsMultiSource runs a parallel BFS from sources over csr. +// If useTranspose is true, traverses csr.T* arrays. Traversal respects +// 'active' mask; returns visited including sources. +// Cancellation: checks ctx between levels. +func bfsMultiSource(ctx context.Context, csr *CSR, sources []int, active *bitset, useTranspose bool, maxWorkers int) *bitset { + if maxWorkers <= 0 { + maxWorkers = runtime.GOMAXPROCS(0) + } + visited := newBitset(csr.N) + frontier := frontierSeed(sources, active, visited) + if len(frontier) == 0 { + return visited + } + + getRow := func(u int) (int, int) { + if useTranspose { + return csr.TRow[u], csr.TRow[u+1] + } + return csr.Row[u], csr.Row[u+1] + } + getCol := func(p int) int { + if useTranspose { + return csr.TCol[p] + } + return csr.Col[p] + } + + for len(frontier) > 0 { + select { + case <-ctx.Done(): + return visited + default: + } + + workers := min(maxWorkers, len(frontier)) + var wg sync.WaitGroup + wg.Add(workers) + + chunkSize := (len(frontier) + workers - 1) / workers + nextBuckets := make([][]int, workers) + + for w := 0; w < workers; w++ { + start := w * chunkSize + end := start + chunkSize + if start >= len(frontier) { + wg.Done() + continue + } + end = min(end, len(frontier)) + + w := w // capture + go func(start, end int) { + defer wg.Done() + local := make([]int, 0, 256) + for i := start; i < end; i++ { + u := frontier[i] + rs, re := getRow(u) + for p := rs; p < re; p++ { + v := getCol(p) + if !active.test(v) { + continue + } + if !visited.testAndSetAtomic(v) { + local = append(local, v) + } + } + } + nextBuckets[w] = local + }(start, end) + } + wg.Wait() + + total := 0 + for _, b := range nextBuckets { + total += len(b) + } + next := make([]int, total) + off := 0 + for _, b := range nextBuckets { + copy(next[off:], b) + off += len(b) + } + frontier = next + } + + return visited +} + +func frontierSeed(sources []int, active, visited *bitset) []int { + out := make([]int, 0, len(sources)) + for _, s := range sources { + if s < 0 { + continue + } + if !active.test(s) { + continue + } + if visited.testAndSetAtomic(s) { + continue + } + out = append(out, s) + } + return out +} + +func firstActive(active *bitset) int { + pivot := -1 + active.forEachSet(func(i int) { + if pivot == -1 { + pivot = i + } + }) + return pivot +} + +// assignComponent writes comp ids for cMask, and clears those vertices from 'active'. +func assignComponent(cMask *bitset, comp []int, nextID *int, active *bitset) { + if cMask.isEmpty() { + return + } + cid := *nextID + *nextID++ + cMask.forEachSet(func(i int) { + comp[i] = cid + active.clearAtomic(i) + }) +} + +// Degree-array pool for trim to reduce allocations for small graphs. +var ( + intSlicePool sync.Pool // *([]int) +) + +func getIntSlice(n int) []int { + p, _ := intSlicePool.Get().(*[]int) + if p == nil || cap(*p) < n { + return make([]int, n) + } + s := (*p)[:n] + for i := range s { + s[i] = 0 + } + return s +} + +func putIntSlice(s []int) { + // avoid keeping very large slices + if cap(s) > 1<<14 { + return + } + intSlicePool.Put(&s) +} + +// trimSingletons peels vertices with restricted in/out degree within 'active'. +// Each peeled vertex becomes its own SCC id. Returns count peeled. +func trimSingletons(csr *CSR, active *bitset, comp []int, nextID *int) int { + n := csr.N + inDeg := getIntSlice(n) + outDeg := getIntSlice(n) + defer func() { putIntSlice(inDeg); putIntSlice(outDeg) }() + + // Out-degree within active. + for u := range n { + if !active.test(u) { + continue + } + rs, re := csr.Row[u], csr.Row[u+1] + d := 0 + for p := rs; p < re; p++ { + v := csr.Col[p] + if active.test(v) { + d++ + } + } + outDeg[u] = d + } + // In-degree within active (via transpose). + for v := range n { + if !active.test(v) { + continue + } + rs, re := csr.TRow[v], csr.TRow[v+1] + d := 0 + for p := rs; p < re; p++ { + u := csr.TCol[p] + if active.test(u) { + d++ + } + } + inDeg[v] = d + } + + // Initialize queue of zeros. + queue := getIntSlice(0) + defer putIntSlice(queue) + for i := range n { + if !active.test(i) { + continue + } + if inDeg[i] == 0 || outDeg[i] == 0 { + queue = append(queue, i) + } + } + + peeled := 0 + for len(queue) > 0 { + u := queue[len(queue)-1] + queue = queue[:len(queue)-1] + if !active.test(u) { + continue + } + if inDeg[u] > 0 && outDeg[u] > 0 { + continue + } + + // assign and remove u + comp[u] = *nextID + *nextID++ + active.clearAtomic(u) + peeled++ + + // Decrement out-neighbors' inDeg + rs, re := csr.Row[u], csr.Row[u+1] + for p := rs; p < re; p++ { + v := csr.Col[p] + if !active.test(v) { + continue + } + if inDeg[v] > 0 { + inDeg[v]-- + if inDeg[v] == 0 { + queue = append(queue, v) + } + } + } + // Decrement in-neighbors' outDeg + rs, re = csr.TRow[u], csr.TRow[u+1] + for p := rs; p < re; p++ { + w := csr.TCol[p] + if !active.test(w) { + continue + } + if outDeg[w] > 0 { + outDeg[w]-- + if outDeg[w] == 0 { + queue = append(queue, w) + } + } + } + } + return peeled +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/test_source.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/test_source.go new file mode 100644 index 00000000..16b5b244 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/scc/test_source.go @@ -0,0 +1,24 @@ +package scc + +// adjSource adapts a map[int]map[int]int adjacency to the Source interface for tests. +type adjSource struct { + adj map[int]map[int]int +} + +func (a adjSource) ForEachNode(fn func(id int) bool) { + for id := range a.adj { + if !fn(id) { + return + } + } +} + +func (a adjSource) ForEachEdgeFrom(src int, fn func(dst int) bool) { + if row, ok := a.adj[src]; ok { + for dst := range row { + if !fn(dst) { + return + } + } + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index eef498c9..7127ba87 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -1362,18 +1362,19 @@ func (s *syncer) SyncGrantExpansion(ctx context.Context) error { } if entitlementGraph.Loaded { - cycle := entitlementGraph.GetFirstCycle() - if cycle != nil { + comps, sccMetrics := entitlementGraph.ComputeCyclicComponents(ctx) + if len(comps) > 0 { + // Log a sample cycle l.Warn( "cycle detected in entitlement graph", - zap.Any("cycle", cycle), + zap.Any("cycle", comps[0]), + zap.Any("scc_metrics", sccMetrics), ) l.Debug("initial graph", zap.Any("initial graph", entitlementGraph)) if dontFixCycles { return fmt.Errorf("cycles detected in entitlement graph") } - - err := entitlementGraph.FixCycles() + err := entitlementGraph.FixCyclesFromComponents(ctx, comps) if err != nil { return err } @@ -2412,7 +2413,7 @@ func (s *syncer) runGrantExpandActions(ctx context.Context) (bool, error) { return false, fmt.Errorf("runGrantExpandActions: error fetching source grants: %w", err) } - var newGrants []*v2.Grant = make([]*v2.Grant, 0) + var newGrants = make([]*v2.Grant, 0) for _, sourceGrant := range sourceGrants.List { // Skip this grant if it is not for a resource type we care about if len(action.ResourceTypeIDs) > 0 { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go index 09ae95c6..000c3aa4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" ) type Compactor struct { @@ -45,8 +47,12 @@ func (c *Compactor) CompactWithSyncID(ctx context.Context, destSyncID string) er if err != nil { return fmt.Errorf("failed to attach databases to destination: %w", err) } + l := ctxzap.Extract(ctx) defer func() { - _, _ = base.DetachFile("base") + _, err := base.DetachFile("base") + if err != nil { + l.Error("failed to detach file", zap.Error(err)) + } }() // Attach both the base and applied databases to the destination @@ -55,7 +61,10 @@ func (c *Compactor) CompactWithSyncID(ctx context.Context, destSyncID string) er return fmt.Errorf("failed to attach databases to destination: %w", err) } defer func() { - _, _ = attached.DetachFile("attached") + _, err := attached.DetachFile("attached") + if err != nil { + l.Error("failed to detach file", zap.Error(err)) + } }() if err := c.processRecords(ctx, attached, destSyncID, baseSyncID, appliedSyncID); err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go index bb8c6821..6fd5300e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/full_sync.go @@ -201,45 +201,45 @@ func uploadDebugLogs(ctx context.Context, helper fullSyncHelpers) error { l.Warn("unable to get the current working directory", zap.Error(err)) } if wd != "" { - l.Warn("no temporal folder found on this system according to our sync helper,"+ + l.Warn("no temporary folder found on this system according to our sync helper,"+ " we may create files in the current working directory by mistake as a result", zap.String("current working directory", wd)) } else { - l.Warn("no temporal folder found on this system according to our sync helper") + l.Warn("no temporary folder found on this system according to our sync helper") } } - debugfilelocation := filepath.Join(tempDir, "debug.log") + debugPath := filepath.Join(tempDir, "debug.log") - _, err := os.Stat(debugfilelocation) + _, err := os.Stat(debugPath) if err != nil { switch { case errors.Is(err, os.ErrNotExist): - l.Warn("debug log file does not exists", zap.Error(err)) + l.Debug("debug log file does not exist", zap.Error(err)) case errors.Is(err, os.ErrPermission): l.Warn("debug log file cannot be stat'd due to lack of permissions", zap.Error(err)) default: l.Warn("cannot stat debug log file", zap.Error(err)) } return nil - } else { - debugfile, err := os.Open(debugfilelocation) - if err != nil { - return err - } - defer debugfile.Close() + } - l.Info("uploading debug logs", zap.String("file", debugfilelocation)) - err = helper.Upload(ctx, debugfile) + debugfile, err := os.Open(debugPath) + if err != nil { + return err + } + defer func() { + err := os.Remove(debugPath) if err != nil { - return err + l.Error("failed to delete file with debug logs", zap.Error(err), zap.String("file", debugPath)) } - defer func() { - err := os.Remove(debugfilelocation) - if err != nil { - l.Error("failed to delete file with debug logs", zap.Error(err), zap.String("file", debugfilelocation)) - } - }() + }() + defer debugfile.Close() - return nil + l.Info("uploading debug logs", zap.String("file", debugPath)) + err = helper.Upload(ctx, debugfile) + if err != nil { + return err } + + return nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go index ad3867e3..21866f93 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/manager.go @@ -132,7 +132,7 @@ func (c *c1ApiTaskManager) Next(ctx context.Context) (*v1.Task, time.Duration, e zap.Stringer("task_type", tasks.GetType(resp.GetTask())), ) - l.Debug("c1_api_task_manager.Next(): got task") + l.Debug("c1_api_task_manager.Next(): got task", zap.Duration("next_poll", nextPoll)) return resp.GetTask(), nextPoll, nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/types.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/types.go index 191c0a9b..fc52f6f4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/types.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/types.go @@ -1,4 +1,4 @@ -package types +package types //nolint:revive // we want this to be "types" import ( "context" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/dbcache.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/dbcache.go index 9fb32424..459990df 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/dbcache.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/dbcache.go @@ -192,7 +192,7 @@ func (d *DBCache) removeDB(ctx context.Context) error { // Get returns cached response (if exists). func (d *DBCache) Get(req *http.Request) (*http.Response, error) { var ( - isFound bool = false + isFound = false resp *http.Response ) key, err := CreateCacheKey(req) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go index 9093baa5..0073f1c1 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go @@ -605,7 +605,7 @@ func WithBearerToken(token string) RequestOption { func (c *BaseHttpClient) NewRequest(ctx context.Context, method string, url *url.URL, options ...RequestOption) (*http.Request, error) { var buffer io.ReadWriter - var headers map[string]string = make(map[string]string) + var headers = make(map[string]string) for _, option := range options { buf, h, err := option() if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/us3/s3.go b/vendor/github.com/conductorone/baton-sdk/pkg/us3/s3.go index 2f9f05c2..4f699815 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/us3/s3.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/us3/s3.go @@ -15,7 +15,6 @@ import ( awsSdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" - _ "github.com/aws/aws-sdk-go-v2/aws/arn" awsConfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/credentials/stscreds" diff --git a/vendor/modules.txt b/vendor/modules.txt index bccf84c4..a15a4963 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -162,7 +162,7 @@ github.com/benbjohnson/clock # github.com/cenkalti/backoff/v4 v4.3.0 ## explicit; go 1.18 github.com/cenkalti/backoff/v4 -# github.com/conductorone/baton-sdk v0.3.48 +# github.com/conductorone/baton-sdk v0.3.51 ## explicit; go 1.23.4 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1 @@ -202,6 +202,7 @@ github.com/conductorone/baton-sdk/pkg/retry github.com/conductorone/baton-sdk/pkg/sdk github.com/conductorone/baton-sdk/pkg/sync github.com/conductorone/baton-sdk/pkg/sync/expand +github.com/conductorone/baton-sdk/pkg/sync/expand/scc github.com/conductorone/baton-sdk/pkg/synccompactor github.com/conductorone/baton-sdk/pkg/synccompactor/attached github.com/conductorone/baton-sdk/pkg/synccompactor/naive