From 6417f87d5bfac82c134b0aaf308feaba13036f10 Mon Sep 17 00:00:00 2001 From: Carolina Roncaglia Date: Tue, 28 Apr 2026 10:13:11 -0300 Subject: [PATCH 1/3] docs: add Billing Accounts capability and OAuth scopes (CXP-455) --- docs/connector.mdx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/connector.mdx b/docs/connector.mdx index 5e4e8da2..ac012ad7 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -19,9 +19,12 @@ sidebarTitle: "Coupa" | Groups | | | | Roles | | | | Licenses | | | +| Billing Accounts ² | | | ¹ Account Groups sync is opt-in. To enable it, select **Account Groups** in the resource types to sync when configuring the connector in C1, and ensure the `core.accounting.read` OAuth scope is added to your Coupa OAuth client. +² Billing Accounts sync requires the `core.accounting.read` OAuth scope. Provisioning (grant/revoke) additionally requires the `core.accounting.write` OAuth scope. + ### Connector actions Connector actions are custom capabilities that extend C1 automations with app-specific operations. You can use connector actions in the [Perform connector action](/product/admin/automations-steps-reference#perform-connector-action) automation step. @@ -77,7 +80,7 @@ A user with **Admin** access in Coupa must perform this task. - openid - profile - **If you also want to sync Account Groups, add:** + **If you also want to sync Account Groups or Billing Accounts, add:** - core.accounting.read **You'll need these scopes to give C1 READ/WRITE access (syncing access data and provisioning access):** @@ -94,8 +97,9 @@ A user with **Admin** access in Coupa must perform this task. - openid - profile - **If you also want to sync and provision Account Groups, add:** + **If you also want to sync and provision Account Groups or Billing Accounts, add:** - core.accounting.read + - core.accounting.write (required for Billing Account provisioning) At the bottom of the page click **Save**. From 1ed7fea3bf32236c835a5805870d9f924faf76a7 Mon Sep 17 00:00:00 2001 From: Carolina Roncaglia Date: Tue, 28 Apr 2026 11:19:28 -0300 Subject: [PATCH 2/3] chore: bump baton-sdk to v0.8.27 to match .versions.yaml Co-Authored-By: Claude Sonnet 4.6 --- go.mod | 2 +- go.sum | 4 +- .../pkg/connectorstore/connectorstore.go | 11 +++ .../baton-sdk/pkg/dotc1z/c1file.go | 44 +++++++++ .../conductorone/baton-sdk/pkg/sdk/version.go | 2 +- .../pkg/sync/progresslog/progresslog.go | 93 +++++++++++++++++-- .../conductorone/baton-sdk/pkg/sync/syncer.go | 23 +++++ vendor/modules.txt | 2 +- 8 files changed, 169 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index a0e586a3..ec02a74f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/conductorone/baton-coupa go 1.25.2 require ( - github.com/conductorone/baton-sdk v0.8.26 + github.com/conductorone/baton-sdk v0.8.27 github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/quasilyte/go-ruleguard/dsl v0.3.23 diff --git a/go.sum b/go.sum index b60c63a2..714d0141 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/conductorone/baton-sdk v0.8.26 h1:/KSdwgDYLrJJqzhiKPytRDQ7kZFwml7d/+zwyG8gOGM= -github.com/conductorone/baton-sdk v0.8.26/go.mod h1:agmFrml6APUw4ZlqMEBrnXYj3aAOGKOJ6gztiNj64h0= +github.com/conductorone/baton-sdk v0.8.27 h1:0eO/belB/c94OHvkPAi+Yd46EeYF8oIm2J+bpZzJdSY= +github.com/conductorone/baton-sdk v0.8.27/go.mod h1:agmFrml6APUw4ZlqMEBrnXYj3aAOGKOJ6gztiNj64h0= github.com/conductorone/dpop v0.2.3 h1:s91U3845GHQ6P6FWrdNr2SEOy1ES/jcFs1JtKSl2S+o= github.com/conductorone/dpop v0.2.3/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= github.com/conductorone/dpop/integrations/dpop_grpc v0.2.3 h1:kLMCNIh0Mo2vbvvkCmJ3ixsPbXEJ6HPcW53Ku9yje3s= diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go index 7ddbf546..6820f428 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go @@ -77,6 +77,17 @@ type LatestFinishedSyncIDFetcher interface { LatestFinishedSyncID(ctx context.Context, syncType SyncType) (string, error) } +// DBSizeProvider is an optional capability for a store that can report its +// current uncompressed working-set size (e.g. dotc1z.C1File stat'ing its +// sqlite file). Consumed by the syncer's ProgressLog to include +// decompressed_bytes and growth delta in the periodic "Expanding grants" +// log during long-running grant expansions — the Expander itself is +// recreated each RunSingleStep by the syncer, so this state cannot live +// there. +type DBSizeProvider interface { + CurrentDBSizeBytes() (int64, error) +} + // GrantUpsertMode controls how grant conflicts are resolved during upsert. type GrantUpsertMode int diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go index 5cb14692..ecff4628 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go @@ -654,6 +654,50 @@ func (c *C1File) OutputFilepath() (string, error) { return c.outputFilePath, nil } +// CurrentDBSizeBytes returns the current total on-disk size of the underlying +// uncompressed sqlite database, including the write-ahead log if present. +// Used by operational tooling (e.g. the grant-expansion progress logger) to +// observe c1z growth during long in-process writes without waiting for +// saveC1z to land a new frame. +// +// The WAL file holds writes that have not yet been checkpointed into the main +// database file; with journal_mode=WAL the main file may stay stable for long +// stretches while the WAL grows into the hundreds of MB. Summing both gives a +// representative "bytes written so far" figure during active expansion. +// +// This is the *uncompressed* size. The post-saveC1z c1z file size (compressed) +// is smaller; for that, see the `c1z: saved` log line emitted by saveC1z. +func (c *C1File) CurrentDBSizeBytes() (int64, error) { + if c.dbFilePath == "" { + return 0, fmt.Errorf("c1file: db file path is empty") + } + fi, err := os.Stat(c.dbFilePath) + if err != nil { + return 0, err + } + total := fi.Size() + // Add the WAL sidecar if it exists. `os.ErrNotExist` is expected (no WAL + // or journal_mode != WAL). Any *other* error — permission, EIO, stale + // handle, etc. — we surface: a silently-underreported WAL would defeat + // the growth-visibility purpose of this method (could hide hundreds of + // MB of pending writes). + switch wal, err := os.Stat(c.dbFilePath + "-wal"); { + case err == nil: + total += wal.Size() + case errors.Is(err, os.ErrNotExist): + // no WAL — fine. + default: + return 0, fmt.Errorf("c1file: stat wal sidecar: %w", err) + } + return total, nil +} + +// Compile-time assertion that *C1File satisfies the DBSizeProvider capability +// that ProgressLog.LogExpandProgress type-asserts against. Catches signature +// drift (e.g. if someone adds a ctx parameter to CurrentDBSizeBytes) at +// compile time instead of silently turning off the expand-log size fields. +var _ connectorstore.DBSizeProvider = (*C1File)(nil) + func (c *C1File) AttachFile(other *C1File, dbName string) (*C1FileAttached, error) { _, err := c.db.Exec(`ATTACH DATABASE ? AS ?`, other.dbFilePath, dbName) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index 1bf3e745..281bcbd5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.8.25" +const Version = "v0.8.26" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go index 2be360c4..b0518687 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" @@ -34,10 +35,25 @@ type ProgressLog struct { lastEntitlementLog map[string]time.Time grantsProgress map[string]int lastGrantLog map[string]time.Time - lastActionLog time.Time mu rwMutex // If noOpMutex, sequential mode is enabled. If sync.RWMutex, parallel mode is enabled. l *zap.Logger maxLogFrequency time.Duration + + // Optional cross-step db-size tracking for LogExpandProgress. Populated + // via WithDBSizeProvider at construction time or SetDBSizeProvider after + // loadStore; nil means "size-free" expand logs. Held here (not on + // Expander) because the syncer recreates the Expander per RunSingleStep, + // so any state on it resets every step and the periodic size log would + // never fire in production. + // + // Guarded by expandMu — a dedicated mutex separate from p.mu so the + // provider's stat I/O doesn't run under the shared hot-path mutex that + // AddResources / AddGrantsProgress contend on. + expandMu sync.Mutex + lastActionLog time.Time // rate-limit timestamp for LogExpandProgress + dbSize connectorstore.DBSizeProvider + lastLoggedDBSize int64 + hasLoggedDBSize bool // false until the first successful size read; gates the delta field } type Option func(*ProgressLog) @@ -68,6 +84,43 @@ func WithLogFrequency(logFrequency time.Duration) Option { } } +// WithDBSizeProvider attaches an optional connectorstore.DBSizeProvider to +// this ProgressLog. When set, LogExpandProgress will include +// decompressed_bytes and decompressed_bytes_delta (growth since the previous +// log) in its output. +// +// This is one of two attachment points — the syncer also calls +// SetDBSizeProvider after loadStore resolves the store, because many callers +// construct NewSyncer via WithC1ZPath (store is nil at NewProgressCounts +// time) and can only be wired once loadStore has run. +// +// nil is a valid value (equivalent to not setting it) and the log falls +// back to the pre-existing action-count-only output. +// +// The log is primarily useful during long grant expansions for catching +// pathological growth before the decoder cap trips on a subsequent read. +// Compare the emitted bytes against BATON_DECODER_MAX_DECODED_SIZE_MB +// (default 3 GiB; see pkg/dotc1z/decoder.go). If a tenant's db size is +// approaching that cap, the per-tenant WithDecoderMaxDecodedSize override +// is the operator's next step. +func WithDBSizeProvider(p connectorstore.DBSizeProvider) Option { + return func(o *ProgressLog) { + o.dbSize = p + } +} + +// SetDBSizeProvider updates the DBSizeProvider attached to this ProgressLog. +// Intended for callers (notably the syncer) that construct the ProgressLog +// before the store is known (e.g. WithC1ZPath path, where the store is +// loaded from disk later by loadStore). Idempotent; safe to call multiple +// times — the last non-nil value wins if called from multiple sites. +// Passing nil clears the provider. +func (p *ProgressLog) SetDBSizeProvider(provider connectorstore.DBSizeProvider) { + p.expandMu.Lock() + defer p.expandMu.Unlock() + p.dbSize = provider +} + func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { p := &ProgressLog{ resources: make(map[string]int), @@ -75,7 +128,6 @@ func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { lastEntitlementLog: make(map[string]time.Time), grantsProgress: make(map[string]int), lastGrantLog: make(map[string]time.Time), - lastActionLog: time.Time{}, l: ctxzap.Extract(ctx), maxLogFrequency: defaultMaxLogFrequency, mu: &noOpMutex{}, // Default to sequential mode for backward compatibility @@ -215,18 +267,45 @@ func (p *ProgressLog) LogGrantsProgress(ctx context.Context, resourceType string } } +// LogExpandProgress emits an Info-level "Expanding grants" log at most once +// per maxLogFrequency window. When a DBSizeProvider is attached (production: +// *dotc1z.C1File), includes the live uncompressed db size and the growth +// since the previous emitted log (`decompressed_bytes_delta`) — the signal +// that surfaces non-linear expansion growth before the next saveC1z. +// +// All expand-log state (lastActionLog, dbSize, lastLoggedDBSize, +// hasLoggedDBSize) is guarded by p.expandMu, a mutex dedicated to this +// function. It is deliberately separate from p.mu so the provider's stat +// calls can run under the lock (simple defer pattern) without serialising +// against the sync-hot-path counters (AddResources / AddGrantsProgress), +// which contend on p.mu. func (p *ProgressLog) LogExpandProgress(ctx context.Context, actions []*expand.EntitlementGraphAction) { - actionsLen := len(actions) + p.expandMu.Lock() + defer p.expandMu.Unlock() - p.mu.Lock() - defer p.mu.Unlock() if time.Since(p.lastActionLog) < p.maxLogFrequency { return } p.lastActionLog = time.Now() - l := ctxzap.Extract(ctx) - l.Info("Expanding grants", zap.Int("actions_remaining", actionsLen)) + fields := []zap.Field{zap.Int("actions_remaining", len(actions))} + + if p.dbSize != nil { + if sz, err := p.dbSize.CurrentDBSizeBytes(); err == nil { + fields = append(fields, zap.Int64("decompressed_bytes", sz)) + // Omit the delta on the very first sample — it would equal the + // full size and create a spurious spike in any Datadog graph of + // growth rate. Subsequent samples compute delta relative to the + // previous emitted size, which is the real signal operators want. + if p.hasLoggedDBSize { + fields = append(fields, zap.Int64("decompressed_bytes_delta", sz-p.lastLoggedDBSize)) + } + p.lastLoggedDBSize = sz + p.hasLoggedDBSize = true + } + } + + ctxzap.Extract(ctx).Info("Expanding grants", fields...) } // Thread-safe methods for parallel syncer diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index ded91536..305b1bfe 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -2599,9 +2599,27 @@ func (s *syncer) loadStore(ctx context.Context) error { } s.store = store + // Now that s.store is populated, wire the expand progress log's size + // provider. NewSyncer could not do this when the caller used + // WithC1ZPath because s.store was still nil at that point. + s.wireCountsDBSizeProvider() + return nil } +// wireCountsDBSizeProvider attaches the store's DBSizeProvider capability +// (if implemented) to s.counts so LogExpandProgress emits decompressed_bytes +// and growth delta during long expansions. Idempotent: may be called from +// both NewSyncer (WithConnectorStore case) and loadStore (WithC1ZPath case). +func (s *syncer) wireCountsDBSizeProvider() { + if s.counts == nil || s.store == nil { + return + } + if sp, ok := s.store.(connectorstore.DBSizeProvider); ok { + s.counts.SetDBSizeProvider(sp) + } +} + // Close closes the datastorage to ensure it is updated on disk. func (s *syncer) Close(ctx context.Context) error { ctx, span := tracer.Start(ctx, "syncer.Close") @@ -2823,6 +2841,11 @@ func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (S progressLogOpts = append(progressLogOpts, progresslog.WithSequentialMode(false)) } s.counts = progresslog.NewProgressCounts(ctx, progressLogOpts...) + // Wire the DBSizeProvider now if the store is already set (WithConnectorStore + // case). For WithC1ZPath, the store is populated later inside loadStore, + // which calls wireCountsDBSizeProvider again. Without this split the feature + // would ship dead for every c1z-path caller — see syncer.loadStore. + s.wireCountsDBSizeProvider() if s.externalResourceC1ZPath != "" { externalC1ZReader, err := dotc1z.NewExternalC1FileReader(ctx, s.tmpDir, s.externalResourceC1ZPath) diff --git a/vendor/modules.txt b/vendor/modules.txt index 37168b04..283ecaa5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -162,7 +162,7 @@ github.com/cenkalti/backoff/v5 # github.com/cespare/xxhash/v2 v2.3.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 -# github.com/conductorone/baton-sdk v0.8.26 +# github.com/conductorone/baton-sdk v0.8.27 ## explicit; go 1.25.2 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1 From 135f36958445bbd6b6ee97a2980873bb938a73bf Mon Sep 17 00:00:00 2001 From: Carolina Roncaglia Date: Tue, 28 Apr 2026 11:52:57 -0300 Subject: [PATCH 3/3] =?UTF-8?q?Revert=20accidental=20SDK=20bump=20?= =?UTF-8?q?=E2=80=94=20docs-only=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- go.mod | 2 +- go.sum | 4 +- .../pkg/connectorstore/connectorstore.go | 11 --- .../baton-sdk/pkg/dotc1z/c1file.go | 44 --------- .../conductorone/baton-sdk/pkg/sdk/version.go | 2 +- .../pkg/sync/progresslog/progresslog.go | 93 ++----------------- .../conductorone/baton-sdk/pkg/sync/syncer.go | 23 ----- vendor/modules.txt | 2 +- 8 files changed, 12 insertions(+), 169 deletions(-) diff --git a/go.mod b/go.mod index ec02a74f..a0e586a3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/conductorone/baton-coupa go 1.25.2 require ( - github.com/conductorone/baton-sdk v0.8.27 + github.com/conductorone/baton-sdk v0.8.26 github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/quasilyte/go-ruleguard/dsl v0.3.23 diff --git a/go.sum b/go.sum index 714d0141..b60c63a2 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/conductorone/baton-sdk v0.8.27 h1:0eO/belB/c94OHvkPAi+Yd46EeYF8oIm2J+bpZzJdSY= -github.com/conductorone/baton-sdk v0.8.27/go.mod h1:agmFrml6APUw4ZlqMEBrnXYj3aAOGKOJ6gztiNj64h0= +github.com/conductorone/baton-sdk v0.8.26 h1:/KSdwgDYLrJJqzhiKPytRDQ7kZFwml7d/+zwyG8gOGM= +github.com/conductorone/baton-sdk v0.8.26/go.mod h1:agmFrml6APUw4ZlqMEBrnXYj3aAOGKOJ6gztiNj64h0= github.com/conductorone/dpop v0.2.3 h1:s91U3845GHQ6P6FWrdNr2SEOy1ES/jcFs1JtKSl2S+o= github.com/conductorone/dpop v0.2.3/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= github.com/conductorone/dpop/integrations/dpop_grpc v0.2.3 h1:kLMCNIh0Mo2vbvvkCmJ3ixsPbXEJ6HPcW53Ku9yje3s= diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go index 6820f428..7ddbf546 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go @@ -77,17 +77,6 @@ type LatestFinishedSyncIDFetcher interface { LatestFinishedSyncID(ctx context.Context, syncType SyncType) (string, error) } -// DBSizeProvider is an optional capability for a store that can report its -// current uncompressed working-set size (e.g. dotc1z.C1File stat'ing its -// sqlite file). Consumed by the syncer's ProgressLog to include -// decompressed_bytes and growth delta in the periodic "Expanding grants" -// log during long-running grant expansions — the Expander itself is -// recreated each RunSingleStep by the syncer, so this state cannot live -// there. -type DBSizeProvider interface { - CurrentDBSizeBytes() (int64, error) -} - // GrantUpsertMode controls how grant conflicts are resolved during upsert. type GrantUpsertMode int diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go index ecff4628..5cb14692 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go @@ -654,50 +654,6 @@ func (c *C1File) OutputFilepath() (string, error) { return c.outputFilePath, nil } -// CurrentDBSizeBytes returns the current total on-disk size of the underlying -// uncompressed sqlite database, including the write-ahead log if present. -// Used by operational tooling (e.g. the grant-expansion progress logger) to -// observe c1z growth during long in-process writes without waiting for -// saveC1z to land a new frame. -// -// The WAL file holds writes that have not yet been checkpointed into the main -// database file; with journal_mode=WAL the main file may stay stable for long -// stretches while the WAL grows into the hundreds of MB. Summing both gives a -// representative "bytes written so far" figure during active expansion. -// -// This is the *uncompressed* size. The post-saveC1z c1z file size (compressed) -// is smaller; for that, see the `c1z: saved` log line emitted by saveC1z. -func (c *C1File) CurrentDBSizeBytes() (int64, error) { - if c.dbFilePath == "" { - return 0, fmt.Errorf("c1file: db file path is empty") - } - fi, err := os.Stat(c.dbFilePath) - if err != nil { - return 0, err - } - total := fi.Size() - // Add the WAL sidecar if it exists. `os.ErrNotExist` is expected (no WAL - // or journal_mode != WAL). Any *other* error — permission, EIO, stale - // handle, etc. — we surface: a silently-underreported WAL would defeat - // the growth-visibility purpose of this method (could hide hundreds of - // MB of pending writes). - switch wal, err := os.Stat(c.dbFilePath + "-wal"); { - case err == nil: - total += wal.Size() - case errors.Is(err, os.ErrNotExist): - // no WAL — fine. - default: - return 0, fmt.Errorf("c1file: stat wal sidecar: %w", err) - } - return total, nil -} - -// Compile-time assertion that *C1File satisfies the DBSizeProvider capability -// that ProgressLog.LogExpandProgress type-asserts against. Catches signature -// drift (e.g. if someone adds a ctx parameter to CurrentDBSizeBytes) at -// compile time instead of silently turning off the expand-log size fields. -var _ connectorstore.DBSizeProvider = (*C1File)(nil) - func (c *C1File) AttachFile(other *C1File, dbName string) (*C1FileAttached, error) { _, err := c.db.Exec(`ATTACH DATABASE ? AS ?`, other.dbFilePath, dbName) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index 281bcbd5..1bf3e745 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.8.26" +const Version = "v0.8.25" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go index b0518687..2be360c4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/progresslog/progresslog.go @@ -5,7 +5,6 @@ import ( "sync" "time" - "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" @@ -35,25 +34,10 @@ type ProgressLog struct { lastEntitlementLog map[string]time.Time grantsProgress map[string]int lastGrantLog map[string]time.Time + lastActionLog time.Time mu rwMutex // If noOpMutex, sequential mode is enabled. If sync.RWMutex, parallel mode is enabled. l *zap.Logger maxLogFrequency time.Duration - - // Optional cross-step db-size tracking for LogExpandProgress. Populated - // via WithDBSizeProvider at construction time or SetDBSizeProvider after - // loadStore; nil means "size-free" expand logs. Held here (not on - // Expander) because the syncer recreates the Expander per RunSingleStep, - // so any state on it resets every step and the periodic size log would - // never fire in production. - // - // Guarded by expandMu — a dedicated mutex separate from p.mu so the - // provider's stat I/O doesn't run under the shared hot-path mutex that - // AddResources / AddGrantsProgress contend on. - expandMu sync.Mutex - lastActionLog time.Time // rate-limit timestamp for LogExpandProgress - dbSize connectorstore.DBSizeProvider - lastLoggedDBSize int64 - hasLoggedDBSize bool // false until the first successful size read; gates the delta field } type Option func(*ProgressLog) @@ -84,43 +68,6 @@ func WithLogFrequency(logFrequency time.Duration) Option { } } -// WithDBSizeProvider attaches an optional connectorstore.DBSizeProvider to -// this ProgressLog. When set, LogExpandProgress will include -// decompressed_bytes and decompressed_bytes_delta (growth since the previous -// log) in its output. -// -// This is one of two attachment points — the syncer also calls -// SetDBSizeProvider after loadStore resolves the store, because many callers -// construct NewSyncer via WithC1ZPath (store is nil at NewProgressCounts -// time) and can only be wired once loadStore has run. -// -// nil is a valid value (equivalent to not setting it) and the log falls -// back to the pre-existing action-count-only output. -// -// The log is primarily useful during long grant expansions for catching -// pathological growth before the decoder cap trips on a subsequent read. -// Compare the emitted bytes against BATON_DECODER_MAX_DECODED_SIZE_MB -// (default 3 GiB; see pkg/dotc1z/decoder.go). If a tenant's db size is -// approaching that cap, the per-tenant WithDecoderMaxDecodedSize override -// is the operator's next step. -func WithDBSizeProvider(p connectorstore.DBSizeProvider) Option { - return func(o *ProgressLog) { - o.dbSize = p - } -} - -// SetDBSizeProvider updates the DBSizeProvider attached to this ProgressLog. -// Intended for callers (notably the syncer) that construct the ProgressLog -// before the store is known (e.g. WithC1ZPath path, where the store is -// loaded from disk later by loadStore). Idempotent; safe to call multiple -// times — the last non-nil value wins if called from multiple sites. -// Passing nil clears the provider. -func (p *ProgressLog) SetDBSizeProvider(provider connectorstore.DBSizeProvider) { - p.expandMu.Lock() - defer p.expandMu.Unlock() - p.dbSize = provider -} - func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { p := &ProgressLog{ resources: make(map[string]int), @@ -128,6 +75,7 @@ func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { lastEntitlementLog: make(map[string]time.Time), grantsProgress: make(map[string]int), lastGrantLog: make(map[string]time.Time), + lastActionLog: time.Time{}, l: ctxzap.Extract(ctx), maxLogFrequency: defaultMaxLogFrequency, mu: &noOpMutex{}, // Default to sequential mode for backward compatibility @@ -267,45 +215,18 @@ func (p *ProgressLog) LogGrantsProgress(ctx context.Context, resourceType string } } -// LogExpandProgress emits an Info-level "Expanding grants" log at most once -// per maxLogFrequency window. When a DBSizeProvider is attached (production: -// *dotc1z.C1File), includes the live uncompressed db size and the growth -// since the previous emitted log (`decompressed_bytes_delta`) — the signal -// that surfaces non-linear expansion growth before the next saveC1z. -// -// All expand-log state (lastActionLog, dbSize, lastLoggedDBSize, -// hasLoggedDBSize) is guarded by p.expandMu, a mutex dedicated to this -// function. It is deliberately separate from p.mu so the provider's stat -// calls can run under the lock (simple defer pattern) without serialising -// against the sync-hot-path counters (AddResources / AddGrantsProgress), -// which contend on p.mu. func (p *ProgressLog) LogExpandProgress(ctx context.Context, actions []*expand.EntitlementGraphAction) { - p.expandMu.Lock() - defer p.expandMu.Unlock() + actionsLen := len(actions) + p.mu.Lock() + defer p.mu.Unlock() if time.Since(p.lastActionLog) < p.maxLogFrequency { return } p.lastActionLog = time.Now() - fields := []zap.Field{zap.Int("actions_remaining", len(actions))} - - if p.dbSize != nil { - if sz, err := p.dbSize.CurrentDBSizeBytes(); err == nil { - fields = append(fields, zap.Int64("decompressed_bytes", sz)) - // Omit the delta on the very first sample — it would equal the - // full size and create a spurious spike in any Datadog graph of - // growth rate. Subsequent samples compute delta relative to the - // previous emitted size, which is the real signal operators want. - if p.hasLoggedDBSize { - fields = append(fields, zap.Int64("decompressed_bytes_delta", sz-p.lastLoggedDBSize)) - } - p.lastLoggedDBSize = sz - p.hasLoggedDBSize = true - } - } - - ctxzap.Extract(ctx).Info("Expanding grants", fields...) + l := ctxzap.Extract(ctx) + l.Info("Expanding grants", zap.Int("actions_remaining", actionsLen)) } // Thread-safe methods for parallel syncer diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index 305b1bfe..ded91536 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -2599,27 +2599,9 @@ func (s *syncer) loadStore(ctx context.Context) error { } s.store = store - // Now that s.store is populated, wire the expand progress log's size - // provider. NewSyncer could not do this when the caller used - // WithC1ZPath because s.store was still nil at that point. - s.wireCountsDBSizeProvider() - return nil } -// wireCountsDBSizeProvider attaches the store's DBSizeProvider capability -// (if implemented) to s.counts so LogExpandProgress emits decompressed_bytes -// and growth delta during long expansions. Idempotent: may be called from -// both NewSyncer (WithConnectorStore case) and loadStore (WithC1ZPath case). -func (s *syncer) wireCountsDBSizeProvider() { - if s.counts == nil || s.store == nil { - return - } - if sp, ok := s.store.(connectorstore.DBSizeProvider); ok { - s.counts.SetDBSizeProvider(sp) - } -} - // Close closes the datastorage to ensure it is updated on disk. func (s *syncer) Close(ctx context.Context) error { ctx, span := tracer.Start(ctx, "syncer.Close") @@ -2841,11 +2823,6 @@ func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (S progressLogOpts = append(progressLogOpts, progresslog.WithSequentialMode(false)) } s.counts = progresslog.NewProgressCounts(ctx, progressLogOpts...) - // Wire the DBSizeProvider now if the store is already set (WithConnectorStore - // case). For WithC1ZPath, the store is populated later inside loadStore, - // which calls wireCountsDBSizeProvider again. Without this split the feature - // would ship dead for every c1z-path caller — see syncer.loadStore. - s.wireCountsDBSizeProvider() if s.externalResourceC1ZPath != "" { externalC1ZReader, err := dotc1z.NewExternalC1FileReader(ctx, s.tmpDir, s.externalResourceC1ZPath) diff --git a/vendor/modules.txt b/vendor/modules.txt index 283ecaa5..37168b04 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -162,7 +162,7 @@ github.com/cenkalti/backoff/v5 # github.com/cespare/xxhash/v2 v2.3.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 -# github.com/conductorone/baton-sdk v0.8.27 +# github.com/conductorone/baton-sdk v0.8.26 ## explicit; go 1.25.2 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1